// Copyright (c) 2025-2026 Advanced Micro Devices, Inc. // SPDX-License-Identifier: MIT /// @file translate_test.cpp /// @brief CPU-only unit tests for the DBT translation pipeline. /// /// Tests encoding correctness, legalization table integrity, and structural /// properties of translated code objects — without requiring a GPU. Covers: /// - Coherency bit remapping (GFX940→GFX12, GFX9→GFX12) /// - Encoding field preservation across SOP1/SOP2/SOPP/SMEM/VOP3 formats /// - Decode-encode round-trip for CDNA4→RDNA4 /// - Legalization table lookup and zero-ILLEGAL invariant across all ISA pairs /// - Waitcnt decode/encode (GFX9 monolithic → GFX12 split counters) /// /// These tests complement the hardware tests in hsa_translate_test.cpp which /// verify correctness on real DBT host GPUs. #include "rocjitsu/code/amdgpu_code_object.h" #include "rocjitsu/code/amdgpu_elf.h" #include "rocjitsu/code/dbt/binary_translator.h" #include "rocjitsu/code/dbt/encoding_translator.h" #include "rocjitsu/code/dbt/generated/encoding_cdna4_to_cdna3.h" #include "rocjitsu/code/dbt/generated/encoding_cdna4_to_rdna4.h" #include "rocjitsu/code/dbt/generated/encoding_fields.h" #include "rocjitsu/code/dbt/generated/legalization_cdna1_to_cdna2.h" #include "rocjitsu/code/dbt/generated/legalization_cdna1_to_cdna3.h" #include "rocjitsu/code/dbt/generated/legalization_cdna1_to_cdna4.h" #include "rocjitsu/code/dbt/generated/legalization_cdna1_to_rdna1.h" #include "rocjitsu/code/dbt/generated/legalization_cdna1_to_rdna2.h" #include "rocjitsu/code/dbt/generated/legalization_cdna1_to_rdna3.h" #include "rocjitsu/code/dbt/generated/legalization_cdna1_to_rdna4.h" #include "rocjitsu/code/dbt/generated/legalization_cdna2_to_cdna3.h" #include "rocjitsu/code/dbt/generated/legalization_cdna2_to_cdna4.h" #include "rocjitsu/code/dbt/generated/legalization_cdna2_to_rdna3.h" #include "rocjitsu/code/dbt/generated/legalization_cdna2_to_rdna4.h" #include "rocjitsu/code/dbt/generated/legalization_cdna3_to_cdna4.h" #include "rocjitsu/code/dbt/generated/legalization_cdna3_to_rdna3.h" #include "rocjitsu/code/dbt/generated/legalization_cdna3_to_rdna4.h" #include "rocjitsu/code/dbt/generated/legalization_cdna4_to_cdna3.h" #include "rocjitsu/code/dbt/generated/legalization_cdna4_to_rdna3.h" #include "rocjitsu/code/dbt/generated/legalization_cdna4_to_rdna4.h" #include "rocjitsu/code/dbt/generated/legalization_rdna1_to_cdna3.h" #include "rocjitsu/code/dbt/generated/legalization_rdna1_to_cdna4.h" #include "rocjitsu/code/dbt/generated/legalization_rdna1_to_rdna2.h" #include "rocjitsu/code/dbt/generated/legalization_rdna1_to_rdna3.h" #include "rocjitsu/code/dbt/generated/legalization_rdna1_to_rdna4.h" #include "rocjitsu/code/dbt/generated/legalization_rdna2_to_rdna3.h" #include "rocjitsu/code/dbt/generated/legalization_rdna2_to_rdna4.h" #include "rocjitsu/code/dbt/generated/legalization_rdna3_5_to_rdna4.h" #include "rocjitsu/code/dbt/generated/legalization_rdna3_to_cdna4.h" #include "rocjitsu/code/dbt/generated/legalization_rdna3_to_rdna4.h" #include "rocjitsu/code/dbt/generated/legalization_rdna4_to_cdna4.h" #include "rocjitsu/code/dbt/generated/legalization_types.h" #include "rocjitsu/code/dbt/kernel_descriptor_translator.h" #include "rocjitsu/code/dbt/semantic/rules.h" #include "rocjitsu/code/dbt/semantic_translator.h" #include "rocjitsu/code/dbt/virtual_lds.h" #include "rocjitsu/code/dbt/waitcnt_translator.h" #include "rocjitsu/code/patch/code_object_patcher.h" #include "rocjitsu/code/patch/instruction_builder.h" #include "rocjitsu/code/patch/kernarg_extension.h" #include "rocjitsu/code/patch/kernel_text_layout.h" #include "rocjitsu/code/patch/sidecar_metadata.h" #include "rocjitsu/code/relocation_function_table.h" #include "rocjitsu/code/rj_code.h" #include "rocjitsu/code/rj_code_internal.h" #include "rocjitsu/isa/arch/amdgpu/cdna3/builders.h" #include "rocjitsu/isa/arch/amdgpu/cdna3/encodings.h" #include "rocjitsu/isa/arch/amdgpu/cdna3/machine_insts.h" #include "rocjitsu/isa/arch/amdgpu/cdna3/opcodes.h" #include "rocjitsu/isa/arch/amdgpu/cdna4/builders.h" #include "rocjitsu/isa/arch/amdgpu/cdna4/encodings.h" #include "rocjitsu/isa/arch/amdgpu/cdna4/machine_insts.h" #include "rocjitsu/isa/arch/amdgpu/cdna4/opcodes.h" #include "rocjitsu/isa/arch/amdgpu/gfx1250/builders.h" #include "rocjitsu/isa/arch/amdgpu/gfx1250/encodings.h" #include "rocjitsu/isa/arch/amdgpu/gfx1250/machine_insts.h" #include "rocjitsu/isa/arch/amdgpu/gfx1250/opcodes.h" #include "rocjitsu/isa/arch/amdgpu/rdna4/encodings.h" #include "rocjitsu/isa/arch/amdgpu/rdna4/opcodes.h" #include "rocjitsu/isa/decoder.h" #include "rocjitsu/isa/instruction.h" #include "rocjitsu/isa/register_set.h" #include "rocjitsu/vm/amdgpu/compute_unit.h" #include "rocjitsu/vm/amdgpu/gpu_memory.h" #include "rocjitsu/vm/amdgpu/l2_cache.h" #include "rocjitsu/vm/amdgpu/wavefront.h" #include "support/elf_test_support.h" #include "util/data_types.h" #include "rocjitsu/base/rj_compiler.h" RJ_DIAGNOSTIC_PUSH RJ_DIAGNOSTIC_IGNORE_PEDANTIC #include "hsa/AMDHSAKernelDescriptor.h" #include "hsa/hsa.h" RJ_DIAGNOSTIC_POP #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace rocjitsu { namespace { uint32_t add_elf_name(std::vector &names, std::string_view name) { const uint32_t offset = static_cast(names.size()); names.resize(names.size() + name.size() + 1); if (!name.empty()) std::memcpy(names.data() + offset, name.data(), name.size()); return offset; } uint64_t align_up_for_test(uint64_t value, uint64_t alignment) { const uint64_t remainder = value % alignment; return remainder == 0 ? value : value + alignment - remainder; } template T read_elf_struct_for_test(const std::vector &image, uint64_t offset) { T value{}; assert(offset <= image.size()); assert(sizeof(T) <= image.size() - offset); std::memcpy(&value, image.data() + offset, sizeof(value)); return value; } template std::vector read_elf_array_for_test(const std::vector &image, uint64_t offset, size_t count) { std::vector values(count); assert(offset <= image.size()); assert(count <= (image.size() - offset) / sizeof(T)); std::memcpy(values.data(), image.data() + offset, count * sizeof(T)); return values; } std::optional find_elf_symbol_for_test(const std::vector &image, std::string_view name) { const auto range_is_in_image = [&](uint64_t offset, uint64_t size) { const uint64_t image_size = image.size(); return offset <= image_size && size <= image_size - offset; }; if (!range_is_in_image(0, sizeof(Elf64_Ehdr))) return std::nullopt; const auto header = read_elf_struct_for_test(image, 0); const uint64_t section_table_size = static_cast(header.e_shnum) * sizeof(Elf64_Shdr); if (!range_is_in_image(header.e_shoff, section_table_size)) return std::nullopt; const auto sections = read_elf_array_for_test(image, header.e_shoff, header.e_shnum); for (const Elf64_Shdr &symtab : sections) { if ((symtab.sh_type != SHT_SYMTAB && symtab.sh_type != SHT_DYNSYM) || symtab.sh_entsize != sizeof(Elf64_Sym) || symtab.sh_size % sizeof(Elf64_Sym) != 0 || symtab.sh_link >= sections.size() || !range_is_in_image(symtab.sh_offset, symtab.sh_size)) { continue; } const Elf64_Shdr &strtab = sections[symtab.sh_link]; if (strtab.sh_type != SHT_STRTAB || !range_is_in_image(strtab.sh_offset, strtab.sh_size)) continue; const auto symbols = read_elf_array_for_test(image, symtab.sh_offset, symtab.sh_size / sizeof(Elf64_Sym)); for (const Elf64_Sym &symbol : symbols) { if (symbol.st_name >= strtab.sh_size) continue; const char *begin = reinterpret_cast(image.data() + strtab.sh_offset + symbol.st_name); const size_t remaining = strtab.sh_size - symbol.st_name; const auto *end = static_cast(std::memchr(begin, '\0', remaining)); if (end != nullptr && std::string_view(begin, static_cast(end - begin)) == name) return symbol; } } return std::nullopt; } template void write_elf_struct_for_test(std::vector &image, uint64_t offset, const T &value) { assert(offset <= image.size()); assert(sizeof(T) <= image.size() - offset); std::memcpy(image.data() + offset, &value, sizeof(value)); } void write_bytes_for_test(std::vector &image, uint64_t offset, const void *src, size_t size) { assert(offset <= image.size()); assert(size <= image.size() - offset); std::memcpy(image.data() + offset, src, size); } template void write_value_for_test(std::vector &image, uint64_t offset, T value) { write_bytes_for_test(image, offset, &value, sizeof(value)); } using TestKernelDescriptor = rocr::llvm::amdhsa::kernel_descriptor_t; constexpr size_t kKernelDescriptorSize = sizeof(TestKernelDescriptor); constexpr size_t kKernelDescriptorEntryOffset = offsetof(TestKernelDescriptor, kernel_code_entry_byte_offset); constexpr uint64_t kKernargPreloadSkipBytes = 256; void write_kernel_descriptor_entry_offset(void *descriptor, int64_t entry_offset) { auto *bytes = static_cast(descriptor); std::memcpy(bytes + kKernelDescriptorEntryOffset, &entry_offset, sizeof(entry_offset)); } int64_t read_kernel_descriptor_entry_offset(const void *descriptor) { const auto *bytes = static_cast(descriptor); int64_t entry_offset = 0; std::memcpy(&entry_offset, bytes + kKernelDescriptorEntryOffset, sizeof(entry_offset)); return entry_offset; } TestKernelDescriptor read_kernel_descriptor_for_test(const void *descriptor) { TestKernelDescriptor kd{}; std::memcpy(&kd, descriptor, sizeof(kd)); return kd; } void write_kernel_descriptor_for_test(void *descriptor, const TestKernelDescriptor &kd) { std::memcpy(descriptor, &kd, sizeof(kd)); } std::vector make_kernel_descriptor_bytes(int64_t entry_offset) { std::vector descriptor(kKernelDescriptorSize, 0); write_kernel_descriptor_entry_offset(descriptor.data(), entry_offset); return descriptor; } std::vector make_minimal_amdgpu_elf_with_text_and_rodata() { constexpr uint64_t text_offset = 0x100; constexpr uint64_t text_size = 8; constexpr uint64_t rodata_size = 4; std::vector shstrtab{'\0'}; const uint32_t text_name = add_elf_name(shstrtab, ".text"); const uint32_t rodata_name = add_elf_name(shstrtab, ".rodata"); const uint32_t shstrtab_name = add_elf_name(shstrtab, ".shstrtab"); const uint64_t rodata_offset = text_offset + text_size; const uint64_t shstrtab_offset = rodata_offset + rodata_size; const uint64_t shoff = align_up_for_test(shstrtab_offset + shstrtab.size(), 8); constexpr uint16_t section_count = 4; std::vector image(shoff + section_count * sizeof(Elf64_Shdr), 0); Elf64_Ehdr ehdr{}; std::memcpy(ehdr.e_ident, EI_MAGIC, EI_MAGIC_SIZE); ehdr.e_ident[EI_CLASS] = ELFCLASS64; ehdr.e_ident[EI_OSABI] = ELFOSABI_AMDGPU_HSA; ehdr.e_type = ET_REL; ehdr.e_machine = EM_AMDGPU; ehdr.e_version = 1; ehdr.e_shoff = shoff; ehdr.e_flags = EF_AMDGPU_MACH_AMDGCN_GFX950; ehdr.e_ehsize = sizeof(Elf64_Ehdr); ehdr.e_shentsize = sizeof(Elf64_Shdr); ehdr.e_shnum = section_count; ehdr.e_shstrndx = 3; std::memcpy(image.data(), &ehdr, sizeof(ehdr)); const std::array text_words = {0xBF800000u, 0xBF800000u}; std::memcpy(image.data() + text_offset, text_words.data(), text_size); const uint32_t rodata_word = 0xA5A55A5Au; std::memcpy(image.data() + rodata_offset, &rodata_word, sizeof(rodata_word)); std::memcpy(image.data() + shstrtab_offset, shstrtab.data(), shstrtab.size()); std::array shdrs{}; shdrs[1].sh_name = text_name; shdrs[1].sh_type = SHT_PROGBITS; shdrs[1].sh_flags = SHF_ALLOC | SHF_EXECINSTR; shdrs[1].sh_offset = text_offset; shdrs[1].sh_size = text_size; shdrs[1].sh_addralign = sizeof(uint32_t); shdrs[2].sh_name = rodata_name; shdrs[2].sh_type = SHT_PROGBITS; shdrs[2].sh_flags = SHF_ALLOC; shdrs[2].sh_offset = rodata_offset; shdrs[2].sh_size = rodata_size; shdrs[2].sh_addralign = sizeof(uint32_t); shdrs[3].sh_name = shstrtab_name; shdrs[3].sh_type = SHT_STRTAB; shdrs[3].sh_offset = shstrtab_offset; shdrs[3].sh_size = shstrtab.size(); shdrs[3].sh_addralign = 1; std::memcpy(image.data() + shoff, shdrs.data(), shdrs.size() * sizeof(Elf64_Shdr)); return image; } std::vector make_minimal_gfx1250_elf_with_empty_text_and_rodata() { auto image = make_minimal_amdgpu_elf_with_text_and_rodata(); auto ehdr = read_elf_struct_for_test(image, 0); auto shdrs = read_elf_array_for_test(image, ehdr.e_shoff, ehdr.e_shnum); assert(shdrs.size() >= 2); ehdr.e_flags = EF_AMDGPU_MACH_AMDGCN_GFX1250; shdrs[1].sh_size = 0; write_elf_struct_for_test(image, 0, ehdr); write_bytes_for_test(image, ehdr.e_shoff, shdrs.data(), shdrs.size() * sizeof(Elf64_Shdr)); return image; } std::vector make_minimal_amdgpu_elf_with_load_segments() { constexpr uint64_t text_offset = 0x100; constexpr uint64_t text_vaddr = 0x1100; constexpr uint64_t text_size = 8; constexpr uint64_t rodata_size = 4; constexpr uint64_t load_align = 0x1000; std::vector shstrtab{'\0'}; const uint32_t text_name = add_elf_name(shstrtab, ".text"); const uint32_t rodata_name = add_elf_name(shstrtab, ".rodata"); const uint32_t symtab_name = add_elf_name(shstrtab, ".symtab"); const uint32_t strtab_name = add_elf_name(shstrtab, ".strtab"); const uint32_t shstrtab_name = add_elf_name(shstrtab, ".shstrtab"); std::vector strtab{'\0'}; const uint32_t rodata_symbol_name = add_elf_name(strtab, "rodata_object"); const uint32_t text_symbol_name = add_elf_name(strtab, "text_start"); const uint64_t rodata_offset = text_offset + text_size; const uint64_t rodata_vaddr = text_vaddr + text_size + load_align; const uint64_t strtab_offset = rodata_offset + rodata_size; const uint64_t symtab_offset = align_up_for_test(strtab_offset + strtab.size(), 8); constexpr size_t sym_count = 3; const uint64_t shstrtab_offset = symtab_offset + sym_count * sizeof(Elf64_Sym); const uint64_t shoff = align_up_for_test(shstrtab_offset + shstrtab.size(), 8); constexpr uint16_t section_count = 6; constexpr uint16_t phdr_count = 2; std::vector image(shoff + section_count * sizeof(Elf64_Shdr), 0); Elf64_Ehdr ehdr{}; std::memcpy(ehdr.e_ident, EI_MAGIC, EI_MAGIC_SIZE); ehdr.e_ident[EI_CLASS] = ELFCLASS64; ehdr.e_ident[EI_OSABI] = ELFOSABI_AMDGPU_HSA; ehdr.e_type = ET_DYN; ehdr.e_machine = EM_AMDGPU; ehdr.e_version = 1; ehdr.e_phoff = sizeof(Elf64_Ehdr); ehdr.e_shoff = shoff; ehdr.e_flags = EF_AMDGPU_MACH_AMDGCN_GFX950; ehdr.e_ehsize = sizeof(Elf64_Ehdr); ehdr.e_phentsize = sizeof(Elf64_Phdr); ehdr.e_phnum = phdr_count; ehdr.e_shentsize = sizeof(Elf64_Shdr); ehdr.e_shnum = section_count; ehdr.e_shstrndx = 5; std::memcpy(image.data(), &ehdr, sizeof(ehdr)); std::array phdrs{}; phdrs[0].p_type = PT_LOAD; phdrs[0].p_flags = 0x5; // PF_R | PF_X phdrs[0].p_offset = text_offset; phdrs[0].p_vaddr = text_vaddr; phdrs[0].p_paddr = text_vaddr; phdrs[0].p_filesz = text_size; phdrs[0].p_memsz = text_size; phdrs[0].p_align = load_align; phdrs[1].p_type = PT_LOAD; phdrs[1].p_flags = 0x4; // PF_R phdrs[1].p_offset = rodata_offset; phdrs[1].p_vaddr = rodata_vaddr; phdrs[1].p_paddr = rodata_vaddr; phdrs[1].p_filesz = rodata_size; phdrs[1].p_memsz = rodata_size; phdrs[1].p_align = load_align; std::memcpy(image.data() + ehdr.e_phoff, phdrs.data(), phdrs.size() * sizeof(Elf64_Phdr)); const std::array text_words = {0xBF800000u, 0xBF800000u}; std::memcpy(image.data() + text_offset, text_words.data(), text_size); const uint32_t rodata_word = 0xA5A55A5Au; std::memcpy(image.data() + rodata_offset, &rodata_word, sizeof(rodata_word)); std::memcpy(image.data() + strtab_offset, strtab.data(), strtab.size()); std::array syms{}; syms[1].st_name = rodata_symbol_name; syms[1].st_shndx = 2; syms[1].st_value = rodata_vaddr; syms[1].st_size = rodata_size; syms[2].st_name = text_symbol_name; syms[2].st_info = elf_symbol_info(kElfSymbolBindGlobal, kElfSymbolTypeFunc); syms[2].st_shndx = 1; syms[2].st_value = text_vaddr; syms[2].st_size = text_size; std::memcpy(image.data() + symtab_offset, syms.data(), syms.size() * sizeof(Elf64_Sym)); std::memcpy(image.data() + shstrtab_offset, shstrtab.data(), shstrtab.size()); std::array shdrs{}; shdrs[1].sh_name = text_name; shdrs[1].sh_type = SHT_PROGBITS; shdrs[1].sh_flags = SHF_ALLOC | SHF_EXECINSTR; shdrs[1].sh_addr = text_vaddr; shdrs[1].sh_offset = text_offset; shdrs[1].sh_size = text_size; shdrs[1].sh_addralign = sizeof(uint32_t); shdrs[2].sh_name = rodata_name; shdrs[2].sh_type = SHT_PROGBITS; shdrs[2].sh_flags = SHF_ALLOC; shdrs[2].sh_addr = rodata_vaddr; shdrs[2].sh_offset = rodata_offset; shdrs[2].sh_size = rodata_size; shdrs[2].sh_addralign = sizeof(uint32_t); shdrs[3].sh_name = symtab_name; shdrs[3].sh_type = SHT_SYMTAB; shdrs[3].sh_offset = symtab_offset; shdrs[3].sh_size = syms.size() * sizeof(Elf64_Sym); shdrs[3].sh_link = 4; shdrs[3].sh_info = 1; shdrs[3].sh_addralign = 8; shdrs[3].sh_entsize = sizeof(Elf64_Sym); shdrs[4].sh_name = strtab_name; shdrs[4].sh_type = SHT_STRTAB; shdrs[4].sh_offset = strtab_offset; shdrs[4].sh_size = strtab.size(); shdrs[4].sh_addralign = 1; shdrs[5].sh_name = shstrtab_name; shdrs[5].sh_type = SHT_STRTAB; shdrs[5].sh_offset = shstrtab_offset; shdrs[5].sh_size = shstrtab.size(); shdrs[5].sh_addralign = 1; std::memcpy(image.data() + shoff, shdrs.data(), shdrs.size() * sizeof(Elf64_Shdr)); return image; } std::vector make_minimal_amdgpu_elf_with_descriptor_after_text( const std::vector &text_words, std::optional text_function_words = std::nullopt, size_t text_function_offset_words = 0, std::optional function_pointer_table_target_words = std::nullopt, bool name_function_pointer_table_with_symbol = true) { if (text_function_words && text_function_offset_words + *text_function_words > text_words.size()) throw std::invalid_argument("text function extent exceeds .text fixture"); if (function_pointer_table_target_words && *function_pointer_table_target_words >= text_words.size()) throw std::invalid_argument("function pointer target exceeds .text fixture"); constexpr uint64_t text_offset = 0x100; constexpr uint64_t text_vaddr = 0x1100; const uint64_t text_size = text_words.size() * sizeof(uint32_t); constexpr uint64_t load_align = 0x1000; constexpr uint64_t rodata_size = kKernelDescriptorSize; const bool has_table = function_pointer_table_target_words.has_value(); constexpr uint64_t table_size = sizeof(uint64_t); std::vector shstrtab{'\0'}; const uint32_t text_name = add_elf_name(shstrtab, ".text"); const uint32_t rodata_name = add_elf_name(shstrtab, ".rodata"); const uint32_t symtab_name = add_elf_name(shstrtab, ".symtab"); const uint32_t strtab_name = add_elf_name(shstrtab, ".strtab"); const uint32_t shstrtab_name = add_elf_name(shstrtab, ".shstrtab"); const uint32_t table_name = has_table ? add_elf_name(shstrtab, ".data.rel.ro") : 0; const uint32_t rela_name = has_table ? add_elf_name(shstrtab, ".rela.dyn") : 0; std::vector strtab{'\0'}; const uint32_t kd_symbol_name = add_elf_name(strtab, "kernel.kd"); const uint32_t text_symbol_name = text_function_words ? add_elf_name(strtab, "kernel") : 0; const uint32_t table_symbol_name = has_table ? add_elf_name(strtab, "function_table") : 0; // The kernel descriptor requires 8-byte alignment (tests reinterpret_cast the // .rodata bytes to TestKernelDescriptor). An odd text_words count leaves // text_offset + text_size only 4-aligned, so pad up to 8. text_offset and // text_vaddr are both 8-aligned and differ by a multiple of load_align, so // padding both keeps the PT_LOAD p_offset == p_vaddr (mod p_align) congruence. const uint64_t rodata_offset = align_up_for_test(text_offset + text_size, 8); const uint64_t rodata_vaddr = align_up_for_test(text_vaddr + text_size, 8) + load_align; const uint64_t table_vaddr = rodata_vaddr + load_align; const uint64_t strtab_offset = rodata_offset + rodata_size; const uint64_t symtab_offset = align_up_for_test(strtab_offset + strtab.size(), 8); const size_t sym_count = (text_function_words ? 3 : 2) + (has_table ? 1 : 0); // The table is SHF_ALLOC, so a real loader only maps it when a PT_LOAD covers it. Give the file // offset the same residue mod load_align as table_vaddr so the extra segment below satisfies the // p_offset == p_vaddr (mod p_align) congruence a loader requires. const uint64_t table_offset = align_up_for_test(symtab_offset + sym_count * sizeof(Elf64_Sym), load_align) + (table_vaddr % load_align); const uint64_t rela_offset = has_table ? table_offset + table_size : table_offset; const uint64_t shstrtab_offset = rela_offset + (has_table ? sizeof(Elf64_Rela) : 0); const uint64_t shoff = align_up_for_test(shstrtab_offset + shstrtab.size(), 8); const uint16_t section_count = has_table ? 8 : 6; const uint16_t phdr_count = has_table ? 3 : 2; std::vector image(shoff + section_count * sizeof(Elf64_Shdr), 0); Elf64_Ehdr ehdr{}; std::memcpy(ehdr.e_ident, EI_MAGIC, EI_MAGIC_SIZE); ehdr.e_ident[EI_CLASS] = ELFCLASS64; ehdr.e_ident[EI_OSABI] = ELFOSABI_AMDGPU_HSA; ehdr.e_type = ET_DYN; ehdr.e_machine = EM_AMDGPU; ehdr.e_version = 1; ehdr.e_phoff = sizeof(Elf64_Ehdr); ehdr.e_shoff = shoff; ehdr.e_flags = EF_AMDGPU_MACH_AMDGCN_GFX950; ehdr.e_ehsize = sizeof(Elf64_Ehdr); ehdr.e_phentsize = sizeof(Elf64_Phdr); ehdr.e_phnum = phdr_count; ehdr.e_shentsize = sizeof(Elf64_Shdr); ehdr.e_shnum = section_count; ehdr.e_shstrndx = 5; std::memcpy(image.data(), &ehdr, sizeof(ehdr)); std::vector phdrs(phdr_count); phdrs[0].p_type = PT_LOAD; phdrs[0].p_flags = 0x5; // PF_R | PF_X phdrs[0].p_offset = text_offset; phdrs[0].p_vaddr = text_vaddr; phdrs[0].p_paddr = text_vaddr; phdrs[0].p_filesz = text_size; phdrs[0].p_memsz = text_size; phdrs[0].p_align = load_align; phdrs[1].p_type = PT_LOAD; phdrs[1].p_flags = 0x4; // PF_R phdrs[1].p_offset = rodata_offset; phdrs[1].p_vaddr = rodata_vaddr; phdrs[1].p_paddr = rodata_vaddr; phdrs[1].p_filesz = rodata_size; phdrs[1].p_memsz = rodata_size; phdrs[1].p_align = load_align; if (has_table) { // Map the table itself, so these cases exercise a slot a loader would place rather than only // section-header discovery. phdrs[2].p_type = PT_LOAD; phdrs[2].p_flags = 0x4; // PF_R phdrs[2].p_offset = table_offset; phdrs[2].p_vaddr = table_vaddr; phdrs[2].p_paddr = table_vaddr; phdrs[2].p_filesz = table_size; phdrs[2].p_memsz = table_size; phdrs[2].p_align = load_align; } std::memcpy(image.data() + ehdr.e_phoff, phdrs.data(), phdrs.size() * sizeof(Elf64_Phdr)); std::memcpy(image.data() + text_offset, text_words.data(), text_size); const auto descriptor = make_kernel_descriptor_bytes(static_cast(text_vaddr) - static_cast(rodata_vaddr)); std::memcpy(image.data() + rodata_offset, descriptor.data(), descriptor.size()); std::memcpy(image.data() + strtab_offset, strtab.data(), strtab.size()); std::vector syms(sym_count); syms[1].st_name = kd_symbol_name; syms[1].st_info = elf_symbol_info(kElfSymbolBindGlobal, kElfSymbolTypeObject); syms[1].st_shndx = 2; syms[1].st_value = rodata_vaddr; syms[1].st_size = kKernelDescriptorSize; if (text_function_words) { syms[2].st_name = text_symbol_name; // Real device functions are LOCAL and appear only in .symtab. Offset zero is the kernel entry, // which function discovery excludes; a non-zero offset names a callee body. syms[2].st_info = elf_symbol_info(text_function_offset_words == 0 ? kElfSymbolBindGlobal : kElfSymbolBindLocal, kElfSymbolTypeFunc); syms[2].st_shndx = 1; syms[2].st_value = text_vaddr + text_function_offset_words * sizeof(uint32_t); syms[2].st_size = *text_function_words * sizeof(uint32_t); } if (has_table) { // A function-pointer table is an STT_OBJECT in a non-executable allocated section, with one // R_AMDGPU_RELATIVE64 slot whose addend is the callee's virtual address. if (name_function_pointer_table_with_symbol) { Elf64_Sym &table_symbol = syms.back(); table_symbol.st_name = table_symbol_name; table_symbol.st_info = elf_symbol_info(kElfSymbolBindGlobal, kElfSymbolTypeObject); table_symbol.st_shndx = 6; table_symbol.st_value = table_vaddr; table_symbol.st_size = table_size; } Elf64_Rela rela{}; rela.r_offset = table_vaddr; rela.r_info = (uint64_t{0} << 32) | rocjitsu::R_AMDGPU_RELATIVE64; rela.r_addend = static_cast(text_vaddr + *function_pointer_table_target_words * sizeof(uint32_t)); std::memcpy(image.data() + rela_offset, &rela, sizeof(rela)); } std::memcpy(image.data() + symtab_offset, syms.data(), syms.size() * sizeof(Elf64_Sym)); std::memcpy(image.data() + shstrtab_offset, shstrtab.data(), shstrtab.size()); std::vector shdrs(section_count); shdrs[1].sh_name = text_name; shdrs[1].sh_type = SHT_PROGBITS; shdrs[1].sh_flags = SHF_ALLOC | SHF_EXECINSTR; shdrs[1].sh_addr = text_vaddr; shdrs[1].sh_offset = text_offset; shdrs[1].sh_size = text_size; shdrs[1].sh_addralign = sizeof(uint32_t); shdrs[2].sh_name = rodata_name; shdrs[2].sh_type = SHT_PROGBITS; shdrs[2].sh_flags = SHF_ALLOC; shdrs[2].sh_addr = rodata_vaddr; shdrs[2].sh_offset = rodata_offset; shdrs[2].sh_size = rodata_size; shdrs[2].sh_addralign = 64; if (has_table) { shdrs[6].sh_name = table_name; shdrs[6].sh_type = SHT_PROGBITS; shdrs[6].sh_flags = SHF_ALLOC; shdrs[6].sh_addr = table_vaddr; shdrs[6].sh_offset = table_offset; shdrs[6].sh_size = table_size; shdrs[6].sh_addralign = 8; shdrs[7].sh_name = rela_name; shdrs[7].sh_type = SHT_RELA; shdrs[7].sh_offset = rela_offset; shdrs[7].sh_size = sizeof(Elf64_Rela); shdrs[7].sh_link = 3; shdrs[7].sh_addralign = 8; shdrs[7].sh_entsize = sizeof(Elf64_Rela); } shdrs[3].sh_name = symtab_name; shdrs[3].sh_type = SHT_SYMTAB; shdrs[3].sh_offset = symtab_offset; shdrs[3].sh_size = syms.size() * sizeof(Elf64_Sym); shdrs[3].sh_link = 4; shdrs[3].sh_info = 1; shdrs[3].sh_addralign = 8; shdrs[3].sh_entsize = sizeof(Elf64_Sym); shdrs[4].sh_name = strtab_name; shdrs[4].sh_type = SHT_STRTAB; shdrs[4].sh_offset = strtab_offset; shdrs[4].sh_size = strtab.size(); shdrs[4].sh_addralign = 1; shdrs[5].sh_name = shstrtab_name; shdrs[5].sh_type = SHT_STRTAB; shdrs[5].sh_offset = shstrtab_offset; shdrs[5].sh_size = shstrtab.size(); shdrs[5].sh_addralign = 1; std::memcpy(image.data() + shoff, shdrs.data(), shdrs.size() * sizeof(Elf64_Shdr)); return image; } std::vector make_minimal_amdgpu_elf_with_descriptor_after_text() { return make_minimal_amdgpu_elf_with_descriptor_after_text({0xBF800000u, 0xBF800000u}); } template std::vector make_gfx1250_image_with_live_sgprs(const std::array &instruction_words, uint16_t live_sgprs) { if (live_sgprs > REGISTER_SET_MAX_SGPRS) { ADD_FAILURE() << "liveness wall exceeds the gfx1250 ordinary SGPR range"; return {}; } constexpr uint16_t kGfx1250M0Operand = 125; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; std::vector words(instruction_words.begin(), instruction_words.end()); for (uint16_t sgpr = 0; sgpr < live_sgprs; ++sgpr) { words.push_back(gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = static_cast(sgpr), .sdst = kGfx1250M0Operand})[0]); } words.push_back(kGfx1250SEndpgm); return make_minimal_amdgpu_elf_with_descriptor_after_text(words); } std::unique_ptr decode_one(uint32_t word, rj_code_arch_t arch) { auto decoder = Decoder::create(arch); if (!decoder) return nullptr; return std::unique_ptr(decoder->decode(&word)); } bool has_error_containing(const TranslatedCodeObject &result, DiagnosticKind kind, std::string_view message) { return std::any_of(result.diagnostics.begin(), result.diagnostics.end(), [&](const TranslationDiagnostic &diagnostic) { return diagnostic.severity == DiagnosticSeverity::Error && diagnostic.kind == kind && diagnostic.message.find(message) != std::string::npos; }); } std::vector make_minimal_amdgpu_elf_with_two_kernel_descriptors( const std::vector &text_words = {0xBF810000u, 0xBF810000u}) { constexpr uint64_t text_offset = 0x100; constexpr uint64_t text_vaddr = 0x1100; const uint64_t text_size = text_words.size() * sizeof(uint32_t); constexpr uint64_t load_align = 0x1000; constexpr uint64_t rodata_size = 2 * kKernelDescriptorSize; std::vector shstrtab{'\0'}; const uint32_t text_name = add_elf_name(shstrtab, ".text"); const uint32_t rodata_name = add_elf_name(shstrtab, ".rodata"); const uint32_t symtab_name = add_elf_name(shstrtab, ".symtab"); const uint32_t strtab_name = add_elf_name(shstrtab, ".strtab"); const uint32_t shstrtab_name = add_elf_name(shstrtab, ".shstrtab"); std::vector strtab{'\0'}; const uint32_t kernel0_name = add_elf_name(strtab, "kernel0.kd"); const uint32_t kernel1_name = add_elf_name(strtab, "kernel1.kd"); // The kernel descriptors require 8-byte alignment (tests reinterpret_cast the // .rodata bytes to TestKernelDescriptor). An odd text_words count leaves // text_offset + text_size only 4-aligned, so pad up to 8. text_offset and // text_vaddr are both 8-aligned and differ by a multiple of load_align, so // padding both keeps the PT_LOAD p_offset == p_vaddr (mod p_align) congruence. const uint64_t rodata_offset = align_up_for_test(text_offset + text_size, 8); const uint64_t rodata_vaddr = align_up_for_test(text_vaddr + text_size, 8) + load_align; const uint64_t strtab_offset = rodata_offset + rodata_size; const uint64_t symtab_offset = align_up_for_test(strtab_offset + strtab.size(), 8); constexpr size_t sym_count = 3; const uint64_t shstrtab_offset = symtab_offset + sym_count * sizeof(Elf64_Sym); const uint64_t shoff = align_up_for_test(shstrtab_offset + shstrtab.size(), 8); constexpr uint16_t section_count = 6; constexpr uint16_t phdr_count = 2; std::vector image(shoff + section_count * sizeof(Elf64_Shdr), 0); Elf64_Ehdr ehdr{}; std::memcpy(ehdr.e_ident, EI_MAGIC, EI_MAGIC_SIZE); ehdr.e_ident[EI_CLASS] = ELFCLASS64; ehdr.e_ident[EI_OSABI] = ELFOSABI_AMDGPU_HSA; ehdr.e_type = ET_DYN; ehdr.e_machine = EM_AMDGPU; ehdr.e_version = 1; ehdr.e_phoff = sizeof(Elf64_Ehdr); ehdr.e_shoff = shoff; ehdr.e_flags = EF_AMDGPU_MACH_AMDGCN_GFX950; ehdr.e_ehsize = sizeof(Elf64_Ehdr); ehdr.e_phentsize = sizeof(Elf64_Phdr); ehdr.e_phnum = phdr_count; ehdr.e_shentsize = sizeof(Elf64_Shdr); ehdr.e_shnum = section_count; ehdr.e_shstrndx = 5; std::memcpy(image.data(), &ehdr, sizeof(ehdr)); std::array phdrs{}; phdrs[0].p_type = PT_LOAD; phdrs[0].p_flags = 0x5; // PF_R | PF_X phdrs[0].p_offset = text_offset; phdrs[0].p_vaddr = text_vaddr; phdrs[0].p_paddr = text_vaddr; phdrs[0].p_filesz = text_size; phdrs[0].p_memsz = text_size; phdrs[0].p_align = load_align; phdrs[1].p_type = PT_LOAD; phdrs[1].p_flags = 0x4; // PF_R phdrs[1].p_offset = rodata_offset; phdrs[1].p_vaddr = rodata_vaddr; phdrs[1].p_paddr = rodata_vaddr; phdrs[1].p_filesz = rodata_size; phdrs[1].p_memsz = rodata_size; phdrs[1].p_align = load_align; std::memcpy(image.data() + ehdr.e_phoff, phdrs.data(), phdrs.size() * sizeof(Elf64_Phdr)); std::memcpy(image.data() + text_offset, text_words.data(), text_size); std::vector descriptors(rodata_size, 0); write_kernel_descriptor_entry_offset(descriptors.data(), static_cast(text_vaddr) - static_cast(rodata_vaddr)); write_kernel_descriptor_entry_offset( descriptors.data() + kKernelDescriptorSize, static_cast(text_vaddr + sizeof(uint32_t)) - static_cast(rodata_vaddr + kKernelDescriptorSize)); std::memcpy(image.data() + rodata_offset, descriptors.data(), descriptors.size()); std::memcpy(image.data() + strtab_offset, strtab.data(), strtab.size()); std::array syms{}; syms[1].st_name = kernel0_name; syms[1].st_info = elf_symbol_info(kElfSymbolBindGlobal, kElfSymbolTypeObject); syms[1].st_shndx = 2; syms[1].st_value = rodata_vaddr; syms[1].st_size = kKernelDescriptorSize; syms[2].st_name = kernel1_name; syms[2].st_info = elf_symbol_info(kElfSymbolBindGlobal, kElfSymbolTypeObject); syms[2].st_shndx = 2; syms[2].st_value = rodata_vaddr + kKernelDescriptorSize; syms[2].st_size = kKernelDescriptorSize; std::memcpy(image.data() + symtab_offset, syms.data(), syms.size() * sizeof(Elf64_Sym)); std::memcpy(image.data() + shstrtab_offset, shstrtab.data(), shstrtab.size()); std::array shdrs{}; shdrs[1].sh_name = text_name; shdrs[1].sh_type = SHT_PROGBITS; shdrs[1].sh_flags = SHF_ALLOC | SHF_EXECINSTR; shdrs[1].sh_addr = text_vaddr; shdrs[1].sh_offset = text_offset; shdrs[1].sh_size = text_size; shdrs[1].sh_addralign = sizeof(uint32_t); shdrs[2].sh_name = rodata_name; shdrs[2].sh_type = SHT_PROGBITS; shdrs[2].sh_flags = SHF_ALLOC; shdrs[2].sh_addr = rodata_vaddr; shdrs[2].sh_offset = rodata_offset; shdrs[2].sh_size = rodata_size; shdrs[2].sh_addralign = 64; shdrs[3].sh_name = symtab_name; shdrs[3].sh_type = SHT_SYMTAB; shdrs[3].sh_offset = symtab_offset; shdrs[3].sh_size = syms.size() * sizeof(Elf64_Sym); shdrs[3].sh_link = 4; shdrs[3].sh_info = 1; shdrs[3].sh_addralign = 8; shdrs[3].sh_entsize = sizeof(Elf64_Sym); shdrs[4].sh_name = strtab_name; shdrs[4].sh_type = SHT_STRTAB; shdrs[4].sh_offset = strtab_offset; shdrs[4].sh_size = strtab.size(); shdrs[4].sh_addralign = 1; shdrs[5].sh_name = shstrtab_name; shdrs[5].sh_type = SHT_STRTAB; shdrs[5].sh_offset = shstrtab_offset; shdrs[5].sh_size = shstrtab.size(); shdrs[5].sh_addralign = 1; std::memcpy(image.data() + shoff, shdrs.data(), shdrs.size() * sizeof(Elf64_Shdr)); return image; } // reloc_type and reloc_addend default to a bare record, which is what the // r_offset tests want. Naming R_AMDGPU_RELATIVE64 with an addend inside the // section that follows .text instead exercises the addend shift: the stored // value is load_bias + r_addend, so a .text that grows past its load alignment // moves the section and the addend has to move with it. std::vector make_minimal_amdgpu_elf_with_relocation_after_text( bool place_reloc_in_text = false, uint32_t reloc_type = 0, int64_t reloc_addend = 0) { constexpr uint64_t text_offset = 0x100; constexpr uint64_t text_vaddr = 0x1100; constexpr uint64_t text_size = 8; constexpr uint64_t data_size = 8; constexpr uint64_t load_align = 0x1000; std::vector shstrtab{'\0'}; const uint32_t text_name = add_elf_name(shstrtab, ".text"); const uint32_t data_name = add_elf_name(shstrtab, ".data"); const uint32_t rela_name = add_elf_name(shstrtab, ".rela.dyn"); const uint32_t shstrtab_name = add_elf_name(shstrtab, ".shstrtab"); const uint64_t data_offset = text_offset + text_size; const uint64_t data_vaddr = text_vaddr + text_size + load_align; const uint64_t rela_offset = data_offset + data_size; constexpr size_t rela_count = 1; const uint64_t shstrtab_offset = rela_offset + rela_count * sizeof(Elf64_Rela); const uint64_t shoff = align_up_for_test(shstrtab_offset + shstrtab.size(), 8); constexpr uint16_t section_count = 5; constexpr uint16_t phdr_count = 2; std::vector image(shoff + section_count * sizeof(Elf64_Shdr), 0); Elf64_Ehdr ehdr{}; std::memcpy(ehdr.e_ident, EI_MAGIC, EI_MAGIC_SIZE); ehdr.e_ident[EI_CLASS] = ELFCLASS64; ehdr.e_ident[EI_OSABI] = ELFOSABI_AMDGPU_HSA; ehdr.e_type = ET_DYN; ehdr.e_machine = EM_AMDGPU; ehdr.e_version = 1; ehdr.e_phoff = sizeof(Elf64_Ehdr); ehdr.e_shoff = shoff; ehdr.e_flags = EF_AMDGPU_MACH_AMDGCN_GFX950; ehdr.e_ehsize = sizeof(Elf64_Ehdr); ehdr.e_phentsize = sizeof(Elf64_Phdr); ehdr.e_phnum = phdr_count; ehdr.e_shentsize = sizeof(Elf64_Shdr); ehdr.e_shnum = section_count; ehdr.e_shstrndx = 4; std::memcpy(image.data(), &ehdr, sizeof(ehdr)); std::array phdrs{}; phdrs[0].p_type = PT_LOAD; phdrs[0].p_flags = 0x5; // PF_R | PF_X phdrs[0].p_offset = text_offset; phdrs[0].p_vaddr = text_vaddr; phdrs[0].p_paddr = text_vaddr; phdrs[0].p_filesz = text_size; phdrs[0].p_memsz = text_size; phdrs[0].p_align = load_align; phdrs[1].p_type = PT_LOAD; phdrs[1].p_flags = 0x6; // PF_R | PF_W phdrs[1].p_offset = data_offset; phdrs[1].p_vaddr = data_vaddr; phdrs[1].p_paddr = data_vaddr; phdrs[1].p_filesz = data_size; phdrs[1].p_memsz = data_size; phdrs[1].p_align = load_align; std::memcpy(image.data() + ehdr.e_phoff, phdrs.data(), phdrs.size() * sizeof(Elf64_Phdr)); const std::array text_words = {0xBF800000u, 0xBF800000u}; std::memcpy(image.data() + text_offset, text_words.data(), text_size); const uint64_t data_word = 0x1234567890ABCDEFull; std::memcpy(image.data() + data_offset, &data_word, sizeof(data_word)); Elf64_Rela rela{}; // By default the relocation place is in .data (safe: DBT shifts it with the // moved section). place_reloc_in_text points it inside .text, which DBT cannot // remap after relocating instructions and must reject. rela.r_offset = place_reloc_in_text ? text_vaddr : data_vaddr; rela.r_info = (uint64_t{0} << 32) | reloc_type; rela.r_addend = reloc_addend; std::memcpy(image.data() + rela_offset, &rela, sizeof(rela)); std::memcpy(image.data() + shstrtab_offset, shstrtab.data(), shstrtab.size()); std::array shdrs{}; shdrs[1].sh_name = text_name; shdrs[1].sh_type = SHT_PROGBITS; shdrs[1].sh_flags = SHF_ALLOC | SHF_EXECINSTR; shdrs[1].sh_addr = text_vaddr; shdrs[1].sh_offset = text_offset; shdrs[1].sh_size = text_size; shdrs[1].sh_addralign = sizeof(uint32_t); shdrs[2].sh_name = data_name; shdrs[2].sh_type = SHT_PROGBITS; shdrs[2].sh_flags = SHF_ALLOC | SHF_WRITE; shdrs[2].sh_addr = data_vaddr; shdrs[2].sh_offset = data_offset; shdrs[2].sh_size = data_size; shdrs[2].sh_addralign = sizeof(uint64_t); shdrs[3].sh_name = rela_name; shdrs[3].sh_type = SHT_RELA; shdrs[3].sh_offset = rela_offset; shdrs[3].sh_size = rela_count * sizeof(Elf64_Rela); shdrs[3].sh_addralign = 8; shdrs[3].sh_entsize = sizeof(Elf64_Rela); shdrs[4].sh_name = shstrtab_name; shdrs[4].sh_type = SHT_STRTAB; shdrs[4].sh_offset = shstrtab_offset; shdrs[4].sh_size = shstrtab.size(); shdrs[4].sh_addralign = 1; std::memcpy(image.data() + shoff, shdrs.data(), shdrs.size() * sizeof(Elf64_Shdr)); return image; } // Build an ET_DYN object whose .data relocation resolves against a symbol of a // chosen type, defined either in .text or .data. Ordinary zero-addend text // symbols can follow their relocated st_value; section symbols and nonzero // addends need relocation-specific reconstruction and remain unsupported. std::vector make_amdgpu_elf_with_symbol_relocation(uint8_t sym_type, bool defined_in_text, int64_t addend = 0) { constexpr uint64_t text_offset = 0x100; constexpr uint64_t text_vaddr = 0x1100; constexpr uint64_t text_size = 8; constexpr uint64_t data_size = 8; constexpr uint64_t load_align = 0x1000; std::vector shstrtab{'\0'}; const uint32_t text_name = add_elf_name(shstrtab, ".text"); const uint32_t data_name = add_elf_name(shstrtab, ".data"); const uint32_t symtab_name = add_elf_name(shstrtab, ".symtab"); const uint32_t strtab_name = add_elf_name(shstrtab, ".strtab"); const uint32_t rela_name = add_elf_name(shstrtab, ".rela.dyn"); const uint32_t shstrtab_name = add_elf_name(shstrtab, ".shstrtab"); std::vector strtab{'\0'}; const uint32_t sym_name = add_elf_name(strtab, "target"); constexpr uint16_t text_index = 1; constexpr uint16_t data_index = 2; const uint64_t data_offset = text_offset + text_size; const uint64_t data_vaddr = text_vaddr + text_size + load_align; const uint64_t symtab_offset = align_up_for_test(data_offset + data_size, 8); constexpr size_t sym_count = 2; // STN_UNDEF + target const uint64_t strtab_offset = symtab_offset + sym_count * sizeof(Elf64_Sym); const uint64_t rela_offset = align_up_for_test(strtab_offset + strtab.size(), 8); constexpr size_t rela_count = 1; const uint64_t shstrtab_offset = rela_offset + rela_count * sizeof(Elf64_Rela); const uint64_t shoff = align_up_for_test(shstrtab_offset + shstrtab.size(), 8); constexpr uint16_t section_count = 7; constexpr uint16_t phdr_count = 2; std::vector image(shoff + section_count * sizeof(Elf64_Shdr), 0); Elf64_Ehdr ehdr{}; std::memcpy(ehdr.e_ident, EI_MAGIC, EI_MAGIC_SIZE); ehdr.e_ident[EI_CLASS] = ELFCLASS64; ehdr.e_ident[EI_OSABI] = ELFOSABI_AMDGPU_HSA; ehdr.e_type = ET_DYN; ehdr.e_machine = EM_AMDGPU; ehdr.e_version = 1; ehdr.e_phoff = sizeof(Elf64_Ehdr); ehdr.e_shoff = shoff; ehdr.e_flags = EF_AMDGPU_MACH_AMDGCN_GFX950; ehdr.e_ehsize = sizeof(Elf64_Ehdr); ehdr.e_phentsize = sizeof(Elf64_Phdr); ehdr.e_phnum = phdr_count; ehdr.e_shentsize = sizeof(Elf64_Shdr); ehdr.e_shnum = section_count; ehdr.e_shstrndx = 6; std::memcpy(image.data(), &ehdr, sizeof(ehdr)); std::array phdrs{}; phdrs[0].p_type = PT_LOAD; phdrs[0].p_flags = 0x5; // PF_R | PF_X phdrs[0].p_offset = text_offset; phdrs[0].p_vaddr = text_vaddr; phdrs[0].p_paddr = text_vaddr; phdrs[0].p_filesz = text_size; phdrs[0].p_memsz = text_size; phdrs[0].p_align = load_align; phdrs[1].p_type = PT_LOAD; phdrs[1].p_flags = 0x6; // PF_R | PF_W phdrs[1].p_offset = data_offset; phdrs[1].p_vaddr = data_vaddr; phdrs[1].p_paddr = data_vaddr; phdrs[1].p_filesz = data_size; phdrs[1].p_memsz = data_size; phdrs[1].p_align = load_align; std::memcpy(image.data() + ehdr.e_phoff, phdrs.data(), phdrs.size() * sizeof(Elf64_Phdr)); const std::array text_words = {0xBF800000u, 0xBF800000u}; std::memcpy(image.data() + text_offset, text_words.data(), text_size); std::array syms{}; syms[1].st_name = sym_name; syms[1].st_info = elf_symbol_info(kElfSymbolBindGlobal, sym_type); syms[1].st_shndx = defined_in_text ? text_index : data_index; syms[1].st_value = defined_in_text ? text_vaddr : data_vaddr; syms[1].st_size = defined_in_text ? text_size : data_size; std::memcpy(image.data() + symtab_offset, syms.data(), syms.size() * sizeof(Elf64_Sym)); std::memcpy(image.data() + strtab_offset, strtab.data(), strtab.size()); Elf64_Rela rela{}; rela.r_offset = data_vaddr; // place in .data, safely shifted with the section rela.r_info = (static_cast(1) << 32) | R_AMDGPU_ABS64; rela.r_addend = addend; std::memcpy(image.data() + rela_offset, &rela, sizeof(rela)); std::memcpy(image.data() + shstrtab_offset, shstrtab.data(), shstrtab.size()); std::array shdrs{}; shdrs[1].sh_name = text_name; shdrs[1].sh_type = SHT_PROGBITS; shdrs[1].sh_flags = SHF_ALLOC | SHF_EXECINSTR; shdrs[1].sh_addr = text_vaddr; shdrs[1].sh_offset = text_offset; shdrs[1].sh_size = text_size; shdrs[1].sh_addralign = sizeof(uint32_t); shdrs[2].sh_name = data_name; shdrs[2].sh_type = SHT_PROGBITS; shdrs[2].sh_flags = SHF_ALLOC | SHF_WRITE; shdrs[2].sh_addr = data_vaddr; shdrs[2].sh_offset = data_offset; shdrs[2].sh_size = data_size; shdrs[2].sh_addralign = sizeof(uint64_t); shdrs[3].sh_name = symtab_name; shdrs[3].sh_type = SHT_SYMTAB; shdrs[3].sh_offset = symtab_offset; shdrs[3].sh_size = syms.size() * sizeof(Elf64_Sym); shdrs[3].sh_link = 4; shdrs[3].sh_info = 1; shdrs[3].sh_addralign = 8; shdrs[3].sh_entsize = sizeof(Elf64_Sym); shdrs[4].sh_name = strtab_name; shdrs[4].sh_type = SHT_STRTAB; shdrs[4].sh_offset = strtab_offset; shdrs[4].sh_size = strtab.size(); shdrs[4].sh_addralign = 1; shdrs[5].sh_name = rela_name; shdrs[5].sh_type = SHT_RELA; shdrs[5].sh_offset = rela_offset; shdrs[5].sh_size = rela_count * sizeof(Elf64_Rela); shdrs[5].sh_link = 3; // .symtab shdrs[5].sh_addralign = 8; shdrs[5].sh_entsize = sizeof(Elf64_Rela); shdrs[6].sh_name = shstrtab_name; shdrs[6].sh_type = SHT_STRTAB; shdrs[6].sh_offset = shstrtab_offset; shdrs[6].sh_size = shstrtab.size(); shdrs[6].sh_addralign = 1; std::memcpy(image.data() + shoff, shdrs.data(), shdrs.size() * sizeof(Elf64_Shdr)); return image; } // Build an ET_DYN object with a symbol-less relocation of the given type in // .rela.dyn (symbol index 0), placed in .data, with the supplied addend. Used to // prove the text-symbol guard also catches an R_AMDGPU_RELATIVE64 whose addend // lands in the source .text virtual-address interval (the loader forms the stored // value from load_bias + r_addend, which DBT would leave pointing at stale PC). // text_vaddr is 0x1100 with size 8, so an addend of 0x1100 is in-text and 0x2108 // (the .data vaddr) is out-of-text. std::vector make_amdgpu_elf_with_relative_relocation(uint32_t reloc_type, int64_t addend) { constexpr uint64_t text_offset = 0x100; constexpr uint64_t text_vaddr = 0x1100; constexpr uint64_t text_size = 8; constexpr uint64_t data_size = 8; constexpr uint64_t load_align = 0x1000; std::vector shstrtab{'\0'}; const uint32_t text_name = add_elf_name(shstrtab, ".text"); const uint32_t data_name = add_elf_name(shstrtab, ".data"); const uint32_t rela_name = add_elf_name(shstrtab, ".rela.dyn"); const uint32_t shstrtab_name = add_elf_name(shstrtab, ".shstrtab"); const uint64_t data_offset = text_offset + text_size; const uint64_t data_vaddr = text_vaddr + text_size + load_align; const uint64_t rela_offset = align_up_for_test(data_offset + data_size, 8); constexpr size_t rela_count = 1; const uint64_t shstrtab_offset = rela_offset + rela_count * sizeof(Elf64_Rela); const uint64_t shoff = align_up_for_test(shstrtab_offset + shstrtab.size(), 8); constexpr uint16_t section_count = 5; constexpr uint16_t phdr_count = 2; std::vector image(shoff + section_count * sizeof(Elf64_Shdr), 0); Elf64_Ehdr ehdr{}; std::memcpy(ehdr.e_ident, EI_MAGIC, EI_MAGIC_SIZE); ehdr.e_ident[EI_CLASS] = ELFCLASS64; ehdr.e_ident[EI_OSABI] = ELFOSABI_AMDGPU_HSA; ehdr.e_type = ET_DYN; ehdr.e_machine = EM_AMDGPU; ehdr.e_version = 1; ehdr.e_phoff = sizeof(Elf64_Ehdr); ehdr.e_shoff = shoff; ehdr.e_flags = EF_AMDGPU_MACH_AMDGCN_GFX950; ehdr.e_ehsize = sizeof(Elf64_Ehdr); ehdr.e_phentsize = sizeof(Elf64_Phdr); ehdr.e_phnum = phdr_count; ehdr.e_shentsize = sizeof(Elf64_Shdr); ehdr.e_shnum = section_count; ehdr.e_shstrndx = 4; std::memcpy(image.data(), &ehdr, sizeof(ehdr)); std::array phdrs{}; phdrs[0].p_type = PT_LOAD; phdrs[0].p_flags = 0x5; // PF_R | PF_X phdrs[0].p_offset = text_offset; phdrs[0].p_vaddr = text_vaddr; phdrs[0].p_paddr = text_vaddr; phdrs[0].p_filesz = text_size; phdrs[0].p_memsz = text_size; phdrs[0].p_align = load_align; phdrs[1].p_type = PT_LOAD; phdrs[1].p_flags = 0x6; // PF_R | PF_W phdrs[1].p_offset = data_offset; phdrs[1].p_vaddr = data_vaddr; phdrs[1].p_paddr = data_vaddr; phdrs[1].p_filesz = data_size; phdrs[1].p_memsz = data_size; phdrs[1].p_align = load_align; std::memcpy(image.data() + ehdr.e_phoff, phdrs.data(), phdrs.size() * sizeof(Elf64_Phdr)); const std::array text_words = {0xBF800000u, 0xBF800000u}; std::memcpy(image.data() + text_offset, text_words.data(), text_size); Elf64_Rela rela{}; rela.r_offset = data_vaddr; // place in .data, safely shifted with the section rela.r_info = static_cast(reloc_type); // symbol index 0, low 32 bits type rela.r_addend = addend; std::memcpy(image.data() + rela_offset, &rela, sizeof(rela)); std::memcpy(image.data() + shstrtab_offset, shstrtab.data(), shstrtab.size()); std::array shdrs{}; shdrs[1].sh_name = text_name; shdrs[1].sh_type = SHT_PROGBITS; shdrs[1].sh_flags = SHF_ALLOC | SHF_EXECINSTR; shdrs[1].sh_addr = text_vaddr; shdrs[1].sh_offset = text_offset; shdrs[1].sh_size = text_size; shdrs[1].sh_addralign = sizeof(uint32_t); shdrs[2].sh_name = data_name; shdrs[2].sh_type = SHT_PROGBITS; shdrs[2].sh_flags = SHF_ALLOC | SHF_WRITE; shdrs[2].sh_addr = data_vaddr; shdrs[2].sh_offset = data_offset; shdrs[2].sh_size = data_size; shdrs[2].sh_addralign = sizeof(uint64_t); shdrs[3].sh_name = rela_name; shdrs[3].sh_type = SHT_RELA; shdrs[3].sh_offset = rela_offset; shdrs[3].sh_size = rela_count * sizeof(Elf64_Rela); shdrs[3].sh_link = 0; // no symtab needed for symbol-zero relocations shdrs[3].sh_addralign = 8; shdrs[3].sh_entsize = sizeof(Elf64_Rela); shdrs[4].sh_name = shstrtab_name; shdrs[4].sh_type = SHT_STRTAB; shdrs[4].sh_offset = shstrtab_offset; shdrs[4].sh_size = shstrtab.size(); shdrs[4].sh_addralign = 1; std::memcpy(image.data() + shoff, shdrs.data(), shdrs.size() * sizeof(Elf64_Shdr)); return image; } std::vector make_large_amdgpu_elf_with_waitcnt_entry() { constexpr uint64_t rodata_offset = 0x100; constexpr uint64_t rodata_vaddr = 0x100; constexpr uint64_t text_offset = 0x1000; constexpr uint64_t text_vaddr = 0x1000; constexpr uint64_t text_size = 0x21000; constexpr uint64_t rodata_size = kKernelDescriptorSize; constexpr uint64_t load_align = 0x1000; std::vector shstrtab{'\0'}; const uint32_t rodata_name = add_elf_name(shstrtab, ".rodata"); const uint32_t text_name = add_elf_name(shstrtab, ".text"); const uint32_t symtab_name = add_elf_name(shstrtab, ".symtab"); const uint32_t strtab_name = add_elf_name(shstrtab, ".strtab"); const uint32_t shstrtab_name = add_elf_name(shstrtab, ".shstrtab"); std::vector strtab{'\0'}; const uint32_t kd_symbol_name = add_elf_name(strtab, "kernel.kd"); const uint64_t strtab_offset = text_offset + text_size; const uint64_t symtab_offset = align_up_for_test(strtab_offset + strtab.size(), 8); constexpr size_t sym_count = 2; const uint64_t shstrtab_offset = symtab_offset + sym_count * sizeof(Elf64_Sym); const uint64_t shoff = align_up_for_test(shstrtab_offset + shstrtab.size(), 8); constexpr uint16_t section_count = 6; constexpr uint16_t phdr_count = 2; std::vector image(shoff + section_count * sizeof(Elf64_Shdr), 0); write_bytes_for_test(image, offsetof(Elf64_Ehdr, e_ident), EI_MAGIC, EI_MAGIC_SIZE); image[offsetof(Elf64_Ehdr, e_ident) + EI_CLASS] = ELFCLASS64; image[offsetof(Elf64_Ehdr, e_ident) + EI_OSABI] = ELFOSABI_AMDGPU_HSA; write_value_for_test(image, offsetof(Elf64_Ehdr, e_type), ET_DYN); write_value_for_test(image, offsetof(Elf64_Ehdr, e_machine), EM_AMDGPU); write_value_for_test(image, offsetof(Elf64_Ehdr, e_version), 1); write_value_for_test(image, offsetof(Elf64_Ehdr, e_phoff), sizeof(Elf64_Ehdr)); write_value_for_test(image, offsetof(Elf64_Ehdr, e_shoff), shoff); write_value_for_test(image, offsetof(Elf64_Ehdr, e_flags), EF_AMDGPU_MACH_AMDGCN_GFX950); write_value_for_test(image, offsetof(Elf64_Ehdr, e_ehsize), sizeof(Elf64_Ehdr)); write_value_for_test(image, offsetof(Elf64_Ehdr, e_phentsize), sizeof(Elf64_Phdr)); write_value_for_test(image, offsetof(Elf64_Ehdr, e_phnum), phdr_count); write_value_for_test(image, offsetof(Elf64_Ehdr, e_shentsize), sizeof(Elf64_Shdr)); write_value_for_test(image, offsetof(Elf64_Ehdr, e_shnum), section_count); write_value_for_test(image, offsetof(Elf64_Ehdr, e_shstrndx), 5); const uint64_t phdr0 = sizeof(Elf64_Ehdr); write_value_for_test(image, phdr0 + offsetof(Elf64_Phdr, p_type), PT_LOAD); write_value_for_test(image, phdr0 + offsetof(Elf64_Phdr, p_flags), 0x4); // PF_R write_value_for_test(image, phdr0 + offsetof(Elf64_Phdr, p_offset), rodata_offset); write_value_for_test(image, phdr0 + offsetof(Elf64_Phdr, p_vaddr), rodata_vaddr); write_value_for_test(image, phdr0 + offsetof(Elf64_Phdr, p_paddr), rodata_vaddr); write_value_for_test(image, phdr0 + offsetof(Elf64_Phdr, p_filesz), rodata_size); write_value_for_test(image, phdr0 + offsetof(Elf64_Phdr, p_memsz), rodata_size); write_value_for_test(image, phdr0 + offsetof(Elf64_Phdr, p_align), load_align); const uint64_t phdr1 = phdr0 + sizeof(Elf64_Phdr); write_value_for_test(image, phdr1 + offsetof(Elf64_Phdr, p_type), PT_LOAD); write_value_for_test(image, phdr1 + offsetof(Elf64_Phdr, p_flags), 0x5); // PF_R | PF_X write_value_for_test(image, phdr1 + offsetof(Elf64_Phdr, p_offset), text_offset); write_value_for_test(image, phdr1 + offsetof(Elf64_Phdr, p_vaddr), text_vaddr); write_value_for_test(image, phdr1 + offsetof(Elf64_Phdr, p_paddr), text_vaddr); write_value_for_test(image, phdr1 + offsetof(Elf64_Phdr, p_filesz), text_size); write_value_for_test(image, phdr1 + offsetof(Elf64_Phdr, p_memsz), text_size); write_value_for_test(image, phdr1 + offsetof(Elf64_Phdr, p_align), load_align); const auto descriptor = make_kernel_descriptor_bytes(static_cast(text_vaddr) - static_cast(rodata_vaddr)); std::memcpy(image.data() + rodata_offset, descriptor.data(), descriptor.size()); std::vector text_words(text_size / sizeof(uint32_t), build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4)); text_words[0] = cdna4::build_sopp(cdna4::kSWaitcntSopp)[0]; // Expands on RDNA4. std::memcpy(image.data() + text_offset, text_words.data(), text_size); std::memcpy(image.data() + strtab_offset, strtab.data(), strtab.size()); const uint64_t sym1 = symtab_offset + sizeof(Elf64_Sym); write_value_for_test(image, sym1 + offsetof(Elf64_Sym, st_name), kd_symbol_name); write_value_for_test(image, sym1 + offsetof(Elf64_Sym, st_info), elf_symbol_info(kElfSymbolBindGlobal, kElfSymbolTypeObject)); write_value_for_test(image, sym1 + offsetof(Elf64_Sym, st_shndx), 1); write_value_for_test(image, sym1 + offsetof(Elf64_Sym, st_value), rodata_vaddr); write_value_for_test(image, sym1 + offsetof(Elf64_Sym, st_size), kKernelDescriptorSize); std::memcpy(image.data() + shstrtab_offset, shstrtab.data(), shstrtab.size()); const auto write_shdr = [&](uint64_t index, uint32_t name, uint32_t type, uint64_t flags, uint64_t addr, uint64_t offset, uint64_t size, uint32_t link, uint32_t info, uint64_t addralign, uint64_t entsize) { const uint64_t base = shoff + index * sizeof(Elf64_Shdr); write_value_for_test(image, base + offsetof(Elf64_Shdr, sh_name), name); write_value_for_test(image, base + offsetof(Elf64_Shdr, sh_type), type); write_value_for_test(image, base + offsetof(Elf64_Shdr, sh_flags), flags); write_value_for_test(image, base + offsetof(Elf64_Shdr, sh_addr), addr); write_value_for_test(image, base + offsetof(Elf64_Shdr, sh_offset), offset); write_value_for_test(image, base + offsetof(Elf64_Shdr, sh_size), size); write_value_for_test(image, base + offsetof(Elf64_Shdr, sh_link), link); write_value_for_test(image, base + offsetof(Elf64_Shdr, sh_info), info); write_value_for_test(image, base + offsetof(Elf64_Shdr, sh_addralign), addralign); write_value_for_test(image, base + offsetof(Elf64_Shdr, sh_entsize), entsize); }; write_shdr(1, rodata_name, SHT_PROGBITS, SHF_ALLOC, rodata_vaddr, rodata_offset, rodata_size, 0, 0, 64, 0); write_shdr(2, text_name, SHT_PROGBITS, SHF_ALLOC | SHF_EXECINSTR, text_vaddr, text_offset, text_size, 0, 0, 256, 0); write_shdr(3, symtab_name, SHT_SYMTAB, 0, 0, symtab_offset, sym_count * sizeof(Elf64_Sym), 4, 1, 8, sizeof(Elf64_Sym)); write_shdr(4, strtab_name, SHT_STRTAB, 0, 0, strtab_offset, strtab.size(), 0, 0, 1, 0); write_shdr(5, shstrtab_name, SHT_STRTAB, 0, 0, shstrtab_offset, shstrtab.size(), 0, 0, 1, 0); return image; } const Section *find_section(const CodeObject &co, std::string_view name) { for (const auto §ion : co.all_sections()) { if (section->name() == name) return section.get(); } return nullptr; } std::optional loaded_vaddr_to_file_offset(const std::vector &image, uint64_t vaddr) { return test_support::loaded_vaddr_to_file_offset(image, vaddr); } void enable_workgroup_id_x_sgpr(std::vector &image) { using KD = rocr::llvm::amdhsa::kernel_descriptor_t; AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(KD)); KD kd{}; std::memcpy(&kd, image.data() + rodata->sectionOffset(), sizeof(kd)); kd.compute_pgm_rsrc2 |= rocr::llvm::amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X; std::memcpy(image.data() + rodata->sectionOffset(), &kd, sizeof(kd)); } void enable_kernarg_segment_ptr_sgpr(std::vector &image, uint32_t kernarg_size = 16) { using KD = rocr::llvm::amdhsa::kernel_descriptor_t; AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(KD)); KD kd{}; std::memcpy(&kd, image.data() + rodata->sectionOffset(), sizeof(kd)); AMDHSA_BITS_SET(kd.compute_pgm_rsrc2, rocr::llvm::amdhsa::COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, 2); AMDHSA_BITS_SET(kd.kernel_code_properties, rocr::llvm::amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR, 1); kd.kernarg_size = kernarg_size; std::memcpy(image.data() + rodata->sectionOffset(), &kd, sizeof(kd)); } TEST(CoherencyRemap, Gfx940ToGfx12AgentScope) { auto coh = remap_gfx940_to_gfx12({1, 0, 0}); EXPECT_EQ(coh.scope, 1); EXPECT_EQ(coh.th, 0); } TEST(CoherencyRemap, Gfx940ToGfx12SystemScope) { auto coh = remap_gfx940_to_gfx12({1, 1, 0}); EXPECT_EQ(coh.scope, 3); EXPECT_EQ(coh.th, 0); } TEST(CoherencyRemap, Gfx940ToGfx12NonTemporal) { auto coh = remap_gfx940_to_gfx12({0, 0, 1}); EXPECT_EQ(coh.scope, 0); EXPECT_EQ(coh.th, 3); } TEST(CoherencyRemap, Gfx9GlcToGfx12) { auto coh_glc1 = remap_gfx9_to_gfx12({1}); EXPECT_EQ(coh_glc1.scope, 2); EXPECT_EQ(coh_glc1.th, 0); auto coh_glc0 = remap_gfx9_to_gfx12({0}); EXPECT_EQ(coh_glc0.scope, 0); EXPECT_EQ(coh_glc0.th, 0); } TEST(EncodingTranslator, Sop1PreservesRegisters) { cdna4::Sop1MachineInst src{}; src.ssrc0 = 42; src.sdst = 17; src.op = cdna4::kSCmovB64Sop1; src.encoding = cdna4::encoding::kSop1; uint32_t w0 = std::bit_cast(src); auto result = cdna4_to_rdna4::translate_encoding_cdna4_to_rdna4(kEnc_SOP1, w0, 0, 0, rdna4::kSBrevB64Sop1); ASSERT_EQ(result.word_count, 1); auto dst = std::bit_cast(result.words[0]); EXPECT_EQ(dst.ssrc0, 42); EXPECT_EQ(dst.sdst, 17); EXPECT_EQ(dst.op, rdna4::kSBrevB64Sop1); EXPECT_EQ(dst.encoding, rdna4::encoding::kSop1); } TEST(EncodingTranslator, Sop2PreservesRegisters) { cdna4::Sop2MachineInst src{}; src.ssrc0 = 10; src.ssrc1 = 20; src.sdst = 30; src.op = cdna4::kSMinU32Sop2; src.encoding = 0x2; uint32_t w0 = std::bit_cast(src); auto result = cdna4_to_rdna4::translate_encoding_cdna4_to_rdna4(kEnc_SOP2, w0, 0, 0, rdna4::kSMinU32Sop2); ASSERT_EQ(result.word_count, 1); auto dst = std::bit_cast(result.words[0]); EXPECT_EQ(dst.ssrc0, 10); EXPECT_EQ(dst.ssrc1, 20); EXPECT_EQ(dst.sdst, 30); EXPECT_EQ(dst.op, rdna4::kSMinU32Sop2); } TEST(InstructionBuilder, Sop2SetsEncodingPrefix) { const uint32_t word = build_s_lshl_b32(1, 2, 3, ROCJITSU_CODE_ARCH_RDNA4); EXPECT_EQ((word >> 30) & 0x3u, 0x2u); } TEST(EncodingTranslator, SoppPreservesSimm16) { cdna4::SoppMachineInst src{}; src.simm16 = 0xABCD; src.op = cdna4::kSWaitcntSopp; src.encoding = cdna4::encoding::kSopp; uint32_t w0 = std::bit_cast(src); auto result = cdna4_to_rdna4::translate_encoding_cdna4_to_rdna4(kEnc_SOPP, w0, 0, 0, rdna4::kSWaitcntSopp); ASSERT_EQ(result.word_count, 1); auto dst = std::bit_cast(result.words[0]); EXPECT_EQ(dst.simm16, 0xABCD); EXPECT_EQ(dst.op, rdna4::kSWaitcntSopp); } TEST(EncodingTranslator, SmemRemapsCoherency) { cdna4::SmemMachineInst src{}; src.sbase = 5; src.sdata = 3; src.glc = 1; src.nv = 0; src.op = cdna4::kSLoadDwordSmem; src.offset = 0x100; src.soffset = 0x7F; src.encoding = 0x3D; uint32_t words[2]; std::memcpy(words, &src, sizeof(src)); auto result = cdna4_to_rdna4::translate_encoding_cdna4_to_rdna4(kEnc_SMEM, words[0], words[1], 0, rdna4::kSLoadB32Smem); ASSERT_EQ(result.word_count, 2); rdna4::SmemMachineInst dst{}; std::memcpy(&dst, result.words, sizeof(dst)); EXPECT_EQ(dst.sbase, 5); EXPECT_EQ(dst.sdata, 3); EXPECT_EQ(dst.scope, 2); EXPECT_EQ(dst.th, 0); EXPECT_EQ(dst.nv, 0); EXPECT_EQ(dst.soffset, 0x7C); // CDNA4 null (0x7F) → RDNA4 null (0x7C) } TEST(EncodingTranslator, Vop3PreservesModifiers) { cdna4::Vop3MachineInst src{}; src.vdst = 10; src.src0 = 100; src.src1 = 200; src.src2 = 50; src.clamp = 1; src.omod = 2; src.neg = 5; src.abs = 3; src.op = cdna4::kVCmpGtF64Vop3; src.encoding = 0x35; uint32_t words[2]; std::memcpy(words, &src, sizeof(src)); auto result = cdna4_to_rdna4::translate_encoding_cdna4_to_rdna4(kEnc_VOP3, words[0], words[1], 0, rdna4::kVCmpGtF64Vop3); ASSERT_EQ(result.word_count, 2); rdna4::Vop3MachineInst dst{}; std::memcpy(&dst, result.words, sizeof(dst)); EXPECT_EQ(dst.vdst, 10); EXPECT_EQ(dst.src0, 100); EXPECT_EQ(dst.src1, 200); EXPECT_EQ(dst.src2, 50); EXPECT_EQ(dst.clamp, 1); EXPECT_EQ(dst.omod, 2); EXPECT_EQ(dst.neg, 5); EXPECT_EQ(dst.abs, 3); } TEST(EncodingTranslator, Cdna4ToCdna3Vop2VectorAddPreservesOperands) { cdna4::Vop2MachineInst src{}; src.src0 = 3; src.vsrc1 = 4; src.vdst = 5; src.op = cdna4::kVAddF32Vop2; src.encoding = 0; // GFX9-family VOP2 prefix. uint32_t w0 = std::bit_cast(src); auto result = cdna4_to_cdna3::translate_encoding_cdna4_to_cdna3(kEnc_VOP2, w0, 0, 0, cdna3::kVAddF32Vop2); ASSERT_EQ(result.word_count, 1); auto dst = std::bit_cast(result.words[0]); EXPECT_EQ(dst.src0, 3); EXPECT_EQ(dst.vsrc1, 4); EXPECT_EQ(dst.vdst, 5); EXPECT_EQ(dst.op, cdna3::kVAddF32Vop2); EXPECT_EQ(dst.encoding, 0); } TEST(EncodingTranslator, UnknownEncodingReturnsEmpty) { auto result = cdna4_to_rdna4::translate_encoding_cdna4_to_rdna4(0xFFFF, 0, 0, 0, 0); EXPECT_EQ(result.word_count, 0); } TEST(EncodingTranslator, DecodeEncodeRoundTrip) { cdna4::Sop1MachineInst src{}; src.ssrc0 = 55; src.sdst = 33; src.op = cdna4::kSNotB32Sop1; src.encoding = cdna4::encoding::kSop1; uint32_t w0 = std::bit_cast(src); auto fields = cdna4_to_rdna4::decode_sop1_cdna4(w0); EXPECT_EQ(fields.ssrc0, 55u); EXPECT_EQ(fields.sdst, 33u); EXPECT_EQ(fields.op, cdna4::kSNotB32Sop1); auto result = cdna4_to_rdna4::encode_sop1_rdna4(fields, rdna4::kSBrevB32Sop1); ASSERT_EQ(result.word_count, 1); auto dst = std::bit_cast(result.words[0]); EXPECT_EQ(dst.ssrc0, 55); EXPECT_EQ(dst.sdst, 33); EXPECT_EQ(dst.op, rdna4::kSBrevB32Sop1); } TEST(LegalizationLookup, FindsKnownInstruction) { const auto *entry = lookup(kLegalization_cdna4_to_rdna4, 0, 0); EXPECT_NE(entry, nullptr); if (entry) { EXPECT_NE(entry->action, Action::Illegal); } } TEST(LegalizationLookup, ReturnsNullForUnknown) { const auto *entry = lookup(kLegalization_cdna4_to_rdna4, 0xFFFF, 0xFFFF); EXPECT_EQ(entry, nullptr); } TEST(LegalizationTable, NoIllegalEntries_Cdna4ToRdna4) { for (const auto &e : kLegalization_cdna4_to_rdna4) { EXPECT_NE(e.action, Action::Illegal) << "ILLEGAL at encoding_id=" << e.src_encoding_id << " opcode=" << e.src_opcode; } } #define CHECK_NO_ILLEGAL(pair) \ TEST(LegalizationTable, NoIllegalEntries_##pair) { \ for (const auto &e : kLegalization_##pair) { \ EXPECT_NE(e.action, Action::Illegal) \ << "ILLEGAL at encoding_id=" << e.src_encoding_id << " opcode=" << e.src_opcode; \ } \ EXPECT_GT(std::size(kLegalization_##pair), 0u) << "table is empty"; \ } CHECK_NO_ILLEGAL(cdna1_to_cdna2) CHECK_NO_ILLEGAL(cdna1_to_cdna3) CHECK_NO_ILLEGAL(cdna1_to_cdna4) CHECK_NO_ILLEGAL(cdna1_to_rdna1) CHECK_NO_ILLEGAL(cdna1_to_rdna2) CHECK_NO_ILLEGAL(cdna1_to_rdna3) CHECK_NO_ILLEGAL(cdna1_to_rdna4) CHECK_NO_ILLEGAL(cdna2_to_cdna3) CHECK_NO_ILLEGAL(cdna2_to_cdna4) CHECK_NO_ILLEGAL(cdna2_to_rdna3) CHECK_NO_ILLEGAL(cdna2_to_rdna4) CHECK_NO_ILLEGAL(cdna3_to_cdna4) CHECK_NO_ILLEGAL(cdna3_to_rdna3) CHECK_NO_ILLEGAL(cdna3_to_rdna4) CHECK_NO_ILLEGAL(cdna4_to_cdna3) CHECK_NO_ILLEGAL(cdna4_to_rdna3) CHECK_NO_ILLEGAL(rdna1_to_cdna3) CHECK_NO_ILLEGAL(rdna1_to_cdna4) CHECK_NO_ILLEGAL(rdna1_to_rdna2) CHECK_NO_ILLEGAL(rdna1_to_rdna3) CHECK_NO_ILLEGAL(rdna1_to_rdna4) CHECK_NO_ILLEGAL(rdna2_to_rdna3) CHECK_NO_ILLEGAL(rdna2_to_rdna4) CHECK_NO_ILLEGAL(rdna3_5_to_rdna4) CHECK_NO_ILLEGAL(rdna3_to_cdna4) CHECK_NO_ILLEGAL(rdna3_to_rdna4) CHECK_NO_ILLEGAL(rdna4_to_cdna4) #undef CHECK_NO_ILLEGAL // Cross-ISA legalization of the literal FMA-K / MAD-K family. The fieldless // OPR_SIMM32 literal now participates in the operand signature (with its size // canonicalized to the 32-bit encoding word), so identical literal FMAs match // faithfully across ISAs. These pin the three outcomes that drives: // - same opcode on both ISAs -> Identity // - same instruction, different opcode -> Substitute (with remapped target) // - the V_FMAAK_F16 case whose per-ISA simm32 size (16 on rdna1/2, 32 on // rdna3/4) would fragment the match without the canonicalization // VOP2 encoding_id is 0 on CDNA guests and 4 on RDNA guests. TEST(LegalizationFmaK, Cdna3ToCdna4MadmkF16IsIdentity) { const auto *e = lookup(kLegalization_cdna3_to_cdna4, /*enc=*/0, /*op=*/36); // v_madmk_f16 ASSERT_NE(e, nullptr); EXPECT_EQ(e->action, Action::Identity); EXPECT_EQ(e->target_opcode, 36); } TEST(LegalizationFmaK, Cdna1ToRdna1MadmkF32IsSubstitute) { // cdna1 v_madmk_f32 (op 23) -> rdna1 v_madmk_f32 (op 32): same instruction, // different opcode number, so the literal-preserving remap is a Substitute. const auto *e = lookup(kLegalization_cdna1_to_rdna1, /*enc=*/0, /*op=*/23); ASSERT_NE(e, nullptr); EXPECT_EQ(e->action, Action::Substitute); EXPECT_EQ(e->target_opcode, 32); } TEST(LegalizationFmaK, Rdna1ToRdna3FmaakF16IsIdentity) { // Regression guard for the simm32-size canonicalization: without it, this // degrades to Lower because the literal's declared size differs (16 vs 32) // for the same instruction across rdna generations. const auto *e = lookup(kLegalization_rdna1_to_rdna3, /*enc=*/4, /*op=*/56); // v_fmaak_f16 ASSERT_NE(e, nullptr); EXPECT_EQ(e->action, Action::Identity); EXPECT_EQ(e->target_opcode, 56); } TEST(BinaryTranslatorE2E, EmptyTextSameArchIsSuccessfulNoOp) { const auto image = make_minimal_gfx1250_elf_with_empty_text_and_rodata(); AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); ASSERT_EQ(source.target_id(), ROCJITSU_CODE_TARGET_GFX1250); ASSERT_FALSE(source.text_sections().empty()); ASSERT_EQ(source.text_sections()[0]->size(), 0u); BinaryTranslatorOptions options; options.input_revision = ProcessorRevision::Gfx1250B0; options.output_revision = ProcessorRevision::Gfx1250A0; BinaryTranslator translator(ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, options); const auto result = translator.translate(source); EXPECT_TRUE(result.ok()); EXPECT_TRUE(result.dispatchable()); EXPECT_EQ(result.elf_bytes, image); const auto warning = std::ranges::find_if(result.diagnostics, [](const auto &diagnostic) { return diagnostic.kind == DiagnosticKind::DataOnly; }); ASSERT_NE(warning, result.diagnostics.end()); EXPECT_EQ(warning->severity, DiagnosticSeverity::Warning); EXPECT_EQ(warning->message, "code object has no executable sections, segments, or callable symbols; leaving " "unchanged"); } TEST(BinaryTranslatorE2E, TruncatedImageFailsBeforeReadingElfHeader) { const std::array image{}; AmdGpuCodeObject source(image.data(), image.size()); ASSERT_FALSE(source.is_valid()); ASSERT_LT(source.image_size(), sizeof(Elf64_Ehdr)); BinaryTranslator translator(ROCJITSU_CODE_ARCH_RDNA4, ROCJITSU_CODE_ARCH_RDNA4); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_TRUE(result.elf_bytes.empty()); EXPECT_TRUE(has_error_containing(result, DiagnosticKind::ResourceLimit, "too small to contain an ELF header")); } TEST(BinaryTranslatorE2E, EmptyTextCrossArchStillFails) { const auto image = make_minimal_gfx1250_elf_with_empty_text_and_rodata(); AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); BinaryTranslator translator(ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_RDNA4); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(has_error_containing(result, DiagnosticKind::ResourceLimit, "does not expose a non-empty .text section")); } TEST(BinaryTranslatorE2E, EmptyTextSameArchDifferentMachineStillFails) { auto image = make_minimal_gfx1250_elf_with_empty_text_and_rodata(); write_value_for_test(image, offsetof(Elf64_Ehdr, e_flags), EF_AMDGPU_MACH_AMDGCN_GFX1200); AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); ASSERT_EQ(source.target_id(), ROCJITSU_CODE_TARGET_GFX1200); BinaryTranslator translator(ROCJITSU_CODE_ARCH_RDNA4, ROCJITSU_CODE_ARCH_RDNA4, EF_AMDGPU_MACH_AMDGCN_GFX1201); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(has_error_containing(result, DiagnosticKind::ResourceLimit, "does not expose a non-empty .text section")); } TEST(BinaryTranslatorE2E, EmptyTextGfx1250StillRequiresRevisions) { const auto image = make_minimal_gfx1250_elf_with_empty_text_and_rodata(); AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); BinaryTranslator translator(ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(has_error_containing(result, DiagnosticKind::Legalization, "requires both input and output silicon revisions")); } TEST(BinaryTranslatorE2E, EmptyTextGfx1250StillRejectsA0ToB0) { const auto image = make_minimal_gfx1250_elf_with_empty_text_and_rodata(); AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); BinaryTranslatorOptions options; options.input_revision = ProcessorRevision::Gfx1250A0; options.output_revision = ProcessorRevision::Gfx1250B0; BinaryTranslator translator(ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, options); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(has_error_containing(result, DiagnosticKind::Legalization, "A0-to-B0 translation is not supported")); } TEST(BinaryTranslatorE2E, DescriptorlessExecutableTextIsSuccessfulNoOp) { auto image = make_minimal_amdgpu_elf_with_text_and_rodata(); write_value_for_test(image, offsetof(Elf64_Ehdr, e_flags), EF_AMDGPU_MACH_AMDGCN_GFX1250); write_value_for_test(image, 0x100, 0xbf850004u); AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); ASSERT_FALSE(source.text_sections().empty()); ASSERT_GT(source.text_sections()[0]->size(), 0u); BinaryTranslatorOptions options; options.input_revision = ProcessorRevision::Gfx1250B0; options.output_revision = ProcessorRevision::Gfx1250A0; BinaryTranslator translator(ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, options); const auto result = translator.translate(source); EXPECT_TRUE(result.ok()); EXPECT_TRUE(result.dispatchable()); EXPECT_EQ(result.elf_bytes, image); const auto warning = std::ranges::find_if(result.diagnostics, [](const auto &diagnostic) { return diagnostic.kind == DiagnosticKind::NothingToTranslate; }); ASSERT_NE(warning, result.diagnostics.end()); EXPECT_EQ(warning->severity, DiagnosticSeverity::Warning); EXPECT_NE(warning->message.find("no kernel descriptors"), std::string::npos); } // A code relocation names a branch destination inside the text being written, so it has to be a // whole instruction there. An offset equal to the size names the byte one past the end, which // would still compute a literal and leave it pointing outside `.text`. TEST(CodeObjectPatcher, ReplaceTextRejectsCodeRelocationTargetPastEndOfText) { const std::array text_words = {0xBF800000u, 0xBF800000u, 0xBF800000u, 0xBF800000u}; const auto bytes = std::span(reinterpret_cast(text_words.data()), sizeof(text_words)); const auto attempt = [&](uint64_t target_text_offset) { auto image = make_minimal_amdgpu_elf_with_text_and_rodata(); AmdGpuCodeObject co(image.data(), image.size()); EXPECT_TRUE(co.is_valid()); CodeObjectPatcher patcher(co); const std::array code_relocations = { PcRelativeTextRelocation{.target_getpc_offset = 0, .target_literal_offset = 4, .target_text_offset = target_text_offset}}; return patcher.replace_text(bytes, {}, {}, code_relocations); }; EXPECT_FALSE(attempt(sizeof(text_words))) << "a target one past the end of .text is not an instruction"; EXPECT_FALSE(attempt(sizeof(text_words) - 2)) << "a target without room for a whole instruction word is not an instruction"; EXPECT_FALSE(attempt(1)) << "an offset interior to the first instruction is not a destination"; EXPECT_FALSE(attempt(sizeof(uint32_t) + 2)) << "an offset interior to a later instruction is not a destination"; EXPECT_TRUE(attempt(sizeof(text_words) - sizeof(uint32_t))) << "the last instruction in .text is a legitimate branch destination"; } TEST(CodeObjectPatcher, ResolvesAllocatedDataPayloadsAndEndpoints) { std::array sections{}; sections[0].sh_addr = 0x1000; sections[0].sh_size = 0x8; sections[1].sh_flags = SHF_ALLOC; sections[1].sh_addr = 0x2000; sections[1].sh_size = 0x10; sections[2].sh_flags = SHF_ALLOC; sections[2].sh_addr = 0x2010; sections[2].sh_size = 0; sections[3].sh_flags = SHF_ALLOC; sections[3].sh_addr = 0x2010; sections[3].sh_size = 0x8; sections[4].sh_flags = SHF_ALLOC | SHF_EXECINSTR; sections[4].sh_addr = 0x3000; sections[4].sh_size = 0x8; struct Case { uint64_t target; size_t section_index; uint64_t section_offset; }; for (const Case &test : {Case{0x2000, 1, 0}, Case{0x200f, 1, 0xf}, Case{0x2011, 3, 1}, Case{0x2018, 3, 8}}) { const auto resolved = resolve_allocated_data_section_address(sections, test.target); ASSERT_TRUE(resolved.has_value()) << "target 0x" << std::hex << test.target; EXPECT_EQ(resolved->section_index, test.section_index); EXPECT_EQ(resolved->section_offset, test.section_offset); } EXPECT_TRUE(resolve_allocated_data_section_address(sections, 0x2010)) << "either nonempty section may own their shared boundary"; std::array empty_section{}; empty_section[0].sh_flags = SHF_ALLOC; empty_section[0].sh_addr = 0x4000; EXPECT_FALSE(resolve_allocated_data_section_address(empty_section, 0x4000)); EXPECT_FALSE(resolve_allocated_data_section_address(sections, 0x1000)); EXPECT_FALSE(resolve_allocated_data_section_address(sections, 0x2019)); EXPECT_FALSE(resolve_allocated_data_section_address(sections, 0x2100)); EXPECT_FALSE(resolve_allocated_data_section_address(sections, 0x3000)); ASSERT_TRUE(resolve_allocated_data_section_address(sections, 0x2010)); EXPECT_FALSE(resolve_pc_relative_data_section_address(sections, 0x2010, 0x2010, 0x8)) << "source text takes precedence over a data section ending at the same address"; } TEST(CodeObjectPatcher, ReplaceTextGrowsTextAndShiftsFollowingSections) { auto image = make_minimal_amdgpu_elf_with_text_and_rodata(); AmdGpuCodeObject co(image.data(), image.size()); ASSERT_TRUE(co.is_valid()); ASSERT_FALSE(co.text_sections().empty()); CodeObjectPatcher patcher(co); const std::array text_words = {0xBF800000u, 0xBF800000u, 0xDEADBEEFu, 0xCAFEBABEu}; const auto *text_bytes = reinterpret_cast(text_words.data()); ASSERT_TRUE(patcher.replace_text({text_bytes, text_words.size() * sizeof(uint32_t)})); auto patched_bytes = patcher.emit(); AmdGpuCodeObject patched(patched_bytes.data(), patched_bytes.size()); ASSERT_TRUE(patched.is_valid()); ASSERT_FALSE(patched.text_sections().empty()); const Section *text = patched.text_sections()[0]; ASSERT_NE(text, nullptr); EXPECT_EQ(text->size(), text_words.size() * sizeof(uint32_t)); ASSERT_EQ(patched.text_sections().size(), 1u); EXPECT_EQ(patched.text_sections()[0]->name(), ".text"); EXPECT_EQ(find_section(patched, ".rj_translations"), nullptr); EXPECT_EQ(std::memcmp(text->data(), text_words.data(), text->size()), 0); const Section *rodata = find_section(patched, ".rodata"); ASSERT_NE(rodata, nullptr); EXPECT_EQ(rodata->sectionOffset(), text->sectionOffset() + text->size()) << "sections following .text must be shifted after the grown text"; uint32_t rodata_word = 0; std::memcpy(&rodata_word, rodata->data(), sizeof(rodata_word)); EXPECT_EQ(rodata_word, 0xA5A55A5Au); } // COMPUTE_PGM_RSRC3 is carried over verbatim only when source and target agree on its layout. // "GFX10 or later" is not a fine enough test: INST_PREF_SIZE is GFX11+, and IMAGE_OP is GFX11+, // so handing a GFX11 or GFX12 word to a GFX10 target would set bits GFX10 requires to be zero. // Equally, a GFX12 word must survive intact on a GFX12 target, including the GFX125-only // NAMED_BAR_CNT and an INST_PREF_SIZE above the 63 that GFX11's narrower field can hold. TEST(CodeObjectPatcher, Rsrc3IsCarriedOnlyBetweenMatchingLayouts) { using namespace rocr::llvm::amdhsa; const auto image = make_minimal_amdgpu_elf_with_descriptor_after_text(); AmdGpuCodeObject probe(image.data(), image.size()); ASSERT_TRUE(probe.is_valid()); const Section *probe_rodata = find_section(probe, ".rodata"); ASSERT_NE(probe_rodata, nullptr); ASSERT_GE(probe_rodata->size(), sizeof(kernel_descriptor_t)); // A GFX12-shaped source word: INST_PREF_SIZE beyond GFX11's 6-bit range, plus a GFX125-only // named-barrier allocation. uint32_t source_rsrc3 = 0; AMDHSA_BITS_SET(source_rsrc3, COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE, 107u); AMDHSA_BITS_SET(source_rsrc3, COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT, 3u); auto patched_rsrc3 = [&](rj_code_arch_t source_arch, rj_code_arch_t target_arch) -> std::optional { auto local_image = image; AmdGpuCodeObject local_co(local_image.data(), local_image.size()); if (!local_co.is_valid()) return std::nullopt; const Section *rodata = find_section(local_co, ".rodata"); if (rodata == nullptr) return std::nullopt; auto descriptor = read_kernel_descriptor_for_test(rodata->data()); descriptor.compute_pgm_rsrc3 = source_rsrc3; write_kernel_descriptor_for_test(local_image.data() + rodata->sectionOffset(), descriptor); AmdGpuCodeObject seeded(local_image.data(), local_image.size()); if (!seeded.is_valid()) return std::nullopt; KdTranslation translation{}; translation.descriptor_file_offset = rodata->sectionOffset(); translation.target_wave_size = 32; translation.source_arch = source_arch; CodeObjectPatcher patcher(seeded); if (!patcher.apply_kernel_descriptor_translation(translation, target_arch)) return std::nullopt; const auto patched_image = patcher.emit(); return read_kernel_descriptor_for_test(patched_image.data() + translation.descriptor_file_offset) .compute_pgm_rsrc3; }; // Matching layout: carried verbatim, both the wide INST_PREF_SIZE and NAMED_BAR_CNT. const auto same_layout = patched_rsrc3(ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250); ASSERT_TRUE(same_layout.has_value()); EXPECT_EQ(*same_layout, source_rsrc3); EXPECT_EQ(AMDHSA_BITS_GET(*same_layout, COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE), 107u); EXPECT_EQ(AMDHSA_BITS_GET(*same_layout, COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT), 3u); // GFX10 targets have no INST_PREF_SIZE and no IMAGE_OP; the GFX12 word must not be inherited. for (const rj_code_arch_t gfx10 : {ROCJITSU_CODE_ARCH_RDNA1, ROCJITSU_CODE_ARCH_RDNA2}) { const auto rebuilt = patched_rsrc3(ROCJITSU_CODE_ARCH_GFX1250, gfx10); ASSERT_TRUE(rebuilt.has_value()); EXPECT_NE(*rebuilt, source_rsrc3); EXPECT_EQ(AMDHSA_BITS_GET(*rebuilt, COMPUTE_PGM_RSRC3_GFX10_PLUS_INST_PREF_SIZE), 0u) << "GFX10 reserves the INST_PREF_SIZE bits and requires them to be zero"; } // A GFX9/CDNA source encodes ACCUM_OFFSET here, so it is rebuilt for a GFX12 target. const auto from_cdna = patched_rsrc3(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_GFX1250); ASSERT_TRUE(from_cdna.has_value()); EXPECT_NE(*from_cdna, source_rsrc3); EXPECT_EQ(AMDHSA_BITS_GET(*from_cdna, COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT), 0u); // The narrow case: GFX11 -> GFX10. Both are "GFX10+" and both use SHARED_VGPR_COUNT, so a // coarse family check treats them as interchangeable -- but GFX11's INST_PREF_SIZE at 9:4 and // IMAGE_OP at 31 are reserved on GFX10 and must not be carried over. uint32_t gfx11_rsrc3 = 0; AMDHSA_BITS_SET(gfx11_rsrc3, COMPUTE_PGM_RSRC3_GFX10_PLUS_INST_PREF_SIZE, 21u); AMDHSA_BITS_SET(gfx11_rsrc3, COMPUTE_PGM_RSRC3_GFX10_PLUS_IMAGE_OP, 1u); source_rsrc3 = gfx11_rsrc3; for (const rj_code_arch_t gfx10 : {ROCJITSU_CODE_ARCH_RDNA1, ROCJITSU_CODE_ARCH_RDNA2}) { const auto rebuilt = patched_rsrc3(ROCJITSU_CODE_ARCH_RDNA3, gfx10); ASSERT_TRUE(rebuilt.has_value()); EXPECT_EQ(AMDHSA_BITS_GET(*rebuilt, COMPUTE_PGM_RSRC3_GFX10_PLUS_INST_PREF_SIZE), 0u) << "GFX10 reserves INST_PREF_SIZE; a GFX11 word must not be carried over"; EXPECT_EQ(AMDHSA_BITS_GET(*rebuilt, COMPUTE_PGM_RSRC3_GFX10_PLUS_IMAGE_OP), 0u) << "IMAGE_OP is GFX11+; GFX10 reserves bit 31"; } // GFX11 -> GFX11 still round-trips, so the split does not over-rebuild. const auto gfx11_same = patched_rsrc3(ROCJITSU_CODE_ARCH_RDNA3, ROCJITSU_CODE_ARCH_RDNA3); ASSERT_TRUE(gfx11_same.has_value()); EXPECT_EQ(*gfx11_same, gfx11_rsrc3); // GFX125 -> GFX120. Both are GFX12, and both use the same 8-bit INST_PREF_SIZE, but LLVM // reserves bits 21:14 on GFX120 while GFX125 fills them with NAMED_BAR_CNT, // ENABLE_DYNAMIC_VGPR, TCP_SPLIT and ENABLE_DIDT_THROTTLE. A shared "GFX12+" bucket would // carry those straight into target-reserved bits. uint32_t gfx125_rsrc3 = 0; AMDHSA_BITS_SET(gfx125_rsrc3, COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE, 107u); AMDHSA_BITS_SET(gfx125_rsrc3, COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT, 3u); AMDHSA_BITS_SET(gfx125_rsrc3, COMPUTE_PGM_RSRC3_GFX125_TCP_SPLIT, 5u); AMDHSA_BITS_SET(gfx125_rsrc3, COMPUTE_PGM_RSRC3_GFX125_ENABLE_DYNAMIC_VGPR, 1u); source_rsrc3 = gfx125_rsrc3; const auto to_gfx120 = patched_rsrc3(ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_RDNA4); ASSERT_TRUE(to_gfx120.has_value()); EXPECT_EQ(AMDHSA_BITS_GET(*to_gfx120, COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT), 0u) << "bits 21:14 are reserved on GFX120 and must not inherit GFX125 state"; EXPECT_EQ(AMDHSA_BITS_GET(*to_gfx120, COMPUTE_PGM_RSRC3_GFX125_TCP_SPLIT), 0u); EXPECT_EQ(AMDHSA_BITS_GET(*to_gfx120, COMPUTE_PGM_RSRC3_GFX125_ENABLE_DYNAMIC_VGPR), 0u); // GFX125 -> GFX125 keeps all of it. const auto gfx125_same = patched_rsrc3(ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250); ASSERT_TRUE(gfx125_same.has_value()); EXPECT_EQ(*gfx125_same, gfx125_rsrc3); } // SHARED_VGPR_COUNT is the one carried field this patcher invalidates itself: it rewrites both // the wave size and the RSRC1 VGPR allocation the field is constrained against. LLVM requires // the count to be zero for wave32, and for wave64 requires // (compute_pgm_rsrc1.vgprs + 1) * 4 + shared_vgpr_count * 8 <= 256. // // The field counts registers the unchanged kernel body uses, so a descriptor that no longer // leaves room for them is refused rather than clamped: clamping would keep the body's demand // while shrinking what the descriptor reserves for it. TEST(CodeObjectPatcher, SharedVgprCountIsReconciledWithTheTargetAllocation) { using namespace rocr::llvm::amdhsa; const auto image = make_minimal_amdgpu_elf_with_descriptor_after_text(); struct PatchOutcome { bool seeded = false; ///< Fixture setup succeeded. bool patched = false; ///< The patcher accepted the request. uint32_t shared_count = 0; }; auto patched = [&](uint32_t source_shared, uint32_t target_wave, uint32_t vgpr_granulated) -> PatchOutcome { auto local_image = image; AmdGpuCodeObject probe(local_image.data(), local_image.size()); if (!probe.is_valid()) return {}; const Section *rodata = find_section(probe, ".rodata"); if (rodata == nullptr) return {}; auto descriptor = read_kernel_descriptor_for_test(rodata->data()); descriptor.compute_pgm_rsrc3 = 0; AMDHSA_BITS_SET(descriptor.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX10_PLUS_SHARED_VGPR_COUNT, source_shared); write_kernel_descriptor_for_test(local_image.data() + rodata->sectionOffset(), descriptor); AmdGpuCodeObject seeded(local_image.data(), local_image.size()); if (!seeded.is_valid()) return {}; KdTranslation translation{}; translation.descriptor_file_offset = rodata->sectionOffset(); translation.target_wave_size = target_wave; translation.target_vgpr_granulated = vgpr_granulated; translation.source_arch = ROCJITSU_CODE_ARCH_RDNA3; PatchOutcome outcome; outcome.seeded = true; CodeObjectPatcher patcher(seeded); outcome.patched = patcher.apply_kernel_descriptor_translation(translation, ROCJITSU_CODE_ARCH_RDNA3); if (!outcome.patched) return outcome; const auto out = patcher.emit(); outcome.shared_count = AMDHSA_BITS_GET( read_kernel_descriptor_for_test(out.data() + translation.descriptor_file_offset) .compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX10_PLUS_SHARED_VGPR_COUNT); return outcome; }; // Wave64 with a small VGPR allocation: the source request still fits, so it survives. const auto roomy = patched(/*source_shared=*/4, /*target_wave=*/64, /*vgpr_granulated=*/1); ASSERT_TRUE(roomy.seeded); ASSERT_TRUE(roomy.patched); EXPECT_EQ(roomy.shared_count, 4u); // Exactly at the limit: (33 + 1) * 4 = 136 arch VGPRs plus 15 * 8 = 120 shared reaches 256. const auto exact = patched(/*source_shared=*/15, /*target_wave=*/64, /*vgpr_granulated=*/33); ASSERT_TRUE(exact.seeded); ASSERT_TRUE(exact.patched); EXPECT_EQ(exact.shared_count, 15u); // One block past the limit is refused rather than trimmed to fit. const auto over = patched(/*source_shared=*/15, /*target_wave=*/64, /*vgpr_granulated=*/34); ASSERT_TRUE(over.seeded); EXPECT_FALSE(over.patched) << "an allocation that crowds out the shared blocks must be refused"; // Wave64 with a large allocation: (63 + 1) * 4 = 256 VGPRs leaves no room at all. Clamping to // zero here used to report success while removing every block the body still uses. const auto tight = patched(/*source_shared=*/9, /*target_wave=*/64, /*vgpr_granulated=*/63); ASSERT_TRUE(tight.seeded); EXPECT_FALSE(tight.patched) << "a 9-to-0 reduction is a resource change, not a translation"; // Wave32 has no shared VGPR blocks at all, so there is nowhere to put the source's request. const auto wave32 = patched(/*source_shared=*/9, /*target_wave=*/32, /*vgpr_granulated=*/1); ASSERT_TRUE(wave32.seeded); EXPECT_FALSE(wave32.patched) << "wave32 cannot carry shared VGPR blocks"; // A source that asked for nothing is unaffected by any of the above. const auto none = patched(/*source_shared=*/0, /*target_wave=*/32, /*vgpr_granulated=*/63); ASSERT_TRUE(none.seeded); ASSERT_TRUE(none.patched); EXPECT_EQ(none.shared_count, 0u); } TEST(CodeObjectPatcher, AppliesArchSpecificWgpModeBit) { using namespace rocr::llvm::amdhsa; const auto image = make_minimal_amdgpu_elf_with_descriptor_after_text(); AmdGpuCodeObject co(image.data(), image.size()); ASSERT_TRUE(co.is_valid()); const Section *rodata = find_section(co, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(kernel_descriptor_t)); auto patched_rsrc1 = [&](rj_code_arch_t arch) -> std::optional { AmdGpuCodeObject local_co(image.data(), image.size()); if (!local_co.is_valid()) return std::nullopt; KdTranslation translation{}; translation.descriptor_file_offset = rodata->sectionOffset(); translation.target_wave_size = 32; CodeObjectPatcher patcher(local_co); if (!patcher.apply_kernel_descriptor_translation(translation, arch)) return std::nullopt; const auto patched_image = patcher.emit(); const auto kd = read_kernel_descriptor_for_test(patched_image.data() + translation.descriptor_file_offset); return kd.compute_pgm_rsrc1; }; const auto cdna3_rsrc1 = patched_rsrc1(ROCJITSU_CODE_ARCH_CDNA3); ASSERT_TRUE(cdna3_rsrc1.has_value()); EXPECT_EQ(AMDHSA_BITS_GET(*cdna3_rsrc1, COMPUTE_PGM_RSRC1_WGP_MODE), 0u); EXPECT_EQ(AMDHSA_BITS_GET(*cdna3_rsrc1, COMPUTE_PGM_RSRC1_MEM_ORDERED), 0u); EXPECT_EQ(AMDHSA_BITS_GET(*cdna3_rsrc1, COMPUTE_PGM_RSRC1_FWD_PROGRESS), 0u); const auto rdna1_rsrc1 = patched_rsrc1(ROCJITSU_CODE_ARCH_RDNA1); ASSERT_TRUE(rdna1_rsrc1.has_value()); EXPECT_EQ(AMDHSA_BITS_GET(*rdna1_rsrc1, COMPUTE_PGM_RSRC1_WGP_MODE), 1u); EXPECT_EQ(AMDHSA_BITS_GET(*rdna1_rsrc1, COMPUTE_PGM_RSRC1_MEM_ORDERED), 1u); EXPECT_EQ(AMDHSA_BITS_GET(*rdna1_rsrc1, COMPUTE_PGM_RSRC1_FWD_PROGRESS), 1u); const auto rdna2_rsrc1 = patched_rsrc1(ROCJITSU_CODE_ARCH_RDNA2); ASSERT_TRUE(rdna2_rsrc1.has_value()); EXPECT_EQ(AMDHSA_BITS_GET(*rdna2_rsrc1, COMPUTE_PGM_RSRC1_WGP_MODE), 1u); EXPECT_EQ(AMDHSA_BITS_GET(*rdna2_rsrc1, COMPUTE_PGM_RSRC1_MEM_ORDERED), 1u); EXPECT_EQ(AMDHSA_BITS_GET(*rdna2_rsrc1, COMPUTE_PGM_RSRC1_FWD_PROGRESS), 1u); const auto rdna3_rsrc1 = patched_rsrc1(ROCJITSU_CODE_ARCH_RDNA3); ASSERT_TRUE(rdna3_rsrc1.has_value()); EXPECT_EQ(AMDHSA_BITS_GET(*rdna3_rsrc1, COMPUTE_PGM_RSRC1_WGP_MODE), 1u); EXPECT_EQ(AMDHSA_BITS_GET(*rdna3_rsrc1, COMPUTE_PGM_RSRC1_MEM_ORDERED), 1u); EXPECT_EQ(AMDHSA_BITS_GET(*rdna3_rsrc1, COMPUTE_PGM_RSRC1_FWD_PROGRESS), 1u); const auto rdna3_5_rsrc1 = patched_rsrc1(ROCJITSU_CODE_ARCH_RDNA3_5); ASSERT_TRUE(rdna3_5_rsrc1.has_value()); EXPECT_EQ(AMDHSA_BITS_GET(*rdna3_5_rsrc1, COMPUTE_PGM_RSRC1_WGP_MODE), 1u); EXPECT_EQ(AMDHSA_BITS_GET(*rdna3_5_rsrc1, COMPUTE_PGM_RSRC1_MEM_ORDERED), 1u); EXPECT_EQ(AMDHSA_BITS_GET(*rdna3_5_rsrc1, COMPUTE_PGM_RSRC1_FWD_PROGRESS), 1u); const auto rdna4_rsrc1 = patched_rsrc1(ROCJITSU_CODE_ARCH_RDNA4); ASSERT_TRUE(rdna4_rsrc1.has_value()); EXPECT_EQ(AMDHSA_BITS_GET(*rdna4_rsrc1, COMPUTE_PGM_RSRC1_WGP_MODE), 1u); EXPECT_EQ(AMDHSA_BITS_GET(*rdna4_rsrc1, COMPUTE_PGM_RSRC1_MEM_ORDERED), 1u); EXPECT_EQ(AMDHSA_BITS_GET(*rdna4_rsrc1, COMPUTE_PGM_RSRC1_FWD_PROGRESS), 1u); const auto gfx1250_rsrc1 = patched_rsrc1(ROCJITSU_CODE_ARCH_GFX1250); ASSERT_TRUE(gfx1250_rsrc1.has_value()); EXPECT_EQ(AMDHSA_BITS_GET(*gfx1250_rsrc1, COMPUTE_PGM_RSRC1_WGP_MODE), 0u); EXPECT_EQ(AMDHSA_BITS_GET(*gfx1250_rsrc1, COMPUTE_PGM_RSRC1_MEM_ORDERED), 1u); EXPECT_EQ(AMDHSA_BITS_GET(*gfx1250_rsrc1, COMPUTE_PGM_RSRC1_FWD_PROGRESS), 1u); } TEST(CodeObjectPatcher, PreservesPrivateEnableForZeroFixedDynamicStack) { using namespace rocr::llvm::amdhsa; auto image = make_minimal_amdgpu_elf_with_descriptor_after_text(); AmdGpuCodeObject source_layout(image.data(), image.size()); ASSERT_TRUE(source_layout.is_valid()); const Section *source_rodata = find_section(source_layout, ".rodata"); ASSERT_NE(source_rodata, nullptr); kernel_descriptor_t source_descriptor{}; AMDHSA_BITS_SET(source_descriptor.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, 1); AMDHSA_BITS_SET(source_descriptor.kernel_code_properties, KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, 1); write_kernel_descriptor_for_test(image.data() + source_rodata->sectionOffset(), source_descriptor); AmdGpuCodeObject code_object(image.data(), image.size()); ASSERT_TRUE(code_object.is_valid()); KdTranslation translation{}; translation.descriptor_file_offset = source_rodata->sectionOffset(); translation.target_private_size = 0; translation.target_wave_size = 64; CodeObjectPatcher patcher(code_object); ASSERT_TRUE(patcher.apply_kernel_descriptor_translation(translation, ROCJITSU_CODE_ARCH_CDNA3)); const auto patched_image = patcher.emit(); const auto patched_descriptor = read_kernel_descriptor_for_test(patched_image.data() + translation.descriptor_file_offset); EXPECT_EQ(patched_descriptor.private_segment_fixed_size, 0u); EXPECT_EQ(AMDHSA_BITS_GET(patched_descriptor.kernel_code_properties, KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK), 1u); EXPECT_EQ(AMDHSA_BITS_GET(patched_descriptor.compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT), 1u); } TEST(CodeObjectPatcher, ReplaceTextPreservesLoadSegmentAlignment) { constexpr uint64_t load_align = 0x1000; constexpr uint64_t padded_file_delta = 2 * load_align; auto image = make_minimal_amdgpu_elf_with_load_segments(); AmdGpuCodeObject co(image.data(), image.size()); ASSERT_TRUE(co.is_valid()); ASSERT_FALSE(co.text_sections().empty()); CodeObjectPatcher patcher(co); const std::vector text_words(load_align / sizeof(uint32_t) + 3, 0xDEADBEEFu); const auto *text_bytes = reinterpret_cast(text_words.data()); ASSERT_TRUE(patcher.replace_text({text_bytes, text_words.size() * sizeof(uint32_t)})); auto patched_bytes = patcher.emit(); AmdGpuCodeObject patched(patched_bytes.data(), patched_bytes.size()); ASSERT_TRUE(patched.is_valid()); ASSERT_FALSE(patched.text_sections().empty()); const Section *text = patched.text_sections()[0]; const Section *rodata = find_section(patched, ".rodata"); ASSERT_NE(rodata, nullptr); EXPECT_EQ(find_section(patched, ".rj_translations"), nullptr); EXPECT_EQ(text->size(), text_words.size() * sizeof(uint32_t)); EXPECT_EQ(rodata->sectionOffset(), text->sectionOffset() + 8u + padded_file_delta) << "file padding should preserve later PT_LOAD p_offset/p_vaddr congruence"; EXPECT_EQ(rodata->vaddr(), text->vaddr() + 8 + load_align + padded_file_delta) << "later allocated sections must move after the expanded RX LOAD segment"; const auto ehdr = read_elf_struct_for_test(patched_bytes, 0); ASSERT_EQ(ehdr.e_phnum, 2u); const auto phdrs = read_elf_array_for_test(patched_bytes, ehdr.e_phoff, ehdr.e_phnum); const auto shdrs = read_elf_array_for_test(patched_bytes, ehdr.e_shoff, ehdr.e_shnum); for (uint16_t i = 0; i < ehdr.e_phnum; ++i) { ASSERT_NE(phdrs[i].p_align, 0u); EXPECT_EQ(phdrs[i].p_offset % phdrs[i].p_align, phdrs[i].p_vaddr % phdrs[i].p_align) << "PT_LOAD " << i << " must remain loader-congruent"; } EXPECT_EQ(phdrs[0].p_filesz, 8u + padded_file_delta); EXPECT_EQ(phdrs[0].p_memsz, 8u + padded_file_delta); EXPECT_EQ(phdrs[1].p_offset, rodata->sectionOffset()); EXPECT_EQ(phdrs[1].p_vaddr, rodata->vaddr()); EXPECT_EQ(phdrs[1].p_paddr, rodata->vaddr()); EXPECT_LE(phdrs[0].p_vaddr + phdrs[0].p_memsz, phdrs[1].p_vaddr) << "expanded RX LOAD must not overlap the following LOAD in virtual memory"; const auto symtab = std::find_if(shdrs.begin(), shdrs.end(), [](const Elf64_Shdr &shdr) { return shdr.sh_type == SHT_SYMTAB; }); ASSERT_NE(symtab, shdrs.end()); ASSERT_EQ(symtab->sh_entsize, sizeof(Elf64_Sym)); ASSERT_GE(symtab->sh_size / symtab->sh_entsize, 3u); const auto symbols = read_elf_array_for_test(patched_bytes, symtab->sh_offset, symtab->sh_size / symtab->sh_entsize); EXPECT_EQ(symbols[1].st_value, rodata->vaddr()) << "defined symbols in moved sections must track the section virtual address"; EXPECT_EQ(symbols[2].st_value, text->vaddr()) << "symbols in unmoved .text must keep their original virtual address"; EXPECT_EQ(symbols[2].st_size, text->size()) << "function symbols spanning old .text must cover appended translated cave code"; } TEST(CodeObjectPatcher, ReplaceTextRelocatesTextSymbolsWithExactOffsetMap) { auto image = make_minimal_amdgpu_elf_with_load_segments(); AmdGpuCodeObject co(image.data(), image.size()); ASSERT_TRUE(co.is_valid()); CodeObjectPatcher patcher(co); const std::array text_words = {0xBF800000u, 0xBF800000u, 0xBF800000u, 0xBF800000u}; const auto *text_bytes = reinterpret_cast(text_words.data()); constexpr std::array relocations = { TextOffsetRelocation{.source_offset = 0, .target_offset = 4}, TextOffsetRelocation{.source_offset = 8, .target_offset = 16}, }; ASSERT_TRUE(patcher.replace_text({text_bytes, sizeof(text_words)}, relocations)); const auto patched_bytes = patcher.emit(); const auto ehdr = read_elf_struct_for_test(patched_bytes, 0); const auto shdrs = read_elf_array_for_test(patched_bytes, ehdr.e_shoff, ehdr.e_shnum); const auto symtab = std::find_if(shdrs.begin(), shdrs.end(), [](const Elf64_Shdr &shdr) { return shdr.sh_type == SHT_SYMTAB; }); ASSERT_NE(symtab, shdrs.end()); const auto symbols = read_elf_array_for_test(patched_bytes, symtab->sh_offset, symtab->sh_size / symtab->sh_entsize); ASSERT_GE(symbols.size(), 3u); EXPECT_EQ(symbols[2].st_value, 0x1104u); EXPECT_EQ(symbols[2].st_size, 12u); } TEST(CodeObjectPatcher, AppendsNonAllocSectionWithoutMovingLoadableSegments) { auto image = make_minimal_amdgpu_elf_with_load_segments(); AmdGpuCodeObject co(image.data(), image.size()); ASSERT_TRUE(co.is_valid()); const Section *original_text = find_section(co, ".text"); const Section *original_rodata = find_section(co, ".rodata"); ASSERT_NE(original_text, nullptr); ASSERT_NE(original_rodata, nullptr); const uint64_t original_text_offset = original_text->sectionOffset(); const uint64_t original_text_vaddr = original_text->vaddr(); const uint64_t original_rodata_offset = original_rodata->sectionOffset(); const uint64_t original_rodata_vaddr = original_rodata->vaddr(); CodeObjectPatcher patcher(co); const std::array payload = {'R', 'J', 'L', 'D', 'S', 1, 2, 3}; ASSERT_TRUE(patcher.append_nonalloc_section(".rocjitsu.lds", payload, 8)); const auto patched_bytes = patcher.emit(); AmdGpuCodeObject patched(patched_bytes.data(), patched_bytes.size()); ASSERT_TRUE(patched.is_valid()); const Section *metadata = find_section(patched, ".rocjitsu.lds"); ASSERT_NE(metadata, nullptr); ASSERT_EQ(metadata->size(), payload.size()); EXPECT_EQ(std::memcmp(metadata->data(), payload.data(), payload.size()), 0); EXPECT_EQ(metadata->flags() & SHF_ALLOC, 0u); const Section *patched_text = find_section(patched, ".text"); const Section *patched_rodata = find_section(patched, ".rodata"); ASSERT_NE(patched_text, nullptr); ASSERT_NE(patched_rodata, nullptr); EXPECT_EQ(patched_text->sectionOffset(), original_text_offset); EXPECT_EQ(patched_text->vaddr(), original_text_vaddr); EXPECT_EQ(patched_rodata->sectionOffset(), original_rodata_offset); EXPECT_EQ(patched_rodata->vaddr(), original_rodata_vaddr); } TEST(CodeObjectPatcher, ReplaceTextPreservesMovedKernelDescriptorEntryAddress) { constexpr uint64_t load_align = 0x1000; constexpr uint64_t padded_file_delta = 2 * load_align; auto image = make_minimal_amdgpu_elf_with_descriptor_after_text(); AmdGpuCodeObject co(image.data(), image.size()); ASSERT_TRUE(co.is_valid()); ASSERT_FALSE(co.text_sections().empty()); const auto *original_rodata = find_section(co, ".rodata"); ASSERT_NE(original_rodata, nullptr); const int64_t original_entry_offset = read_kernel_descriptor_entry_offset(original_rodata->data()); CodeObjectPatcher patcher(co); const std::vector text_words(load_align / sizeof(uint32_t) + 3, 0xDEADBEEFu); const auto *text_bytes = reinterpret_cast(text_words.data()); ASSERT_TRUE(patcher.replace_text({text_bytes, text_words.size() * sizeof(uint32_t)})); auto patched_bytes = patcher.emit(); AmdGpuCodeObject patched(patched_bytes.data(), patched_bytes.size()); ASSERT_TRUE(patched.is_valid()); const auto *patched_text = patched.text_sections()[0]; const auto *patched_rodata = find_section(patched, ".rodata"); ASSERT_NE(patched_rodata, nullptr); const int64_t patched_entry_offset = read_kernel_descriptor_entry_offset(patched_rodata->data()); EXPECT_EQ(patched_rodata->vaddr(), original_rodata->vaddr() + padded_file_delta); EXPECT_EQ( static_cast(static_cast(patched_rodata->vaddr()) + patched_entry_offset), patched_text->vaddr()) << "KERNEL_CODE_ENTRY_BYTE_OFFSET is relative to the descriptor address"; EXPECT_EQ(patched_entry_offset, original_entry_offset - static_cast(padded_file_delta)); } TEST(CodeObjectPatcher, ReplaceTextUpdatesRelocationOffsetsIntoMovedSections) { constexpr uint64_t load_align = 0x1000; auto image = make_minimal_amdgpu_elf_with_relocation_after_text(); AmdGpuCodeObject co(image.data(), image.size()); ASSERT_TRUE(co.is_valid()); ASSERT_FALSE(co.text_sections().empty()); CodeObjectPatcher patcher(co); const std::vector text_words(load_align / sizeof(uint32_t) + 3, 0xDEADBEEFu); const auto *text_bytes = reinterpret_cast(text_words.data()); ASSERT_TRUE(patcher.replace_text({text_bytes, text_words.size() * sizeof(uint32_t)})); auto patched_bytes = patcher.emit(); AmdGpuCodeObject patched(patched_bytes.data(), patched_bytes.size()); ASSERT_TRUE(patched.is_valid()); const auto *data = find_section(patched, ".data"); const auto *rela_dyn = find_section(patched, ".rela.dyn"); ASSERT_NE(data, nullptr); ASSERT_NE(rela_dyn, nullptr); ASSERT_EQ(rela_dyn->size(), sizeof(Elf64_Rela)); Elf64_Rela rela{}; std::memcpy(&rela, rela_dyn->data(), sizeof(rela)); EXPECT_EQ(rela.r_offset, data->vaddr()) << "ET_DYN relocation r_offset is the relocated storage address"; } // The r_offset test above moves the relocation's storage. This one moves what the // relocation names: an R_AMDGPU_RELATIVE64 addend pointing into .data resolves to // load_bias + r_addend, so growing .text past its load alignment relocates .data // and the addend has to follow it or the pointer lands short of the moved section. TEST(CodeObjectPatcher, ReplaceTextShiftsRelativeAddendsIntoMovedSections) { constexpr uint64_t load_align = 0x1000; constexpr uint64_t kTextVaddr = 0x1100; constexpr uint64_t kTextSize = 8; constexpr uint64_t kOriginalDataVaddr = kTextVaddr + kTextSize + load_align; auto image = make_minimal_amdgpu_elf_with_relocation_after_text( /*place_reloc_in_text=*/false, rocjitsu::R_AMDGPU_RELATIVE64, static_cast(kOriginalDataVaddr)); AmdGpuCodeObject co(image.data(), image.size()); ASSERT_TRUE(co.is_valid()); ASSERT_FALSE(co.text_sections().empty()); CodeObjectPatcher patcher(co); const std::vector text_words(load_align / sizeof(uint32_t) + 3, 0xDEADBEEFu); const auto *text_bytes = reinterpret_cast(text_words.data()); ASSERT_TRUE(patcher.replace_text({text_bytes, text_words.size() * sizeof(uint32_t)})); auto patched_bytes = patcher.emit(); AmdGpuCodeObject patched(patched_bytes.data(), patched_bytes.size()); ASSERT_TRUE(patched.is_valid()); const auto *data = find_section(patched, ".data"); const auto *rela_dyn = find_section(patched, ".rela.dyn"); ASSERT_NE(data, nullptr); ASSERT_NE(rela_dyn, nullptr); ASSERT_EQ(rela_dyn->size(), sizeof(Elf64_Rela)); ASSERT_GT(data->vaddr(), kOriginalDataVaddr) << "test only proves anything if .data moved"; Elf64_Rela rela{}; std::memcpy(&rela, rela_dyn->data(), sizeof(rela)); EXPECT_EQ(static_cast(rela.r_addend), data->vaddr()) << "RELATIVE64 addend naming a section above .text must move with that section"; } TEST(CodeObjectPatcher, DetectsRelocationsWithinText) { auto safe_image = make_minimal_amdgpu_elf_with_relocation_after_text(/*place_reloc_in_text=*/false); AmdGpuCodeObject safe_co(safe_image.data(), safe_image.size()); ASSERT_TRUE(safe_co.is_valid()); CodeObjectPatcher safe_patcher(safe_co); EXPECT_FALSE(safe_patcher.has_relocations_within_text()); auto text_image = make_minimal_amdgpu_elf_with_relocation_after_text(/*place_reloc_in_text=*/true); AmdGpuCodeObject text_co(text_image.data(), text_image.size()); ASSERT_TRUE(text_co.is_valid()); CodeObjectPatcher text_patcher(text_co); EXPECT_TRUE(text_patcher.has_relocations_within_text()); } TEST(BinaryTranslatorE2E, RejectsRelocationTargetingText) { // A relocation whose place is inside .text cannot be remapped after DBT // relocates instructions, so translation must fail closed and leave the code // object unchanged rather than apply the relocation to the wrong bytes. auto image = make_minimal_amdgpu_elf_with_relocation_after_text(/*place_reloc_in_text=*/true); AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_RDNA4); auto result = translator.translate(source); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(rocjitsu::has_error_containing(result, rocjitsu::DiagnosticKind::Legalization, "relocation place inside .text")); } TEST(CodeObjectPatcher, ClassifiesTextSymbolRelocationCompatibility) { for (uint8_t sym_type : {kElfSymbolTypeFunc, kElfSymbolTypeNone, kElfSymbolTypeObject}) { auto text_image = make_amdgpu_elf_with_symbol_relocation(sym_type, /*defined_in_text=*/true); AmdGpuCodeObject text_co(text_image.data(), text_image.size()); ASSERT_TRUE(text_co.is_valid()) << "sym_type=" << static_cast(sym_type); CodeObjectPatcher text_patcher(text_co); EXPECT_FALSE(text_patcher.has_unsupported_relocation_to_text()) << "zero-addend ordinary text sym_type=" << static_cast(sym_type) << " can follow its relocated st_value"; auto data_image = make_amdgpu_elf_with_symbol_relocation(sym_type, /*defined_in_text=*/false); AmdGpuCodeObject data_co(data_image.data(), data_image.size()); ASSERT_TRUE(data_co.is_valid()) << "sym_type=" << static_cast(sym_type); CodeObjectPatcher data_patcher(data_co); EXPECT_FALSE(data_patcher.has_unsupported_relocation_to_text()) << "data-defined sym_type=" << static_cast(sym_type) << " must be accepted"; } auto section_image = make_amdgpu_elf_with_symbol_relocation(kElfSymbolTypeSection, /*defined_in_text=*/true); AmdGpuCodeObject section_co(section_image.data(), section_image.size()); ASSERT_TRUE(section_co.is_valid()); CodeObjectPatcher section_patcher(section_co); EXPECT_TRUE(section_patcher.has_unsupported_relocation_to_text()) << "STT_SECTION keeps its in-text offset in the relocation addend"; auto addend_image = make_amdgpu_elf_with_symbol_relocation( kElfSymbolTypeFunc, /*defined_in_text=*/true, /*addend=*/4); AmdGpuCodeObject addend_co(addend_image.data(), addend_image.size()); ASSERT_TRUE(addend_co.is_valid()); CodeObjectPatcher addend_patcher(addend_co); EXPECT_TRUE(addend_patcher.has_unsupported_relocation_to_text()) << "a named symbol plus addend can target an independently moved instruction"; // Convert the fixture's explicit-addend RELA record into an implicit-addend // REL record. Without relocation-type-specific decoding, DBT cannot prove // what source text offset is stored at the relocation place. auto rel_image = make_amdgpu_elf_with_symbol_relocation(kElfSymbolTypeFunc, /*defined_in_text=*/true); const auto rel_ehdr = read_elf_struct_for_test(rel_image, 0); auto rel_shdrs = read_elf_array_for_test(rel_image, rel_ehdr.e_shoff, rel_ehdr.e_shnum); const auto rela = std::ranges::find_if( rel_shdrs, [](const Elf64_Shdr §ion) { return section.sh_type == SHT_RELA; }); ASSERT_NE(rela, rel_shdrs.end()); const size_t rela_index = static_cast(rela - rel_shdrs.begin()); const auto rela_record = read_elf_struct_for_test(rel_image, rela->sh_offset); const Elf64_Rel rel_record{.r_offset = rela_record.r_offset, .r_info = rela_record.r_info}; std::memcpy(rel_image.data() + rela->sh_offset, &rel_record, sizeof(rel_record)); rel_shdrs[rela_index].sh_type = SHT_REL; rel_shdrs[rela_index].sh_size = sizeof(Elf64_Rel); rel_shdrs[rela_index].sh_entsize = sizeof(Elf64_Rel); std::memcpy(rel_image.data() + rel_ehdr.e_shoff, rel_shdrs.data(), rel_shdrs.size() * sizeof(Elf64_Shdr)); AmdGpuCodeObject rel_co(rel_image.data(), rel_image.size()); ASSERT_TRUE(rel_co.is_valid()); CodeObjectPatcher rel_patcher(rel_co); EXPECT_TRUE(rel_patcher.has_unsupported_relocation_to_text()) << "REL keeps an implicit addend that generic text-symbol remapping cannot inspect"; } TEST(CodeObjectPatcher, RelocatesReferencedTextSymbolWithExactOffsetMap) { auto image = make_amdgpu_elf_with_symbol_relocation(kElfSymbolTypeFunc, /*defined_in_text=*/true); AmdGpuCodeObject object(image.data(), image.size()); ASSERT_TRUE(object.is_valid()); CodeObjectPatcher patcher(object); const std::array expanded_text = {0xBF800000u, 0xBF800000u, 0xBF800000u, 0xBF800000u}; constexpr std::array mappings = { TextOffsetRelocation{.source_offset = 0, .target_offset = 4}, TextOffsetRelocation{.source_offset = 8, .target_offset = 16}, }; const auto bytes = std::span( reinterpret_cast(expanded_text.data()), sizeof(expanded_text)); ASSERT_TRUE(patcher.replace_text(bytes, mappings)); const auto patched = patcher.emit(); const auto ehdr = read_elf_struct_for_test(patched, 0); const auto shdrs = read_elf_array_for_test(patched, ehdr.e_shoff, ehdr.e_shnum); const auto symtab = std::ranges::find_if( shdrs, [](const Elf64_Shdr §ion) { return section.sh_type == SHT_SYMTAB; }); ASSERT_NE(symtab, shdrs.end()); const auto symbols = read_elf_array_for_test(patched, symtab->sh_offset, symtab->sh_size / sizeof(Elf64_Sym)); ASSERT_GE(symbols.size(), 2u); EXPECT_EQ(symbols[1].st_value, 0x1104u); EXPECT_EQ(symbols[1].st_size, 12u); } TEST(CodeObjectPatcher, RejectsReferencedTextSymbolWithoutExactOffsetMap) { auto image = make_amdgpu_elf_with_symbol_relocation(kElfSymbolTypeFunc, /*defined_in_text=*/true); AmdGpuCodeObject object(image.data(), image.size()); ASSERT_TRUE(object.is_valid()); CodeObjectPatcher patcher(object); const std::array expanded_text = {0xBF800000u, 0xBF800000u, 0xBF800000u}; constexpr std::array incomplete_mapping = { TextOffsetRelocation{.source_offset = 4, .target_offset = 8}, }; const auto bytes = std::span( reinterpret_cast(expanded_text.data()), sizeof(expanded_text)); EXPECT_FALSE(patcher.replace_text(bytes, incomplete_mapping)); } TEST(BinaryTranslatorE2E, RejectsRelocationToTextSectionSymbol) { auto image = make_amdgpu_elf_with_symbol_relocation(kElfSymbolTypeSection, /*defined_in_text=*/true); AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_RDNA4); auto result = translator.translate(source); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(rocjitsu::has_error_containing(result, rocjitsu::DiagnosticKind::Legalization, "unsupported relocation referencing .text")); } // Independently pin the AMDGPU ELF ABI value so the tests below cannot mask a // regression by constructing the same wrong relocation type the patcher checks // for. The fixtures use the production constant (so they never drift from it), // but this assertion anchors that constant to the authoritative ABI number. static_assert(rocjitsu::R_AMDGPU_RELATIVE64 == 13, "R_AMDGPU_RELATIVE64 must be the AMDGPU ELF ABI value 13"); // R_AMDGPU_RELATIVE64 carries symbol index 0, so the target is entirely in the // explicit addend. DBT can remap an in-text addend through its exact offset map. TEST(CodeObjectPatcher, SupportsRelative64AddendIntoText) { // Reference the production ABI constant so the test can never drift from it. constexpr uint32_t kRelative64 = rocjitsu::R_AMDGPU_RELATIVE64; constexpr int64_t kInTextAddend = 0x1100; // == text_vaddr constexpr int64_t kOutOfTextAddend = 0x2108; // == data_vaddr auto in_text = make_amdgpu_elf_with_relative_relocation(kRelative64, kInTextAddend); AmdGpuCodeObject in_text_co(in_text.data(), in_text.size()); ASSERT_TRUE(in_text_co.is_valid()); CodeObjectPatcher in_text_patcher(in_text_co); EXPECT_FALSE(in_text_patcher.has_unsupported_relocation_to_text()) << "RELATIVE64 addend inside .text has explicit remapping semantics"; // The addend place is in .data, so the in-text-place guard must NOT be what // catches it -- the symbol-zero addend check is the code path under test. EXPECT_FALSE(in_text_patcher.has_relocations_within_text()); auto out_of_text = make_amdgpu_elf_with_relative_relocation(kRelative64, kOutOfTextAddend); AmdGpuCodeObject out_co(out_of_text.data(), out_of_text.size()); ASSERT_TRUE(out_co.is_valid()); CodeObjectPatcher out_patcher(out_co); EXPECT_FALSE(out_patcher.has_unsupported_relocation_to_text()) << "RELATIVE64 addend outside .text must be accepted"; } TEST(BinaryTranslator, InlineExpansionAvoidsCaveBranchOverflow) { auto image = make_large_amdgpu_elf_with_waitcnt_entry(); AmdGpuCodeObject co(image.data(), image.size()); ASSERT_TRUE(co.is_valid()); ASSERT_FALSE(co.text_sections().empty()); BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_RDNA4); auto result = translator.translate(co); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); const bool diagnosed = std::any_of( result.diagnostics.begin(), result.diagnostics.end(), [](const TranslationDiagnostic &diagnostic) { return diagnostic.severity == DiagnosticSeverity::Error && diagnostic.kind == DiagnosticKind::ResourceLimit && diagnostic.message.find("branch range") != std::string::npos && diagnostic.message.find("leaving code object unchanged") != std::string::npos; }); EXPECT_FALSE(diagnosed); } TEST(BinaryTranslator, InlineExpansionIgnoresUnreachableTextTail) { auto image = make_large_amdgpu_elf_with_waitcnt_entry(); AmdGpuCodeObject source_layout(image.data(), image.size()); ASSERT_TRUE(source_layout.is_valid()); ASSERT_FALSE(source_layout.text_sections().empty()); const auto *source_text = source_layout.text_sections()[0]; auto *source_words = reinterpret_cast(image.data() + source_text->sectionOffset()); source_words[1] = 0xBF810000u; // CDNA4 s_endpgm; the remaining large tail is unreachable. AmdGpuCodeObject co(image.data(), image.size()); ASSERT_TRUE(co.is_valid()); BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_RDNA4); auto result = translator.translate(co); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); EXPECT_EQ(find_section(translated, ".rj_translations"), nullptr); const auto expected_waitcnt = encode_waitcnt_gfx12(decode_waitcnt_gfx9(0)); ASSERT_FALSE(expected_waitcnt.empty()); const auto *text = translated.text_sections()[0]; ASSERT_GE(text->size(), (expected_waitcnt.size() + 1) * sizeof(uint32_t)); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); for (size_t i = 0; i < expected_waitcnt.size(); ++i) EXPECT_EQ(target_words[i], expected_waitcnt[i]); EXPECT_EQ(target_words[expected_waitcnt.size()], build_s_endpgm(ROCJITSU_CODE_ARCH_RDNA4)); } TEST(BinaryTranslator, SynthesizesKernargPreloadEntrySkipWindow) { auto image = make_large_amdgpu_elf_with_waitcnt_entry(); AmdGpuCodeObject source_layout(image.data(), image.size()); ASSERT_TRUE(source_layout.is_valid()); ASSERT_FALSE(source_layout.text_sections().empty()); const auto *source_rodata = find_section(source_layout, ".rodata"); ASSERT_NE(source_rodata, nullptr); ASSERT_GE(source_rodata->size(), sizeof(rocr::llvm::amdhsa::kernel_descriptor_t)); auto source_kd = read_kernel_descriptor_for_test(image.data() + source_rodata->sectionOffset()); AMDHSA_BITS_SET(source_kd.kernarg_preload, rocr::llvm::amdhsa::KERNARG_PRELOAD_SPEC_LENGTH, 1); write_kernel_descriptor_for_test(image.data() + source_rodata->sectionOffset(), source_kd); const auto *source_text = source_layout.text_sections()[0]; auto *source_words = reinterpret_cast(image.data() + source_text->sectionOffset()); source_words[0] = build_s_branch(63, ROCJITSU_CODE_ARCH_CDNA4); source_words[64] = 0xBF810000u; // CDNA4 s_endpgm at the post-preload body entry. AmdGpuCodeObject co(image.data(), image.size()); ASSERT_TRUE(co.is_valid()); BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(co); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *text = translated.text_sections()[0]; ASSERT_GT(text->size(), kKernargPreloadSkipBytes); const auto *target_words = reinterpret_cast(text->data()); EXPECT_EQ(target_words[0], build_s_branch(64, ROCJITSU_CODE_ARCH_CDNA3)) << "old firmware enters the synthesized compatibility stub, which branches to the " "translated compatibility source entry"; for (size_t i = 1; i < 64; ++i) EXPECT_EQ(target_words[i], build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA3)) << "the synthesized launch window must keep the compatible firmware entry exactly 256 " "bytes after the descriptor entry"; EXPECT_EQ(target_words[64], build_s_branch(1, ROCJITSU_CODE_ARCH_CDNA3)) << "compatible firmware enters at descriptor entry + 256 and branches to the translated " "preloaded-kernarg source entry"; EXPECT_EQ(target_words[65], build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA3)) << "the original compatibility source block is translated in the compact body"; EXPECT_EQ(target_words[66], 0xBF810000u) << "the original compatible-firmware source entry is translated in the compact body"; const auto *target_rodata = find_section(translated, ".rodata"); ASSERT_NE(target_rodata, nullptr); ASSERT_GE(target_rodata->size(), sizeof(rocr::llvm::amdhsa::kernel_descriptor_t)); const auto target_kd = read_kernel_descriptor_for_test(translated.image_data() + target_rodata->sectionOffset()); EXPECT_EQ(target_kd.kernel_code_entry_byte_offset, source_kd.kernel_code_entry_byte_offset) << "the descriptor is redirected to the synthesized compatibility entry; compatible " "firmware still reaches the synthesized +256 entry by adding the ABI skip"; } TEST(BinaryTranslator, SynthesizesKernargPreloadEntrySkipWindowWithDescriptorPrologue) { constexpr uint64_t kSourceEntryBytes = 512; constexpr size_t kSourceEntryWord = kSourceEntryBytes / sizeof(uint32_t); constexpr size_t kSourcePreloadEntryWord = (kSourceEntryBytes + kKernargPreloadSkipBytes) / sizeof(uint32_t); constexpr uint16_t kScalarOperandTtmpBase = 108; constexpr uint16_t kTtmpRdna4GridX = 9; std::vector words(kSourcePreloadEntryWord + 1, build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4)); words[0] = build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4); words[kSourceEntryWord] = build_s_branch(63, ROCJITSU_CODE_ARCH_CDNA4); words[kSourcePreloadEntryWord] = build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4); auto image = make_minimal_amdgpu_elf_with_descriptor_after_text(words); enable_workgroup_id_x_sgpr(image); AmdGpuCodeObject source_layout(image.data(), image.size()); ASSERT_TRUE(source_layout.is_valid()); ASSERT_FALSE(source_layout.text_sections().empty()); const auto *source_text = source_layout.text_sections()[0]; const auto *source_rodata = find_section(source_layout, ".rodata"); ASSERT_NE(source_rodata, nullptr); ASSERT_GE(source_rodata->size(), sizeof(rocr::llvm::amdhsa::kernel_descriptor_t)); auto source_kd = read_kernel_descriptor_for_test(image.data() + source_rodata->sectionOffset()); AMDHSA_BITS_SET(source_kd.kernarg_preload, rocr::llvm::amdhsa::KERNARG_PRELOAD_SPEC_LENGTH, 1); source_kd.kernel_code_entry_byte_offset = static_cast(source_text->vaddr() + kSourceEntryBytes) - static_cast(source_rodata->vaddr()); write_kernel_descriptor_for_test(image.data() + source_rodata->sectionOffset(), source_kd); AmdGpuCodeObject co(image.data(), image.size()); ASSERT_TRUE(co.is_valid()); BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_RDNA4); auto result = translator.translate(co); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *text = translated.text_sections()[0]; ASSERT_GT(text->size(), kKernargPreloadSkipBytes + 3 * sizeof(uint32_t)); const auto *target_words = reinterpret_cast(text->data()); const uint32_t workgroup_id_x_prologue = build_s_mov_b32(0, kScalarOperandTtmpBase + kTtmpRdna4GridX, ROCJITSU_CODE_ARCH_RDNA4); const uint32_t prologue_delay = build_s_delay_alu(kDelayAluSaluDep1, ROCJITSU_CODE_ARCH_RDNA4); const auto expect_launch_stub = [&](size_t word_index, int16_t branch_offset) { EXPECT_EQ(target_words[word_index], workgroup_id_x_prologue) << "the synthesized kernarg-preload launch stub must materialize descriptor ABI SGPRs " "before branching into the relocated body"; EXPECT_EQ(target_words[word_index + 1], prologue_delay) << "the synthesized launch stub must preserve scalar producer/consumer hazards"; EXPECT_EQ(target_words[word_index + 2], build_s_branch(branch_offset, ROCJITSU_CODE_ARCH_RDNA4)) << "the synthesized launch stub branches only after the descriptor ABI prologue"; }; expect_launch_stub(0, 64); expect_launch_stub(kKernargPreloadSkipBytes / sizeof(uint32_t), 1); EXPECT_EQ(target_words[67], build_s_branch(0, ROCJITSU_CODE_ARCH_RDNA4)) << "the original compatibility source entry is translated in the compact body"; EXPECT_EQ(target_words[68], build_s_endpgm(ROCJITSU_CODE_ARCH_RDNA4)) << "the original preloaded-kernarg source entry is translated in the compact body"; const auto *target_rodata = find_section(translated, ".rodata"); ASSERT_NE(target_rodata, nullptr); ASSERT_GE(target_rodata->size(), sizeof(rocr::llvm::amdhsa::kernel_descriptor_t)); const auto target_kd = read_kernel_descriptor_for_test(translated.image_data() + target_rodata->sectionOffset()); const int64_t target_entry_text_offset = static_cast(target_rodata->vaddr()) + target_kd.kernel_code_entry_byte_offset - static_cast(text->vaddr()); EXPECT_EQ(target_entry_text_offset, 0) << "the descriptor must be redirected from the moved source entry to the synthesized " "compatibility launch stub"; EXPECT_NE(target_kd.kernel_code_entry_byte_offset, source_kd.kernel_code_entry_byte_offset) << "the source descriptor entry is deliberately nonzero, so this assertion proves the " "descriptor was repointed rather than passing because both entries were zero"; } TEST(InstructionBuilder, PatchPcrelBranchOffsetInRange) { const uint32_t source_word = build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA4); auto inst = decode_one(source_word, ROCJITSU_CODE_ARCH_CDNA4); ASSERT_NE(inst, nullptr); std::array words = {build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA3)}; EXPECT_TRUE(patch_pcrel_branch_offset(*inst, words, -8, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(words[0], build_s_branch(-2, ROCJITSU_CODE_ARCH_CDNA3)); } TEST(InstructionBuilder, PatchPcrelBranchOffsetRejectsOutOfRange) { const uint32_t source_word = build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA4); auto inst = decode_one(source_word, ROCJITSU_CODE_ARCH_CDNA4); ASSERT_NE(inst, nullptr); std::array words = {build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA3)}; EXPECT_FALSE(patch_pcrel_branch_offset(*inst, words, 32768LL * 4, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(words[0], build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_FALSE(patch_pcrel_branch_offset(*inst, words, -32769LL * 4, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(words[0], build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA3)); } TEST(InstructionBuilder, PatchPcrelBranchOffsetRejectsNonBranch) { const uint32_t source_word = build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4); auto inst = decode_one(source_word, ROCJITSU_CODE_ARCH_CDNA4); ASSERT_NE(inst, nullptr); std::array words = {build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA3)}; EXPECT_FALSE(patch_pcrel_branch_offset(*inst, words, 4, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(words[0], build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA3)); } TEST(InstructionBuilder, PatchPcrelBranchOffsetRejectsMisalignedDelta) { const uint32_t source_word = build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA4); auto inst = decode_one(source_word, ROCJITSU_CODE_ARCH_CDNA4); ASSERT_NE(inst, nullptr); std::array words = {build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA3)}; EXPECT_FALSE(patch_pcrel_branch_offset(*inst, words, 2, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(words[0], build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA3)); } /// @brief One incomplete-consumer body with a second, independent PC builder. /// /// @details The consumer's own fact is incomplete: one path builds a concrete PC /// in s[8:9], the other reaches the setpc with the pair unconstrained. Whether /// the translation may keep that dynamic transfer depends entirely on the second /// builder in s[10:11]: @p bypass_builder_delta decides whether its value lands /// on a relocatable block start or on an address DBT cannot move. std::vector make_incomplete_indirect_consumer_body(uint32_t bypass_builder_delta) { constexpr uint16_t kPcSreg = 8; constexpr uint16_t kBypassSreg = 10; constexpr uint32_t kLiteralOperand = 255; constexpr uint32_t kInlineInt0 = 128; return { pack_sopp(5, 5), // 0x00: cbranch scc0 -> bypass at 0x18. pack_sop1(0x1c, kPcSreg, 0), // 0x04: s_getpc_b64. pack_sop2(0, kPcSreg, kPcSreg, kLiteralOperand), // 0x08: s_add_u32. 40, // 0x0c: target delta -> 0x30. pack_sop2(4, kPcSreg + 1, kPcSreg + 1, kInlineInt0), // 0x10: s_addc_u32. build_s_branch(5, ROCJITSU_CODE_ARCH_CDNA4), // 0x14 -> consumer at 0x2c. pack_sop1(0x1c, kBypassSreg, 0), // 0x18: bypass-path s_getpc_b64. pack_sop2(0, kBypassSreg, kBypassSreg, kLiteralOperand), // 0x1c: s_add_u32. bypass_builder_delta, // 0x20. pack_sop2(4, kBypassSreg + 1, kBypassSreg + 1, kInlineInt0), // 0x24: s_addc_u32. build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x28: closes the chain. pack_sop1(0x1d, 0, kPcSreg), // 0x2c: joined consumer setpc. build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), // 0x30: builder target. }; } TEST(BinaryTranslatorE2E, IncompleteIndirectConsumerFailsClosed) { // The bypass builder computes 0x1c + 0x100000, far outside .text. That value // is a PC-derived address DBT cannot rewrite to a relocated one, so the scope // still contains a potential stale PC. The unconstrained path into the setpc // may hold exactly such a value, so the whole translation must fail closed // rather than relocate only the known builder and leave the dynamic transfer // pointing at stale bytes. auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( make_incomplete_indirect_consumer_body(0x100000)); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); EXPECT_FALSE(result.ok()); ASSERT_FALSE(result.diagnostics.empty()); EXPECT_NE(result.diagnostics.front().message.find("unconstrained predecessor path"), std::string::npos) << result.diagnostics.front().message; EXPECT_EQ(result.elf_bytes, image) << "fail-closed must leave the object unchanged"; } TEST(BinaryTranslatorE2E, IncompleteIndirectConsumerFailsClosedOnOpenPcBuilderChain) { constexpr uint16_t kPcSreg = 8; constexpr uint16_t kOpenSreg = 10; constexpr uint32_t kLiteralOperand = 255; constexpr uint32_t kInlineInt0 = 128; // The s[10:11] getpc at 0x14 is the last instruction of its block, so nothing // in this block proves what its value becomes: this is exactly the shape whose // delta add lives in a successor, where an unmodeled write would leave the // original delta in place. The scope therefore cannot be proven free of stale // PC values and the incomplete consumer must keep failing closed. std::vector words = { pack_sopp(5, 5), // 0x00: cbranch scc0 -> bypass at 0x18. pack_sop1(0x1c, kPcSreg, 0), // 0x04: s_getpc_b64. pack_sop2(0, kPcSreg, kPcSreg, kLiteralOperand), // 0x08: s_add_u32. 36, // 0x0c: target delta -> 0x2c. pack_sop2(4, kPcSreg + 1, kPcSreg + 1, kInlineInt0), // 0x10: s_addc_u32 (chain closed). pack_sop1(0x1c, kOpenSreg, 0), // 0x14: bare s_getpc_b64 at block end. pack_sop2(0, kOpenSreg, kOpenSreg, kLiteralOperand), // 0x18: its s_add_u32, next block. 12, // 0x1c. pack_sop2(4, kOpenSreg + 1, kOpenSreg + 1, kInlineInt0), // 0x20: s_addc_u32. pack_sop1(0x1d, 0, kPcSreg), // 0x24: joined consumer setpc. build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x28: not a target. build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), // 0x2c: builder target. }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); EXPECT_FALSE(result.ok()); ASSERT_FALSE(result.diagnostics.empty()); EXPECT_NE(result.diagnostics.front().message.find("unconstrained predecessor path"), std::string::npos) << result.diagnostics.front().message; EXPECT_EQ(result.elf_bytes, image) << "fail-closed must leave the object unchanged"; } TEST(BinaryTranslatorE2E, IncompleteIndirectConsumerTranslatesWhenScopeHasNoStalePcValues) { constexpr uint16_t kPcSreg = 8; constexpr uint16_t kBypassSreg = 10; // Same body, except the bypass builder now targets 0x1c + 0x14 = 0x30, the // same relocatable block start as the consumed builder. Every PC-derived value // in the scope can therefore be rewritten to its relocated address, so no path // into the setpc can deliver a stale one and the dynamic transfer is safe to // keep even though the consumer's own fact stays incomplete. auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( make_incomplete_indirect_consumer_body(0x14)); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); // Assert the relocated deltas structurally instead of by absolute index: for a // rewritten builder, getpc_next + delta must land on the relocated s_endpgm. const auto expect_builder_targets_endpgm = [&](uint16_t pc_sreg) { const uint32_t getpc = pack_sop1(0x1c, pc_sreg, 0); size_t found = 0; for (size_t i = 0; i + 2 < word_count; ++i) { if (target_words[i] != getpc) continue; ++found; const uint64_t target = (i + 1) * sizeof(uint32_t) + target_words[i + 2]; ASSERT_EQ(target % sizeof(uint32_t), 0u); ASSERT_LT(target / sizeof(uint32_t), word_count); EXPECT_EQ(target_words[target / sizeof(uint32_t)], build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA3)) << "builder for s[" << pc_sreg << ":" << pc_sreg + 1 << "] was not rewritten to its relocated target"; } EXPECT_EQ(found, 1u); }; expect_builder_targets_endpgm(kPcSreg); // The unconsumed builder must be relocated too: it is the producer whose // stale value the fail-closed path exists to prevent. expect_builder_targets_endpgm(kBypassSreg); EXPECT_NE(std::find(target_words, target_words + word_count, pack_sop1(0x1d, 0, kPcSreg)), target_words + word_count) << "an incomplete consumer must keep its dynamic transfer, not become a direct window"; } TEST(BinaryTranslatorE2E, IncompleteIndirectConsumerFailsClosedOnNonContiguousBuilderRange) { constexpr uint16_t kPcSreg = 8; constexpr uint16_t kBypassSreg = 10; constexpr uint16_t kUnrelatedSreg = 20; constexpr uint32_t kLiteralOperand = 255; constexpr uint32_t kInlineInt0 = 128; // The bypass builder in s[10:11] targets the relocatable block start at 0x34, // so on its own it would satisfy the whole-scope proof. But an unrelated // s_mov_b32 sits between its s_add_u32 and s_addc_u32: the pair survives that // move (it writes s20, not the pair), so the recovered builder range spans it. // patch_recovered_builder_fixups NOPs the whole range, which would erase the // move. The proof must fail closed on the non-contiguous range, keeping the // incomplete consumer's original refusal and leaving the object unchanged. std::vector words = { pack_sopp(5, 6), // 0x00: cbranch scc0 -> bypass at 0x1c. pack_sop1(0x1c, kPcSreg, 0), // 0x04: s_getpc_b64. pack_sop2(0, kPcSreg, kPcSreg, kLiteralOperand), // 0x08: s_add_u32. 44, // 0x0c: target delta -> 0x34. pack_sop2(4, kPcSreg + 1, kPcSreg + 1, kInlineInt0), // 0x10: s_addc_u32. build_s_branch(6, ROCJITSU_CODE_ARCH_CDNA4), // 0x14 -> consumer at 0x30. build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x18: padding before bypass. pack_sop1(0x1c, kBypassSreg, 0), // 0x1c: bypass s_getpc_b64. pack_sop2(0, kBypassSreg, kBypassSreg, kLiteralOperand), // 0x20: s_add_u32. 20, // 0x24: -> 0x20 + 20 = 0x34. build_s_mov_b32(kUnrelatedSreg, 0, ROCJITSU_CODE_ARCH_CDNA4), // 0x28: unrelated write, in range. pack_sop2(4, kBypassSreg + 1, kBypassSreg + 1, kInlineInt0), // 0x2c: s_addc_u32. pack_sop1(0x1d, 0, kPcSreg), // 0x30: joined consumer setpc. build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), // 0x34: builder target. }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); EXPECT_FALSE(result.ok()); ASSERT_FALSE(result.diagnostics.empty()); EXPECT_NE(result.diagnostics.front().message.find("unconstrained predecessor path"), std::string::npos) << result.diagnostics.front().message; EXPECT_EQ(result.elf_bytes, image) << "a non-contiguous builder range must fail closed, preserving the unrelated instruction"; } TEST(BinaryTranslatorE2E, IncompleteConsumerFailsClosedAtKernargPreloadFirmwareEntry) { // The same incomplete-consumer body that translates when rooted at the ordinary // kernel entry (IncompleteIndirectConsumerTranslatesWhenScopeHasNoStalePcValues) // must instead fail closed when its scope root is the kernarg-preload firmware // entry (descriptor entry + 256). Before that entry runs, the command processor // copies caller-controlled kernarg words into user SGPRs, so the pair the // consumer reads on its unconstrained path can hold an original .text pointer // that no in-scope builder or relocation rewrites. The entry-state gate must not // treat the +256 entry as a safe root. constexpr size_t kPreloadEntryWord = kKernargPreloadSkipBytes / sizeof(uint32_t); // Descriptor entry (word 0) is a trivial ordinary path. The incomplete-consumer // body is placed at the +256 preload firmware entry, so it is reachable only // from that root. std::vector words(kPreloadEntryWord, build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4)); words[0] = build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4); const auto body = make_incomplete_indirect_consumer_body(0x14); words.insert(words.end(), body.begin(), body.end()); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); enable_workgroup_id_x_sgpr(image); rocjitsu::AmdGpuCodeObject source_layout(image.data(), image.size()); ASSERT_TRUE(source_layout.is_valid()); const auto *source_rodata = find_section(source_layout, ".rodata"); ASSERT_NE(source_rodata, nullptr); auto source_kd = read_kernel_descriptor_for_test(image.data() + source_rodata->sectionOffset()); AMDHSA_BITS_SET(source_kd.kernarg_preload, rocr::llvm::amdhsa::KERNARG_PRELOAD_SPEC_LENGTH, 1); write_kernel_descriptor_for_test(image.data() + source_rodata->sectionOffset(), source_kd); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); EXPECT_FALSE(result.ok()); ASSERT_FALSE(result.diagnostics.empty()); EXPECT_NE(result.diagnostics.front().message.find("unconstrained predecessor path"), std::string::npos) << result.diagnostics.front().message; EXPECT_EQ(result.elf_bytes, image) << "fail-closed must leave the object unchanged"; } // The external-entry soundness gate (rejecting relocation-table-dispatched // callees as unconstrained roots) is unit-tested directly in // analysis/liveness_test.cpp (BinaryTranslatorInternal.ScopeRootsReject // RelocationTableCallee): reaching its rejecting branch end-to-end would require // a descriptor-bearing relocation-table dispatch fixture, whereas the pure gate // exercises both the accept and reject paths precisely. The entry-state ACCEPT // path is already covered end-to-end by // IncompleteIndirectConsumerTranslatesWhenScopeHasNoStalePcValues above. } // namespace } // namespace rocjitsu namespace cdna3 = rocjitsu::cdna3; namespace cdna4 = rocjitsu::cdna4; namespace gfx1250 = rocjitsu::gfx1250; namespace rdna4 = rocjitsu::rdna4; constexpr uint16_t kGfx1250WmmaCompletionWaitImmediate = 0x0f9f; constexpr auto kGfx1250WmmaCompletionWait = gfx1250::build_sopp(gfx1250::kSWaitAluSopp, {.simm16 = kGfx1250WmmaCompletionWaitImmediate}); /// @brief Build explicit gfx1250 revision options without partial aggregate /// initializers, which are rejected by the test suite's warning policy. rocjitsu::BinaryTranslatorOptions gfx1250_revision_options(rocjitsu::ProcessorRevision input_revision, rocjitsu::ProcessorRevision output_revision) { rocjitsu::BinaryTranslatorOptions options; options.input_revision = input_revision; options.output_revision = output_revision; return options; } constexpr uint16_t kAnyExpectedField = 0xffff; enum class ExpectedCdna3Kind { Vop3, Vop3p, Vop2, Vop1, Sop2, Sop1, Vop3pMfma, Ds, Mubuf, Sopp, }; struct ExpectedCdna3Inst { ExpectedCdna3Kind kind = ExpectedCdna3Kind::Vop3; uint16_t op = 0; uint16_t vdst = kAnyExpectedField; uint16_t acc = kAnyExpectedField; uint16_t acc_cd = kAnyExpectedField; uint16_t src0 = kAnyExpectedField; uint16_t src1 = kAnyExpectedField; uint16_t src2 = kAnyExpectedField; uint16_t data0 = kAnyExpectedField; uint16_t vdata = kAnyExpectedField; }; struct Cdna4ToCdna3SemanticRuleCase { const char *name = ""; uint16_t encoding_id = 0; uint16_t opcode = 0; std::array words{}; std::vector expected{}; size_t word_count = 2; }; ExpectedCdna3Inst expect_vop3(uint16_t op) { ExpectedCdna3Inst inst{}; inst.kind = ExpectedCdna3Kind::Vop3; inst.op = op; return inst; } ExpectedCdna3Inst expect_vop3(uint16_t op, uint16_t vdst, uint16_t src0, uint16_t src1, uint16_t src2 = kAnyExpectedField) { ExpectedCdna3Inst inst{}; inst.kind = ExpectedCdna3Kind::Vop3; inst.op = op; inst.vdst = vdst; inst.src0 = src0; inst.src1 = src1; inst.src2 = src2; return inst; } ExpectedCdna3Inst expect_vop3p(uint16_t op) { ExpectedCdna3Inst inst{}; inst.kind = ExpectedCdna3Kind::Vop3p; inst.op = op; return inst; } ExpectedCdna3Inst expect_vop2(uint16_t op) { ExpectedCdna3Inst inst{}; inst.kind = ExpectedCdna3Kind::Vop2; inst.op = op; return inst; } ExpectedCdna3Inst expect_vop1(uint16_t op) { ExpectedCdna3Inst inst{}; inst.kind = ExpectedCdna3Kind::Vop1; inst.op = op; return inst; } ExpectedCdna3Inst expect_sop2(uint16_t op) { ExpectedCdna3Inst inst{}; inst.kind = ExpectedCdna3Kind::Sop2; inst.op = op; return inst; } ExpectedCdna3Inst expect_sop1(uint16_t op) { ExpectedCdna3Inst inst{}; inst.kind = ExpectedCdna3Kind::Sop1; inst.op = op; return inst; } ExpectedCdna3Inst expect_mfma(uint16_t op, uint16_t vdst, uint16_t acc_cd, uint16_t src0, uint16_t src1, uint16_t src2, uint16_t acc = 0) { ExpectedCdna3Inst inst{}; inst.kind = ExpectedCdna3Kind::Vop3pMfma; inst.op = op; inst.vdst = vdst; inst.acc = acc; inst.acc_cd = acc_cd; inst.src0 = src0; inst.src1 = src1; inst.src2 = src2; return inst; } ExpectedCdna3Inst expect_ds(uint16_t op, uint16_t data0 = kAnyExpectedField) { ExpectedCdna3Inst inst{}; inst.kind = ExpectedCdna3Kind::Ds; inst.op = op; inst.data0 = data0; return inst; } ExpectedCdna3Inst expect_mubuf(uint16_t op, uint16_t vdata = kAnyExpectedField) { ExpectedCdna3Inst inst{}; inst.kind = ExpectedCdna3Kind::Mubuf; inst.op = op; inst.vdata = vdata; return inst; } ExpectedCdna3Inst expect_sopp(uint16_t op) { ExpectedCdna3Inst inst{}; inst.kind = ExpectedCdna3Kind::Sopp; inst.op = op; return inst; } std::vector expected_cdna3_bitop3_sequence(bool b16) { // Truth table 0xde lowers to S2 ^ S1 ^ (S1 & S2) ^ S0 ^ (S0 & S1). std::vector expected = { expect_vop3(cdna3::kVMovB32Vop3), // v_mov_b32 expect_vop3(cdna3::kVXorB32Vop3), // v_xor_b32 expect_vop3(cdna3::kVAndB32Vop3), // v_and_b32 expect_vop3(cdna3::kVXorB32Vop3), // v_xor_b32 expect_vop3(cdna3::kVXorB32Vop3), // v_xor_b32 expect_vop3(cdna3::kVAndB32Vop3), // v_and_b32 expect_vop3(cdna3::kVXorB32Vop3), // v_xor_b32 }; if (b16) { expected.push_back(expect_vop3(cdna3::kVLshlrevB32Vop3)); // v_lshlrev_b32 expected.push_back(expect_vop3(cdna3::kVLshrrevB32Vop3)); // v_lshrrev_b32 } expected.push_back( expect_vop3(cdna3::kVMovB32Vop3)); // v_mov_b32 copy scratch accumulator to vdst. return expected; } std::vector expected_cdna3_bitop3_fast_sequence(uint16_t op, bool b16) { std::vector expected = {expect_vop3(op)}; if (b16) { // Compact V_BITOP3_B16 fast paths still need the same high-half cleanup as // the generic ANF expansion; otherwise the CDNA3 B32 op leaves stale bits // above the architectural 16-bit result. expected.push_back(expect_vop3(cdna3::kVLshlrevB32Vop3)); // v_lshlrev_b32 expected.push_back(expect_vop3(cdna3::kVLshrrevB32Vop3)); // v_lshrrev_b32 } return expected; } std::vector expected_cdna3_mfma_sequence(uint16_t narrow_op, uint16_t src2 = 128, uint16_t source_acc = 0) { // Wide-K MFMA lowering is intentionally high-half first, then low-half. The // order is part of the layout contract verified against MI300X Qwen q_proj. return { expect_mfma(narrow_op, 0, 1, 258, 262, src2, source_acc), expect_mfma(narrow_op, 0, 1, 256, 260, 256, source_acc), }; } std::vector expected_cdna3_buffer_load_lds_sequence(uint16_t mubuf_op, uint16_t ds_op, uint16_t scratch_data = kAnyExpectedField) { return { expect_sop1(cdna3::kSMovB64), // s_mov_b64 save EXEC. expect_sop1(cdna3::kSMovB32), // s_mov_b32 exec_lo, -1. expect_sop1(cdna3::kSMovB32), // s_mov_b32 exec_hi, -1. expect_vop3(cdna3::kVMbcntLoU32B32Vop3), // v_mbcnt_lo_u32_b32 expect_vop3(cdna3::kVMbcntHiU32B32Vop3), // v_mbcnt_hi_u32_b32 expect_vop3(cdna3::kVLshlrevB32Vop3), // v_lshlrev_b32 lane_id, 4 expect_vop3(cdna3::kVAddU32Vop3), // v_add_u32 m0, lane_offset expect_sop1(cdna3::kSMovB64), // s_mov_b64 restore EXEC. expect_mubuf(mubuf_op, scratch_data), // buffer_load_dword{x3,x4} into scratch VGPRs. expect_sopp(cdna3::kSWaitcntSopp), // s_waitcnt 0 before consuming VMEM data. expect_ds(ds_op, scratch_data), // ds_write_b32/b96/b128 expect_sopp(cdna3::kSWaitcntSopp), // s_waitcnt lgkmcnt(0) for the explicit DS write. }; } std::vector expected_cdna3_permlane32_swap_sequence() { return { expect_sop1(cdna3::kSMovB64), // s_mov_b64 save EXEC. expect_sop1(cdna3::kSMovB32), // s_mov_b32 exec_lo, -1. expect_sop1(cdna3::kSMovB32), // s_mov_b32 exec_hi, -1. expect_vop3(cdna3::kVMbcntLoU32B32Vop3), // v_mbcnt_lo_u32_b32 expect_vop3(cdna3::kVMbcntHiU32B32Vop3), // v_mbcnt_hi_u32_b32 expect_vop3(cdna3::kVXorB32Vop3), // v_xor_b32 lane, 32. expect_vop3(cdna3::kVLshlrevB32Vop3), // v_lshlrev_b32 byte address. expect_ds(cdna3::kDsBpermuteB32Ds), // ds_bpermute_b32 from old vdst high half. expect_ds(cdna3::kDsBpermuteB32Ds), // ds_bpermute_b32 from old src low half. expect_sopp(cdna3::kSWaitcntSopp), // s_waitcnt lgkmcnt(0). expect_sop1(cdna3::kSMovB32), // s_mov_b32 exec_lo, low-half mask. expect_sop1(cdna3::kSMovB32), // s_mov_b32 exec_hi, low-half mask. expect_vop3(cdna3::kVMovB32Vop3), // v_mov_b32 src <- old vdst high. expect_sop1(cdna3::kSMovB32), // s_mov_b32 exec_lo, high-half mask. expect_sop1(cdna3::kSMovB32), // s_mov_b32 exec_hi, high-half mask. expect_vop3(cdna3::kVMovB32Vop3), // v_mov_b32 vdst <- old src low. expect_sop1(cdna3::kSMovB64), // s_mov_b64 restore EXEC. }; } std::vector expected_cdna3_raw_b16_pack_sequence() { return { expect_vop3(cdna3::kVMovB32Vop3), // v_mov_b32 -1 expect_vop3(cdna3::kVLshrrevB32Vop3), // v_lshrrev_b32 16, mask expect_vop3(cdna3::kVAndB32Vop3), // v_and_b32 low half expect_vop3(cdna3::kVAndB32Vop3), // v_and_b32 high half expect_vop3(cdna3::kVLshlrevB32Vop3), // v_lshlrev_b32 16, high half expect_vop3(cdna3::kVOrB32Vop3), // v_or_b32 }; } std::vector expected_cdna3_cvt_pk_f16_f32_sequence() { return { expect_vop3(cdna3::kVCvtF16F32Vop3), // v_cvt_f16_f32 low half into scratch. expect_vop3(cdna3::kVCvtF16F32Vop3), // v_cvt_f16_f32 high half into scratch. expect_vop3(cdna3::kVLshlrevB32Vop3), // v_lshlrev_b32 16, high half. expect_vop3(cdna3::kVOrB32Vop3), // v_or_b32 pack low/high halves into vdst. }; } std::vector expected_cdna3_cvt_pk_bf16_f32_sequence() { // Per-half NaN-safe RNE lowering (emit_cdna3_f32_to_bf16_rne): compute the // rounded value and the NaN-preserving value in parallel, then blend with an // exponent-all-ones mask via v_bfi_b32. See semantic/cdna4_to_cdna3.cpp. auto half = []() { return std::vector{ expect_vop3(cdna3::kVMovB32Vop3), // f = source bits into scratch. expect_vop3(cdna3::kVLshrrevB32Vop3), // (f >> 16) expect_vop3(cdna3::kVAndB32Vop3), // & 1 -> rounding lsb expect_vop2(cdna3::kVAddU32Vop2), // + 0x7fff (rounding bias) expect_vop3(cdna3::kVAddU32Vop3), // f + bias -> rounded (t0) expect_vop3(cdna3::kVBfeU32Vop3), // f & 0xffff -> low bits expect_vop3(cdna3::kVMinU32Vop3), // min(1, low) -> nonzero flag expect_vop3(cdna3::kVLshlrevB32Vop3), // flag << 16 expect_vop3(cdna3::kVOrB32Vop3), // f | (flag<<16) -> NaN-preserving (t1) expect_vop2(cdna3::kVAndB32Vop2), // f & 0x7f800000 (exponent field) expect_vop2(cdna3::kVXorB32Vop2), // ^ 0x7f800000 -> 0 iff exp all-ones expect_vop3(cdna3::kVSubU32Vop3), // - 1 expect_vop3(cdna3::kVLshrrevB32Vop3), // >> 31 expect_vop3(cdna3::kVSubU32Vop3), // 0 - x -> all-ones mask (t2) expect_vop3(cdna3::kVBfiB32Vop3), // bfi(mask, t1, t0) -> select expect_vop3(cdna3::kVLshrrevB32Vop3), // >> 16 -> BF16 half }; }; std::vector expected = half(); const std::vector high = half(); expected.insert(expected.end(), high.begin(), high.end()); expected.push_back(expect_vop3(cdna3::kVLshlrevB32Vop3)); // high half into position. expected.push_back(expect_vop3(cdna3::kVOrB32Vop3)); // pack low/high into vdst. return expected; } std::vector expected_cdna3_cvt_f32_bf16_sequence() { return { expect_vop3( cdna3::kVLshlrevB32Vop3), // v_lshlrev_b32 16, src; BF16 bits become FP32 high half. }; } std::vector expected_cdna3_cvt_f32_bf16_sdwa_word0_sequence() { return { expect_vop3(cdna3::kVLshlrevB32Vop3, 4, rocjitsu::scalar_positive_inline_u32(16), 256 + 4), }; } std::vector expected_cdna3_cvt_f32_bf16_sdwa_word1_sequence() { return { expect_vop3(cdna3::kVLshrrevB32Vop3, 4, rocjitsu::scalar_positive_inline_u32(16), 256 + 76), expect_vop3(cdna3::kVLshlrevB32Vop3, 4, rocjitsu::scalar_positive_inline_u32(16), 256 + 4), }; } std::vector expected_cdna3_dot2_f32_bf16_sequence(uint8_t vdst, uint16_t src0, uint16_t src1, uint16_t src2, uint8_t scratch_base, uint8_t op_sel = 0, uint8_t op_sel_hi = 0, uint8_t neg = 0, uint8_t neg_hi = 0, uint8_t op_sel_hi_2 = 0, bool clamp = false) { const uint8_t a_lo = scratch_base; const uint8_t b_lo = static_cast(scratch_base + 1); const uint8_t a_hi = static_cast(scratch_base + 2); const uint8_t b_hi = static_cast(scratch_base + 3); std::vector expected; auto expect_widen = [&](uint8_t dst, uint16_t src, bool high_half, bool negate) { if (high_half) { expected.push_back( expect_vop3(cdna3::kVLshrrevB32Vop3, dst, rocjitsu::scalar_positive_inline_u32(16), src)); expected.push_back(expect_vop3(cdna3::kVLshlrevB32Vop3, dst, rocjitsu::scalar_positive_inline_u32(16), 256 + dst)); } else { expected.push_back( expect_vop3(cdna3::kVLshlrevB32Vop3, dst, rocjitsu::scalar_positive_inline_u32(16), src)); } if (negate) expected.push_back(expect_vop2(cdna3::kVXorB32Vop2)); // v_xor_b32 literal flips FP32 sign. }; expect_widen(a_lo, src0, (op_sel & 0x1u) != 0, (neg & 0x1u) != 0); expect_widen(b_lo, src1, (op_sel & 0x2u) != 0, (neg & 0x2u) != 0); expect_widen(a_hi, src0, op_sel_hi_2 != 0, (neg_hi & 0x1u) != 0); expect_widen(b_hi, src1, (op_sel_hi & 0x2u) != 0, (neg_hi & 0x2u) != 0); expected.push_back(expect_vop3(cdna3::kVMulF32Vop3, a_lo, 256 + a_lo, 256 + b_lo)); expected.push_back(expect_vop3(cdna3::kVMulF32Vop3, a_hi, 256 + a_hi, 256 + b_hi)); expected.push_back(expect_vop3(cdna3::kVAddF32Vop3, a_lo, 256 + a_lo, 256 + a_hi)); expected.push_back(expect_vop3(cdna3::kVAddF32Vop3, vdst, 256 + a_lo, src2)); if (clamp) expected.push_back( expect_vop3(cdna3::kVMed3F32Vop3, vdst, 256 + vdst, 128, 242)); // v_med3_f32 clamps [0,1]. return expected; } std::vector expected_cdna3_ds_read_b64_tr_b16_sequence(bool acc_dst = false) { std::vector expected = { expect_sop1(cdna3::kSMovB64), // s_mov_b64 save EXEC. expect_sop1(cdna3::kSMovB32), // s_mov_b32 exec_lo, all lanes. expect_sop1(cdna3::kSMovB32), // s_mov_b32 exec_hi, all lanes. expect_ds(cdna3::kDsReadB64Ds), // ds_read_b64 expect_sopp(cdna3::kSWaitcntSopp), // s_waitcnt lgkmcnt(0) expect_vop3(cdna3::kVMbcntLoU32B32Vop3), // v_mbcnt_lo_u32_b32 expect_vop3(cdna3::kVMbcntHiU32B32Vop3), // v_mbcnt_hi_u32_b32 expect_vop3(cdna3::kVAndB32Vop3), // v_and_b32 expect_vop3(cdna3::kVLshlrevB32Vop3), // v_lshlrev_b32 expect_vop3(cdna3::kVAndB32Vop3), // v_and_b32 expect_vop3(cdna3::kVLshlrevB32Vop3), // v_lshlrev_b32 expect_vop3(cdna3::kVAndB32Vop3), // v_and_b32 expect_vop3(cdna3::kVOrB32Vop3), // v_or_b32 expect_vop3(cdna3::kVAddU32Vop3), // v_add_u32 expect_vop3(cdna3::kVLshlrevB32Vop3), // v_lshlrev_b32 expect_vop3(cdna3::kVOrB32Vop3), // v_or_b32 expect_ds(cdna3::kDsBpermuteB32Ds), // ds_bpermute_b32 expect_ds(cdna3::kDsBpermuteB32Ds), // ds_bpermute_b32 expect_sopp(cdna3::kSWaitcntSopp), // s_waitcnt lgkmcnt(0) expect_vop3(cdna3::kVPermB32Vop3), // v_perm_b32 expect_vop3(cdna3::kVAddU32Vop3), // v_add_u32 expect_ds(cdna3::kDsBpermuteB32Ds), expect_ds(cdna3::kDsBpermuteB32Ds), expect_sopp(cdna3::kSWaitcntSopp), expect_vop3(cdna3::kVPermB32Vop3), expect_vop3(cdna3::kVAddU32Vop3), expect_ds(cdna3::kDsBpermuteB32Ds), expect_ds(cdna3::kDsBpermuteB32Ds), expect_sopp(cdna3::kSWaitcntSopp), expect_vop3(cdna3::kVPermB32Vop3), expect_vop3(cdna3::kVAddU32Vop3), expect_ds(cdna3::kDsBpermuteB32Ds), expect_ds(cdna3::kDsBpermuteB32Ds), expect_sopp(cdna3::kSWaitcntSopp), expect_vop3(cdna3::kVPermB32Vop3), expect_sop1(cdna3::kSMovB64), // s_mov_b64 restore EXEC. }; auto first_pack = expected_cdna3_raw_b16_pack_sequence(); expected.insert(expected.end(), first_pack.begin(), first_pack.end()); auto second_pack = expected_cdna3_raw_b16_pack_sequence(); expected.insert(expected.end(), second_pack.begin(), second_pack.end()); if (acc_dst) { expected.push_back(expect_vop3p(cdna3::kVAccvgprWriteVop3p)); // v_accvgpr_write_b32 low dword. expected.push_back(expect_vop3p(cdna3::kVAccvgprWriteVop3p)); // v_accvgpr_write_b32 high dword. } return expected; } template std::array encode_two_word_inst(const MachineInst &inst) { std::array words{}; std::memcpy(words.data(), &inst, sizeof(inst)); return words; } std::array make_cdna4_bitop3_words(uint16_t opcode, uint8_t vdst, uint8_t truth_table = 0xde) { rocjitsu::cdna4::Vop3MachineInst inst{}; inst.encoding = 0x34; inst.op = opcode; inst.vdst = vdst; inst.src0 = static_cast(256 + vdst + 1); inst.src1 = static_cast(256 + vdst + 2); inst.src2 = static_cast(256 + vdst + 3); inst.omod = truth_table >> 6; inst.abs = (truth_table >> 3) & 0x7; inst.neg = truth_table & 0x7; return encode_two_word_inst(inst); } std::array make_cdna4_v_lshl_add_u64_words(uint8_t vdst, uint16_t src0, uint16_t src1, uint16_t src2) { rocjitsu::cdna4::Vop3MachineInst inst{}; inst.encoding = 0x34; inst.op = cdna4::kVLshlAddU64Vop3; inst.vdst = vdst; inst.src0 = src0; inst.src1 = src1; inst.src2 = src2; return encode_two_word_inst(inst); } std::array make_cdna4_bitop3_b16_unsupported_op_sel_words() { rocjitsu::cdna4::Vop3MachineInst inst{}; inst.encoding = 0x34; inst.op = cdna4::kVBitop3B16Vop3; inst.vdst = 8; inst.src0 = 256 + 9; inst.src1 = 256 + 10; inst.src2 = 256 + 11; inst.op_sel = 1; inst.omod = 1; return encode_two_word_inst(inst); } std::array make_cdna4_cvt_pk_f16_f32_words() { rocjitsu::cdna4::Vop3MachineInst inst{}; inst.encoding = 0x34; inst.op = cdna4::kVCvtPkF16F32Vop3; inst.vdst = 0; inst.src0 = 256 + 1; inst.src1 = 256 + 2; return encode_two_word_inst(inst); } std::array make_cdna4_cvt_pk_bf16_f32_words(uint8_t vdst = 0, uint16_t src0 = 256 + 1, uint16_t src1 = 256 + 2) { rocjitsu::cdna4::Vop3MachineInst inst{}; inst.encoding = 0x34; inst.op = cdna4::kVCvtPkBf16F32Vop3; inst.vdst = vdst; inst.src0 = src0; inst.src1 = src1; return encode_two_word_inst(inst); } std::array make_cdna4_cvt_f32_bf16_words(uint16_t encoding_id) { rocjitsu::cdna4::Vop1MachineInst inst{}; // As with the VOP1 permlane cases below, semantic-rule lookup sees generated // primary-decode ids 0xfc..0xff while the raw instruction keeps the hardware // VOP1 selector in bits 31:25. Vary VDST high bits to exercise each id. inst.encoding = 0x3f; inst.op = cdna4::kVCvtF32Bf16Vop1; inst.vdst = static_cast((encoding_id - 0xFCu) << 6); inst.src0 = 256 + 1; return encode_two_word_inst(inst); } std::array make_cdna4_dot2_f32_bf16_words(uint8_t vdst, uint16_t src0, uint16_t src1, uint16_t src2, uint8_t op_sel = 0, uint8_t op_sel_hi = 0, uint8_t neg = 0, uint8_t neg_hi = 0, uint8_t op_sel_hi_2 = 0, bool clamp = false) { rocjitsu::cdna4::Vop3pMachineInst inst{}; inst.encoding = cdna4::encoding::kVop3p; inst.op = cdna4::kVDot2F32Bf16Vop3p; inst.vdst = vdst; inst.op_sel = op_sel; inst.op_sel_hi = op_sel_hi; inst.op_sel_hi_2 = op_sel_hi_2; inst.clamp = clamp ? 1 : 0; inst.neg = neg; inst.neg_hi = neg_hi; inst.src0 = src0; inst.src1 = src1; inst.src2 = src2; return encode_two_word_inst(inst); } std::array make_cdna4_cvt_f32_bf16_sdwa_word0_words() { rocjitsu::cdna4::Vop1VopSdwaMachineInst inst{}; inst.encoding = 0x3f; inst.op = cdna4::kVCvtF32Bf16Vop1; inst.vdst = 4; inst.src0 = 249; inst.vsrc0 = 4; inst.dst_sel = 6; inst.dst_unused = 2; inst.src0_sel = 4; return encode_two_word_inst(inst); } std::array make_cdna4_cvt_f32_bf16_sdwa_word1_words() { rocjitsu::cdna4::Vop1VopSdwaMachineInst inst{}; inst.encoding = 0x3f; inst.op = cdna4::kVCvtF32Bf16Vop1; inst.vdst = 4; inst.src0 = 249; inst.vsrc0 = 76; inst.dst_sel = 6; inst.dst_unused = 2; inst.src0_sel = 5; return encode_two_word_inst(inst); } std::array make_cdna4_permlane_swap_b32_words(uint16_t encoding_id, uint16_t opcode) { rocjitsu::cdna4::Vop1MachineInst inst{}; // The legalization table's VOP1 encoding ids (0xfc..0xff) are the generated // primary-decode ids, not the raw 7-bit VOP1 selector. Primary decode looks // at bits 31:23, so VOP1 contributes its fixed selector in bits 31:25 and // VDST[7:6] in bits 24:23. Keep the real VOP1 selector at 0x3f and vary // VDST's high bits to exercise each generated semantic rule. inst.encoding = 0x3f; inst.op = opcode & 0xFF; inst.vdst = static_cast((encoding_id - cdna4::encoding::kVop1) << 6); inst.src0 = 256 + 1; return encode_two_word_inst(inst); } std::array make_cdna4_permlane16_swap_b32_words(uint16_t encoding_id) { return make_cdna4_permlane_swap_b32_words(encoding_id, cdna4::kVPermlane16SwapB32Vop1); } std::array make_cdna4_permlane32_swap_b32_words(uint16_t encoding_id) { return make_cdna4_permlane_swap_b32_words(encoding_id, cdna4::kVPermlane32SwapB32Vop1); } std::array make_cdna4_mfma_words(uint16_t opcode, uint8_t vdst, uint16_t src0, uint16_t src1, uint16_t src2 = 128, uint8_t acc = 0) { rocjitsu::cdna4::Vop3pMfmaMachineInst inst{}; inst.encoding = cdna4::encoding::kVop3p; inst.op = opcode & 0x7F; inst.vdst = vdst; inst.acc_cd = 1; inst.src0 = src0; inst.src1 = src1; inst.src2 = src2; inst.acc = acc; return encode_two_word_inst(inst); } std::array make_cdna4_mfma_vgpr_dst_alias_words() { rocjitsu::cdna4::Vop3pMfmaMachineInst inst{}; inst.encoding = cdna4::encoding::kVop3pMfma; inst.op = cdna4::kVMfmaF3216x16x32F16Vop3pMfma; inst.vdst = 0; inst.acc_cd = 0; // Ordinary-VGPR destination v[0:3] overlaps the first wide source window. // The lowering must therefore place the first narrow MFMA's partial result in // scratch and report that scratch through TranslationContext::require_vgprs(). inst.src0 = 256; inst.src1 = 260; inst.src2 = 128; return encode_two_word_inst(inst); } std::array make_cdna4_dot2c_unimplemented_expand_words() { // v_dot2c_f32_bf16 is present in CDNA4 and not CDNA3. The generated // legalization table marks raw encoding-id 88/opcode 22 as EXPAND, but no // handwritten semantic rule exists yet. return {0x2C000000U, 0x00000000U}; } std::array make_cdna4_ds_read_b64_tr_b16_words(uint16_t byte_offset = 0, uint8_t addr = 2, bool acc = false) { rocjitsu::cdna4::DsMachineInst inst{}; inst.encoding = 0x36; inst.op = cdna4::kDsReadB64TrB16Ds; inst.offset0 = byte_offset & 0xFFu; inst.offset1 = byte_offset >> 8; inst.addr = addr; inst.vdst = 0; inst.acc = acc ? 1 : 0; return encode_two_word_inst(inst); } std::array make_cdna4_ds_read_b32_words(uint16_t byte_offset = 0x0134, uint8_t addr = 4, uint8_t vdst = 7) { rocjitsu::cdna4::DsMachineInst inst{}; inst.encoding = 0x36; inst.op = cdna4::kDsReadB32Ds; inst.offset0 = byte_offset & 0xFFu; inst.offset1 = byte_offset >> 8; inst.addr = addr; inst.vdst = vdst; return encode_two_word_inst(inst); } std::array make_cdna4_ds_read2_b32_words(uint8_t offset0 = 1, uint8_t offset1 = 2, uint8_t addr = 12, uint8_t vdst = 20) { rocjitsu::cdna4::DsMachineInst inst{}; inst.encoding = 0x36; inst.op = cdna4::kDsRead2B32Ds; inst.offset0 = offset0; inst.offset1 = offset1; inst.addr = addr; inst.vdst = vdst; return encode_two_word_inst(inst); } std::array make_cdna4_ds_read2st64_b32_words(uint8_t offset0 = 2, uint8_t offset1 = 3) { rocjitsu::cdna4::DsMachineInst inst{}; inst.encoding = 0x36; inst.op = cdna4::kDsRead2st64B32Ds; inst.offset0 = offset0; inst.offset1 = offset1; inst.addr = 12; inst.vdst = 20; return encode_two_word_inst(inst); } std::array make_cdna4_ds_read2_b64_words(uint8_t offset0 = 3, uint8_t offset1 = 68, uint8_t addr = 58, uint8_t vdst = 66) { rocjitsu::cdna4::DsMachineInst inst{}; inst.encoding = 0x36; inst.op = cdna4::kDsRead2B64Ds; inst.offset0 = offset0; inst.offset1 = offset1; inst.addr = addr; inst.vdst = vdst; return encode_two_word_inst(inst); } std::array make_cdna4_ds_read2st64_b64_words(uint8_t offset0 = 2, uint8_t offset1 = 3, uint8_t addr = 98, uint8_t vdst = 112) { rocjitsu::cdna4::DsMachineInst inst{}; inst.encoding = 0x36; inst.op = cdna4::kDsRead2st64B64Ds; inst.offset0 = offset0; inst.offset1 = offset1; inst.addr = addr; inst.vdst = vdst; return encode_two_word_inst(inst); } std::array make_cdna4_ds_read_b64_words(uint8_t addr = 4, uint8_t vdst = 8, uint16_t byte_offset = 0) { rocjitsu::cdna4::DsMachineInst inst{}; inst.encoding = 0x36; inst.op = cdna4::kDsReadB64Ds; inst.offset0 = byte_offset & 0xFFu; inst.offset1 = byte_offset >> 8; inst.addr = addr; inst.vdst = vdst; return encode_two_word_inst(inst); } std::array make_cdna4_ds_read_b128_words(uint8_t addr = 4, uint8_t vdst = 8, uint16_t byte_offset = 0, bool acc = false) { rocjitsu::cdna4::DsMachineInst inst{}; inst.encoding = 0x36; inst.op = cdna4::kDsReadB128Ds; inst.offset0 = byte_offset & 0xFFu; inst.offset1 = byte_offset >> 8; inst.addr = addr; inst.vdst = vdst; inst.acc = acc ? 1 : 0; return encode_two_word_inst(inst); } std::array make_cdna4_ds_read_u16_words() { rocjitsu::cdna4::DsMachineInst inst{}; inst.encoding = 0x36; inst.op = cdna4::kDsReadU16Ds; inst.offset0 = 0x20; inst.addr = 4; inst.vdst = 7; return encode_two_word_inst(inst); } std::array make_cdna4_ds_read_u8_words() { rocjitsu::cdna4::DsMachineInst inst{}; inst.encoding = 0x36; inst.op = cdna4::kDsReadU8Ds; inst.offset0 = 0x20; inst.addr = 8; inst.vdst = 12; return encode_two_word_inst(inst); } std::array make_cdna4_ds_read_i8_words() { rocjitsu::cdna4::DsMachineInst inst{}; inst.encoding = 0x36; inst.op = cdna4::kDsReadI8Ds; inst.offset0 = 0x20; inst.addr = 8; inst.vdst = 12; return encode_two_word_inst(inst); } std::array make_cdna4_ds_write_b8_words() { rocjitsu::cdna4::DsMachineInst inst{}; inst.encoding = 0x36; inst.op = cdna4::kDsWriteB8Ds; inst.offset0 = 0x10; inst.addr = 4; inst.data0 = 7; return encode_two_word_inst(inst); } std::array make_cdna4_ds_write_b8_d16_hi_words() { rocjitsu::cdna4::DsMachineInst inst{}; inst.encoding = 0x36; inst.op = cdna4::kDsWriteB8D16HiDs; inst.offset0 = 0x10; inst.addr = 4; inst.data0 = 7; return encode_two_word_inst(inst); } std::array make_cdna4_ds_write_b32_words(uint8_t addr = 4, uint8_t data = 7, uint16_t byte_offset = 0) { rocjitsu::cdna4::DsMachineInst inst{}; inst.encoding = 0x36; inst.op = cdna4::kDsWriteB32Ds; inst.offset0 = byte_offset & 0xFFu; inst.offset1 = byte_offset >> 8; inst.addr = addr; inst.data0 = data; return encode_two_word_inst(inst); } std::array make_cdna4_ds_write_b96_words(uint8_t addr = 4, uint8_t data = 8, uint16_t byte_offset = 0) { rocjitsu::cdna4::DsMachineInst inst{}; inst.encoding = 0x36; inst.op = cdna4::kDsWriteB96Ds; inst.offset0 = byte_offset & 0xFFu; inst.offset1 = byte_offset >> 8; inst.addr = addr; inst.data0 = data; return encode_two_word_inst(inst); } std::array make_cdna4_ds_write_b64_words(uint8_t addr = 4, uint8_t data = 8, uint16_t byte_offset = 0) { rocjitsu::cdna4::DsMachineInst inst{}; inst.encoding = 0x36; inst.op = cdna4::kDsWriteB64Ds; inst.offset0 = byte_offset & 0xFFu; inst.offset1 = byte_offset >> 8; inst.addr = addr; inst.data0 = data; return encode_two_word_inst(inst); } std::array make_cdna4_ds_write_b128_words(uint8_t addr = 4, uint8_t data = 8, uint16_t byte_offset = 0, bool acc = false) { rocjitsu::cdna4::DsMachineInst inst{}; inst.encoding = 0x36; inst.op = cdna4::kDsWriteB128Ds; inst.offset0 = byte_offset & 0xFFu; inst.offset1 = byte_offset >> 8; inst.addr = addr; inst.data0 = data; inst.acc = acc ? 1 : 0; return encode_two_word_inst(inst); } std::array make_cdna4_ds_write2_b32_words() { rocjitsu::cdna4::DsMachineInst inst{}; inst.encoding = 0x36; inst.op = cdna4::kDsWrite2B32Ds; inst.offset0 = 1; inst.offset1 = 2; inst.addr = 4; inst.data0 = 7; inst.data1 = 9; return encode_two_word_inst(inst); } std::array make_cdna4_ds_write2_b64_words(uint8_t addr = 4, uint8_t data0 = 8, uint8_t data1 = 12) { rocjitsu::cdna4::DsMachineInst inst{}; inst.encoding = 0x36; inst.op = cdna4::kDsWrite2B64Ds; inst.offset0 = 1; inst.offset1 = 2; inst.addr = addr; inst.data0 = data0; inst.data1 = data1; return encode_two_word_inst(inst); } std::array make_cdna4_ds_write2st64_b64_words() { rocjitsu::cdna4::DsMachineInst inst{}; inst.encoding = 0x36; inst.op = cdna4::kDsWrite2st64B64Ds; inst.offset0 = 1; inst.offset1 = 2; inst.addr = 4; inst.data0 = 8; inst.data1 = 12; return encode_two_word_inst(inst); } std::array make_cdna4_ds_add_u32_words() { rocjitsu::cdna4::DsMachineInst inst{}; inst.encoding = 0x36; inst.op = cdna4::kDsAddU32Ds; inst.addr = 3; inst.data0 = 7; return encode_two_word_inst(inst); } std::array make_cdna4_buffer_load_lds_words(uint16_t op, uint8_t vaddr = 2) { rocjitsu::cdna4::MubufMachineInst inst{}; inst.encoding = 0x38; inst.op = op & 0x7F; inst.lds = 1; inst.offen = 1; inst.vaddr = vaddr; inst.vdata = 0; inst.srsrc = 4; inst.soffset = 0; return encode_two_word_inst(inst); } std::vector cdna4_to_cdna3_semantic_rule_cases() { return { {"VBitop3B16", cdna4::encoding::kVop3OpHi4, cdna4::kVBitop3B16Vop3, make_cdna4_bitop3_words(cdna4::kVBitop3B16Vop3, 8), expected_cdna3_bitop3_sequence(true)}, {"VBitop3B16FastEc", cdna4::encoding::kVop3OpHi4, cdna4::kVBitop3B16Vop3, make_cdna4_bitop3_words(cdna4::kVBitop3B16Vop3, 8, 0xec), expected_cdna3_bitop3_fast_sequence(cdna3::kVAndOrB32Vop3, true)}, {"VBitop3B16FastF8", cdna4::encoding::kVop3OpHi4, cdna4::kVBitop3B16Vop3, make_cdna4_bitop3_words(cdna4::kVBitop3B16Vop3, 8, 0xf8), expected_cdna3_bitop3_fast_sequence(cdna3::kVAndOrB32Vop3, true)}, {"VBitop3B16FastFe", cdna4::encoding::kVop3OpHi4, cdna4::kVBitop3B16Vop3, make_cdna4_bitop3_words(cdna4::kVBitop3B16Vop3, 8, 0xfe), expected_cdna3_bitop3_fast_sequence(cdna3::kVOr3B32Vop3, true)}, {"VBitop3B32", cdna4::encoding::kVop3OpHi4, cdna4::kVBitop3B32Vop3, make_cdna4_bitop3_words(cdna4::kVBitop3B32Vop3, 16), expected_cdna3_bitop3_sequence(false)}, {"VCvtPkF16F32", cdna4::encoding::kVop3OpHi4, cdna4::kVCvtPkF16F32Vop3, make_cdna4_cvt_pk_f16_f32_words(), expected_cdna3_cvt_pk_f16_f32_sequence()}, {"VCvtPkBf16F32", cdna4::encoding::kVop3OpHi4, cdna4::kVCvtPkBf16F32Vop3, make_cdna4_cvt_pk_bf16_f32_words(), expected_cdna3_cvt_pk_bf16_f32_sequence()}, {"VCvtF32Bf16E32", cdna4::encoding::kVop1, cdna4::kVCvtF32Bf16Vop1, make_cdna4_cvt_f32_bf16_words(cdna4::encoding::kVop1), expected_cdna3_cvt_f32_bf16_sequence(), 1}, {"VCvtF32Bf16E32Hi1", cdna4::encoding::kVop1Hi1, cdna4::kVCvtF32Bf16Vop1, make_cdna4_cvt_f32_bf16_words(cdna4::encoding::kVop1Hi1), expected_cdna3_cvt_f32_bf16_sequence(), 1}, {"VCvtF32Bf16E32Hi2", cdna4::encoding::kVop1Hi2, cdna4::kVCvtF32Bf16Vop1, make_cdna4_cvt_f32_bf16_words(cdna4::encoding::kVop1Hi2), expected_cdna3_cvt_f32_bf16_sequence(), 1}, {"VCvtF32Bf16E32Hi3", cdna4::encoding::kVop1Hi3, cdna4::kVCvtF32Bf16Vop1, make_cdna4_cvt_f32_bf16_words(cdna4::encoding::kVop1Hi3), expected_cdna3_cvt_f32_bf16_sequence(), 1}, {"VCvtF32Bf16SdwaWord0", cdna4::encoding::kVop1, cdna4::kVCvtF32Bf16Vop1, make_cdna4_cvt_f32_bf16_sdwa_word0_words(), expected_cdna3_cvt_f32_bf16_sdwa_word0_sequence(), 2}, {"VCvtF32Bf16SdwaWord1", cdna4::encoding::kVop1, cdna4::kVCvtF32Bf16Vop1, make_cdna4_cvt_f32_bf16_sdwa_word1_words(), expected_cdna3_cvt_f32_bf16_sdwa_word1_sequence(), 2}, {"VDot2F32Bf16AccumAlias", cdna4::encoding::kVop3p, cdna4::kVDot2F32Bf16Vop3p, make_cdna4_dot2_f32_bf16_words(/*vdst=*/0, /*src0=*/256 + 1, /*src1=*/256 + 2, /*src2=*/256), expected_cdna3_dot2_f32_bf16_sequence(/*vdst=*/0, /*src0=*/256 + 1, /*src1=*/256 + 2, /*src2=*/256, /*scratch_base=*/3)}, {"VDot2F32Bf16OpSelNeg", cdna4::encoding::kVop3p, cdna4::kVDot2F32Bf16Vop3p, make_cdna4_dot2_f32_bf16_words(/*vdst=*/0, /*src0=*/256 + 1, /*src1=*/256 + 2, /*src2=*/256, /*op_sel=*/1, /*op_sel_hi=*/2, /*neg=*/1, /*neg_hi=*/2), expected_cdna3_dot2_f32_bf16_sequence(/*vdst=*/0, /*src0=*/256 + 1, /*src1=*/256 + 2, /*src2=*/256, /*scratch_base=*/3, /*op_sel=*/1, /*op_sel_hi=*/2, /*neg=*/1, /*neg_hi=*/2)}, {"VDot2F32Bf16OpSelHi2Clamp", cdna4::encoding::kVop3p, cdna4::kVDot2F32Bf16Vop3p, make_cdna4_dot2_f32_bf16_words(/*vdst=*/0, /*src0=*/256 + 1, /*src1=*/256 + 2, /*src2=*/256, /*op_sel=*/0, /*op_sel_hi=*/0, /*neg=*/0, /*neg_hi=*/0, /*op_sel_hi_2=*/1, /*clamp=*/true), expected_cdna3_dot2_f32_bf16_sequence(/*vdst=*/0, /*src0=*/256 + 1, /*src1=*/256 + 2, /*src2=*/256, /*scratch_base=*/3, /*op_sel=*/0, /*op_sel_hi=*/0, /*neg=*/0, /*neg_hi=*/0, /*op_sel_hi_2=*/1, /*clamp=*/true)}, {"VPermlane16SwapB32E32", cdna4::encoding::kVop1, cdna4::kVPermlane16SwapB32Vop1, make_cdna4_permlane16_swap_b32_words(cdna4::encoding::kVop1), expected_cdna3_permlane32_swap_sequence(), 1}, {"VPermlane16SwapB32E32Hi1", cdna4::encoding::kVop1Hi1, cdna4::kVPermlane16SwapB32Vop1, make_cdna4_permlane16_swap_b32_words(cdna4::encoding::kVop1Hi1), expected_cdna3_permlane32_swap_sequence(), 1}, {"VPermlane16SwapB32E32Hi2", cdna4::encoding::kVop1Hi2, cdna4::kVPermlane16SwapB32Vop1, make_cdna4_permlane16_swap_b32_words(cdna4::encoding::kVop1Hi2), expected_cdna3_permlane32_swap_sequence(), 1}, {"VPermlane16SwapB32E32Hi3", cdna4::encoding::kVop1Hi3, cdna4::kVPermlane16SwapB32Vop1, make_cdna4_permlane16_swap_b32_words(cdna4::encoding::kVop1Hi3), expected_cdna3_permlane32_swap_sequence(), 1}, {"VPermlane32SwapB32E32", cdna4::encoding::kVop1, cdna4::kVPermlane32SwapB32Vop1, make_cdna4_permlane32_swap_b32_words(cdna4::encoding::kVop1), expected_cdna3_permlane32_swap_sequence(), 1}, {"VPermlane32SwapB32E32Hi1", cdna4::encoding::kVop1Hi1, cdna4::kVPermlane32SwapB32Vop1, make_cdna4_permlane32_swap_b32_words(cdna4::encoding::kVop1Hi1), expected_cdna3_permlane32_swap_sequence(), 1}, {"VPermlane32SwapB32E32Hi2", cdna4::encoding::kVop1Hi2, cdna4::kVPermlane32SwapB32Vop1, make_cdna4_permlane32_swap_b32_words(cdna4::encoding::kVop1Hi2), expected_cdna3_permlane32_swap_sequence(), 1}, {"VPermlane32SwapB32E32Hi3", cdna4::encoding::kVop1Hi3, cdna4::kVPermlane32SwapB32Vop1, make_cdna4_permlane32_swap_b32_words(cdna4::encoding::kVop1Hi3), expected_cdna3_permlane32_swap_sequence(), 1}, {"MfmaF32_16x16x32Bf16", cdna4::encoding::kVop3p, cdna4::kVMfmaF3216x16x32Bf16Vop3pMfma, make_cdna4_mfma_words(cdna4::kVMfmaF3216x16x32Bf16Vop3pMfma, 0, 256, 260), expected_cdna3_mfma_sequence(cdna3::kVMfmaF3216x16x16Bf16Vop3pMfma)}, {"MfmaF32_32x32x16Bf16", cdna4::encoding::kVop3p, cdna4::kVMfmaF3232x32x16Bf16Vop3pMfma, make_cdna4_mfma_words(cdna4::kVMfmaF3232x32x16Bf16Vop3pMfma, 0, 256, 260), expected_cdna3_mfma_sequence(cdna3::kVMfmaF3232x32x8Bf16Vop3pMfma)}, {"MfmaF32_32x32x16Bf16Source1Acc", cdna4::encoding::kVop3p, cdna4::kVMfmaF3232x32x16Bf16Vop3pMfma, make_cdna4_mfma_words(cdna4::kVMfmaF3232x32x16Bf16Vop3pMfma, 0, 256, 260, 272, 2), expected_cdna3_mfma_sequence(cdna3::kVMfmaF3232x32x8Bf16Vop3pMfma, 272, 2)}, {"MfmaF32_16x16x32F16", cdna4::encoding::kVop3p, cdna4::kVMfmaF3216x16x32F16Vop3pMfma, make_cdna4_mfma_words(cdna4::kVMfmaF3216x16x32F16Vop3pMfma, 0, 256, 260), expected_cdna3_mfma_sequence(cdna3::kVMfmaF3216x16x16F16Vop3pMfma)}, {"MfmaF32_32x32x16F16", cdna4::encoding::kVop3p, cdna4::kVMfmaF3232x32x16F16Vop3pMfma, make_cdna4_mfma_words(cdna4::kVMfmaF3232x32x16F16Vop3pMfma, 0, 256, 260), expected_cdna3_mfma_sequence(cdna3::kVMfmaF3232x32x8F16Vop3pMfma)}, {"MfmaF32_16x16x32F16AccumVgpr", cdna4::encoding::kVop3p, cdna4::kVMfmaF3216x16x32F16Vop3pMfma, make_cdna4_mfma_words(cdna4::kVMfmaF3216x16x32F16Vop3pMfma, 0, 256, 260, 272), expected_cdna3_mfma_sequence(cdna3::kVMfmaF3216x16x16F16Vop3pMfma, 272)}, {"MfmaF32_32x32x16F16AccumVgpr", cdna4::encoding::kVop3p, cdna4::kVMfmaF3232x32x16F16Vop3pMfma, make_cdna4_mfma_words(cdna4::kVMfmaF3232x32x16F16Vop3pMfma, 0, 256, 260, 272), expected_cdna3_mfma_sequence(cdna3::kVMfmaF3232x32x8F16Vop3pMfma, 272)}, {"DsReadB64TrB16", cdna4::encoding::kDsHi3, cdna4::kDsReadB64TrB16Ds, make_cdna4_ds_read_b64_tr_b16_words(), expected_cdna3_ds_read_b64_tr_b16_sequence()}, {"DsReadB64TrB16Acc", cdna4::encoding::kDsHi7, cdna4::kDsReadB64TrB16Ds, make_cdna4_ds_read_b64_tr_b16_words(/*byte_offset=*/0, /*addr=*/2, /*acc=*/true), expected_cdna3_ds_read_b64_tr_b16_sequence(/*acc_dst=*/true)}, {"BufferLoadDwordLds", cdna4::encoding::kMubuf, cdna4::kBufferLoadDwordMubuf, make_cdna4_buffer_load_lds_words(cdna4::kBufferLoadDwordMubuf), expected_cdna3_buffer_load_lds_sequence(cdna3::kBufferLoadDwordMubuf, cdna3::kDsWriteB32Ds)}, {"BufferLoadDwordx3Lds", cdna4::encoding::kMubuf, cdna4::kBufferLoadDwordx3Mubuf, make_cdna4_buffer_load_lds_words(cdna4::kBufferLoadDwordx3Mubuf), expected_cdna3_buffer_load_lds_sequence(cdna3::kBufferLoadDwordx3Mubuf, cdna3::kDsWriteB96Ds)}, {"BufferLoadDwordx3LdsEvenScratch", cdna4::encoding::kMubuf, cdna4::kBufferLoadDwordx3Mubuf, make_cdna4_buffer_load_lds_words(cdna4::kBufferLoadDwordx3Mubuf, /*vaddr=*/0), expected_cdna3_buffer_load_lds_sequence(cdna3::kBufferLoadDwordx3Mubuf, cdna3::kDsWriteB96Ds, /*scratch_data=*/2)}, {"BufferLoadDwordx4Lds", cdna4::encoding::kMubuf, cdna4::kBufferLoadDwordx4Mubuf, make_cdna4_buffer_load_lds_words(cdna4::kBufferLoadDwordx4Mubuf), expected_cdna3_buffer_load_lds_sequence(cdna3::kBufferLoadDwordx4Mubuf, cdna3::kDsWriteB128Ds)}, {"BufferLoadDwordx4LdsEvenScratch", cdna4::encoding::kMubuf, cdna4::kBufferLoadDwordx4Mubuf, make_cdna4_buffer_load_lds_words(cdna4::kBufferLoadDwordx4Mubuf, /*vaddr=*/0), expected_cdna3_buffer_load_lds_sequence(cdna3::kBufferLoadDwordx4Mubuf, cdna3::kDsWriteB128Ds, /*scratch_data=*/2)}, }; } bool has_cdna4_to_cdna3_semantic_rule(uint16_t encoding_id, uint16_t opcode) { for (const auto &rule : rocjitsu::semantic_expand_rules_cdna4_to_cdna3()) { if (rule.src_encoding_id == encoding_id && rule.src_opcode == opcode) return true; } return false; } bool has_cdna4_to_cdna3_semantic_rule_case(uint16_t encoding_id, uint16_t opcode) { for (const auto &test_case : cdna4_to_cdna3_semantic_rule_cases()) { if (test_case.encoding_id == encoding_id && test_case.opcode == opcode) return true; } return false; } void expect_field_matches(uint16_t expected, uint16_t actual, std::string_view field_name) { if (expected != kAnyExpectedField) { EXPECT_EQ(actual, expected) << field_name; } } void expect_cdna3_instruction_matches(const rocjitsu::Instruction &inst, const ExpectedCdna3Inst &expected) { const uint32_t *raw = inst.raw_encoding(); ASSERT_NE(raw, nullptr); switch (expected.kind) { case ExpectedCdna3Kind::Vop3: { rocjitsu::cdna3::Vop3MachineInst actual{}; std::memcpy(&actual, raw, sizeof(actual)); EXPECT_EQ(actual.encoding, 0x34u); EXPECT_EQ(actual.op, expected.op); expect_field_matches(expected.vdst, static_cast(actual.vdst), "vdst"); expect_field_matches(expected.src0, static_cast(actual.src0), "src0"); expect_field_matches(expected.src1, static_cast(actual.src1), "src1"); expect_field_matches(expected.src2, static_cast(actual.src2), "src2"); break; } case ExpectedCdna3Kind::Vop3p: { rocjitsu::cdna3::Vop3pMachineInst actual{}; std::memcpy(&actual, raw, sizeof(actual)); EXPECT_EQ(actual.encoding, cdna3::encoding::kVop3p); EXPECT_EQ(actual.op, expected.op); expect_field_matches(expected.vdst, static_cast(actual.vdst), "vdst"); expect_field_matches(expected.src0, static_cast(actual.src0), "src0"); expect_field_matches(expected.src1, static_cast(actual.src1), "src1"); expect_field_matches(expected.src2, static_cast(actual.src2), "src2"); break; } case ExpectedCdna3Kind::Vop2: { rocjitsu::cdna3::Vop2MachineInst actual{}; std::memcpy(&actual, raw, sizeof(actual)); EXPECT_EQ(actual.op, expected.op); break; } case ExpectedCdna3Kind::Vop1: { rocjitsu::cdna3::Vop1MachineInst actual{}; std::memcpy(&actual, raw, sizeof(actual)); EXPECT_EQ(actual.op, expected.op); break; } case ExpectedCdna3Kind::Sop2: { rocjitsu::cdna3::Sop2MachineInst actual{}; std::memcpy(&actual, raw, sizeof(actual)); EXPECT_EQ(actual.encoding, 0x2u); EXPECT_EQ(actual.op, expected.op); break; } case ExpectedCdna3Kind::Sop1: { rocjitsu::cdna3::Sop1MachineInst actual{}; std::memcpy(&actual, raw, sizeof(actual)); EXPECT_EQ(actual.encoding, cdna3::encoding::kSop1); EXPECT_EQ(actual.op, expected.op); break; } case ExpectedCdna3Kind::Vop3pMfma: { rocjitsu::cdna3::Vop3pMfmaMachineInst actual{}; std::memcpy(&actual, raw, sizeof(actual)); EXPECT_EQ(actual.encoding, cdna3::encoding::kVop3pMfma); EXPECT_EQ(actual.op, expected.op); expect_field_matches(expected.vdst, static_cast(actual.vdst), "vdst"); expect_field_matches(expected.acc, static_cast(actual.acc), "acc"); expect_field_matches(expected.acc_cd, static_cast(actual.acc_cd), "acc_cd"); expect_field_matches(expected.src0, static_cast(actual.src0), "src0"); expect_field_matches(expected.src1, static_cast(actual.src1), "src1"); expect_field_matches(expected.src2, static_cast(actual.src2), "src2"); break; } case ExpectedCdna3Kind::Ds: { rocjitsu::cdna3::DsMachineInst actual{}; std::memcpy(&actual, raw, sizeof(actual)); EXPECT_EQ(actual.encoding, 0x36u); EXPECT_EQ(actual.op, expected.op); expect_field_matches(expected.vdst, static_cast(actual.vdst), "vdst"); expect_field_matches(expected.data0, static_cast(actual.data0), "data0"); break; } case ExpectedCdna3Kind::Mubuf: { rocjitsu::cdna3::MubufMachineInst actual{}; std::memcpy(&actual, raw, sizeof(actual)); EXPECT_EQ(actual.encoding, 0x38u); EXPECT_EQ(actual.op, expected.op); EXPECT_EQ(actual.lds, 0u); expect_field_matches(expected.vdata, static_cast(actual.vdata), "vdata"); break; } case ExpectedCdna3Kind::Sopp: { rocjitsu::cdna3::SoppMachineInst actual{}; std::memcpy(&actual, raw, sizeof(actual)); EXPECT_EQ(actual.encoding, cdna3::encoding::kSopp); EXPECT_EQ(actual.op, expected.op); break; } } } void expect_cdna3_text_matches(const rocjitsu::Section &text, const std::vector &expected, bool allow_non_nop_tail = false) { ASSERT_EQ(text.size() % sizeof(uint32_t), 0u); auto decoder = rocjitsu::Decoder::create(ROCJITSU_CODE_ARCH_CDNA3); ASSERT_NE(decoder, nullptr); const auto *words = reinterpret_cast(text.data()); const size_t word_count = text.size() / sizeof(uint32_t); std::vector> actual; for (size_t pc = 0; pc < word_count;) { SCOPED_TRACE(pc); auto inst = std::unique_ptr(decoder->decode(&words[pc])); ASSERT_NE(inst, nullptr); ASSERT_GT(inst->size(), 0u); ASSERT_EQ(inst->size() % sizeof(uint32_t), 0u); ASSERT_LE(pc + inst->size() / sizeof(uint32_t), word_count); pc += inst->size() / sizeof(uint32_t); actual.push_back(std::move(inst)); } ASSERT_GE(actual.size(), expected.size()); for (size_t i = 0; i < expected.size(); ++i) { SCOPED_TRACE(i); expect_cdna3_instruction_matches(*actual[i], expected[i]); } if (allow_non_nop_tail) return; for (size_t i = expected.size(); i < actual.size(); ++i) { SCOPED_TRACE(i); const uint32_t *raw = actual[i]->raw_encoding(); ASSERT_NE(raw, nullptr); EXPECT_EQ(*raw, rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA3)); } } void expect_cdna3_translated_descriptor_vgprs_at_least(const std::vector &image, uint32_t expected_minimum) { rocjitsu::AmdGpuCodeObject translated(image.data(), image.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); rocjitsu::KernelDescriptorTranslator parser(ROCJITSU_CODE_ARCH_CDNA3, ROCJITSU_CODE_ARCH_CDNA3); const auto infos = parser.translate_image(image, translated.text_sections()[0]->sectionOffset(), translated.text_sections()[0]->size(), rocjitsu::KernelDescriptorTranslationOptions{}); ASSERT_EQ(infos.size(), 1u); EXPECT_GE(infos[0].target_vgpr_count, expected_minimum); } std::optional find_vop2_literal_add(const uint32_t *words, size_t word_count, uint8_t vsrc1, uint32_t literal) { for (size_t i = 0; i + 1 < word_count; ++i) { rocjitsu::cdna3::Vop2InstLiteralMachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); if (actual.encoding == 0u && actual.op == 52u && actual.src0 == 0xFFu && actual.vsrc1 == vsrc1 && actual.simm32 == literal) return static_cast(actual.vdst); } return std::nullopt; } size_t count_cdna3_s_mov_b32_literal(const uint32_t *words, size_t word_count, uint8_t sdst, uint32_t literal) { size_t count = 0; for (size_t i = 0; i + 1 < word_count; ++i) { rocjitsu::cdna3::Sop1MachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); if (actual.encoding == 0x17Du && actual.op == 0u && actual.sdst == sdst && actual.ssrc0 == 0xFFu && words[i + 1] == literal) { ++count; } } return count; } bool contains_vop3_mov_b32(const uint32_t *words, size_t word_count, uint8_t vdst, uint16_t src0) { for (size_t i = 0; i + 1 < word_count; ++i) { rocjitsu::cdna3::Vop3MachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); if (actual.encoding == 0x34u && actual.op == 321u && actual.vdst == vdst && actual.src0 == src0) return true; } return false; } bool contains_flat_global_load(const uint32_t *words, size_t word_count, uint8_t op, uint8_t vdst, uint8_t addr, uint16_t offset) { for (size_t i = 0; i + 1 < word_count; ++i) { rocjitsu::cdna3::FlatMachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); if (actual.encoding == 0x37u && actual.op == op && actual.seg == 2u && actual.vdst == vdst && actual.addr == addr && actual.offset == (offset & 0x0FFFu) && actual.pad_12 == ((offset >> 12) & 0x1u) && actual.sc0 == 1u && actual.sc1 == 0u && actual.saddr != 0x7Fu) return true; } return false; } bool contains_flat_global_load_acc(const uint32_t *words, size_t word_count, uint8_t op, uint8_t vdst, uint8_t addr, uint16_t offset) { for (size_t i = 0; i + 1 < word_count; ++i) { rocjitsu::cdna3::FlatMachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); if (actual.encoding == 0x37u && actual.op == op && actual.seg == 2u && actual.vdst == vdst && actual.addr == addr && actual.offset == (offset & 0x0FFFu) && actual.pad_12 == ((offset >> 12) & 0x1u) && actual.sc0 == 1u && actual.sc1 == 0u && actual.acc == 1u && actual.saddr != 0x7Fu) return true; } return false; } std::optional find_flat_global_load_saddr(const uint32_t *words, size_t word_count, uint8_t op, uint8_t vdst, uint8_t addr, uint16_t offset) { for (size_t i = 0; i + 1 < word_count; ++i) { rocjitsu::cdna3::FlatMachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); if (actual.encoding == 0x37u && actual.op == op && actual.seg == 2u && actual.vdst == vdst && actual.addr == addr && actual.offset == (offset & 0x0FFFu) && actual.pad_12 == ((offset >> 12) & 0x1u) && actual.sc0 == 1u && actual.sc1 == 0u && actual.saddr != 0x7Fu) return static_cast(actual.saddr); } return std::nullopt; } bool contains_flat_global_load_addr(const uint32_t *words, size_t word_count, uint8_t op, uint8_t addr, uint16_t offset) { for (size_t i = 0; i + 1 < word_count; ++i) { rocjitsu::cdna3::FlatMachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); if (actual.encoding == 0x37u && actual.op == op && actual.seg == 2u && actual.addr == addr && actual.offset == (offset & 0x0FFFu) && actual.pad_12 == ((offset >> 12) & 0x1u) && actual.sc0 == 1u && actual.sc1 == 0u && actual.saddr != 0x7Fu) return true; } return false; } std::optional find_flat_global_load_vdst(const uint32_t *words, size_t word_count, uint8_t op, uint8_t addr, uint16_t offset) { for (size_t i = 0; i + 1 < word_count; ++i) { rocjitsu::cdna3::FlatMachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); if (actual.encoding == 0x37u && actual.op == op && actual.seg == 2u && actual.addr == addr && actual.offset == (offset & 0x0FFFu) && actual.pad_12 == ((offset >> 12) & 0x1u) && actual.sc0 == 1u && actual.sc1 == 0u && actual.saddr != 0x7Fu) return static_cast(actual.vdst); } return std::nullopt; } std::optional find_flat_global_load_addr_for_vdst(const uint32_t *words, size_t word_count, uint8_t op, uint8_t vdst, uint16_t offset) { for (size_t i = 0; i + 1 < word_count; ++i) { rocjitsu::cdna3::FlatMachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); if (actual.encoding == 0x37u && actual.op == op && actual.seg == 2u && actual.vdst == vdst && actual.offset == (offset & 0x0FFFu) && actual.pad_12 == ((offset >> 12) & 0x1u) && actual.sc0 == 1u && actual.sc1 == 0u && actual.saddr != 0x7Fu) return static_cast(actual.addr); } return std::nullopt; } bool contains_flat_global_store(const uint32_t *words, size_t word_count, uint8_t op, uint8_t data, uint8_t addr, uint16_t offset) { for (size_t i = 0; i + 1 < word_count; ++i) { rocjitsu::cdna3::FlatMachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); if (actual.encoding == 0x37u && actual.op == op && actual.seg == 2u && actual.data == data && actual.addr == addr && actual.offset == (offset & 0x0FFFu) && actual.pad_12 == ((offset >> 12) & 0x1u) && actual.sc0 == 1u && actual.sc1 == 0u && actual.saddr != 0x7Fu) return true; } return false; } bool contains_flat_global_store_acc(const uint32_t *words, size_t word_count, uint8_t op, uint8_t data, uint8_t addr, uint16_t offset) { for (size_t i = 0; i + 1 < word_count; ++i) { rocjitsu::cdna3::FlatMachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); if (actual.encoding == 0x37u && actual.op == op && actual.seg == 2u && actual.data == data && actual.addr == addr && actual.offset == (offset & 0x0FFFu) && actual.pad_12 == ((offset >> 12) & 0x1u) && actual.sc0 == 1u && actual.sc1 == 0u && actual.acc == 1u && actual.saddr != 0x7Fu) return true; } return false; } std::optional find_flat_global_store_data(const uint32_t *words, size_t word_count, uint8_t op, uint8_t addr, uint16_t offset) { for (size_t i = 0; i + 1 < word_count; ++i) { rocjitsu::cdna3::FlatMachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); if (actual.encoding == 0x37u && actual.op == op && actual.seg == 2u && actual.addr == addr && actual.offset == (offset & 0x0FFFu) && actual.pad_12 == ((offset >> 12) & 0x1u) && actual.sc0 == 1u && actual.sc1 == 0u && actual.saddr != 0x7Fu) return static_cast(actual.data); } return std::nullopt; } std::optional find_flat_global_store_addr_for_data(const uint32_t *words, size_t word_count, uint8_t op, uint8_t data, uint16_t offset) { for (size_t i = 0; i + 1 < word_count; ++i) { rocjitsu::cdna3::FlatMachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); if (actual.encoding == 0x37u && actual.op == op && actual.seg == 2u && actual.data == data && actual.offset == (offset & 0x0FFFu) && actual.pad_12 == ((offset >> 12) & 0x1u) && actual.sc0 == 1u && actual.sc1 == 0u && actual.saddr != 0x7Fu) return static_cast(actual.addr); } return std::nullopt; } bool contains_flat_global_store_op(const uint32_t *words, size_t word_count, uint8_t op) { for (size_t i = 0; i + 1 < word_count; ++i) { rocjitsu::cdna3::FlatMachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); if (actual.encoding == 0x37u && actual.op == op && actual.seg == 2u && actual.lds == 0u && actual.saddr != 0x7Fu) return true; } return false; } bool contains_flat_global_op_with_null_saddr(const uint32_t *words, size_t word_count, uint8_t op) { for (size_t i = 0; i + 1 < word_count; ++i) { rocjitsu::cdna3::FlatMachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); if (actual.encoding == 0x37u && actual.op == op && actual.seg == 2u && actual.lds == 0u && actual.saddr == 0x7Fu) return true; } return false; } bool contains_mubuf_lds_op(const uint32_t *words, size_t word_count, uint8_t op) { for (size_t i = 0; i + 1 < word_count; ++i) { rocjitsu::cdna3::MubufMachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); if (actual.encoding == 0x38u && actual.op == op && actual.lds != 0u) return true; } return false; } bool contains_flat_scratch_dword(const uint32_t *words, size_t word_count, uint8_t op, uint8_t vgpr, uint16_t offset, bool is_load) { for (size_t i = 0; i + 1 < word_count; ++i) { rocjitsu::cdna3::FlatScratchMachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); const uint8_t encoded_vgpr = is_load ? static_cast(actual.vdst) : static_cast(actual.data); if (actual.encoding == 0x37u && actual.op == op && actual.seg == 1u && actual.saddr == 0x7Fu && encoded_vgpr == vgpr && actual.offset == offset) return true; } return false; } bool contains_flat_scratch_dword_offset(const uint32_t *words, size_t word_count, uint8_t op, uint16_t offset, bool is_load) { for (size_t i = 0; i + 1 < word_count; ++i) { rocjitsu::cdna3::FlatScratchMachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); if (actual.encoding == 0x37u && actual.op == op && actual.seg == 1u && actual.saddr == 0x7Fu && actual.offset == offset) { const bool decoded_load = actual.op == 20u; if (decoded_load == is_load) return true; } } return false; } bool contains_cdna3_vop1(const uint32_t *words, size_t word_count, uint8_t op, uint8_t vdst, uint16_t src0) { for (size_t i = 0; i < word_count; ++i) { rocjitsu::cdna3::Vop1MachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); if (actual.encoding == 0x3Fu && actual.op == op && actual.vdst == vdst && actual.src0 == src0) return true; } return false; } size_t count_cdna3_vop1_writes(const uint32_t *words, size_t word_count, uint8_t op, uint8_t vdst) { size_t count = 0; for (size_t i = 0; i < word_count; ++i) { rocjitsu::cdna3::Vop1MachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); if (actual.encoding == 0x3Fu && actual.op == op && actual.vdst == vdst) ++count; } return count; } bool contains_cdna3_vop3p(const uint32_t *words, size_t word_count, uint8_t op, uint8_t vdst, uint16_t src0) { for (size_t i = 0; i + 1 < word_count; ++i) { rocjitsu::cdna3::Vop3pMachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); if (actual.encoding == 0x1A7u && actual.op == op && actual.vdst == vdst && actual.src0 == src0) return true; } return false; } std::optional find_cdna3_vop1_vdst(const uint32_t *words, size_t word_count, uint8_t op, uint16_t src0) { for (size_t i = 0; i < word_count; ++i) { rocjitsu::cdna3::Vop1MachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); if (actual.encoding == 0x3Fu && actual.op == op && actual.src0 == src0) return static_cast(actual.vdst); } return std::nullopt; } bool contains_sopp(const uint32_t *words, size_t word_count, uint8_t op) { for (size_t i = 0; i < word_count; ++i) { rocjitsu::cdna3::SoppMachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); if (actual.encoding == 0x17Fu && actual.op == op) return true; } return false; } bool contains_smem_load_dwordx2_with_wait(const uint32_t *words, size_t word_count, uint8_t sdata, uint8_t sbase_sgpr, uint32_t offset) { for (size_t i = 0; i + 2 < word_count; ++i) { rocjitsu::cdna3::SmemMachineInst actual{}; std::memcpy(&actual, words + i, sizeof(actual)); if (actual.encoding == 0x30u && actual.op == 1u && actual.sdata == sdata && actual.sbase == (sbase_sgpr / 2u) && actual.imm == 1u && actual.offset == offset && words[i + 2] == rocjitsu::pack_sopp(cdna3::kSWaitcntSopp, 0)) return true; } return false; } void expect_cdna3_translated_descriptor_sgprs_eq(const std::vector &image, uint32_t expected) { rocjitsu::AmdGpuCodeObject translated(image.data(), image.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); rocjitsu::KernelDescriptorTranslator parser(ROCJITSU_CODE_ARCH_CDNA3, ROCJITSU_CODE_ARCH_CDNA3); const auto infos = parser.translate_image(image, translated.text_sections()[0]->sectionOffset(), translated.text_sections()[0]->size(), rocjitsu::KernelDescriptorTranslationOptions{}); ASSERT_EQ(infos.size(), 1u); EXPECT_EQ(infos[0].target_sgpr_count, expected); } struct InstructionWordsView { const uint32_t *words = nullptr; size_t word_count = 0; }; uint32_t cdna3_descriptor_vgpr_allocation_count(const rocjitsu::TestKernelDescriptor &descriptor) { const uint32_t granulated = AMDHSA_BITS_GET(descriptor.compute_pgm_rsrc1, rocr::llvm::amdhsa::COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT); return (granulated + 1u) * 8u; } uint32_t cdna3_descriptor_sgpr_count(const rocjitsu::TestKernelDescriptor &descriptor) { const uint32_t granulated = AMDHSA_BITS_GET(descriptor.compute_pgm_rsrc1, rocr::llvm::amdhsa::COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT); return (granulated + 1u) * 8u; } std::optional find_virtual_lds_metadata_record_for_test(const std::vector &image, std::string_view kernel_name = "kernel") { rocjitsu::AmdGpuCodeObject translated(image.data(), image.size()); if (!translated.is_valid()) { ADD_FAILURE() << "translated code object is invalid"; return std::nullopt; } const auto *metadata_section = rocjitsu::find_section(translated, rocjitsu::kVirtualLdsMetadataSectionName); if (metadata_section == nullptr) { ADD_FAILURE() << "missing " << rocjitsu::kVirtualLdsMetadataSectionName << " section"; return std::nullopt; } const auto parsed = rocjitsu::parse_virtual_lds_metadata( {reinterpret_cast(metadata_section->data()), metadata_section->size()}); if (!parsed.has_value()) { ADD_FAILURE() << "could not parse virtual-LDS metadata"; return std::nullopt; } const auto it = std::ranges::find_if(*parsed, [&](const rocjitsu::VirtualLdsKernelMetadata &record) { return record.kernel_name == kernel_name; }); if (it == parsed->end()) { ADD_FAILURE() << "missing virtual-LDS metadata for " << kernel_name; return std::nullopt; } return *it; } std::optional find_sidecar_metadata_record_for_test(const std::vector &image, std::string_view kernel_name = "kernel") { rocjitsu::AmdGpuCodeObject translated(image.data(), image.size()); const auto *section = rocjitsu::find_section(translated, rocjitsu::kSidecarMetadataSectionName); if (section == nullptr) return std::nullopt; const auto parsed = rocjitsu::parse_sidecar_metadata( {reinterpret_cast(section->data()), section->size()}); if (!parsed) return std::nullopt; const auto record = std::ranges::find_if(*parsed, [&](const auto &candidate) { return candidate.kernel_name == kernel_name && candidate.variant_name == rocjitsu::kVirtualLdsSidecarVariantName; }); return record == parsed->end() ? std::nullopt : std::optional{*record}; } std::optional find_kernarg_extension_record_for_test(const std::vector &image, std::string_view kernel_name = "kernel") { rocjitsu::AmdGpuCodeObject translated(image.data(), image.size()); const auto *section = rocjitsu::find_section(translated, rocjitsu::kKernargExtensionMetadataSectionName); if (section == nullptr) return std::nullopt; const auto parsed = rocjitsu::parse_kernarg_extension_metadata( {reinterpret_cast(section->data()), section->size()}); if (!parsed) return std::nullopt; const auto record = std::ranges::find_if(*parsed, [&](const auto &candidate) { return candidate.kernel_name == kernel_name && candidate.variant_name == rocjitsu::kVirtualLdsSidecarVariantName; }); return record == parsed->end() ? std::nullopt : std::optional{*record}; } std::optional make_kernarg_extension_layout_for_test(const rocjitsu::KernargExtensionMetadata &metadata) { std::vector payloads; for (const auto &payload : metadata.payloads) payloads.push_back({.size = payload.size, .alignment = payload.alignment}); return rocjitsu::make_kernarg_extension_layout(metadata.original_kernarg_size, payloads); } std::optional read_descriptor_at_loaded_vaddr_for_test(const std::vector &image, uint64_t vaddr) { const auto descriptor_file_offset = rocjitsu::loaded_vaddr_to_file_offset(image, vaddr); if (!descriptor_file_offset.has_value()) { ADD_FAILURE() << "could not map descriptor vaddr 0x" << std::hex << vaddr << std::dec; return std::nullopt; } if (*descriptor_file_offset > image.size() || sizeof(rocjitsu::TestKernelDescriptor) > image.size() - *descriptor_file_offset) { ADD_FAILURE() << "descriptor vaddr 0x" << std::hex << vaddr << std::dec << " maps outside the ELF image"; return std::nullopt; } return rocjitsu::read_elf_struct_for_test( image, *descriptor_file_offset); } std::optional read_virtual_lds_sidecar_descriptor_for_test(const std::vector &image, std::string_view kernel_name = "kernel") { const auto record = find_sidecar_metadata_record_for_test(image, kernel_name); if (!record.has_value()) return std::nullopt; return read_descriptor_at_loaded_vaddr_for_test(image, record->variant_descriptor_vaddr); } std::optional virtual_lds_sidecar_entry_words_for_test(const std::vector &image, std::string_view kernel_name = "kernel") { rocjitsu::AmdGpuCodeObject translated(image.data(), image.size()); if (!translated.is_valid()) { ADD_FAILURE() << "translated code object is invalid"; return std::nullopt; } const auto record = find_sidecar_metadata_record_for_test(image, kernel_name); const auto descriptor = read_virtual_lds_sidecar_descriptor_for_test(image, kernel_name); if (!record.has_value() || !descriptor.has_value()) return std::nullopt; const int64_t entry_delta = rocjitsu::read_kernel_descriptor_entry_offset(&*descriptor); const int64_t entry_vaddr_signed = static_cast(record->variant_descriptor_vaddr) + entry_delta; if (entry_vaddr_signed < 0) { ADD_FAILURE() << "sidecar entry resolves before address zero"; return std::nullopt; } const uint64_t entry_vaddr = static_cast(entry_vaddr_signed); const auto entry_file_offset = rocjitsu::loaded_vaddr_to_file_offset(image, entry_vaddr); if (!entry_file_offset.has_value()) { ADD_FAILURE() << "could not map sidecar entry vaddr 0x" << std::hex << entry_vaddr << std::dec; return std::nullopt; } for (const auto *text : translated.text_sections()) { const uint64_t text_begin = text->sectionOffset(); const uint64_t text_end = text_begin + text->size(); if (*entry_file_offset < text_begin || *entry_file_offset >= text_end) continue; const uint64_t byte_count = text_end - *entry_file_offset; return InstructionWordsView{ reinterpret_cast(image.data() + *entry_file_offset), static_cast(byte_count / sizeof(uint32_t)), }; } ADD_FAILURE() << "sidecar entry does not point inside a text section"; return std::nullopt; } void expect_cdna3_sidecar_descriptor_sgprs_eq(const std::vector &image, uint32_t expected) { const auto sidecar_kd = read_virtual_lds_sidecar_descriptor_for_test(image); ASSERT_TRUE(sidecar_kd.has_value()); EXPECT_EQ(cdna3_descriptor_sgpr_count(*sidecar_kd), expected); } void expect_cdna3_sidecar_descriptor_vgprs_at_least(const std::vector &image, uint32_t expected_minimum) { const auto sidecar_kd = read_virtual_lds_sidecar_descriptor_for_test(image); ASSERT_TRUE(sidecar_kd.has_value()); EXPECT_GE(cdna3_descriptor_vgpr_allocation_count(*sidecar_kd), expected_minimum); } uint32_t build_s_getpc_b64(uint16_t sdst, rj_code_arch_t arch) { switch (arch) { case ROCJITSU_CODE_ARCH_RDNA3: case ROCJITSU_CODE_ARCH_RDNA3_5: case ROCJITSU_CODE_ARCH_RDNA4: return rocjitsu::build_sop1_encoding(arch, 0x47, sdst, 0); default: return rocjitsu::build_sop1_encoding(arch, 0x1c, sdst, 0); } } uint32_t build_s_setpc_b64(uint16_t ssrc0, rj_code_arch_t arch) { switch (arch) { case ROCJITSU_CODE_ARCH_RDNA3: case ROCJITSU_CODE_ARCH_RDNA3_5: case ROCJITSU_CODE_ARCH_RDNA4: return rocjitsu::build_sop1_encoding(arch, 0x48, 0, ssrc0); default: return rocjitsu::build_sop1_encoding(arch, 0x1d, 0, ssrc0); } } uint32_t build_s_swappc_b64(uint16_t sdst, uint16_t ssrc0, rj_code_arch_t arch) { switch (arch) { case ROCJITSU_CODE_ARCH_RDNA3: case ROCJITSU_CODE_ARCH_RDNA3_5: case ROCJITSU_CODE_ARCH_RDNA4: return rocjitsu::build_sop1_encoding(arch, 0x49, sdst, ssrc0); default: return rocjitsu::build_sop1_encoding(arch, 0x1e, sdst, ssrc0); } } uint32_t build_s_call_b64(uint16_t sdst, int16_t simm16) { return cdna4::build_sopk(cdna4::kSCallB64Sopk, {.simm16 = static_cast(simm16), .sdst = static_cast(sdst)})[0]; } uint32_t build_s_trap(uint16_t simm16) { // CDNA1-4 encode S_TRAP at SOPP opcode 0x12. This helper is intentionally // local to the CDNA4->CDNA3 tests below; RDNA3+ uses a different SOPP opcode. return cdna4::build_sopp(cdna4::kSTrapSopp, {.simm16 = simm16})[0]; } uint32_t build_s_add_u32(uint16_t sdst, uint16_t ssrc0, uint16_t ssrc1) { return cdna4::build_sop2(cdna4::kSAddU32Sop2, {.ssrc0 = static_cast(ssrc0), .ssrc1 = static_cast(ssrc1), .sdst = static_cast(sdst)})[0]; } uint32_t build_s_addc_u32(uint16_t sdst, uint16_t ssrc0, uint16_t ssrc1) { return cdna4::build_sop2(cdna4::kSAddcU32Sop2, {.ssrc0 = static_cast(ssrc0), .ssrc1 = static_cast(ssrc1), .sdst = static_cast(sdst)})[0]; } std::array build_cdna4_smem_load(uint8_t op, uint8_t sdata, uint8_t sbase, uint32_t byte_offset) { rocjitsu::cdna4::SmemMachineInst inst{}; inst.encoding = 0x30; inst.op = op; inst.sbase = (sbase / 2) & 0x3f; inst.sdata = sdata & 0x7f; inst.imm = 1; inst.offset = byte_offset & 0x1fffff; std::array words{}; std::memcpy(words.data(), &inst, sizeof(inst)); return words; } std::vector> decode_text_instructions(const rocjitsu::Section &text, rj_code_arch_t arch) { std::vector> decoded; auto decoder = rocjitsu::Decoder::create(arch); if (!decoder) return decoded; const auto *words = reinterpret_cast(text.data()); const size_t word_count = text.size() / sizeof(rj_code_binary_inst_t); size_t word_offset = 0; while (word_offset < word_count) { std::unique_ptr inst( decoder->decode(words + word_offset, word_offset * sizeof(rj_code_binary_inst_t))); if (!inst) break; word_offset += static_cast(inst->size()) / sizeof(rj_code_binary_inst_t); decoded.push_back(std::move(inst)); } return decoded; } // --- Synthetic BinaryTranslator integration tests --- TEST(BinaryTranslatorE2E, TranslatesMultiKernelCodeObject) { auto image = rocjitsu::make_minimal_amdgpu_elf_with_two_kernel_descriptors(); rocjitsu::AmdGpuCodeObject co(image.data(), image.size()); ASSERT_TRUE(co.is_valid()); ASSERT_FALSE(co.text_sections().empty()); const auto *text = co.text_sections()[0]; const auto *rodata = rocjitsu::find_section(co, ".rodata"); ASSERT_NE(rodata, nullptr); rocjitsu::KernelDescriptorTranslator original_parser(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_RDNA4); const auto original_infos = original_parser.translate_image( image, text->sectionOffset(), text->size(), rocjitsu::KernelDescriptorTranslationOptions{}); ASSERT_EQ(original_infos.size(), 2u); std::vector original_entries; std::vector original_descriptor_offsets; for (const auto &info : original_infos) { original_entries.push_back(info.entry_text_offset); original_descriptor_offsets.push_back(info.descriptor_file_offset); } std::ranges::sort(original_entries); std::ranges::sort(original_descriptor_offsets); EXPECT_EQ(original_entries, (std::vector{0, sizeof(uint32_t)})); EXPECT_EQ(original_descriptor_offsets, (std::vector{rodata->sectionOffset(), rodata->sectionOffset() + rocjitsu::kKernelDescriptorSize})); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_RDNA4); auto result = translator.translate(co); ASSERT_FALSE(result.elf_bytes.empty()); EXPECT_TRUE(result.ok()); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); EXPECT_EQ(translated.text_sections()[0]->size(), text->size()); EXPECT_EQ(rocjitsu::find_section(translated, ".rj_translations"), nullptr) << "this fixture should exercise multi-kernel descriptor handling without code caves"; const auto *translated_header = reinterpret_cast(result.elf_bytes.data()); EXPECT_EQ(translated_header->e_flags & rocjitsu::EF_AMDGPU_MACH, rocjitsu::EF_AMDGPU_MACH_AMDGCN_GFX1200); rocjitsu::KernelDescriptorTranslator translated_parser(ROCJITSU_CODE_ARCH_RDNA4, ROCJITSU_CODE_ARCH_RDNA4); const auto translated_infos = translated_parser.translate_image( result.elf_bytes, translated.text_sections()[0]->sectionOffset(), translated.text_sections()[0]->size(), rocjitsu::KernelDescriptorTranslationOptions{}); ASSERT_EQ(translated_infos.size(), 2u); std::vector translated_entries; std::vector translated_descriptor_offsets; for (const auto &info : translated_infos) { translated_entries.push_back(info.entry_text_offset); translated_descriptor_offsets.push_back(info.descriptor_file_offset); } std::ranges::sort(translated_entries); std::ranges::sort(translated_descriptor_offsets); EXPECT_EQ(translated_entries, (std::vector{0, sizeof(uint32_t)})); EXPECT_EQ(translated_descriptor_offsets, original_descriptor_offsets); } TEST(BinaryTranslatorE2E, SkipFailedKernelKeepsIndependentKernelTranslating) { constexpr uint16_t kReturnSreg = 0; constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const std::vector words = { rocjitsu::build_s_setpc_b64(kReturnSreg, ROCJITSU_CODE_ARCH_CDNA4), kCdna4SEndpgm, }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_two_kernel_descriptors(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; options.skip_failed_kernels = true; options.debug_continue_after_failure = true; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); // A skipped kernel is only a warning, so ok() stays true, but the artifact is // NOT dispatchable: its s_endpgm stub would silently complete if dispatched. Code // object output paths (CLI, API consumers) must gate on dispatchable(). EXPECT_FALSE(result.dispatchable()); ASSERT_FALSE(result.elf_bytes.empty()); const auto skipped = std::ranges::find_if(result.diagnostics, [](const auto &diagnostic) { return diagnostic.kind == rocjitsu::DiagnosticKind::KernelSkipped; }); ASSERT_NE(skipped, result.diagnostics.end()); EXPECT_EQ(skipped->severity, rocjitsu::DiagnosticSeverity::Warning); EXPECT_EQ(skipped->guest_offset, std::optional(0)); EXPECT_NE(skipped->message.find("kernel0"), std::string::npos); EXPECT_NE(skipped->message.find("indirect branch"), std::string::npos); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *translated_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(translated_words[0], rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA3)) << "the failed kernel body exits safely if it is dispatched"; EXPECT_NE(skipped->message.find("S_ENDPGM"), std::string::npos); EXPECT_NE(skipped->message.find("INVALID OUTPUTS"), std::string::npos); rocjitsu::KernelDescriptorTranslator parser(ROCJITSU_CODE_ARCH_CDNA3, ROCJITSU_CODE_ARCH_CDNA3); const auto infos = parser.translate_image( result.elf_bytes, translated.text_sections()[0]->sectionOffset(), translated.text_sections()[0]->size(), rocjitsu::KernelDescriptorTranslationOptions{}); ASSERT_EQ(infos.size(), 2u); std::vector entries; for (const auto &info : infos) entries.push_back(info.entry_text_offset); std::ranges::sort(entries); ASSERT_EQ(entries.size(), 2u); EXPECT_EQ(entries[0], 0u); EXPECT_EQ(entries[1], sizeof(uint32_t)) << "the independent kernel follows the one-instruction skipped body"; ASSERT_EQ(entries[1] % sizeof(uint32_t), 0u); ASSERT_LT(entries[1], translated.text_sections()[0]->size()); EXPECT_EQ(translated_words[entries[1] / sizeof(uint32_t)], rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA3)) << "the independent kernel still translates instead of using the skipped-kernel trap"; } TEST(BinaryTranslatorE2E, OversizedTargetLdsDescriptorEmitsVirtualVariantInsteadOfSkipping) { auto image = rocjitsu::make_minimal_amdgpu_elf_with_two_kernel_descriptors(); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), 2 * rocjitsu::kKernelDescriptorSize); // gfx950/CDNA4 can advertise 160 KiB LDS kernels, but gfx942/CDNA3 can only // dispatch 64 KiB per workgroup in the checked-in topology. Keep the normal // descriptor for launches that fit, and emit a virtual-LDS sidecar descriptor // for runtime packet rewriting when static plus dynamic LDS exceeds the host. rocjitsu::write_value_for_test( image, rodata->sectionOffset() + offsetof(rocjitsu::TestKernelDescriptor, group_segment_fixed_size), 105600u); rocjitsu::write_value_for_test( image, rodata->sectionOffset() + offsetof(rocjitsu::TestKernelDescriptor, kernarg_size), 16u); uint32_t source_rsrc2 = 0; AMDHSA_BITS_SET(source_rsrc2, rocr::llvm::amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1); AMDHSA_BITS_SET(source_rsrc2, rocr::llvm::amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_Y, 1); AMDHSA_BITS_SET(source_rsrc2, rocr::llvm::amdhsa::COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, 2); // COMPUTE_PGM_RSRC2.LDS_SIZE is programmed by CP from the dispatch packet, not // by the code-object descriptor. Seed a stale guest value here so the test // proves translated normal and virtual descriptors do not preserve it. AMDHSA_BITS_SET(source_rsrc2, rocr::llvm::amdhsa::COMPUTE_PGM_RSRC2_GRANULATED_LDS_SIZE, 22); rocjitsu::write_value_for_test( image, rodata->sectionOffset() + offsetof(rocjitsu::TestKernelDescriptor, compute_pgm_rsrc2), source_rsrc2); uint16_t source_properties = 0; AMDHSA_BITS_SET(source_properties, rocr::llvm::amdhsa::KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR, 1); rocjitsu::write_value_for_test( image, rodata->sectionOffset() + offsetof(rocjitsu::TestKernelDescriptor, kernel_code_properties), source_properties); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; options.skip_failed_kernels = true; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); ASSERT_FALSE(result.elf_bytes.empty()); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); const auto *translated_rodata = rocjitsu::find_section(translated, ".rodata"); ASSERT_NE(translated_rodata, nullptr); const auto normal_kd = rocjitsu::read_elf_struct_for_test( result.elf_bytes, translated_rodata->sectionOffset()); EXPECT_EQ(normal_kd.group_segment_fixed_size, 105600u); EXPECT_EQ(normal_kd.kernarg_size, 16u); EXPECT_EQ(AMDHSA_BITS_GET(normal_kd.compute_pgm_rsrc2, rocr::llvm::amdhsa::COMPUTE_PGM_RSRC2_GRANULATED_LDS_SIZE), 0u); const auto *metadata_section = rocjitsu::find_section(translated, rocjitsu::kVirtualLdsMetadataSectionName); ASSERT_NE(metadata_section, nullptr); const auto parsed = rocjitsu::parse_virtual_lds_metadata( {reinterpret_cast(metadata_section->data()), metadata_section->size()}); ASSERT_TRUE(parsed.has_value()); ASSERT_EQ(parsed->size(), 1u); const auto &record = parsed->front(); const auto sidecar = find_sidecar_metadata_record_for_test(result.elf_bytes, "kernel0"); const auto extension = find_kernarg_extension_record_for_test(result.elf_bytes, "kernel0"); ASSERT_TRUE(sidecar.has_value()); ASSERT_TRUE(extension.has_value()); const auto wrapper_layout = make_kernarg_extension_layout_for_test(*extension); ASSERT_TRUE(wrapper_layout.has_value()); EXPECT_EQ(record.kernel_name, "kernel0"); EXPECT_EQ(sidecar->normal_descriptor_vaddr, translated.kernel_descriptor_offset("kernel0")); EXPECT_NE(sidecar->variant_descriptor_vaddr, sidecar->normal_descriptor_vaddr); EXPECT_EQ(record.static_lds_bytes, 105600u); EXPECT_EQ(extension->original_kernarg_size, 16u); ASSERT_EQ(wrapper_layout->payload_offsets.size(), 1u); EXPECT_EQ(wrapper_layout->payload_offsets.front(), 24u); EXPECT_NE(record.virtual_lds_base_sgpr, 0u); EXPECT_NE(record.flags & rocjitsu::kVirtualLdsFlagRuntimeStateBlock, 0u); EXPECT_NE(record.flags & rocjitsu::kVirtualLdsFlagWorkgroupIdX, 0u); EXPECT_NE(record.flags & rocjitsu::kVirtualLdsFlagWorkgroupIdY, 0u); EXPECT_EQ(record.flags & rocjitsu::kVirtualLdsFlagWorkgroupIdZ, 0u); const auto virtual_descriptor_offset = rocjitsu::loaded_vaddr_to_file_offset(result.elf_bytes, sidecar->variant_descriptor_vaddr); ASSERT_TRUE(virtual_descriptor_offset.has_value()); const auto virtual_kd = rocjitsu::read_elf_struct_for_test( result.elf_bytes, *virtual_descriptor_offset); EXPECT_EQ(virtual_kd.group_segment_fixed_size, 0u); EXPECT_EQ(virtual_kd.kernarg_size, 48u); EXPECT_EQ(AMDHSA_BITS_GET(virtual_kd.compute_pgm_rsrc2, rocr::llvm::amdhsa::COMPUTE_PGM_RSRC2_GRANULATED_LDS_SIZE), 0u); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarPreservesReservedBytesAfterTextGrowth) { // Regression for the sidecar being copied from a stale pre-.text-growth file // offset. The DS-read fixture lowers to a flat-global load, which GROWS .text // and thus shifts the descriptor section that follows it. The sidecar template // must come from a snapshot of the source descriptor, not from re-reading the // now-shifted offset (which would read relocated instruction bytes). Seed the // reserved regions with recognizable patterns and assert the emitted sidecar // carries them unchanged. constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read_b32_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); // Seed reserved1[20] with a distinctive byte pattern. reserved1 is a field DBT // never rewrites, so it is only correct in the sidecar if the template was // copied from the (pre-growth) source descriptor snapshot. const uint64_t reserved1_off = rodata->sectionOffset() + offsetof(rocjitsu::TestKernelDescriptor, reserved1); std::array reserved1_pattern{}; for (size_t i = 0; i < reserved1_pattern.size(); ++i) reserved1_pattern[i] = static_cast(0xA0 + i); for (size_t i = 0; i < reserved1_pattern.size(); ++i) image[reserved1_off + i] = reserved1_pattern[i]; rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, rocjitsu::BinaryTranslatorOptions{}); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; // Confirm .text actually grew (otherwise the regression could not occur). rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); EXPECT_GT(translated.text_sections()[0]->size(), 3u * sizeof(uint32_t)); const auto sidecar_kd = read_virtual_lds_sidecar_descriptor_for_test(result.elf_bytes, "kernel"); ASSERT_TRUE(sidecar_kd.has_value()); EXPECT_TRUE(std::equal(std::begin(sidecar_kd->reserved1), std::end(sidecar_kd->reserved1), reserved1_pattern.begin())) << "sidecar reserved1 was not copied from the source descriptor snapshot"; } TEST(BinaryTranslatorE2E, VirtualLdsWrapperIgnoresInlineKernargLoadsBeyondDescriptorSize) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; constexpr uint16_t kKernargSgpr = 0; constexpr uint16_t kInlineInt0 = 128; constexpr uint16_t kInlineInt16 = 144; constexpr uint32_t kDescriptorKernargSize = 48; constexpr uint32_t kExpectedStateOffset = 56; constexpr uint32_t kExpectedWrapperBytes = 80; const auto load_header = build_cdna4_smem_load(/*op=*/0, /*sdata=*/16, kKernargSgpr, /*byte_offset=*/0); const auto load_userargs = build_cdna4_smem_load(/*op=*/4, /*sdata=*/20, kKernargSgpr, /*byte_offset=*/0x40); std::vector words = { load_header[0], load_header[1], build_s_add_u32(kKernargSgpr, kKernargSgpr, kInlineInt16), build_s_addc_u32(kKernargSgpr + 1, kKernargSgpr + 1, kInlineInt0), load_userargs[0], load_userargs[1], kCdna4SEndpgm, }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::enable_kernarg_segment_ptr_sgpr(image, kDescriptorKernargSize); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); rocjitsu::write_value_for_test( image, rodata->sectionOffset() + offsetof(rocjitsu::TestKernelDescriptor, group_segment_fixed_size), 6144u); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; options.skip_failed_kernels = true; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); ASSERT_FALSE(result.elf_bytes.empty()); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); const auto *translated_rodata = rocjitsu::find_section(translated, ".rodata"); ASSERT_NE(translated_rodata, nullptr); const auto normal_kd = rocjitsu::read_elf_struct_for_test( result.elf_bytes, translated_rodata->sectionOffset()); EXPECT_EQ(normal_kd.kernarg_size, kDescriptorKernargSize); const auto *metadata_section = rocjitsu::find_section(translated, rocjitsu::kVirtualLdsMetadataSectionName); ASSERT_NE(metadata_section, nullptr); const auto parsed = rocjitsu::parse_virtual_lds_metadata( {reinterpret_cast(metadata_section->data()), metadata_section->size()}); ASSERT_TRUE(parsed.has_value()); ASSERT_EQ(parsed->size(), 1u); const auto sidecar = find_sidecar_metadata_record_for_test(result.elf_bytes); const auto extension = find_kernarg_extension_record_for_test(result.elf_bytes); ASSERT_TRUE(sidecar.has_value()); ASSERT_TRUE(extension.has_value()); const auto wrapper_layout = make_kernarg_extension_layout_for_test(*extension); ASSERT_TRUE(wrapper_layout.has_value()); EXPECT_EQ(extension->original_kernarg_size, kDescriptorKernargSize); ASSERT_EQ(wrapper_layout->payload_offsets.size(), 1u); EXPECT_EQ(wrapper_layout->payload_offsets.front(), kExpectedStateOffset); const auto virtual_descriptor_offset = rocjitsu::loaded_vaddr_to_file_offset(result.elf_bytes, sidecar->variant_descriptor_vaddr); ASSERT_TRUE(virtual_descriptor_offset.has_value()); const auto virtual_kd = rocjitsu::read_elf_struct_for_test( result.elf_bytes, *virtual_descriptor_offset); EXPECT_EQ(virtual_kd.kernarg_size, kExpectedWrapperBytes); } TEST(BinaryTranslatorE2E, VirtualLdsWrapperIgnoresDirectKernargLoadsInSharedHelperBlocks) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; constexpr uint16_t kKernargSgpr = 0; constexpr uint32_t kDescriptorKernargSize = 0x90; constexpr uint32_t kExpectedStateOffset = 0x98; constexpr uint32_t kExpectedWrapperBytes = 0xB0; const auto load_helper_tail = build_cdna4_smem_load(/*op=*/0, /*sdata=*/16, kKernargSgpr, /*byte_offset=*/0xA0); // Model Tensile-style helper code that is present in the code object but not // reached by the local descriptor CFG. The virtual-LDS dispatcher still must // not size its wrapper prefix from a decoded direct kernarg load range from // the same ABI kernarg SGPR pair. std::vector words = { rocjitsu::pack_sopp(/*op=*/5, /*simm16=*/2), load_helper_tail[0], load_helper_tail[1], kCdna4SEndpgm, }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::enable_kernarg_segment_ptr_sgpr(image, kDescriptorKernargSize); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); rocjitsu::write_value_for_test( image, rodata->sectionOffset() + offsetof(rocjitsu::TestKernelDescriptor, group_segment_fixed_size), 6144u); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; options.skip_failed_kernels = true; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); ASSERT_FALSE(result.elf_bytes.empty()); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); const auto *translated_rodata = rocjitsu::find_section(translated, ".rodata"); ASSERT_NE(translated_rodata, nullptr); const auto normal_kd = rocjitsu::read_elf_struct_for_test( result.elf_bytes, translated_rodata->sectionOffset()); EXPECT_EQ(normal_kd.kernarg_size, kDescriptorKernargSize); const auto *metadata_section = rocjitsu::find_section(translated, rocjitsu::kVirtualLdsMetadataSectionName); ASSERT_NE(metadata_section, nullptr); const auto parsed = rocjitsu::parse_virtual_lds_metadata( {reinterpret_cast(metadata_section->data()), metadata_section->size()}); ASSERT_TRUE(parsed.has_value()); ASSERT_EQ(parsed->size(), 1u); const auto sidecar = find_sidecar_metadata_record_for_test(result.elf_bytes); const auto extension = find_kernarg_extension_record_for_test(result.elf_bytes); ASSERT_TRUE(sidecar.has_value()); ASSERT_TRUE(extension.has_value()); const auto wrapper_layout = make_kernarg_extension_layout_for_test(*extension); ASSERT_TRUE(wrapper_layout.has_value()); EXPECT_EQ(extension->original_kernarg_size, kDescriptorKernargSize); ASSERT_EQ(wrapper_layout->payload_offsets.size(), 1u); EXPECT_EQ(wrapper_layout->payload_offsets.front(), kExpectedStateOffset); const auto virtual_descriptor_offset = rocjitsu::loaded_vaddr_to_file_offset(result.elf_bytes, sidecar->variant_descriptor_vaddr); ASSERT_TRUE(virtual_descriptor_offset.has_value()); const auto virtual_kd = rocjitsu::read_elf_struct_for_test( result.elf_bytes, *virtual_descriptor_offset); EXPECT_EQ(virtual_kd.kernarg_size, kExpectedWrapperBytes); } TEST(BinaryTranslatorE2E, Gfx1250LongDirectBranchGrowthIsIdempotent) { // Each selected DS2 becomes two two-word operations plus a wait. Choose the // count so the first target starts one dword below SOPP's forward limit. // Growing the already-out-of-range second branch then shifts that near target // and forces a second monotonic layout round; both branches must end up long. constexpr size_t kSoppMaxForwardDwords = 0x7fff; constexpr size_t kExpandedDs2Words = 5; constexpr size_t kNearExpansionCount = (kSoppMaxForwardDwords - 2) / kExpandedDs2Words; static_assert(1 + kNearExpansionCount * kExpandedDs2Words == kSoppMaxForwardDwords - 1); constexpr size_t kExpansionCount = kNearExpansionCount + 1; constexpr size_t kNearTargetWord = 2 + kNearExpansionCount * 2; constexpr size_t kFarTargetWord = 2 + kExpansionCount * 2; constexpr auto ds2 = gfx1250::build_vds(gfx1250::kDsStore2addrB32Vds, {.offset0 = 3, .offset1 = 5, .addr = 20, .data0 = 30, .data1 = 40}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; std::vector words; words.reserve(kFarTargetWord + 1); words.push_back(gfx1250::build_sopp(gfx1250::kSCbranchScc0Sopp, {.simm16 = static_cast(kNearTargetWord - 1)})[0]); words.push_back(gfx1250::build_sopp(gfx1250::kSCbranchScc0Sopp, {.simm16 = static_cast(kFarTargetWord - 2)})[0]); for (size_t i = 0; i < kExpansionCount; ++i) { words.push_back(ds2[0]); words.push_back(ds2[1]); } words.push_back(kGfx1250SEndpgm); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const uint32_t marker = rocjitsu::build_s_nop(rocjitsu::kLongDirectBranchMarkerNopImmediate, ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ((target_words[0] >> 16) & 0x7fu, gfx1250::kSCbranchScc1Sopp); EXPECT_EQ(target_words[1], marker); const auto translated_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); EXPECT_EQ(std::count(target_words, target_words + translated_word_count, marker), 2); rocjitsu::BinaryTranslator verifier( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto second = verifier.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250LongBranchReportsSgprExhaustionAfterSemanticExpansion) { using namespace rocr::llvm::amdhsa; constexpr size_t kTargetWord = 0x8000; constexpr uint16_t kLiveSgprs = 104; constexpr auto conversion = gfx1250::build_vop3(gfx1250::kVCvtF32Fp8Vop3, {.vdst = 30, .clamp = 1, .src0 = 256 + 22}); constexpr uint32_t kGfx1250SNop = 0xBF800000u; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; std::vector words; words.reserve(kTargetWord + 1); words.push_back(gfx1250::build_sopp(gfx1250::kSCbranchScc0Sopp, {.simm16 = static_cast(kTargetWord - 1u)})[0]); words.insert(words.end(), conversion.begin(), conversion.end()); for (uint16_t sgpr = 0; sgpr < kLiveSgprs; ++sgpr) { words.push_back(gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = static_cast(sgpr), .sdst = 125})[0]); } words.resize(kTargetWord, kGfx1250SNop); words.push_back(kGfx1250SEndpgm); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); auto descriptor = rocjitsu::read_kernel_descriptor_for_test(rodata->data()); AMDHSA_BITS_SET(descriptor.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, 12); rocjitsu::write_kernel_descriptor_for_test(image.data() + rodata->sectionOffset(), descriptor); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(rocjitsu::has_error_containing( result, rocjitsu::DiagnosticKind::ResourceLimit, "long direct branch requires an additional descriptor-backed SGPR pair after semantic " "expansion")); const auto diagnostic = std::ranges::find_if(result.diagnostics, [](const auto &item) { return item.kind == rocjitsu::DiagnosticKind::ResourceLimit; }); ASSERT_NE(diagnostic, result.diagnostics.end()); EXPECT_EQ(diagnostic->guest_offset, std::optional(0)) << "the resource diagnostic must identify the branch that could not be expanded"; } TEST(BinaryTranslatorE2E, Gfx1250CompactConditionalLayoutIsIdempotent) { constexpr size_t kTargetWord = 20000; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; std::vector words; words.reserve(kTargetWord + 1); words.push_back( rocjitsu::pack_sopp(gfx1250::kSCbranchExecnzSopp, static_cast(kTargetWord - 1))); for (size_t i = 1; i < kTargetWord; ++i) words.push_back(rocjitsu::pack_sopp(gfx1250::kSCbranchScc0Sopp, 0)); words.push_back(kGfx1250SEndpgm); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[0], rocjitsu::pack_sopp(gfx1250::kSCbranchExecnzSopp, static_cast(kTargetWord - 1))); rocjitsu::BinaryTranslator verifier( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto second = verifier.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250CompilerLongJumpCompactsIdempotentlyAfterPaddingNop) { constexpr uint16_t kPcSreg = 4; constexpr uint16_t kGfx1250SAddNcU64Opcode = 83; constexpr uint16_t kLiteralOperand = 255; constexpr uint32_t kTargetOffset = 140000; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; std::vector words = { rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_GFX1250), rocjitsu::build_s_getpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_GFX1250), rocjitsu::build_sop2_encoding(ROCJITSU_CODE_ARCH_GFX1250, kGfx1250SAddNcU64Opcode, kPcSreg, kPcSreg, kLiteralOperand), kTargetOffset - 2 * sizeof(uint32_t), rocjitsu::build_s_setpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_GFX1250), }; words.resize(kTargetOffset / sizeof(uint32_t), rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_GFX1250)); words.push_back(kGfx1250SEndpgm); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[0], rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_GFX1250)); EXPECT_EQ((target_words[4] >> 16) & 0x7fu, gfx1250::kSBranchSopp); rocjitsu::BinaryTranslator verifier( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto second = verifier.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250MarkedLongJumpStaysLongWhenTargetIsDirectlyReachable) { constexpr uint16_t kPcSreg = 4; constexpr uint16_t kGfx1250SAddNcU64Opcode = 83; constexpr uint16_t kLiteralOperand = 255; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; constexpr uint32_t kTargetOffset = 5 * sizeof(uint32_t); const uint32_t marker = rocjitsu::build_s_nop(rocjitsu::kLongDirectBranchMarkerNopImmediate, ROCJITSU_CODE_ARCH_GFX1250); const std::vector words = { marker, rocjitsu::build_s_getpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_GFX1250), rocjitsu::build_sop2_encoding(ROCJITSU_CODE_ARCH_GFX1250, kGfx1250SAddNcU64Opcode, kPcSreg, kPcSreg, kLiteralOperand), kTargetOffset - 2 * sizeof(uint32_t), rocjitsu::build_s_setpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_GFX1250), kGfx1250SEndpgm, }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[0], marker); EXPECT_EQ(target_words[4], rocjitsu::build_s_setpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_GFX1250)); rocjitsu::BinaryTranslator verifier( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto second = verifier.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250DoesNotTreatLiteralAsLongJumpMarker) { constexpr uint16_t kPcSreg = 4; constexpr uint16_t kUnrelatedSgpr = 20; constexpr uint16_t kGfx1250SAddNcU64Opcode = 83; constexpr uint16_t kLiteralOperand = 255; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; constexpr uint32_t kTargetOffset = 7 * sizeof(uint32_t); const uint32_t marker = rocjitsu::build_s_nop(rocjitsu::kLongDirectBranchMarkerNopImmediate, ROCJITSU_CODE_ARCH_GFX1250); const std::vector words = { rocjitsu::build_sop2_encoding(ROCJITSU_CODE_ARCH_GFX1250, 0, kUnrelatedSgpr, kUnrelatedSgpr, kLiteralOperand), marker, rocjitsu::build_s_getpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_GFX1250), rocjitsu::build_sop2_encoding(ROCJITSU_CODE_ARCH_GFX1250, kGfx1250SAddNcU64Opcode, kPcSreg, kPcSreg, kLiteralOperand), kTargetOffset - 3 * sizeof(uint32_t), rocjitsu::build_s_setpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_GFX1250), rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_GFX1250), kGfx1250SEndpgm, }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "s_setpc_b64"; }), 0); const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250DoesNotPreserveLongJumpMarkerAcrossBlockBoundary) { constexpr uint16_t kPcSreg = 4; constexpr uint16_t kGfx1250SAddNcU64Opcode = 83; constexpr uint16_t kLiteralOperand = 255; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; constexpr uint32_t kTargetOffset = 7 * sizeof(uint32_t); const uint32_t marker = rocjitsu::build_s_nop(rocjitsu::kLongDirectBranchMarkerNopImmediate, ROCJITSU_CODE_ARCH_GFX1250); const std::vector words = { gfx1250::build_sopp(gfx1250::kSCbranchScc0Sopp, {.simm16 = 1})[0], marker, rocjitsu::build_s_getpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_GFX1250), rocjitsu::build_sop2_encoding(ROCJITSU_CODE_ARCH_GFX1250, kGfx1250SAddNcU64Opcode, kPcSreg, kPcSreg, kLiteralOperand), kTargetOffset - 3 * sizeof(uint32_t), rocjitsu::build_s_setpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_GFX1250), rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_GFX1250), kGfx1250SEndpgm, }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "s_setpc_b64"; }), 0); const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250DoesNotPreserveLongJumpMarkerAcrossInteriorBlockEntry) { constexpr uint16_t kPcSreg = 4; constexpr uint16_t kGfx1250SAddNcU64Opcode = 83; constexpr uint16_t kLiteralOperand = 255; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; constexpr uint32_t kTargetOffset = 8 * sizeof(uint32_t); const uint32_t marker = rocjitsu::build_s_nop(rocjitsu::kLongDirectBranchMarkerNopImmediate, ROCJITSU_CODE_ARCH_GFX1250); const std::vector words = { marker, rocjitsu::build_s_getpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_GFX1250), // The branch retains the getpc result while making the add an interior // block entry in the recovered window. rocjitsu::build_s_branch(1, ROCJITSU_CODE_ARCH_GFX1250), rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_GFX1250), rocjitsu::build_sop2_encoding(ROCJITSU_CODE_ARCH_GFX1250, kGfx1250SAddNcU64Opcode, kPcSreg, kPcSreg, kLiteralOperand), kTargetOffset - 2 * sizeof(uint32_t), rocjitsu::build_s_setpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_GFX1250), rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_GFX1250), kGfx1250SEndpgm, }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE( rocjitsu::has_error_containing(result, rocjitsu::DiagnosticKind::Legalization, "indirect branch or call target recovery is not implemented")); } TEST(BinaryTranslatorE2E, Gfx1250DoesNotPreserveLongJumpMarkerAtConsumerBlockEntry) { constexpr uint16_t kPcSreg = 4; constexpr uint16_t kGfx1250SAddNcU64Opcode = 83; constexpr uint16_t kLiteralOperand = 255; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; constexpr uint32_t kTargetOffset = 5 * sizeof(uint32_t); const uint32_t marker = rocjitsu::build_s_nop(rocjitsu::kLongDirectBranchMarkerNopImmediate, ROCJITSU_CODE_ARCH_GFX1250); const std::vector words = { marker, rocjitsu::build_s_getpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_GFX1250), rocjitsu::build_sop2_encoding(ROCJITSU_CODE_ARCH_GFX1250, kGfx1250SAddNcU64Opcode, kPcSreg, kPcSreg, kLiteralOperand), kTargetOffset - 2 * sizeof(uint32_t), rocjitsu::build_s_setpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_GFX1250), // The recovered target branches back to the consumer, making the // consumer itself a second block entry into the marked window. rocjitsu::build_s_branch(-2, ROCJITSU_CODE_ARCH_GFX1250), kGfx1250SEndpgm, }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "s_setpc_b64"; }), 0); const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, LongDirectBranchInvertsEveryGfx1250ConditionalPair) { struct TestCase { uint16_t source_opcode; uint16_t inverse_opcode; }; constexpr std::array test_cases = { TestCase{gfx1250::kSCbranchScc0Sopp, gfx1250::kSCbranchScc1Sopp}, TestCase{gfx1250::kSCbranchScc1Sopp, gfx1250::kSCbranchScc0Sopp}, TestCase{gfx1250::kSCbranchVcczSopp, gfx1250::kSCbranchVccnzSopp}, TestCase{gfx1250::kSCbranchVccnzSopp, gfx1250::kSCbranchVcczSopp}, TestCase{gfx1250::kSCbranchExeczSopp, gfx1250::kSCbranchExecnzSopp}, TestCase{gfx1250::kSCbranchExecnzSopp, gfx1250::kSCbranchExeczSopp}, }; constexpr uint64_t kTargetOffset = 140000; for (const TestCase &test_case : test_cases) { const uint32_t branch_word = rocjitsu::pack_sopp(test_case.source_opcode, 0); const auto branch = rocjitsu::decode_one(branch_word, ROCJITSU_CODE_ARCH_GFX1250); ASSERT_NE(branch, nullptr); std::vector text(kTargetOffset + sizeof(uint32_t), 0); rocjitsu::KernelTextLayout layout; layout.body_end = text.size(); layout.long_branch_sgpr = 8; layout.blocks.push_back( {.source_start = 0, .source_end = sizeof(uint32_t), .target_start = 0, .target_end = rocjitsu::kMaxDirectBranchTransferWords * sizeof(uint32_t)}); layout.blocks.push_back({.source_start = sizeof(uint32_t), .source_end = 2 * sizeof(uint32_t), .target_start = kTargetOffset, .target_end = kTargetOffset + sizeof(uint32_t)}); layout.branch_fixups.push_back( {.inst = branch.get(), .source_inst_offset = 0, .source_target_offset = sizeof(uint32_t), .target_inst_offset = 0, .target_window_bytes = rocjitsu::kMaxDirectBranchTransferWords * sizeof(uint32_t), .translated_words = {branch_word}}); const auto result = rocjitsu::patch_direct_branch_fixups(text, layout, ROCJITSU_CODE_ARCH_GFX1250); ASSERT_TRUE(result.ok) << result.message; uint32_t inverted = 0; std::memcpy(&inverted, text.data(), sizeof(inverted)); EXPECT_EQ((inverted >> 16) & 0x7fu, test_case.inverse_opcode); } } TEST(BinaryTranslatorE2E, RebaseTextOffsetShiftsOffsetsAtInsertionBoundaries) { constexpr uint64_t kWord = sizeof(uint32_t); const std::array insertions = { rocjitsu::TextLayoutInsertion{.offset = kWord, .size = kWord}, rocjitsu::TextLayoutInsertion{.offset = 2 * kWord, .size = 2 * kWord}, }; uint64_t before = kWord - 1; uint64_t at_first = kWord; uint64_t between = 2 * kWord - 1; uint64_t at_second = 2 * kWord; rocjitsu::rebase_text_offset(before, insertions); rocjitsu::rebase_text_offset(at_first, insertions); rocjitsu::rebase_text_offset(between, insertions); rocjitsu::rebase_text_offset(at_second, insertions); EXPECT_EQ(before, kWord - 1); EXPECT_EQ(at_first, 2 * kWord); EXPECT_EQ(between, 3 * kWord - 1); EXPECT_EQ(at_second, 5 * kWord); } TEST(BinaryTranslatorE2E, GrowDirectBranchWindowsRebasesEveryLaterLayoutOffset) { const uint32_t branch_word = rocjitsu::build_s_branch(0, ROCJITSU_CODE_ARCH_GFX1250); const auto branch = rocjitsu::decode_one(branch_word, ROCJITSU_CODE_ARCH_GFX1250); ASSERT_NE(branch, nullptr); constexpr uint64_t kWord = sizeof(uint32_t); std::vector text(8 * kWord, 0); rocjitsu::KernelTextLayout layout; layout.body_end = text.size(); layout.blocks.push_back( {.source_start = 0, .source_end = kWord, .target_start = 0, .target_end = kWord}); layout.blocks.push_back({.source_start = kWord, .source_end = 8 * kWord, .target_start = kWord, .target_end = 8 * kWord}); layout.branch_fixups.push_back({.inst = branch.get(), .source_inst_offset = 0, .source_target_offset = kWord, .target_inst_offset = 0, .target_window_bytes = kWord, .translated_words = {branch_word}}); layout.branch_fixups.push_back({.inst = branch.get(), .source_inst_offset = 2 * kWord, .source_target_offset = 0, .target_inst_offset = 2 * kWord, .target_window_bytes = kWord, .translated_words = {branch_word}}); layout.recovered_indirect_fixups.push_back({.source_call_offset = 3 * kWord, .target_window_offset = 3 * kWord, .target_window_bytes = 3 * kWord}); layout.recovered_builder_fixups.push_back({.target_getpc_offset = 4 * kWord, .target_recovery_begin_offset = 5 * kWord, .target_recovery_end_offset = 6 * kWord}); layout.branch_island_slots.push_back(7 * kWord); const std::array requirements = { rocjitsu::ControlFlowWindowRequirement{.source_inst_offset = 0, .required_window_bytes = 3 * kWord}, rocjitsu::ControlFlowWindowRequirement{.kind = rocjitsu::ControlFlowWindowKind::RecoveredIndirect, .source_inst_offset = 3 * kWord, .required_window_bytes = 5 * kWord}, }; const auto insertions = rocjitsu::grow_control_flow_windows(text, layout, requirements, ROCJITSU_CODE_ARCH_GFX1250); ASSERT_TRUE(insertions.has_value()); ASSERT_EQ(insertions->size(), 2u); EXPECT_EQ(insertions->at(0).offset, kWord); EXPECT_EQ(insertions->at(0).size, 2 * kWord); EXPECT_EQ(insertions->at(1).offset, 6 * kWord); EXPECT_EQ(insertions->at(1).size, 2 * kWord); EXPECT_EQ(text.size(), 12 * kWord); EXPECT_EQ(layout.body_end, text.size()); EXPECT_EQ(layout.blocks[0].target_end, 3 * kWord); EXPECT_EQ(layout.blocks[1].target_start, 3 * kWord); EXPECT_EQ(layout.blocks[1].target_end, 12 * kWord); EXPECT_EQ(layout.branch_fixups[0].target_inst_offset, 0u); EXPECT_EQ(layout.branch_fixups[0].target_window_bytes, 3 * kWord); EXPECT_EQ(layout.branch_fixups[1].target_inst_offset, 4 * kWord); EXPECT_EQ(layout.recovered_indirect_fixups[0].target_window_offset, 5 * kWord); EXPECT_EQ(layout.recovered_indirect_fixups[0].target_window_bytes, 5 * kWord); EXPECT_EQ(layout.recovered_builder_fixups[0].target_getpc_offset, 6 * kWord); EXPECT_EQ(layout.recovered_builder_fixups[0].target_recovery_begin_offset, 7 * kWord); EXPECT_EQ(layout.recovered_builder_fixups[0].target_recovery_end_offset, 10 * kWord); EXPECT_EQ(layout.branch_island_slots[0], 11 * kWord); const uint32_t expected_nop = rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_GFX1250); uint32_t inserted_word = 0; std::memcpy(&inserted_word, text.data() + kWord, sizeof(inserted_word)); EXPECT_EQ(inserted_word, expected_nop); std::memcpy(&inserted_word, text.data() + 2 * kWord, sizeof(inserted_word)); EXPECT_EQ(inserted_word, expected_nop); std::memcpy(&inserted_word, text.data() + 8 * kWord, sizeof(inserted_word)); EXPECT_EQ(inserted_word, expected_nop); std::memcpy(&inserted_word, text.data() + 9 * kWord, sizeof(inserted_word)); EXPECT_EQ(inserted_word, expected_nop); } TEST(BinaryTranslatorE2E, FixedDirectBranchWindowRejectsGrowthWithoutMutation) { constexpr uint64_t kWord = sizeof(uint32_t); constexpr uint64_t kTargetOffset = 140000; const uint32_t branch_word = rocjitsu::build_s_branch(0, ROCJITSU_CODE_ARCH_GFX1250); const auto branch = rocjitsu::decode_one(branch_word, ROCJITSU_CODE_ARCH_GFX1250); ASSERT_NE(branch, nullptr); std::vector text(kTargetOffset + kWord, 0); rocjitsu::KernelTextLayout layout; layout.body_end = text.size(); layout.long_branch_sgpr = 8; layout.blocks.push_back( {.source_start = 0, .source_end = kWord, .target_start = 0, .target_end = kWord}); layout.blocks.push_back({.source_start = kWord, .source_end = 2 * kWord, .target_start = kTargetOffset, .target_end = kTargetOffset + kWord}); layout.branch_fixups.push_back({.inst = branch.get(), .source_inst_offset = 0, .source_target_offset = kWord, .target_inst_offset = 0, .target_window_bytes = kWord, .allow_window_growth = false, .translated_words = {branch_word}}); const auto original_text = text; const auto result = rocjitsu::patch_direct_branch_fixups(text, layout, ROCJITSU_CODE_ARCH_GFX1250); EXPECT_FALSE(result.ok); EXPECT_EQ(result.reason, rocjitsu::TextLayoutFailureReason::BranchOutOfRange); EXPECT_TRUE(result.required_windows.empty()); EXPECT_NE(result.message.find("fixed-size"), std::string::npos); EXPECT_EQ(text, original_text); } TEST(BinaryTranslatorE2E, DirectBranchWindowsRejectInvalidGrowthWithoutMutation) { const uint32_t branch_word = rocjitsu::build_s_branch(0, ROCJITSU_CODE_ARCH_GFX1250); const auto branch = rocjitsu::decode_one(branch_word, ROCJITSU_CODE_ARCH_GFX1250); ASSERT_NE(branch, nullptr); auto rejects_without_mutation = [&](std::vector requirements) { std::vector text(sizeof(uint32_t), 0); rocjitsu::KernelTextLayout layout; layout.body_end = text.size(); layout.branch_fixups.push_back({.inst = branch.get(), .source_inst_offset = 0, .source_target_offset = sizeof(uint32_t), .target_inst_offset = 0, .target_window_bytes = sizeof(uint32_t), .translated_words = {branch_word}}); const auto original_text = text; const auto result = rocjitsu::grow_control_flow_windows(text, layout, requirements, ROCJITSU_CODE_ARCH_GFX1250); EXPECT_FALSE(result.has_value()); EXPECT_EQ(text, original_text); EXPECT_EQ(layout.body_end, sizeof(uint32_t)); EXPECT_EQ(layout.branch_fixups.front().target_inst_offset, 0u); EXPECT_EQ(layout.branch_fixups.front().target_window_bytes, sizeof(uint32_t)); }; rejects_without_mutation( {{.source_inst_offset = 0, .required_window_bytes = sizeof(uint32_t) + 1}}); rejects_without_mutation( {{.source_inst_offset = 0x1000, .required_window_bytes = 2 * sizeof(uint32_t)}}); rejects_without_mutation({{.source_inst_offset = 0, .required_window_bytes = sizeof(uint32_t)}}); rejects_without_mutation( {{.source_inst_offset = 0, .required_window_bytes = 2 * sizeof(uint32_t)}, {.source_inst_offset = 0, .required_window_bytes = 3 * sizeof(uint32_t)}}); { std::vector text(sizeof(uint32_t), 0); rocjitsu::KernelTextLayout layout; layout.body_end = text.size(); layout.recovered_indirect_fixups.push_back({.source_call_offset = 0, .target_window_offset = 0, .target_window_bytes = sizeof(uint32_t)}); const std::array requirements = {rocjitsu::ControlFlowWindowRequirement{ .kind = rocjitsu::ControlFlowWindowKind::RecoveredIndirect, .source_inst_offset = 0x1000, .required_window_bytes = 2 * sizeof(uint32_t), }}; const auto original_text = text; const auto result = rocjitsu::grow_control_flow_windows(text, layout, requirements, ROCJITSU_CODE_ARCH_GFX1250); EXPECT_FALSE(result.has_value()); EXPECT_EQ(text, original_text); EXPECT_EQ(layout.body_end, sizeof(uint32_t)); EXPECT_EQ(layout.recovered_indirect_fixups.front().target_window_bytes, sizeof(uint32_t)); } } TEST(BinaryTranslatorE2E, PatchesOutOfRangeConditionalThroughIslandSlotWithoutSgpr) { constexpr uint64_t kWord = sizeof(uint32_t); constexpr uint64_t kTargetOffset = 140000; constexpr uint64_t kIslandOffset = 70000; const uint32_t branch_word = rocjitsu::pack_sopp(gfx1250::kSCbranchScc1Sopp, static_cast(0)); const auto branch = rocjitsu::decode_one(branch_word, ROCJITSU_CODE_ARCH_GFX1250); ASSERT_NE(branch, nullptr); std::vector text(kTargetOffset + kWord, 0); rocjitsu::KernelTextLayout layout; layout.body_end = text.size(); layout.blocks.push_back( {.source_start = 0, .source_end = kWord, .target_start = 0, .target_end = kWord}); layout.blocks.push_back({.source_start = kWord, .source_end = 2 * kWord, .target_start = kTargetOffset, .target_end = kTargetOffset + kWord}); layout.branch_fixups.push_back({.inst = branch.get(), .source_inst_offset = 0, .source_target_offset = kWord, .target_inst_offset = 0, .target_window_bytes = kWord, .translated_words = {branch_word}}); layout.branch_island_slots.push_back(kIslandOffset); const auto before_growth = rocjitsu::patch_direct_branch_fixups(text, layout, ROCJITSU_CODE_ARCH_GFX1250); ASSERT_FALSE(before_growth.ok); ASSERT_EQ(before_growth.required_windows.size(), 1u); EXPECT_EQ(before_growth.required_windows.front().required_window_bytes, 2 * kWord); const auto insertions = rocjitsu::grow_control_flow_windows( text, layout, before_growth.required_windows, ROCJITSU_CODE_ARCH_GFX1250); ASSERT_TRUE(insertions.has_value()); ASSERT_TRUE(rocjitsu::patch_direct_branch_fixups(text, layout, ROCJITSU_CODE_ARCH_GFX1250).ok); const auto *target_words = reinterpret_cast(text.data()); EXPECT_EQ(target_words[0], rocjitsu::pack_sopp(gfx1250::kSCbranchScc0Sopp, 1)); const uint64_t rebased_island_offset = kIslandOffset + kWord; const auto source_to_island = rocjitsu::compute_sopp_branch_simm16(kWord, rebased_island_offset); ASSERT_TRUE(source_to_island.has_value()); EXPECT_EQ(target_words[1], rocjitsu::build_s_branch(*source_to_island, ROCJITSU_CODE_ARCH_GFX1250)); uint32_t island_word = 0; std::memcpy(&island_word, text.data() + rebased_island_offset, sizeof(island_word)); const auto island_to_target = rocjitsu::compute_sopp_branch_simm16(rebased_island_offset, kTargetOffset + kWord); ASSERT_TRUE(island_to_target.has_value()); EXPECT_EQ(island_word, rocjitsu::build_s_branch(*island_to_target, ROCJITSU_CODE_ARCH_GFX1250)); } TEST(BinaryTranslatorE2E, FullSgprConditionalPreservesPoolAfterUnconditionalBranch) { using namespace rocr::llvm::amdhsa; constexpr size_t kExpansionCount = 16000; const auto cvt = make_cdna4_cvt_f32_bf16_words(cdna4::encoding::kVop1); constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; std::vector words; words.reserve(2 * kExpansionCount + 2); for (size_t i = 0; i < kExpansionCount; ++i) { words.push_back(cvt[0]); // The first pass inserts each pool after one of these source blocks. Once // relocated, the direct branch skips that pool's header while the // out-of-range branch below still reaches one of its private slots. words.push_back(rocjitsu::build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA4)); } const int16_t back_to_entry = static_cast(-static_cast(words.size() + 1)); words.push_back(cdna4::build_sopp(cdna4::kSCbranchScc1Sopp, {.simm16 = static_cast(back_to_entry)})[0]); words.push_back(kCdna4SEndpgm); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, 13); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t translated_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); const uint32_t marker = rocjitsu::build_s_nop(rocjitsu::kBranchIslandPoolMarkerNopImmediate, ROCJITSU_CODE_ARCH_CDNA3); const uint32_t skip_pool = rocjitsu::build_s_branch(16, ROCJITSU_CODE_ARCH_CDNA3); const uint32_t skip_over_pool = rocjitsu::build_s_branch(static_cast(rocjitsu::kGeneratedIslandPoolHeaderWords + rocjitsu::kDirectBranchIslandPoolSlots), ROCJITSU_CODE_ARCH_CDNA3); const uint32_t unused_slot = rocjitsu::build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA3); bool found_live_skipped_pool = false; for (size_t i = 1; i + rocjitsu::kGeneratedIslandPoolHeaderWords + rocjitsu::kDirectBranchIslandPoolSlots <= translated_word_count; ++i) { if (target_words[i] != marker || target_words[i - 1] != skip_over_pool || target_words[i + 1] != skip_pool) { continue; } const size_t first_slot = i + rocjitsu::kGeneratedIslandPoolHeaderWords; const size_t past_slots = first_slot + rocjitsu::kDirectBranchIslandPoolSlots; if (std::any_of(target_words + first_slot, target_words + past_slots, [&](uint32_t word) { return word != unused_slot; })) { found_live_skipped_pool = true; break; } } EXPECT_TRUE(found_live_skipped_pool); bool found_island_source = false; const uint32_t inverted = cdna3::build_sopp(cdna3::kSCbranchScc0Sopp, {.simm16 = 1})[0]; for (size_t i = 0; i + 1 < translated_word_count; ++i) { if (target_words[i] == inverted && ((target_words[i + 1] >> 23) & 0x1ffu) == 0x17fu) { found_island_source = true; break; } } EXPECT_TRUE(found_island_source); expect_cdna3_translated_descriptor_sgprs_eq(result.elf_bytes, 112); rocjitsu::BinaryTranslator verifier(ROCJITSU_CODE_ARCH_CDNA3, ROCJITSU_CODE_ARCH_CDNA3); const auto second = verifier.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250RelocatesReachableGeneratedIslandPoolSlotsIdempotently) { constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; std::vector words = { // The conditional reaches the first private slot while fallthrough // reaches the pool header and skip. gfx1250::build_sopp(gfx1250::kSCbranchScc0Sopp, {.simm16 = 2})[0], rocjitsu::build_s_nop(rocjitsu::kBranchIslandPoolMarkerNopImmediate, ROCJITSU_CODE_ARCH_GFX1250), rocjitsu::build_s_branch(static_cast(rocjitsu::kDirectBranchIslandPoolSlots), ROCJITSU_CODE_ARCH_GFX1250), }; words.insert(words.end(), rocjitsu::kDirectBranchIslandPoolSlots, rocjitsu::build_s_branch(0, ROCJITSU_CODE_ARCH_GFX1250)); words.push_back(kGfx1250SEndpgm); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto first = translator.translate(source); ASSERT_TRUE(first.ok()) << (first.diagnostics.empty() ? "" : first.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(first.elf_bytes.data(), first.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, first.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250RejectsGeneratedIslandPoolSlotTargetingOutsideText) { constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; std::vector words = { // Reach the first private slot so it is live in this kernel scope. gfx1250::build_sopp(gfx1250::kSCbranchScc0Sopp, {.simm16 = 2})[0], rocjitsu::build_s_nop(rocjitsu::kBranchIslandPoolMarkerNopImmediate, ROCJITSU_CODE_ARCH_GFX1250), rocjitsu::build_s_branch(static_cast(rocjitsu::kDirectBranchIslandPoolSlots), ROCJITSU_CODE_ARCH_GFX1250), }; words.insert(words.end(), rocjitsu::kDirectBranchIslandPoolSlots, rocjitsu::build_s_branch(0, ROCJITSU_CODE_ARCH_GFX1250)); words[rocjitsu::kGeneratedIslandPoolHeaderWords + 1] = rocjitsu::build_s_branch(30000, ROCJITSU_CODE_ARCH_GFX1250); words.push_back(kGfx1250SEndpgm); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_TRUE(rocjitsu::has_error_containing( result, rocjitsu::DiagnosticKind::Legalization, "generated direct branch island pool targets outside source .text")); } TEST(BinaryTranslatorE2E, Gfx1250RejectsNearMissGeneratedIslandPool) { constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; std::vector words = { rocjitsu::build_s_nop(rocjitsu::kBranchIslandPoolMarkerNopImmediate, ROCJITSU_CODE_ARCH_GFX1250), rocjitsu::build_s_branch(static_cast(rocjitsu::kDirectBranchIslandPoolSlots), ROCJITSU_CODE_ARCH_GFX1250), }; words.insert(words.end(), rocjitsu::kDirectBranchIslandPoolSlots, rocjitsu::build_s_branch(0, ROCJITSU_CODE_ARCH_GFX1250)); words[2 + rocjitsu::kDirectBranchIslandPoolSlots / 2] = rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_GFX1250); words.push_back(kGfx1250SEndpgm); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto first = translator.translate(source); ASSERT_TRUE(first.ok()) << (first.diagnostics.empty() ? "" : first.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(first.elf_bytes.data(), first.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ(std::ranges::count_if(decoded, [](const auto &inst) { return inst->mnemonic() == "s_branch"; }), 1); const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, first.elf_bytes); } TEST(BinaryTranslatorE2E, DuplicatesSharedReachableBlocksPerKernel) { constexpr uint32_t kCdna4SEndpgm = cdna4::build_sopp(cdna4::kSEndpgmSopp)[0]; const std::vector words = { rocjitsu::build_s_branch(1, ROCJITSU_CODE_ARCH_CDNA4), // kernel0: 0x00 -> helper 0x08. rocjitsu::build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA4), // kernel1: 0x04 -> helper 0x08. kCdna4SEndpgm, // Shared source helper. }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_two_kernel_descriptors(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *text = translated.text_sections()[0]; rocjitsu::KernelDescriptorTranslator parser(ROCJITSU_CODE_ARCH_CDNA3, ROCJITSU_CODE_ARCH_CDNA3); const auto infos = parser.translate_image(result.elf_bytes, text->sectionOffset(), text->size(), rocjitsu::KernelDescriptorTranslationOptions{}); ASSERT_EQ(infos.size(), 2u); std::vector translated_entries; for (const auto &info : infos) translated_entries.push_back(info.entry_text_offset); std::ranges::sort(translated_entries); ASSERT_EQ(translated_entries[0], 0u); ASSERT_GT(translated_entries[1], 2 * sizeof(uint32_t)); ASSERT_LE(translated_entries[1] + 2 * sizeof(uint32_t), text->size()); const auto *target_words = reinterpret_cast(text->data()); const uint64_t second_entry_word = translated_entries[1] / sizeof(uint32_t); EXPECT_EQ(target_words[0], rocjitsu::build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[1], kCdna4SEndpgm); EXPECT_EQ(target_words[second_entry_word], rocjitsu::build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[second_entry_word + 1], kCdna4SEndpgm); } TEST(BinaryTranslatorE2E, Cdna4ToCdna3SemanticExpandRulesHaveTranslationFixtures) { const auto test_cases = cdna4_to_cdna3_semantic_rule_cases(); const auto rules = rocjitsu::semantic_expand_rules_cdna4_to_cdna3(); for (const auto &rule : rules) { EXPECT_TRUE(has_cdna4_to_cdna3_semantic_rule_case(rule.src_encoding_id, rule.src_opcode)) << "missing fixture for CDNA4->CDNA3 semantic rule encoding=0x" << std::hex << rule.src_encoding_id << " opcode=" << rule.src_opcode << std::dec; } for (const auto &test_case : test_cases) { EXPECT_TRUE(has_cdna4_to_cdna3_semantic_rule(test_case.encoding_id, test_case.opcode)) << "test fixture has no CDNA4->CDNA3 semantic rule: " << test_case.name; } } class Cdna4ToCdna3SemanticRuleTranslationTest : public ::testing::TestWithParam {}; TEST_P(Cdna4ToCdna3SemanticRuleTranslationTest, TranslatesSingleInstruction) { const auto &test_case = GetParam(); SCOPED_TRACE(test_case.name); ASSERT_LE(test_case.word_count, test_case.words.size()); const std::vector source_words(test_case.words.begin(), test_case.words.begin() + test_case.word_count); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(source_words); rocjitsu::AmdGpuCodeObject source_layout(image.data(), image.size()); ASSERT_TRUE(source_layout.is_valid()); ASSERT_FALSE(source_layout.text_sections().empty()); const auto *source_text = source_layout.text_sections()[0]; ASSERT_EQ(source_text->size(), source_words.size() * sizeof(uint32_t)); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); EXPECT_EQ(rocjitsu::find_section(translated, ".rj_translations"), nullptr); const bool has_sidecar = rocjitsu::find_section(translated, rocjitsu::kVirtualLdsMetadataSectionName) != nullptr; expect_cdna3_text_matches(*translated.text_sections()[0], test_case.expected, has_sidecar); } INSTANTIATE_TEST_SUITE_P(ImplementedRules, Cdna4ToCdna3SemanticRuleTranslationTest, ::testing::ValuesIn(cdna4_to_cdna3_semantic_rule_cases()), [](const ::testing::TestParamInfo &info) { return std::string(info.param.name); }); TEST(BinaryTranslatorE2E, LshlAddU64ForRdnaEmitsShiftAndCarryAdd) { // v[4:5] = (v[0:1] << 2) + v[2:3]. vdst does not alias the addend, so the shift // goes straight into vdst. Verify a v_lshlrev_b64 (the shift the old lowering // dropped) precedes the carry-add pair. const auto words = make_cdna4_v_lshl_add_u64_words(/*vdst=*/4, /*src0=*/256 + 0, /*src1=*/2, /*src2=*/256 + 2); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {words[0], words[1], 0xBF810000u}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_RDNA4); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_RDNA4); const bool has_shift = std::any_of(decoded.begin(), decoded.end(), [](const auto &inst) { return inst->mnemonic() == "v_lshlrev_b64"; }); EXPECT_TRUE(has_shift) << "lowering must materialize the shift, not drop it"; const bool has_add = std::any_of(decoded.begin(), decoded.end(), [](const auto &inst) { return inst->mnemonic() == "v_add_co_u32"; }); EXPECT_TRUE(has_add) << "lowering must emit the 64-bit carry add"; } TEST(BinaryTranslatorE2E, LshlAddU64ForRdnaHandlesDestinationAliasingAddend) { // v[2:3] = (v[0:1] << 2) + v[2:3]: the destination aliases the VGPR addend, the // common address-computation pattern emitted by real kernels (e.g. vector_add). // The lowering must shift into a dead scratch pair rather than clobbering the // addend, and must still succeed (the earlier fail-closed guard wrongly rejected // this). const auto words = make_cdna4_v_lshl_add_u64_words(/*vdst=*/2, /*src0=*/256 + 0, /*src1=*/2, /*src2=*/256 + 2); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {words[0], words[1], 0xBF810000u}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_RDNA4); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_RDNA4); const auto shift = std::find_if(decoded.begin(), decoded.end(), [](const auto &inst) { return inst->mnemonic() == "v_lshlrev_b64"; }); ASSERT_NE(shift, decoded.end()) << "lowering must materialize the shift"; // The shift destination must not be v2/v3 (the aliased addend); it must land in // a separate scratch pair so the following add can still read the addend. ASSERT_GE((*shift)->num_dst_operands(), 1); const auto shift_dst = (*shift)->dst_operand(0)->to_register_ref(); ASSERT_TRUE(shift_dst.has_value()); EXPECT_NE(shift_dst->index, 2u) << "shift result must not overwrite the aliased addend v[2:3] before the add reads it"; } TEST(BinaryTranslatorE2E, LshlAddU64ForRdnaRejectsInlineConstantAddend) { // v[4:5] = (v[0:1] << 2) + . The 64-bit addend is encoded as // a single inline-constant operand (128), so the high half cannot be derived as // src2 + 1 (129 == constant 1). The lowering must fail closed rather than // silently add 1 to the high word. constexpr uint16_t kInlineConstZero = 128; const auto words = make_cdna4_v_lshl_add_u64_words(/*vdst=*/4, /*src0=*/256 + 0, /*src1=*/2, /*src2=*/kInlineConstZero); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {words[0], words[1], 0xBF810000u}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_RDNA4); auto result = translator.translate(source); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(rocjitsu::has_error_containing(result, rocjitsu::DiagnosticKind::ExpandFailed, "non-register 64-bit addend")); } TEST(BinaryTranslatorE2E, LshlAddU64ForRdnaRejectsV255Addend) { // v[4:5] = (v[0:1] << 2) + v[255:256]. src2=v255 (selector 511) has no valid // high half: the derived src2+1 selector is 512, which does not encode a VGPR. // The lowering must fail closed rather than emit an add reading an invalid // operand. constexpr uint16_t kV255 = 256 + 255; const auto words = make_cdna4_v_lshl_add_u64_words(/*vdst=*/4, /*src0=*/256 + 0, /*src1=*/2, /*src2=*/kV255); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {words[0], words[1], 0xBF810000u}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_RDNA4); auto result = translator.translate(source); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(rocjitsu::has_error_containing(result, rocjitsu::DiagnosticKind::ExpandFailed, "non-register 64-bit addend")); } TEST(BinaryTranslatorE2E, LshlAddU64ForRdnaCarryUsesScalarSgprNotVcc) { // v_lshl_add_u64 defines no carry output and VCC is not liveness-tracked, so // the carry chain must target a dead ordinary SGPR pair, never VCC (which could // be live across the instruction). Verify the emitted v_add_co_u32 / // v_add_co_ci_u32 write a scalar SDST that is not VCC. const auto words = make_cdna4_v_lshl_add_u64_words(/*vdst=*/4, /*src0=*/256 + 0, /*src1=*/2, /*src2=*/256 + 2); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {words[0], words[1], 0xBF810000u}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_RDNA4); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_RDNA4); // VCC_LO is scalar selector 106. The carry SDST must be an ordinary SGPR. constexpr uint32_t kVccLo = 106; bool saw_add = false; for (const auto &inst : decoded) { const auto mnemonic = inst->mnemonic(); if (mnemonic != "v_add_co_u32" && mnemonic != "v_add_co_ci_u32") continue; saw_add = true; ASSERT_GE(inst->num_dst_operands(), 2) << mnemonic << " must expose an explicit scalar carry destination"; const auto sdst = inst->dst_operand(1)->to_register_ref(); ASSERT_TRUE(sdst.has_value()) << mnemonic << " carry SDST must be a register"; EXPECT_NE(sdst->index, kVccLo) << mnemonic << " must not clobber VCC as its carry destination"; } EXPECT_TRUE(saw_add) << "lowering must emit the carry-add pair"; } TEST(BinaryTranslatorE2E, VirtualLdsSidecarLowersDsReadB32ToFlatGlobalLoad) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read_b32_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); const auto saddr = find_flat_global_load_saddr(target_words, target_word_count, /*op=*/20, /*vdst=*/7, /*addr=*/4, /*offset=*/0x134); ASSERT_TRUE(saddr.has_value()); EXPECT_TRUE(contains_smem_load_dwordx2_with_wait(target_words, target_word_count, *saddr, /*sbase_sgpr=*/0, /*offset=*/24)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarLowersMubufDwordLdsToFlatGlobalStore) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto mubuf = make_cdna4_buffer_load_lds_words(/*op=*/20); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {mubuf[0], mubuf[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); // A virtual-LDS sidecar cannot leave the MUBUF LDS bit set: the side effect // would target zero-sized hardware LDS instead of rocjitsu's backing buffer. EXPECT_FALSE(contains_mubuf_lds_op(target_words, target_word_count, /*op=*/20)); EXPECT_TRUE(contains_flat_global_store_op(target_words, target_word_count, /*op=*/28)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarReservesFreshSgprsBelowCdnaSpecialTail) { using namespace rocr::llvm::amdhsa; constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read_b32_words(); const std::vector words = { rocjitsu::build_s_mov_b32(/*sdst=*/47, /*ssrc0=*/128, ROCJITSU_CODE_ARCH_CDNA4), ds[0], ds[1], kCdna4SEndpgm, }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); // A HipKittens-style descriptor can allocate more SGPRs than the ordinary // guest body names because VCC/flat-scratch/XNACK and granularity padding are // included in COMPUTE_PGM_RSRC1. Virtual LDS must not treat that tail as // ordinary scratch space. AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, 6); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); const auto *metadata_section = rocjitsu::find_section(translated, rocjitsu::kVirtualLdsMetadataSectionName); ASSERT_NE(metadata_section, nullptr); const auto parsed = rocjitsu::parse_virtual_lds_metadata( {reinterpret_cast(metadata_section->data()), metadata_section->size()}); ASSERT_TRUE(parsed.has_value()); ASSERT_EQ(parsed->size(), 1u); EXPECT_EQ(parsed->front().virtual_lds_base_sgpr, 48u); expect_cdna3_sidecar_descriptor_sgprs_eq(result.elf_bytes, 64); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarKeepsDescriptorSpecialTailFree) { using namespace rocr::llvm::amdhsa; constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read_b32_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, 13); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; expect_cdna3_translated_descriptor_sgprs_eq(result.elf_bytes, 112); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *metadata_section = rocjitsu::find_section(translated, rocjitsu::kVirtualLdsMetadataSectionName); ASSERT_NE(metadata_section, nullptr); const auto parsed = rocjitsu::parse_virtual_lds_metadata( {reinterpret_cast(metadata_section->data()), metadata_section->size()}); ASSERT_TRUE(parsed.has_value()); ASSERT_EQ(parsed->size(), 1u); EXPECT_EQ(parsed->front().virtual_lds_base_sgpr, 2u); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); // A large SGPR allocation does not imply all of those numbers are ordinary // scratchable registers. VCC/flat-scratch/XNACK live in the descriptor tail, // so a guest body that only names low SGPRs should place virtual-LDS scratch // below that tail instead of borrowing a high pair with spill-per-use. EXPECT_FALSE(contains_sopp(target_words, target_word_count, /*op=*/8)); EXPECT_TRUE(contains_smem_load_dwordx2_with_wait(target_words, target_word_count, /*sdata=*/2, /*sbase_sgpr=*/0, /*offset=*/24)); rocjitsu::cdna3::FlatMachineInst actual{}; bool found_load = false; for (size_t i = 0; i + 1 < target_word_count; ++i) { std::memcpy(&actual, target_words + i, sizeof(actual)); if (actual.encoding == 0x37u && actual.op == 20u && actual.seg == 2u && actual.addr == 4u && actual.vdst == 7u && actual.sc0 == 1u && actual.sc1 == 0u && actual.saddr == 2u) { found_load = true; break; } } EXPECT_TRUE(found_load); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarSpillsTouchedHighSgprPairWhenDescriptorSgprsAreFull) { using namespace rocr::llvm::amdhsa; constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read_b32_words(); std::vector words; words.reserve(102 + 3); for (uint16_t sgpr = 0; sgpr < 102; ++sgpr) words.push_back(rocjitsu::build_s_mov_b32(sgpr, /*ssrc0=*/128, ROCJITSU_CODE_ARCH_CDNA4)); words.push_back(ds[0]); words.push_back(ds[1]); words.push_back(kCdna4SEndpgm); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, 13); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; expect_cdna3_translated_descriptor_sgprs_eq(result.elf_bytes, 112); const auto entry = virtual_lds_sidecar_entry_words_for_test(result.elf_bytes); ASSERT_TRUE(entry.has_value()); const auto *target_words = entry->words; const size_t target_word_count = entry->word_count; EXPECT_TRUE(contains_sopp(target_words, target_word_count, /*op=*/8)); const auto saved_lo = find_cdna3_vop1_vdst(target_words, target_word_count, /*op=*/1, /*src0=*/100); const auto saved_hi = find_cdna3_vop1_vdst(target_words, target_word_count, /*op=*/1, /*src0=*/101); ASSERT_TRUE(saved_lo.has_value()); ASSERT_TRUE(saved_hi.has_value()); EXPECT_NE(*saved_lo, *saved_hi); EXPECT_TRUE(contains_smem_load_dwordx2_with_wait(target_words, target_word_count, /*sdata=*/100, /*sbase_sgpr=*/0, /*offset=*/24)); // Spill-per-use lowering first installs the runtime backing pointer in the // borrowed SGPR pair, then restores the guest scalar values after the lowered // memory access. The two operations can use different VGPR temps because the // entry prologue saves the backing pointer in persistent scratch. EXPECT_GE(count_cdna3_vop1_writes(target_words, target_word_count, /*op=*/2, /*vdst=*/100), 2u); EXPECT_GE(count_cdna3_vop1_writes(target_words, target_word_count, /*op=*/2, /*vdst=*/101), 2u); } TEST(BinaryTranslatorE2E, VirtualLdsMubufSemanticRuleUsesSpillPerUseAccessEmitter) { using namespace rocr::llvm::amdhsa; constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto mubuf = make_cdna4_buffer_load_lds_words(/*op=*/20); std::vector words; words.reserve(102 + 3); for (uint16_t sgpr = 0; sgpr < 102; ++sgpr) words.push_back(rocjitsu::build_s_mov_b32(sgpr, /*ssrc0=*/128, ROCJITSU_CODE_ARCH_CDNA4)); words.push_back(mubuf[0]); words.push_back(mubuf[1]); words.push_back(kCdna4SEndpgm); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, 13); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; const auto entry = virtual_lds_sidecar_entry_words_for_test(result.elf_bytes); ASSERT_TRUE(entry.has_value()); EXPECT_TRUE(contains_flat_global_op_with_null_saddr(entry->words, entry->word_count, /*op=*/28)); EXPECT_TRUE(contains_sopp(entry->words, entry->word_count, /*op=*/8)); EXPECT_GE(count_cdna3_vop1_writes(entry->words, entry->word_count, /*op=*/2, /*vdst=*/100), 2u); EXPECT_GE(count_cdna3_vop1_writes(entry->words, entry->word_count, /*op=*/2, /*vdst=*/101), 2u); } TEST(BinaryTranslatorE2E, VirtualLdsTransposeSemanticRuleUsesSpillPerUseAccessEmitter) { using namespace rocr::llvm::amdhsa; constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read_b64_tr_b16_words(); std::vector words; words.reserve(102 + 3); for (uint16_t sgpr = 0; sgpr < 102; ++sgpr) words.push_back(rocjitsu::build_s_mov_b32(sgpr, /*ssrc0=*/128, ROCJITSU_CODE_ARCH_CDNA4)); words.push_back(ds[0]); words.push_back(ds[1]); words.push_back(kCdna4SEndpgm); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, 13); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; const auto entry = virtual_lds_sidecar_entry_words_for_test(result.elf_bytes); ASSERT_TRUE(entry.has_value()); EXPECT_TRUE(contains_flat_global_op_with_null_saddr(entry->words, entry->word_count, /*op=*/21)); EXPECT_TRUE(contains_sopp(entry->words, entry->word_count, /*op=*/8)); EXPECT_GE(count_cdna3_vop1_writes(entry->words, entry->word_count, /*op=*/2, /*vdst=*/100), 2u); EXPECT_GE(count_cdna3_vop1_writes(entry->words, entry->word_count, /*op=*/2, /*vdst=*/101), 2u); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarSpillPerUseGrowsDescriptorForBorrowedHighSgprs) { using namespace rocr::llvm::amdhsa; constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read_b32_words(); const std::vector words = { rocjitsu::build_s_mov_b32(/*sdst=*/100, /*ssrc0=*/128, ROCJITSU_CODE_ARCH_CDNA4), ds[0], ds[1], kCdna4SEndpgm, }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, 2); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; expect_cdna3_sidecar_descriptor_sgprs_eq(result.elf_bytes, 104); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); const auto *metadata_section = rocjitsu::find_section(translated, rocjitsu::kVirtualLdsMetadataSectionName); ASSERT_NE(metadata_section, nullptr); const auto parsed = rocjitsu::parse_virtual_lds_metadata( {reinterpret_cast(metadata_section->data()), metadata_section->size()}); ASSERT_TRUE(parsed.has_value()); ASSERT_EQ(parsed->size(), 1u); EXPECT_EQ(parsed->front().virtual_lds_base_sgpr, 98u); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarSpillPerUseCanGrowTinyNoAgprKernelsPastAccumOffset) { using namespace rocr::llvm::amdhsa; constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_write_b32_words(/*addr=*/0, /*data=*/1); const std::vector words = { rocjitsu::build_s_mov_b32(/*sdst=*/100, /*ssrc0=*/128, ROCJITSU_CODE_ARCH_CDNA4), ds[0], ds[1], kCdna4SEndpgm, }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, 2); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); bool found_full_vaddr_store = false; for (size_t i = 0; i + 1 < target_word_count; ++i) { rocjitsu::cdna3::FlatMachineInst actual{}; std::memcpy(&actual, target_words + i, sizeof(actual)); if (actual.encoding == 0x37u && actual.op == 28u && actual.seg == 2u && actual.addr == 0u && actual.saddr == 0x7Fu) { found_full_vaddr_store = true; break; } } EXPECT_TRUE(found_full_vaddr_store); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarRaisesDescriptorForExplicitHighDsVgprs) { using namespace rocr::llvm::amdhsa; constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds_write = make_cdna4_ds_write_b32_words(/*addr=*/62, /*data=*/0, /*byte_offset=*/256); const auto ds_read = make_cdna4_ds_read_b32_words(/*byte_offset=*/256, /*addr=*/62, /*vdst=*/116); const std::vector words = { rocjitsu::build_s_mov_b32(/*sdst=*/100, /*ssrc0=*/128, ROCJITSU_CODE_ARCH_CDNA4), ds_write[0], ds_write[1], ds_read[0], ds_read[1], kCdna4SEndpgm, }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, 2); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; // The source descriptor intentionally under-declares the inline-assembly // VGPRs. Virtual-LDS lowering must grow the target descriptor for both the // synthetic GLOBAL address high half v63 and the explicit read destination // v116 before the runtime dispatches the translated body. expect_cdna3_sidecar_descriptor_vgprs_at_least(result.elf_bytes, 117); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarSpillPerUseCapturesWrapperBeforeGuestClobber) { using namespace rocr::llvm::amdhsa; constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read_b32_words(); const uint32_t touch_s100 = rocjitsu::build_s_mov_b32(/*sdst=*/100, /*ssrc0=*/128, ROCJITSU_CODE_ARCH_CDNA4); const uint32_t touch_s101 = rocjitsu::build_s_mov_b32(/*sdst=*/101, /*ssrc0=*/128, ROCJITSU_CODE_ARCH_CDNA4); const uint32_t clobber_dispatch_ptr = rocjitsu::build_s_mov_b32(/*sdst=*/0, /*ssrc0=*/128, ROCJITSU_CODE_ARCH_CDNA4); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {touch_s100, touch_s101, clobber_dispatch_ptr, ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); source_kd->group_segment_fixed_size = 6144; source_kd->kernarg_size = 0; AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, 13); AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, 2); AMDHSA_BITS_SET(source_kd->kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, 1); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; const auto entry = virtual_lds_sidecar_entry_words_for_test(result.elf_bytes); ASSERT_TRUE(entry.has_value()); const auto *entry_words = entry->words; const size_t entry_word_count = std::min(entry->word_count, size_t{48}); // The descriptor entry must consume the wrapper kernarg pointer before the // guest body executes `s_mov_b32 s0, 0`. Spill-per-use virtual LDS then // reloads the backing pointer from private scratch, not from a guest-owned // ABI pointer SGPR. EXPECT_TRUE(contains_smem_load_dwordx2_with_wait(entry_words, entry_word_count, /*sdata=*/100, /*sbase_sgpr=*/2, /*offset=*/8)); EXPECT_TRUE(contains_flat_scratch_dword_offset(entry_words, entry_word_count, /*op=*/28, /*offset=*/0, /*is_load=*/false)); EXPECT_TRUE(contains_flat_scratch_dword_offset(entry_words, entry_word_count, /*op=*/28, /*offset=*/4, /*is_load=*/false)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarSpillPerUseKeepsEntryPrologueStableAfterVgprGrowth) { using namespace rocr::llvm::amdhsa; constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read_b32_words(); const auto grow_vgprs = make_cdna4_permlane32_swap_b32_words(/*encoding_id=*/cdna4::encoding::kVop1Hi3); const uint32_t touch_s100 = rocjitsu::build_s_mov_b32(/*sdst=*/100, /*ssrc0=*/128, ROCJITSU_CODE_ARCH_CDNA4); const uint32_t touch_s101 = rocjitsu::build_s_mov_b32(/*sdst=*/101, /*ssrc0=*/128, ROCJITSU_CODE_ARCH_CDNA4); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {touch_s100, touch_s101, ds[0], ds[1], grow_vgprs[0], grow_vgprs[1], kCdna4SEndpgm}); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); source_kd->group_segment_fixed_size = 6144; source_kd->kernarg_size = 0; AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, 13); AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, 2); AMDHSA_BITS_SET(source_kd->kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, 1); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; options.debug_min_free_vgpr = 193; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; expect_cdna3_sidecar_descriptor_vgprs_at_least(result.elf_bytes, 196); const auto entry = virtual_lds_sidecar_entry_words_for_test(result.elf_bytes); ASSERT_TRUE(entry.has_value()); const auto *entry_words = entry->words; const size_t entry_word_count = std::min(entry->word_count, size_t{48}); // The test-only liveness floor makes the permlane lowering grow the final // target VGPR count to at least v195 after the entry prologue has already // been emitted. Spill-per-use virtual LDS must still reuse the original low // entry temps during descriptor recomputation instead of regenerating the // prologue with v194/v195. const auto saved_lo = find_cdna3_vop1_vdst(entry_words, entry_word_count, /*op=*/1, /*src0=*/100); const auto saved_hi = find_cdna3_vop1_vdst(entry_words, entry_word_count, /*op=*/1, /*src0=*/101); ASSERT_TRUE(saved_lo.has_value()); ASSERT_TRUE(saved_hi.has_value()); EXPECT_LT(*saved_lo, 194u); EXPECT_LT(*saved_hi, 194u); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarEmitsRuntimeMetadataSection) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read_b32_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); source_kd->group_segment_fixed_size = 6144; rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); const auto *metadata_section = rocjitsu::find_section(translated, rocjitsu::kVirtualLdsMetadataSectionName); ASSERT_NE(metadata_section, nullptr); const auto parsed = rocjitsu::parse_virtual_lds_metadata( {reinterpret_cast(metadata_section->data()), metadata_section->size()}); ASSERT_TRUE(parsed.has_value()); ASSERT_EQ(parsed->size(), 1u); const auto &record = parsed->front(); const auto sidecar = find_sidecar_metadata_record_for_test(result.elf_bytes); const auto extension = find_kernarg_extension_record_for_test(result.elf_bytes); ASSERT_TRUE(sidecar.has_value()); ASSERT_TRUE(extension.has_value()); const auto wrapper_layout = make_kernarg_extension_layout_for_test(*extension); ASSERT_TRUE(wrapper_layout.has_value()); EXPECT_EQ(record.kernel_name, "kernel"); EXPECT_EQ(sidecar->normal_descriptor_vaddr, translated.kernel_descriptor_offset("kernel")); EXPECT_NE(sidecar->variant_descriptor_vaddr, sidecar->normal_descriptor_vaddr); EXPECT_EQ(record.static_lds_bytes, 6144u); EXPECT_EQ(extension->original_kernarg_size, 16u); ASSERT_EQ(wrapper_layout->payload_offsets.size(), 1u); EXPECT_EQ(wrapper_layout->payload_offsets.front(), 24u); EXPECT_NE(record.virtual_lds_base_sgpr, 0u); EXPECT_NE(record.flags & rocjitsu::kVirtualLdsFlagRuntimeStateBlock, 0u); const auto normal_kd = read_descriptor_at_loaded_vaddr_for_test(result.elf_bytes, sidecar->normal_descriptor_vaddr); ASSERT_TRUE(normal_kd.has_value()); EXPECT_EQ(normal_kd->kernarg_size, 16u); const auto sidecar_kd = read_descriptor_at_loaded_vaddr_for_test(result.elf_bytes, sidecar->variant_descriptor_vaddr); ASSERT_TRUE(sidecar_kd.has_value()); EXPECT_EQ(sidecar_kd->kernarg_size, 48u); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarZeroKernargUsesWrapperKernarg) { using namespace rocr::llvm::amdhsa; constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read_b32_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); source_kd->group_segment_fixed_size = 6144; source_kd->kernarg_size = 0; AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, 2); AMDHSA_BITS_SET(source_kd->kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, 1); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); const auto saddr = find_flat_global_load_saddr(target_words, target_word_count, /*op=*/20, /*vdst=*/7, /*addr=*/4, /*offset=*/0x134); ASSERT_TRUE(saddr.has_value()); EXPECT_TRUE(contains_smem_load_dwordx2_with_wait(target_words, target_word_count, *saddr, /*sbase_sgpr=*/2, /*offset=*/8)); const auto *metadata_section = rocjitsu::find_section(translated, rocjitsu::kVirtualLdsMetadataSectionName); ASSERT_NE(metadata_section, nullptr); const auto parsed = rocjitsu::parse_virtual_lds_metadata( {reinterpret_cast(metadata_section->data()), metadata_section->size()}); ASSERT_TRUE(parsed.has_value()); ASSERT_EQ(parsed->size(), 1u); const auto &record = parsed->front(); const auto sidecar = find_sidecar_metadata_record_for_test(result.elf_bytes); const auto extension = find_kernarg_extension_record_for_test(result.elf_bytes); ASSERT_TRUE(sidecar.has_value()); ASSERT_TRUE(extension.has_value()); const auto wrapper_layout = make_kernarg_extension_layout_for_test(*extension); ASSERT_TRUE(wrapper_layout.has_value()); EXPECT_EQ(extension->original_kernarg_size, 0u); ASSERT_EQ(wrapper_layout->payload_offsets.size(), 1u); EXPECT_EQ(wrapper_layout->payload_offsets.front(), 8u); EXPECT_NE(record.flags & rocjitsu::kVirtualLdsFlagRuntimeStateBlock, 0u); const auto normal_kd = read_descriptor_at_loaded_vaddr_for_test(result.elf_bytes, sidecar->normal_descriptor_vaddr); const auto sidecar_kd = read_descriptor_at_loaded_vaddr_for_test(result.elf_bytes, sidecar->variant_descriptor_vaddr); ASSERT_TRUE(normal_kd.has_value()); ASSERT_TRUE(sidecar_kd.has_value()); EXPECT_EQ(record.normal_private_segment_size, normal_kd->private_segment_fixed_size); EXPECT_EQ(record.virtual_private_segment_size, sidecar_kd->private_segment_fixed_size); EXPECT_EQ(sidecar_kd->kernarg_size, 32u); EXPECT_EQ(AMDHSA_BITS_GET(sidecar_kd->kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_KERNARG_SEGMENT_PTR), 1u) << "kernel_code_properties=0x" << std::hex << sidecar_kd->kernel_code_properties << std::dec; EXPECT_EQ(AMDHSA_BITS_GET(sidecar_kd->compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_USER_SGPR_COUNT), 4u); } TEST(BinaryTranslatorE2E, LdsKernelEmitsNormalAndVirtualDescriptorVariants) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read_b32_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); source_kd->group_segment_fixed_size = 6144; rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); ASSERT_FALSE(result.elf_bytes.empty()); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); const auto *metadata_section = rocjitsu::find_section(translated, rocjitsu::kVirtualLdsMetadataSectionName); ASSERT_NE(metadata_section, nullptr); const auto parsed = rocjitsu::parse_virtual_lds_metadata( {reinterpret_cast(metadata_section->data()), metadata_section->size()}); ASSERT_TRUE(parsed.has_value()); ASSERT_EQ(parsed->size(), 1u); const auto &record = parsed->front(); const auto sidecar = find_sidecar_metadata_record_for_test(result.elf_bytes); const auto extension = find_kernarg_extension_record_for_test(result.elf_bytes); ASSERT_TRUE(sidecar.has_value()); ASSERT_TRUE(extension.has_value()); const auto wrapper_layout = make_kernarg_extension_layout_for_test(*extension); ASSERT_TRUE(wrapper_layout.has_value()); EXPECT_EQ(record.kernel_name, "kernel"); EXPECT_EQ(sidecar->normal_descriptor_vaddr, translated.kernel_descriptor_offset("kernel")); EXPECT_NE(sidecar->variant_descriptor_vaddr, sidecar->normal_descriptor_vaddr); EXPECT_EQ(record.static_lds_bytes, 6144u); EXPECT_EQ(extension->original_kernarg_size, 16u); ASSERT_EQ(wrapper_layout->payload_offsets.size(), 1u); EXPECT_EQ(wrapper_layout->payload_offsets.front(), 24u); EXPECT_NE(record.virtual_lds_base_sgpr, 0u); EXPECT_NE(record.flags & rocjitsu::kVirtualLdsFlagRuntimeStateBlock, 0u); const auto *translated_rodata = rocjitsu::find_section(translated, ".rodata"); ASSERT_NE(translated_rodata, nullptr); const auto normal_kd = rocjitsu::read_elf_struct_for_test( result.elf_bytes, translated_rodata->sectionOffset()); EXPECT_EQ(normal_kd.group_segment_fixed_size, 6144u); EXPECT_EQ(normal_kd.kernarg_size, 16u); const auto virtual_descriptor_offset = rocjitsu::loaded_vaddr_to_file_offset(result.elf_bytes, sidecar->variant_descriptor_vaddr); ASSERT_TRUE(virtual_descriptor_offset.has_value()); const auto virtual_kd = rocjitsu::read_elf_struct_for_test( result.elf_bytes, *virtual_descriptor_offset); EXPECT_EQ(virtual_kd.group_segment_fixed_size, 0u); EXPECT_EQ(virtual_kd.kernarg_size, 48u); } TEST(BinaryTranslatorE2E, MubufLdsKernelEmitsNormalAndVirtualDescriptorVariants) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto mubuf = make_cdna4_buffer_load_lds_words(/*op=*/23); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {mubuf[0], mubuf[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); source_kd->group_segment_fixed_size = 6144; rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); // MUBUF instructions encode the LDS side effect in a modifier bit, not in the // mnemonic. The sidecar detector must inspect the decoded machine fields, or // MUBUF-only LDS kernels would keep only the normal hardware-LDS descriptor. rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); ASSERT_FALSE(result.elf_bytes.empty()); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); const auto *metadata_section = rocjitsu::find_section(translated, rocjitsu::kVirtualLdsMetadataSectionName); ASSERT_NE(metadata_section, nullptr); const auto parsed = rocjitsu::parse_virtual_lds_metadata( {reinterpret_cast(metadata_section->data()), metadata_section->size()}); ASSERT_TRUE(parsed.has_value()); ASSERT_EQ(parsed->size(), 1u); const auto &record = parsed->front(); const auto sidecar = find_sidecar_metadata_record_for_test(result.elf_bytes); const auto extension = find_kernarg_extension_record_for_test(result.elf_bytes); ASSERT_TRUE(sidecar.has_value()); ASSERT_TRUE(extension.has_value()); const auto wrapper_layout = make_kernarg_extension_layout_for_test(*extension); ASSERT_TRUE(wrapper_layout.has_value()); EXPECT_EQ(record.kernel_name, "kernel"); EXPECT_EQ(sidecar->normal_descriptor_vaddr, translated.kernel_descriptor_offset("kernel")); EXPECT_NE(sidecar->variant_descriptor_vaddr, sidecar->normal_descriptor_vaddr); EXPECT_EQ(record.static_lds_bytes, 6144u); EXPECT_EQ(extension->original_kernarg_size, 16u); ASSERT_EQ(wrapper_layout->payload_offsets.size(), 1u); EXPECT_EQ(wrapper_layout->payload_offsets.front(), 24u); EXPECT_NE(record.virtual_lds_base_sgpr, 0u); EXPECT_NE(record.flags & rocjitsu::kVirtualLdsFlagRuntimeStateBlock, 0u); const auto *translated_rodata = rocjitsu::find_section(translated, ".rodata"); ASSERT_NE(translated_rodata, nullptr); const auto normal_kd = rocjitsu::read_elf_struct_for_test( result.elf_bytes, translated_rodata->sectionOffset()); EXPECT_EQ(normal_kd.group_segment_fixed_size, 6144u); EXPECT_EQ(normal_kd.kernarg_size, 16u); const auto virtual_descriptor_offset = rocjitsu::loaded_vaddr_to_file_offset(result.elf_bytes, sidecar->variant_descriptor_vaddr); ASSERT_TRUE(virtual_descriptor_offset.has_value()); const auto virtual_kd = rocjitsu::read_elf_struct_for_test( result.elf_bytes, *virtual_descriptor_offset); EXPECT_EQ(virtual_kd.group_segment_fixed_size, 0u); EXPECT_EQ(virtual_kd.kernarg_size, 48u); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarMaterializesLargeDsReadB32Offset) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; constexpr uint16_t kLargeDsOffset = 0x1234; const auto ds = make_cdna4_ds_read_b32_words(kLargeDsOffset); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); EXPECT_TRUE(contains_flat_global_load(target_words, target_word_count, /*op=*/20, /*vdst=*/7, /*addr=*/4, /*offset=*/0)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarLowersDsReadU16ToFlatGlobalLoadUshort) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read_u16_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); EXPECT_TRUE(contains_flat_global_load(target_words, target_word_count, /*op=*/18, /*vdst=*/7, /*addr=*/4, /*offset=*/0x20)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarLowersDsReadU8ToFlatGlobalLoadUbyte) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read_u8_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); EXPECT_TRUE(contains_flat_global_load(target_words, target_word_count, /*op=*/16, /*vdst=*/12, /*addr=*/8, /*offset=*/0x20)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarLowersDsReadI8ToFlatGlobalLoadSbyte) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read_i8_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); EXPECT_TRUE(contains_flat_global_load(target_words, target_word_count, /*op=*/17, /*vdst=*/12, /*addr=*/8, /*offset=*/0x20)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarLowersDsWriteB8ToFlatGlobalStoreByte) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_write_b8_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); EXPECT_TRUE(contains_flat_global_store(target_words, target_word_count, /*op=*/24, /*data=*/7, /*addr=*/4, /*offset=*/0x10)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarLowersDsWriteB8D16HiToFlatGlobalStoreByteD16Hi) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_write_b8_d16_hi_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); EXPECT_TRUE(contains_flat_global_store(target_words, target_word_count, /*op=*/25, /*data=*/7, /*addr=*/4, /*offset=*/0x10)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarStagesDsWriteB96DataOverlappingAddressPair) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_write_b96_words(/*addr=*/4, /*data=*/4); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); const auto staged_data = find_flat_global_store_data(target_words, target_word_count, /*op=*/30, /*addr=*/4, /*offset=*/0); ASSERT_TRUE(staged_data.has_value()); EXPECT_NE(*staged_data, 4u); EXPECT_TRUE(contains_cdna3_vop1(target_words, target_word_count, /*op=*/1, *staged_data, /*src0=*/256 + 4)); EXPECT_TRUE(contains_cdna3_vop1(target_words, target_word_count, /*op=*/1, static_cast(*staged_data + 1), /*src0=*/256 + 5)); EXPECT_TRUE(contains_cdna3_vop1(target_words, target_word_count, /*op=*/1, static_cast(*staged_data + 2), /*src0=*/256 + 6)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarStagesOddDsAddressIntoEvenGlobalAddressPair) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_write_b128_words(/*addr=*/7, /*data=*/58); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); const auto staged_addr = find_flat_global_store_addr_for_data(target_words, target_word_count, /*op=*/31, /*data=*/58, /*offset=*/0); ASSERT_TRUE(staged_addr.has_value()); EXPECT_EQ(*staged_addr % 2, 0u); EXPECT_NE(*staged_addr, 7u); EXPECT_EQ(find_vop2_literal_add(target_words, target_word_count, /*vsrc1=*/7, /*literal=*/0), staged_addr); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarLowersAccDsWriteB128ToAccFlatGlobalStore) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_write_b128_words(/*addr=*/28, /*data=*/30, /*byte_offset=*/0x40, /*acc=*/true); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); EXPECT_TRUE(contains_flat_global_store_acc(target_words, target_word_count, /*op=*/31, /*data=*/30, /*addr=*/28, /*offset=*/0x40)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarStagesOddWideStoreDataIntoEvenTuple) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_write_b64_words(/*addr=*/4, /*data=*/7); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); const auto staged_data = find_flat_global_store_data(target_words, target_word_count, /*op=*/29, /*addr=*/4, /*offset=*/0); ASSERT_TRUE(staged_data.has_value()); EXPECT_EQ(*staged_data % 2, 0u); EXPECT_NE(*staged_data, 7u); EXPECT_TRUE(contains_cdna3_vop1(target_words, target_word_count, /*op=*/1, *staged_data, /*src0=*/256 + 7)); EXPECT_TRUE(contains_cdna3_vop1(target_words, target_word_count, /*op=*/1, static_cast(*staged_data + 1), /*src0=*/256 + 8)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarStagesOddWideLoadResultIntoEvenTuple) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read_b64_words(/*addr=*/4, /*vdst=*/7); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); const auto staged_vdst = find_flat_global_load_vdst(target_words, target_word_count, /*op=*/21, /*addr=*/4, /*offset=*/0); ASSERT_TRUE(staged_vdst.has_value()); EXPECT_EQ(*staged_vdst % 2, 0u); EXPECT_NE(*staged_vdst, 7u); EXPECT_TRUE(contains_cdna3_vop1(target_words, target_word_count, /*op=*/1, /*vdst=*/7, /*src0=*/static_cast(256u + *staged_vdst))); EXPECT_TRUE(contains_cdna3_vop1(target_words, target_word_count, /*op=*/1, /*vdst=*/8, /*src0=*/static_cast(256u + *staged_vdst + 1))); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarLowersDsReadB128ToFlatGlobalLoad) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read_b128_words(/*addr=*/28, /*vdst=*/30, /*byte_offset=*/0x40); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); EXPECT_TRUE(contains_flat_global_load(target_words, target_word_count, /*op=*/23, /*vdst=*/30, /*addr=*/28, /*offset=*/0x40)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarLowersAccDsReadB128ToAccFlatGlobalLoad) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read_b128_words(/*addr=*/28, /*vdst=*/30, /*byte_offset=*/0x40, /*acc=*/true); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); EXPECT_TRUE(contains_flat_global_load_acc(target_words, target_word_count, /*op=*/23, /*vdst=*/30, /*addr=*/28, /*offset=*/0x40)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarStagesOddDsReadB128AddressIntoEvenGlobalPair) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read_b128_words(/*addr=*/27, /*vdst=*/30); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); const auto staged_addr = find_flat_global_load_addr_for_vdst(target_words, target_word_count, /*op=*/23, /*vdst=*/30, /*offset=*/0); ASSERT_TRUE(staged_addr.has_value()); EXPECT_EQ(*staged_addr % 2, 0u); EXPECT_NE(*staged_addr, 27u); EXPECT_EQ(find_vop2_literal_add(target_words, target_word_count, /*vsrc1=*/27, /*literal=*/0), staged_addr); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarStagesOddDsReadB128ResultIntoEvenTuple) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read_b128_words(/*addr=*/28, /*vdst=*/31); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); const auto staged_vdst = find_flat_global_load_vdst(target_words, target_word_count, /*op=*/23, /*addr=*/28, /*offset=*/0); ASSERT_TRUE(staged_vdst.has_value()); EXPECT_EQ(*staged_vdst % 2, 0u); EXPECT_NE(*staged_vdst, 31u); EXPECT_TRUE(contains_cdna3_vop1(target_words, target_word_count, /*op=*/1, /*vdst=*/31, /*src0=*/static_cast(256u + *staged_vdst))); EXPECT_TRUE(contains_cdna3_vop1(target_words, target_word_count, /*op=*/1, /*vdst=*/32, /*src0=*/static_cast(256u + *staged_vdst + 1))); EXPECT_TRUE(contains_cdna3_vop1(target_words, target_word_count, /*op=*/1, /*vdst=*/33, /*src0=*/static_cast(256u + *staged_vdst + 2))); EXPECT_TRUE(contains_cdna3_vop1(target_words, target_word_count, /*op=*/1, /*vdst=*/34, /*src0=*/static_cast(256u + *staged_vdst + 3))); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarLowersDsWrite2B32ToFlatGlobalStores) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_write2_b32_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); EXPECT_TRUE(contains_flat_global_store(target_words, target_word_count, /*op=*/28, /*data=*/7, /*addr=*/4, /*offset=*/4)); EXPECT_TRUE(contains_flat_global_store(target_words, target_word_count, /*op=*/28, /*data=*/9, /*addr=*/4, /*offset=*/8)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarLowersDsWrite2B64ToFlatGlobalStores) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_write2_b64_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); EXPECT_TRUE(contains_flat_global_store(target_words, target_word_count, /*op=*/29, /*data=*/8, /*addr=*/4, /*offset=*/8)); EXPECT_TRUE(contains_flat_global_store(target_words, target_word_count, /*op=*/29, /*data=*/12, /*addr=*/4, /*offset=*/16)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarStagesDsWrite2B64DataOverlappingAddressPair) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_write2_b64_words(/*addr=*/8, /*data0=*/8, /*data1=*/12); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); const auto staged_data = find_flat_global_store_data(target_words, target_word_count, /*op=*/29, /*addr=*/8, /*offset=*/8); ASSERT_TRUE(staged_data.has_value()); EXPECT_NE(*staged_data, 8u); EXPECT_TRUE(contains_cdna3_vop1(target_words, target_word_count, /*op=*/1, *staged_data, /*src0=*/256 + 8)); EXPECT_TRUE(contains_cdna3_vop1(target_words, target_word_count, /*op=*/1, /*vdst=*/static_cast(*staged_data + 1), /*src0=*/256 + 9)); EXPECT_TRUE(contains_flat_global_store(target_words, target_word_count, /*op=*/29, /*data=*/12, /*addr=*/8, /*offset=*/16)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarLowersDsWrite2st64B64ToFlatGlobalStores) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_write2st64_b64_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); EXPECT_TRUE(contains_flat_global_store(target_words, target_word_count, /*op=*/29, /*data=*/8, /*addr=*/4, /*offset=*/512)); EXPECT_TRUE(contains_flat_global_store(target_words, target_word_count, /*op=*/29, /*data=*/12, /*addr=*/4, /*offset=*/1024)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarLowersDsRead2B32ToFlatGlobalLoads) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read2_b32_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); EXPECT_TRUE(contains_flat_global_load(target_words, target_word_count, /*op=*/20, /*vdst=*/20, /*addr=*/12, /*offset=*/4)); EXPECT_TRUE(contains_flat_global_load(target_words, target_word_count, /*op=*/20, /*vdst=*/21, /*addr=*/12, /*offset=*/8)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarStagesDsRead2B32ResultsOverlappingAddressPair) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read2_b32_words(/*offset0=*/1, /*offset1=*/2, /*addr=*/20, /*vdst=*/20); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); const auto temp0 = find_flat_global_load_vdst(target_words, target_word_count, /*op=*/20, /*addr=*/20, /*offset=*/4); const auto temp1 = find_flat_global_load_vdst(target_words, target_word_count, /*op=*/20, /*addr=*/20, /*offset=*/8); ASSERT_TRUE(temp0.has_value()); ASSERT_TRUE(temp1.has_value()); EXPECT_NE(*temp0, 20u); EXPECT_NE(*temp1, 21u); EXPECT_EQ(*temp1, static_cast(*temp0 + 1)); EXPECT_TRUE(contains_cdna3_vop1(target_words, target_word_count, /*op=*/1, /*vdst=*/20, /*src0=*/static_cast(256u + *temp0))); EXPECT_TRUE(contains_cdna3_vop1(target_words, target_word_count, /*op=*/1, /*vdst=*/21, /*src0=*/static_cast(256u + *temp1))); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarLowersDsRead2st64B32ToFlatGlobalLoads) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read2st64_b32_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); EXPECT_TRUE(contains_flat_global_load(target_words, target_word_count, /*op=*/20, /*vdst=*/20, /*addr=*/12, /*offset=*/512)); EXPECT_TRUE(contains_flat_global_load(target_words, target_word_count, /*op=*/20, /*vdst=*/21, /*addr=*/12, /*offset=*/768)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarLowersDsRead2B64ToFlatGlobalLoads) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read2_b64_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); EXPECT_TRUE(contains_flat_global_load(target_words, target_word_count, /*op=*/21, /*vdst=*/66, /*addr=*/58, /*offset=*/24)); EXPECT_TRUE(contains_flat_global_load(target_words, target_word_count, /*op=*/21, /*vdst=*/68, /*addr=*/58, /*offset=*/544)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarStagesOddWideRead2ResultsIntoEvenTuple) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read2_b64_words(/*offset0=*/3, /*offset1=*/68, /*addr=*/58, /*vdst=*/67); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); const auto staged_vdst = find_flat_global_load_vdst(target_words, target_word_count, /*op=*/21, /*addr=*/58, /*offset=*/24); ASSERT_TRUE(staged_vdst.has_value()); EXPECT_EQ(*staged_vdst % 2, 0u); EXPECT_NE(*staged_vdst, 67u); EXPECT_TRUE(contains_cdna3_vop1(target_words, target_word_count, /*op=*/1, /*vdst=*/67, /*src0=*/static_cast(256u + *staged_vdst))); EXPECT_TRUE(contains_cdna3_vop1(target_words, target_word_count, /*op=*/1, /*vdst=*/68, /*src0=*/static_cast(256u + *staged_vdst + 1))); EXPECT_TRUE(contains_cdna3_vop1(target_words, target_word_count, /*op=*/1, /*vdst=*/69, /*src0=*/static_cast(256u + *staged_vdst + 2))); EXPECT_TRUE(contains_cdna3_vop1(target_words, target_word_count, /*op=*/1, /*vdst=*/70, /*src0=*/static_cast(256u + *staged_vdst + 3))); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarLowersDsRead2st64B64ToFlatGlobalLoads) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read2st64_b64_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); EXPECT_TRUE(contains_flat_global_load(target_words, target_word_count, /*op=*/21, /*vdst=*/112, /*addr=*/98, /*offset=*/1024)); EXPECT_TRUE(contains_flat_global_load(target_words, target_word_count, /*op=*/21, /*vdst=*/114, /*addr=*/98, /*offset=*/1536)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarUsesDistinctSpillSlotsForTwoAddressTemps) { using namespace rocr::llvm::amdhsa; constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; // vdst aliases addr, so the replacement must preserve the original address // VGPR. Offsets 9 and 10 exceed the flat/global immediate range after B64 ST64 // scaling. The descriptor is already at the ordinary VGPR ceiling and the body // touches s[100:101], so virtual LDS must use the spill-per-use path: the // preserved address temp and two backing-pointer VGPR temps are live together // and must receive distinct private scratch slots. const auto ds = make_cdna4_ds_read2st64_b64_words(/*offset0=*/9, /*offset1=*/10, /*addr=*/4, /*vdst=*/4); const uint32_t touch_s100 = rocjitsu::build_s_mov_b32(/*sdst=*/100, /*ssrc0=*/128, ROCJITSU_CODE_ARCH_CDNA4); const uint32_t touch_s101 = rocjitsu::build_s_mov_b32(/*sdst=*/101, /*ssrc0=*/128, ROCJITSU_CODE_ARCH_CDNA4); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {touch_s100, touch_s101, ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, 63); AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, 13); AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, 63); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); EXPECT_TRUE(contains_flat_scratch_dword_offset(target_words, target_word_count, /*op=*/28, /*offset=*/16, /*is_load=*/false)); EXPECT_TRUE(contains_flat_scratch_dword_offset(target_words, target_word_count, /*op=*/28, /*offset=*/20, /*is_load=*/false)); EXPECT_TRUE(contains_flat_scratch_dword_offset(target_words, target_word_count, /*op=*/28, /*offset=*/24, /*is_load=*/false)); EXPECT_TRUE(contains_flat_scratch_dword_offset(target_words, target_word_count, /*op=*/20, /*offset=*/16, /*is_load=*/true)); EXPECT_TRUE(contains_flat_scratch_dword_offset(target_words, target_word_count, /*op=*/20, /*offset=*/20, /*is_load=*/true)); EXPECT_TRUE(contains_flat_scratch_dword_offset(target_words, target_word_count, /*op=*/20, /*offset=*/24, /*is_load=*/true)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarLowersDsReadB64TrB16LoadToFlatGlobal) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read_b64_tr_b16_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); const auto staged_addr = find_vop2_literal_add(target_words, target_word_count, /*vsrc1=*/2, /*literal=*/0); ASSERT_TRUE(staged_addr.has_value()); EXPECT_EQ(*staged_addr % 2, 0u); EXPECT_TRUE(contains_vop3_mov_b32(target_words, target_word_count, static_cast(*staged_addr + 1), rocjitsu::scalar_positive_inline_u32(0))); EXPECT_TRUE(contains_flat_global_load_addr(target_words, target_word_count, /*op=*/21, *staged_addr, /*offset=*/0)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarStagesOddDsReadB64TrB16AddressIntoEvenGlobalPair) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_read_b64_tr_b16_words(/*byte_offset=*/0, /*addr=*/7); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); const auto staged_addr = find_vop2_literal_add(target_words, target_word_count, /*vsrc1=*/7, /*literal=*/0); ASSERT_TRUE(staged_addr.has_value()); EXPECT_EQ(*staged_addr % 2, 0u); EXPECT_NE(*staged_addr, 7u); EXPECT_TRUE(contains_vop3_mov_b32(target_words, target_word_count, static_cast(*staged_addr + 1), rocjitsu::scalar_positive_inline_u32(0))); EXPECT_TRUE(contains_flat_global_load_addr(target_words, target_word_count, /*op=*/21, *staged_addr, /*offset=*/0)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarMaterializesLargeDsReadB64TrB16Offset) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; constexpr uint16_t kLargeDsOffset = 0x1234; const auto ds = make_cdna4_ds_read_b64_tr_b16_words(kLargeDsOffset); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_FALSE(result.elf_bytes.empty()); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); const auto temp = find_vop2_literal_add(target_words, target_word_count, /*vsrc1=*/2, kLargeDsOffset); ASSERT_TRUE(temp.has_value()); EXPECT_EQ(*temp % 2, 0u); EXPECT_TRUE(contains_vop3_mov_b32(target_words, target_word_count, static_cast(*temp + 1), rocjitsu::scalar_positive_inline_u32(0))); EXPECT_TRUE(contains_flat_global_load_addr(target_words, target_word_count, /*op=*/21, *temp, /*offset=*/0)); } TEST(BinaryTranslatorE2E, VirtualLdsSidecarRejectsUnsupportedDsMemoryOpcode) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_add_u32_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); rocjitsu::write_value_for_test( image, rodata->sectionOffset() + offsetof(rocjitsu::TestKernelDescriptor, group_segment_fixed_size), 105600u); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); EXPECT_FALSE(result.ok()); const auto diagnostic = std::ranges::find_if(result.diagnostics, [](const auto &d) { return d.message.find("virtual LDS lowering does not support this DS opcode") != std::string::npos; }); EXPECT_NE(diagnostic, result.diagnostics.end()); } TEST(BinaryTranslatorE2E, SkipFailedVirtualLdsSidecarKeepsNormalDescriptor) { // Static LDS that fits the host limit: the sidecar is only needed for // *dynamic* LDS overflow, so the normal hardware-LDS descriptor is a valid // launch target on its own. A sidecar lowering failure must therefore leave // the normal descriptor intact. constexpr uint32_t kBelowHostLdsBytes = 1024u; constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_add_u32_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); rocjitsu::write_value_for_test( image, rodata->sectionOffset() + offsetof(rocjitsu::TestKernelDescriptor, group_segment_fixed_size), kBelowHostLdsBytes); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; options.skip_failed_kernels = true; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); ASSERT_FALSE(result.elf_bytes.empty()); const auto skipped = std::ranges::find_if(result.diagnostics, [](const auto &diagnostic) { return diagnostic.kind == rocjitsu::DiagnosticKind::KernelSkipped && diagnostic.message.find("virtual LDS lowering does not support this DS opcode") != std::string::npos; }); ASSERT_NE(skipped, result.diagnostics.end()); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *translated_text = translated.text_sections()[0]; const auto *translated_rodata = rocjitsu::find_section(translated, ".rodata"); ASSERT_NE(translated_rodata, nullptr); ASSERT_GE(translated_rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); const auto normal_kd = rocjitsu::read_elf_struct_for_test( result.elf_bytes, translated_rodata->sectionOffset()); const int64_t normal_entry_vaddr = static_cast(translated_rodata->vaddr()) + rocjitsu::read_kernel_descriptor_entry_offset(&normal_kd); ASSERT_GE(normal_entry_vaddr, 0); const auto normal_entry_file_offset = rocjitsu::loaded_vaddr_to_file_offset( result.elf_bytes, static_cast(normal_entry_vaddr)); ASSERT_TRUE(normal_entry_file_offset.has_value()); // The optional virtual-LDS sidecar failed, but below-threshold launches still // use the normal descriptor. Its entry must therefore continue to point at the // translated hardware-LDS body, not at the skipped-kernel stub appended for // the sidecar variant. EXPECT_EQ(*normal_entry_file_offset, translated_text->sectionOffset()); EXPECT_EQ(rocjitsu::find_section(translated, rocjitsu::kVirtualLdsMetadataSectionName), nullptr); } TEST(BinaryTranslatorE2E, SkipFailedVirtualLdsSidecarStubsOversizedNormalDescriptor) { // Static LDS that exceeds the host limit: the normal descriptor advertises // more hardware LDS than the host has, so it is only launchable through its // virtual sidecar. When the sidecar lowering fails and is skipped, the normal // descriptor must be stubbed too — leaving it dispatchable would fault the // host at launch. This is the regression guard for the sidecar-dependent case. constexpr uint32_t kOverHostLdsBytes = 105600u; constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto ds = make_cdna4_ds_add_u32_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({ds[0], ds[1], kCdna4SEndpgm}); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); rocjitsu::write_value_for_test( image, rodata->sectionOffset() + offsetof(rocjitsu::TestKernelDescriptor, group_segment_fixed_size), kOverHostLdsBytes); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; options.skip_failed_kernels = true; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); ASSERT_FALSE(result.elf_bytes.empty()); const auto skipped = std::ranges::find_if(result.diagnostics, [](const auto &diagnostic) { return diagnostic.kind == rocjitsu::DiagnosticKind::KernelSkipped && diagnostic.message.find("virtual LDS lowering does not support this DS opcode") != std::string::npos; }); ASSERT_NE(skipped, result.diagnostics.end()); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); const auto *translated_rodata = rocjitsu::find_section(translated, ".rodata"); ASSERT_NE(translated_rodata, nullptr); ASSERT_GE(translated_rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); const auto normal_kd = rocjitsu::read_elf_struct_for_test( result.elf_bytes, translated_rodata->sectionOffset()); const int64_t normal_entry_vaddr = static_cast(translated_rodata->vaddr()) + rocjitsu::read_kernel_descriptor_entry_offset(&normal_kd); ASSERT_GE(normal_entry_vaddr, 0); const auto normal_entry_file_offset = rocjitsu::loaded_vaddr_to_file_offset( result.elf_bytes, static_cast(normal_entry_vaddr)); ASSERT_TRUE(normal_entry_file_offset.has_value()); // The normal descriptor's advertised static LDS (105600) exceeds the CDNA3 // host limit, so it could only ever launch via the failed sidecar. Its entry // must now point at the skipped-kernel endpgm stub, and the descriptor must // advertise no fixed LDS so a launch attempt cannot fault the host. const auto *entry_words = reinterpret_cast(result.elf_bytes.data() + *normal_entry_file_offset); EXPECT_EQ(entry_words[0], rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(normal_kd.group_segment_fixed_size, 0u); EXPECT_EQ(rocjitsu::find_section(translated, rocjitsu::kVirtualLdsMetadataSectionName), nullptr); } TEST(BinaryTranslatorE2E, Cdna4ToCdna3KernargPreloadFirmwareEntryKeepsMaskedDefLive) { using namespace rocr::llvm::amdhsa; constexpr uint16_t kScratchFloor = 120; auto image = rocjitsu::make_large_amdgpu_elf_with_waitcnt_entry(); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_FALSE(layout.text_sections().empty()); auto source_kd = rocjitsu::read_kernel_descriptor_for_test(image.data() + rodata->sectionOffset()); AMDHSA_BITS_SET(source_kd.kernarg_preload, KERNARG_PRELOAD_SPEC_LENGTH, 1); // Declare a small VGPR file so the globally-unused search is exhausted at the // scratch floor, forcing the allocator onto the per-point find_free_run path // (the only path the firmware EXEC pin can influence). AMDHSA_BITS_SET(source_kd.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, 15); rocjitsu::write_kernel_descriptor_for_test(image.data() + rodata->sectionOffset(), source_kd); auto *words = reinterpret_cast(image.data() + layout.text_sections()[0]->sectionOffset()); words[0] = 0xBEFE01C1u; // s_mov_b64 exec, -1 (ordinary path establishes Full). const auto cvt = make_cdna4_cvt_pk_f16_f32_words(); // Needs one scratch temp. words[64] = cvt[0]; words[65] = cvt[1]; // Masked vector def of the floor VGPR, read afterwards. Under the firmware // entry's pinned-Unknown EXEC this def is not a whole-register kill, so v120 is // live across the cvt and find_free_run must not reuse it. words[66] = cdna4::build_vop1(cdna4::kVMovB32Vop1, {.src0 = 0, .vdst = kScratchFloor})[0]; words[67] = cdna4::build_vop1( cdna4::kVMovB32Vop1, {.src0 = static_cast(256 + kScratchFloor), .vdst = 20})[0]; words[68] = cdna4::build_sopp(cdna4::kSEndpgmSopp)[0]; rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; options.debug_min_free_vgpr = kScratchFloor; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); // The packed-F16 lowering converts the high half into a scratch temp picked by // find_free_run (globally-unused VGPRs are exhausted at the floor). With the // firmware entry pinned Unknown, v120's masked def is not a kill, so v120 is // live across the cvt and the temp must skip it (v121). If the firmware entry // wrongly inherited Full, v120 would look dead and be reused as the temp (v120), // clobbering the value the kernel reads after the preload window. const auto writes_cvt_f16_to = [&](uint16_t vdst) { for (size_t i = 0; i + 1 < target_word_count; ++i) { rocjitsu::cdna3::Vop3MachineInst v{}; std::memcpy(&v, target_words + i, sizeof(v)); if (v.encoding == 0x34u && v.op == cdna3::kVCvtF16F32Vop3 && v.vdst == vdst) return true; } return false; }; EXPECT_TRUE(writes_cvt_f16_to(kScratchFloor + 1)) << "temp must skip the live firmware-entry def register v" << kScratchFloor; EXPECT_FALSE(writes_cvt_f16_to(kScratchFloor)) << "temp reused v" << kScratchFloor << ", clobbering the live firmware-entry def"; } TEST(BinaryTranslatorE2E, Cdna4ToCdna3Bitop3ScratchGrowsDescriptor) { constexpr uint16_t kScratchFloor = 120; const auto words = make_cdna4_bitop3_words(cdna4::kVBitop3B32Vop3, 16); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({words[0], words[1]}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; options.debug_min_free_vgpr = kScratchFloor; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; ASSERT_FALSE(result.elf_bytes.empty()); // This fixture's LUT needs a two-VGPR scratch run. The conservative liveness // floor forces that run above the descriptor's original allocation, so missing // require_vgprs() feedback would leave the patched descriptor too small. expect_cdna3_translated_descriptor_vgprs_at_least(result.elf_bytes, kScratchFloor + 2); } TEST(BinaryTranslatorE2E, Cdna4ToCdna3Bitop3UsesSpillBackedScratchWhenVgprsAreFull) { using namespace rocr::llvm::amdhsa; const auto words = make_cdna4_bitop3_words(cdna4::kVBitop3B32Vop3, 16); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({words[0], words[1]}); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, 63); AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, 63); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; options.debug_min_free_vgpr = 256; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; ASSERT_FALSE(result.elf_bytes.empty()); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); // With find_free_run() forced above the physical VGPR namespace, the bitop3 // lowering must borrow descriptor-backed v0/v1, preserve them in the reusable // per-lane semantic spill window, and restore them after writing VDST. The // forbidden set excludes v16..v19 so the borrowed window cannot clobber the // destination or any source while the LUT expression is being synthesized. EXPECT_TRUE(contains_flat_scratch_dword(target_words, target_word_count, /*op=*/28, /*vgpr=*/0, /*offset=*/0, /*is_load=*/false)); EXPECT_TRUE(contains_flat_scratch_dword(target_words, target_word_count, /*op=*/28, /*vgpr=*/1, /*offset=*/4, /*is_load=*/false)); EXPECT_TRUE(contains_flat_scratch_dword(target_words, target_word_count, /*op=*/20, /*vgpr=*/0, /*offset=*/0, /*is_load=*/true)); EXPECT_TRUE(contains_flat_scratch_dword(target_words, target_word_count, /*op=*/20, /*vgpr=*/1, /*offset=*/4, /*is_load=*/true)); } TEST(BinaryTranslatorE2E, Cdna4ToCdna3CvtPkBf16SpillScratchAvoidsVgprSources) { using namespace rocr::llvm::amdhsa; const auto words = make_cdna4_cvt_pk_bf16_f32_words(/*vdst=*/0, /*src0=*/256 + 4, /*src1=*/256 + 1); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({words[0], words[1]}); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, 1); AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX90A_ACCUM_OFFSET, 3); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; options.debug_min_free_vgpr = 256; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; ASSERT_FALSE(result.elf_bytes.empty()); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); // Force liveness to report no dead physical VGPR window. The BF16 lowering // must then spill a borrowed three-VGPR window, but that window cannot include // VDST or the still-needed high source. The old allocator borrowed v[1:3] // here and clobbered SRC1 before converting the high packed half. EXPECT_FALSE(contains_flat_scratch_dword(target_words, target_word_count, /*op=*/28, /*vgpr=*/1, /*offset=*/0, /*is_load=*/false)); EXPECT_TRUE(contains_flat_scratch_dword(target_words, target_word_count, /*op=*/28, /*vgpr=*/2, /*offset=*/0, /*is_load=*/false)); EXPECT_TRUE(contains_flat_scratch_dword(target_words, target_word_count, /*op=*/28, /*vgpr=*/3, /*offset=*/4, /*is_load=*/false)); EXPECT_TRUE(contains_flat_scratch_dword(target_words, target_word_count, /*op=*/28, /*vgpr=*/4, /*offset=*/8, /*is_load=*/false)); EXPECT_TRUE(contains_flat_scratch_dword(target_words, target_word_count, /*op=*/20, /*vgpr=*/2, /*offset=*/0, /*is_load=*/true)); EXPECT_TRUE(contains_flat_scratch_dword(target_words, target_word_count, /*op=*/20, /*vgpr=*/3, /*offset=*/4, /*is_load=*/true)); EXPECT_TRUE(contains_flat_scratch_dword(target_words, target_word_count, /*op=*/20, /*vgpr=*/4, /*offset=*/8, /*is_load=*/true)); } TEST(BinaryTranslatorE2E, Cdna4ToCdna3MfmaPartialScratchGrowsDescriptor) { constexpr uint16_t kScratchFloor = 121; constexpr uint16_t kAlignedScratch = 122; const auto words = make_cdna4_mfma_vgpr_dst_alias_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({words[0], words[1]}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; options.debug_min_free_vgpr = kScratchFloor; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; ASSERT_FALSE(result.elf_bytes.empty()); // The 16x16x32 F16 lowering uses a four-VGPR partial accumulator when an // ordinary destination overlaps the still-needed wide A/B source window. expect_cdna3_translated_descriptor_vgprs_at_least(result.elf_bytes, kAlignedScratch + 4); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); std::optional partial_mfma; for (size_t i = 0; i + 1 < target_word_count; ++i) { rocjitsu::cdna3::Vop3pMfmaMachineInst actual{}; std::memcpy(&actual, target_words + i, sizeof(actual)); if (actual.encoding == 0x1A7u && actual.op == 77u && actual.acc_cd == 0u) { partial_mfma = actual; break; } } ASSERT_TRUE(partial_mfma.has_value()); EXPECT_EQ(partial_mfma->vdst, kAlignedScratch); EXPECT_EQ(partial_mfma->vdst % 2u, 0u); } TEST(BinaryTranslatorE2E, Cdna4ToCdna3Dot2ScratchGrowsDescriptor) { constexpr uint16_t kScratchFloor = 121; const auto words = make_cdna4_dot2_f32_bf16_words(/*vdst=*/0, /*src0=*/256 + 1, /*src1=*/256 + 2, /*src2=*/256); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({words[0], words[1]}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; options.debug_min_free_vgpr = kScratchFloor; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; ASSERT_FALSE(result.elf_bytes.empty()); // The BF16 dot2 fallback widens four packed halves into ordinary VGPR // temporaries before issuing scalar FP32 arithmetic. If liveness selects a // scratch run above the guest allocation, the descriptor must grow to cover // all four generated VGPRs. expect_cdna3_translated_descriptor_vgprs_at_least(result.elf_bytes, kScratchFloor + 4); } TEST(BinaryTranslatorE2E, Rdna4ScratchAllocationDoesNotWrapPastV255) { const auto words = make_cdna4_mfma_words(cdna4::kVMfmaF3216x16x16F16Vop3pMfma, 0, 256, 260); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({words[0], words[1]}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; options.debug_min_free_vgpr = 256; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_RDNA4, 0, options); auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(rocjitsu::has_error_containing( result, rocjitsu::DiagnosticKind::ExpandFailed, "MFMA lowering could not find a free VGPR for ds_bpermute addresses")); } TEST(BinaryTranslatorE2E, Cdna4ToCdna3PermlaneRejectsDescriptorFullExecSave) { using namespace rocr::llvm::amdhsa; const auto permlane = make_cdna4_permlane32_swap_b32_words(/*encoding_id=*/cdna4::encoding::kVop1); std::vector words = {permlane[0], permlane[1]}; for (uint16_t sgpr = 0; sgpr < 102; ++sgpr) { // Keep every ordinary SGPR live after the permlane replacement point. With // the descriptor already full, the EXEC-save helper must then fail instead // of borrowing and clobbering one of these guest scalar values. words.push_back(rocjitsu::build_s_mov_b32(sgpr, sgpr, ROCJITSU_CODE_ARCH_CDNA4)); } auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, 12); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); EXPECT_FALSE(result.ok()); const auto diagnostic = std::ranges::find_if(result.diagnostics, [](const auto &d) { return d.kind == rocjitsu::DiagnosticKind::ExpandFailed && d.message.find("No free descriptor-backed SGPR pair for EXEC save/restore") != std::string::npos; }); EXPECT_NE(diagnostic, result.diagnostics.end()); } TEST(BinaryTranslatorE2E, Cdna4ToCdna3PermlaneExecSaveReservesSpecialSgprTail) { using namespace rocr::llvm::amdhsa; constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto permlane = make_cdna4_permlane32_swap_b32_words(/*encoding_id=*/cdna4::encoding::kVop1); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {permlane[0], permlane[1], kCdna4SEndpgm}); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); // This mirrors the Qwen SDPA `attn_fwd` shape: the source descriptor allocates // 64 SGPRs, while the semantic permlane lowering introduces an EXEC-save pair // starting at s64. Growing only through s65 leaves no room for the // architecture-owned VCC/flat-scratch/XNACK tail, so DBT must reserve that // tail explicitly when materializing the generated ordinary SGPR pair. AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WAVEFRONT_SGPR_COUNT, 7); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); ASSERT_FALSE(result.elf_bytes.empty()); expect_cdna3_translated_descriptor_sgprs_eq(result.elf_bytes, 80); } TEST(BinaryTranslatorE2E, Cdna4ToCdna3Permlane16SwapWritesBothRowPairs) { constexpr uint32_t kCdna4SEndpgm = 0xBF810000u; const auto permlane = make_cdna4_permlane16_swap_b32_words(/*encoding_id=*/cdna4::encoding::kVop1); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {permlane[0], permlane[1], kCdna4SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); ASSERT_FALSE(result.elf_bytes.empty()); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); // V_PERMLANE16_SWAP_B32 swaps both 16-lane row pairs: lanes 0..15 with // 16..31, and lanes 32..47 with 48..63. The lowering therefore emits the // same low/high row masks into EXEC_LO and EXEC_HI. constexpr uint8_t kExecLoSgpr = 126; constexpr uint8_t kExecHiSgpr = 127; EXPECT_EQ( count_cdna3_s_mov_b32_literal(target_words, target_word_count, kExecLoSgpr, 0x0000ffffu), 1u); EXPECT_EQ( count_cdna3_s_mov_b32_literal(target_words, target_word_count, kExecLoSgpr, 0xffff0000u), 1u); EXPECT_EQ( count_cdna3_s_mov_b32_literal(target_words, target_word_count, kExecHiSgpr, 0x0000ffffu), 1u); EXPECT_EQ( count_cdna3_s_mov_b32_literal(target_words, target_word_count, kExecHiSgpr, 0xffff0000u), 1u); } TEST(BinaryTranslatorE2E, RelocatedKernelCompactsReachableBodyAndPatchesBranches) { constexpr uint32_t kCdna4SEndpgm = cdna4::build_sopp(cdna4::kSEndpgmSopp)[0]; constexpr uint32_t kCdna4SCbranchScc1ToSourceTarget = cdna4::build_sopp(cdna4::kSCbranchScc1Sopp, {.simm16 = 4})[0]; const std::vector words = { rocjitsu::build_s_branch(2, ROCJITSU_CODE_ARCH_CDNA4), // 0x00 -> source 0x0c. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x04 unreachable. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x08 unreachable. kCdna4SCbranchScc1ToSourceTarget, // 0x0c -> 0x20, else 0x10. rocjitsu::build_s_branch(4, ROCJITSU_CODE_ARCH_CDNA4), // 0x10 -> source 0x24. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x14 unreachable. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x18 unreachable. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x1c unreachable. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x20 conditional target. rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), // 0x24 fallthrough-branch target. }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); // This kernel does not use the kernarg-preload compatibility entry path, so // relocation emits only the reachable CFG body. Source gaps disappear and // branch immediates are patched against the compact target layout. const std::vector expected = { rocjitsu::build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA3), cdna3::build_sopp(cdna3::kSCbranchScc1Sopp, {.simm16 = 1})[0], rocjitsu::build_s_branch(1, ROCJITSU_CODE_ARCH_CDNA3), rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA3), kCdna4SEndpgm, }; for (size_t i = 0; i < expected.size(); ++i) { SCOPED_TRACE(i); EXPECT_EQ(target_words[i], expected[i]); } } TEST(BinaryTranslatorE2E, RelocatedKernelCompactsReachableBlocksAfterEntry) { std::vector words(74, rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4)); words[0] = rocjitsu::build_s_branch(63, ROCJITSU_CODE_ARCH_CDNA4); // 0x00 -> 0x100. words[64] = rocjitsu::build_s_branch(7, ROCJITSU_CODE_ARCH_CDNA4); // 0x100 -> 0x120. words[72] = rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4); // Reachable target. auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source_layout(image.data(), image.size()); ASSERT_TRUE(source_layout.is_valid()); const auto *source_rodata = rocjitsu::find_section(source_layout, ".rodata"); ASSERT_NE(source_rodata, nullptr); ASSERT_GE(source_rodata->size(), sizeof(rocr::llvm::amdhsa::kernel_descriptor_t)); auto source_kd = rocjitsu::read_kernel_descriptor_for_test(image.data() + source_rodata->sectionOffset()); AMDHSA_BITS_SET(source_kd.kernarg_preload, rocr::llvm::amdhsa::KERNARG_PRELOAD_SPEC_LENGTH, 1); rocjitsu::write_kernel_descriptor_for_test(image.data() + source_rodata->sectionOffset(), source_kd); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *text = translated.text_sections()[0]; ASSERT_GE(text->size(), words.size() * sizeof(uint32_t)); const auto *target_words = reinterpret_cast(text->data()); // The synthesized preload launch window occupies words 0 and 64. The compact // relocated body starts after that protected window, and the source 0x120 // target lands immediately after the source 0x100 branch instead of remaining // at the original word 72. EXPECT_EQ(target_words[0], rocjitsu::build_s_branch(64, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[64], rocjitsu::build_s_branch(1, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[65], rocjitsu::build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[66], rocjitsu::build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[67], rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[72], rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA3)); } TEST(BinaryTranslatorE2E, PatchesRecoveredSetpcTargetAfterRelocation) { constexpr uint16_t kPcSreg = 8; constexpr uint32_t kLiteralOperand = 255; constexpr uint32_t kInlineInt0 = 128; constexpr uint32_t kOriginalGetpcDelta = 20; const std::vector words = { build_s_getpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_CDNA4), // 0x00. build_s_add_u32(kPcSreg, kPcSreg, kLiteralOperand), // 0x04. kOriginalGetpcDelta, // 0x08. build_s_addc_u32(kPcSreg + 1, kPcSreg + 1, kInlineInt0), // 0x0c. build_s_setpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_CDNA4), // 0x10. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x14 unreachable. rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), // 0x18 target. }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); ASSERT_GE(translated.text_sections()[0]->size(), 6 * sizeof(uint32_t)); // Recovered indirect jumps now patch the consumer site rather than rewriting // the source-side getpc/add builder. Its consumer starts as one compact word // and grows only if final placement requires the canonical long form. EXPECT_EQ(target_words[0], build_s_getpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[1], build_s_add_u32(kPcSreg, kPcSreg, kLiteralOperand)); EXPECT_EQ(target_words[2], kOriginalGetpcDelta); EXPECT_EQ(target_words[3], build_s_addc_u32(kPcSreg + 1, kPcSreg + 1, kInlineInt0)); EXPECT_EQ(target_words[4], rocjitsu::build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[5], rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA3)); } TEST(BinaryTranslatorE2E, PatchesRecoveredSwappcTargetAfterRelocation) { constexpr uint16_t kPcSreg = 10; constexpr uint16_t kReturnSreg = 20; constexpr uint32_t kLiteralOperand = 255; constexpr uint32_t kInlineInt0 = 128; constexpr uint32_t kOriginalGetpcDelta = 24; const std::vector words = { build_s_getpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_CDNA4), // 0x00. build_s_add_u32(kPcSreg, kPcSreg, kLiteralOperand), // 0x04. kOriginalGetpcDelta, // 0x08. build_s_addc_u32(kPcSreg + 1, kPcSreg + 1, kInlineInt0), // 0x0c. build_s_swappc_b64(kReturnSreg, kPcSreg, ROCJITSU_CODE_ARCH_CDNA4), // 0x10. rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), // 0x14 unreachable continuation. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x18 unreachable. rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), // 0x1c target. }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); ASSERT_GE(translated.text_sections()[0]->size(), 6 * sizeof(uint32_t)); EXPECT_EQ(target_words[0], build_s_getpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[1], build_s_add_u32(kPcSreg, kPcSreg, kLiteralOperand)); EXPECT_EQ(target_words[2], kOriginalGetpcDelta); EXPECT_EQ(target_words[3], build_s_addc_u32(kPcSreg + 1, kPcSreg + 1, kInlineInt0)); // The target is non-returning, so relocation drops the dead continuation // and compacts the target immediately after the call. EXPECT_EQ(target_words[4], build_s_call_b64(kReturnSreg, 0)); EXPECT_EQ(target_words[5], rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA3)); } TEST(BinaryTranslatorE2E, TranslatesDirectSCallWithSetpcReturn) { constexpr uint16_t kReturnSreg = 30; const std::vector words = { rocjitsu::build_s_branch(2, ROCJITSU_CODE_ARCH_CDNA4), // 0x00 -> call block at 0x0c. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x04 unreachable gap. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x08 unreachable gap. build_s_call_b64(kReturnSreg, 2), // 0x0c -> callee at 0x18. rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), // 0x10 call continuation. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x14 unreachable gap. build_s_setpc_b64(kReturnSreg, ROCJITSU_CODE_ARCH_CDNA4), // 0x18 callee return. }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[0], rocjitsu::build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[1], build_s_call_b64(kReturnSreg, 1)) << "the direct call target must be recomputed after unreachable source gaps are compacted"; EXPECT_EQ(target_words[2], rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[3], build_s_setpc_b64(kReturnSreg, ROCJITSU_CODE_ARCH_CDNA3)); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_CDNA3); ASSERT_GE(decoded.size(), 4u); ASSERT_EQ(decoded[1]->mnemonic(), "s_call_b64"); ASSERT_TRUE(decoded[1]->branch_offset_bytes().has_value()); EXPECT_EQ(*decoded[1]->branch_offset_bytes(), 4) << "translated call should branch from word 1 to the relocated return block at word 3"; } TEST(BinaryTranslatorE2E, TranslatesDirectSCallWhenCalleeBranchesToSetpcReturn) { constexpr uint16_t kReturnSreg = 30; const std::vector words = { rocjitsu::build_s_branch(2, ROCJITSU_CODE_ARCH_CDNA4), // 0x00 -> call block at 0x0c. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x04 unreachable gap. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x08 unreachable gap. build_s_call_b64(kReturnSreg, 2), // 0x0c -> callee at 0x18. rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), // 0x10 call continuation. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x14 unreachable gap. rocjitsu::build_s_branch(1, ROCJITSU_CODE_ARCH_CDNA4), // 0x18 callee -> return. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x1c unreachable gap. build_s_setpc_b64(kReturnSreg, ROCJITSU_CODE_ARCH_CDNA4), // 0x20 callee return. }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[0], rocjitsu::build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[1], build_s_call_b64(kReturnSreg, 1)); EXPECT_EQ(target_words[2], rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[3], rocjitsu::build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[4], build_s_setpc_b64(kReturnSreg, ROCJITSU_CODE_ARCH_CDNA3)); } TEST(BinaryTranslatorE2E, TranslatesNestedCallReturningThroughOuterPair) { constexpr uint16_t kOuterReturnSreg = 30; constexpr uint16_t kInnerReturnSreg = 28; const std::vector words = { build_s_call_b64(kOuterReturnSreg, 1), // 0x00 -> outer callee at 0x08. rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), // 0x04 outer continuation. build_s_call_b64(kInnerReturnSreg, 1), // 0x08 -> inner callee at 0x10. rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), // 0x0c inner continuation. rocjitsu::pack_sopp(5, 1), // 0x10 -> outer return at 0x18. build_s_setpc_b64(kInnerReturnSreg, ROCJITSU_CODE_ARCH_CDNA4), // 0x14 inner return. build_s_setpc_b64(kOuterReturnSreg, ROCJITSU_CODE_ARCH_CDNA4), // 0x18 outer return. }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_CDNA3); ASSERT_GE(decoded.size(), words.size()); const auto count_mnemonic = [&](std::string_view mnemonic) { return std::ranges::count_if( decoded, [&](const auto &inst) { return inst != nullptr && inst->mnemonic() == mnemonic; }); }; EXPECT_EQ(count_mnemonic("s_call_b64"), 2); EXPECT_EQ(count_mnemonic("s_setpc_b64"), 2); } TEST(BinaryTranslatorE2E, TranslatesSwappcCallWhenCalleeSetpcBranchesToReturn) { constexpr uint16_t kCallTargetSreg = 10; constexpr uint16_t kReturnTargetSreg = 12; constexpr uint16_t kReturnSreg = 30; constexpr uint32_t kLiteralOperand = 255; constexpr uint32_t kInlineInt0 = 128; constexpr uint32_t kOriginalCallTargetDelta = 28; constexpr uint32_t kOriginalReturnTargetDelta = 20; const std::vector words = { build_s_getpc_b64(kCallTargetSreg, ROCJITSU_CODE_ARCH_CDNA4), // 0x00. build_s_add_u32(kCallTargetSreg, kCallTargetSreg, kLiteralOperand), // 0x04. kOriginalCallTargetDelta, // 0x08. build_s_addc_u32(kCallTargetSreg + 1, kCallTargetSreg + 1, kInlineInt0), // 0x0c. build_s_swappc_b64(kReturnSreg, kCallTargetSreg, ROCJITSU_CODE_ARCH_CDNA4), // 0x10. rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), // 0x14 continuation. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x18 unreachable. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x1c unreachable. build_s_getpc_b64(kReturnTargetSreg, ROCJITSU_CODE_ARCH_CDNA4), // 0x20 callee. build_s_add_u32(kReturnTargetSreg, kReturnTargetSreg, kLiteralOperand), // 0x24. kOriginalReturnTargetDelta, // 0x28. build_s_addc_u32(kReturnTargetSreg + 1, kReturnTargetSreg + 1, kInlineInt0), // 0x2c. build_s_setpc_b64(kReturnTargetSreg, ROCJITSU_CODE_ARCH_CDNA4), // 0x30 -> return. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x34 unreachable. build_s_setpc_b64(kReturnSreg, ROCJITSU_CODE_ARCH_CDNA4), // 0x38 return. }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); ASSERT_GE(translated.text_sections()[0]->size(), 12 * sizeof(uint32_t)); EXPECT_EQ(target_words[0], build_s_getpc_b64(kCallTargetSreg, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[2], kOriginalCallTargetDelta); EXPECT_EQ(target_words[4], build_s_call_b64(kReturnSreg, 1)) << "the swappc call window should patch directly to the compact callee body"; EXPECT_EQ(target_words[5], rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[6], build_s_getpc_b64(kReturnTargetSreg, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[8], kOriginalReturnTargetDelta); EXPECT_EQ(target_words[10], rocjitsu::build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA3)) << "the callee's recovered setpc window is what reaches the return block"; EXPECT_EQ(target_words[11], build_s_setpc_b64(kReturnSreg, ROCJITSU_CODE_ARCH_CDNA3)); } TEST(BinaryTranslatorE2E, PatchesOneRecoveredBuilderUsedByTwoSetpcConsumers) { constexpr uint16_t kPcSreg = 12; constexpr uint32_t kLiteralOperand = 255; constexpr uint32_t kInlineInt0 = 128; constexpr uint32_t kOriginalGetpcDelta = 28; const std::vector words = { build_s_getpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_CDNA4), // 0x00. build_s_add_u32(kPcSreg, kPcSreg, kLiteralOperand), // 0x04. kOriginalGetpcDelta, // 0x08. build_s_addc_u32(kPcSreg + 1, kPcSreg + 1, kInlineInt0), // 0x0c. cdna4::build_sopp(cdna4::kSCbranchScc1Sopp, {.simm16 = 1})[0], // 0x10 -> second consumer. build_s_setpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_CDNA4), // 0x14 first consumer. build_s_setpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_CDNA4), // 0x18 carried consumer. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x1c unreachable gap. rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), // 0x20 shared target. }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); ASSERT_GE(translated.text_sections()[0]->size(), 8 * sizeof(uint32_t)); EXPECT_EQ(target_words[0], build_s_getpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[1], build_s_add_u32(kPcSreg, kPcSreg, kLiteralOperand)); EXPECT_EQ(target_words[2], kOriginalGetpcDelta); EXPECT_EQ(target_words[3], build_s_addc_u32(kPcSreg + 1, kPcSreg + 1, kInlineInt0)); EXPECT_EQ(target_words[4], cdna3::build_sopp(cdna3::kSCbranchScc1Sopp, {.simm16 = 1})[0]); EXPECT_EQ(target_words[5], rocjitsu::build_s_branch(1, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[6], rocjitsu::build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[7], rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA3)); } TEST(BinaryTranslatorE2E, RewritesDistinctBuildersForOneMultiTargetSetpcConsumer) { constexpr uint16_t kPcSreg = 8; constexpr uint32_t kLiteralOperand = 255; constexpr uint32_t kInlineInt0 = 128; constexpr uint32_t kOriginalTargetADelta = 44; constexpr uint32_t kOriginalTargetBDelta = 28; constexpr uint32_t kRelocatedTargetADelta = 40; constexpr uint32_t kRelocatedTargetBDelta = 24; const std::vector words = { cdna4::build_sopp(cdna4::kSCbranchScc1Sopp, {.simm16 = 5})[0], // 0x00 -> builder B. build_s_getpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_CDNA4), // 0x04 builder A. build_s_add_u32(kPcSreg, kPcSreg, kLiteralOperand), // 0x08. kOriginalTargetADelta, // 0x0c -> target A. build_s_addc_u32(kPcSreg + 1, kPcSreg + 1, kInlineInt0), // 0x10. rocjitsu::build_s_branch(5, ROCJITSU_CODE_ARCH_CDNA4), // 0x14 -> consumer. build_s_getpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_CDNA4), // 0x18 builder B. build_s_add_u32(kPcSreg, kPcSreg, kLiteralOperand), // 0x1c. kOriginalTargetBDelta, // 0x20 -> target B. build_s_addc_u32(kPcSreg + 1, kPcSreg + 1, kInlineInt0), // 0x24. rocjitsu::build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x28 -> consumer. build_s_setpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_CDNA4), // 0x2c multi-target consumer. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x30 unreachable. rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), // 0x34 target A. rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), // 0x38 target B. }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); ASSERT_GE(translated.text_sections()[0]->size(), 15 * sizeof(uint32_t)); EXPECT_EQ(target_words[3], kRelocatedTargetADelta) << "builder A should be rewritten once for its relocated target"; EXPECT_EQ(target_words[8], kRelocatedTargetBDelta) << "builder B should be rewritten once for its relocated target"; EXPECT_EQ(target_words[11], build_s_setpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_CDNA3)) << "one consumer with multiple possible targets must stay indirect"; EXPECT_EQ(target_words[12], rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[13], rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA3)); } TEST(BinaryTranslatorE2E, RelocatesDirectCallReturnAcrossShiftedOffsets) { constexpr uint16_t kReturnSreg = 28; std::vector words = { rocjitsu::build_s_branch(4, ROCJITSU_CODE_ARCH_CDNA4), // 0x00 -> call block at 0x14. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x04 unreachable. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x08 unreachable. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x0c unreachable. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x10 unreachable. build_s_call_b64(kReturnSreg, 6), // 0x14 -> callee at 0x30. rocjitsu::build_s_branch(6, ROCJITSU_CODE_ARCH_CDNA4), // 0x18 continuation -> 0x34. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x1c unreachable. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x20 unreachable. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x24 unreachable. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x28 unreachable. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x2c unreachable. build_s_setpc_b64(kReturnSreg, ROCJITSU_CODE_ARCH_CDNA4), // 0x30 callee return. rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), // 0x34 final continuation. }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[0], rocjitsu::build_s_branch(0, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[1], build_s_call_b64(kReturnSreg, 1)) << "source call target 0x30 should relocate to compact word 3"; EXPECT_EQ(target_words[2], rocjitsu::build_s_branch(1, ROCJITSU_CODE_ARCH_CDNA3)) << "the call continuation branch should relocate to compact word 4"; EXPECT_EQ(target_words[3], build_s_setpc_b64(kReturnSreg, ROCJITSU_CODE_ARCH_CDNA3)); EXPECT_EQ(target_words[4], rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA3)); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_CDNA3); ASSERT_GE(decoded.size(), 5u); ASSERT_TRUE(decoded[1]->branch_offset_bytes().has_value()); EXPECT_EQ(*decoded[1]->branch_offset_bytes(), 4); } TEST(BinaryTranslatorE2E, EndpgmAfterTrapTerminatesCfgBeforeFollowingFunction) { // A resumable S_TRAP is not a CFG terminator: on hardware it transfers to the // trap handler and may return to the next instruction. A real terminator // (S_ENDPGM) must follow it to end the block. The S_ENDPGM terminates the CFG // so the following ELF function bytes stay unreachable and the unrecovered // S_SETPC_B64 below is never decoded into the CFG. const std::vector words = { rocjitsu::build_s_branch(1, ROCJITSU_CODE_ARCH_CDNA4), // 0x00 -> trap block. rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), // 0x04 unreachable gap. build_s_trap(3), // 0x08 falls through to endpgm. rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), // 0x0c terminates the block. build_s_setpc_b64(30, ROCJITSU_CODE_ARCH_CDNA4), // 0x10 next function body. rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), // 0x14 unreachable. }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_CDNA3); ASSERT_GE(decoded.size(), 3u); EXPECT_EQ(decoded[0]->mnemonic(), "s_branch"); EXPECT_EQ(decoded[1]->mnemonic(), "s_trap"); EXPECT_EQ(decoded[2]->mnemonic(), "s_endpgm"); // The trailing S_ENDPGM (a real terminator) prevents a bogus fallthrough into // the following ELF function bytes, so the unrecovered S_SETPC_B64 is never // reached and translation does not emit it. EXPECT_TRUE(std::none_of(decoded.begin(), decoded.end(), [](const auto &inst) { return inst->mnemonic() == "s_setpc_b64"; })); } TEST(BinaryTranslatorE2E, ResumableTrapFallsThroughAndDoesNotHideFollowingCode) { // Trap IDs other than ROCr's assertion/abort trap remain resumable, so code // after them is reachable. Here the fallthrough reaches an unrecovered // S_SETPC_B64, which the translator must surface as an error. const std::vector words = { build_s_trap(3), // 0x00 falls through. build_s_setpc_b64(30, ROCJITSU_CODE_ARCH_CDNA4), // 0x04 reachable, unrecovered. rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE( rocjitsu::has_error_containing(result, rocjitsu::DiagnosticKind::Legalization, "indirect branch or call target recovery is not implemented")); } TEST(BinaryTranslatorE2E, RocrAbortTrapTerminatesCfgBeforeFollowingCode) { // ROCr reserves trap ID 2 for assertion/abort handling. That path does not // return, so bytes after S_TRAP 2 must not be pulled into the current CFG. const std::vector words = { build_s_trap(2), // 0x00 terminates the program path. build_s_setpc_b64(30, ROCJITSU_CODE_ARCH_CDNA4), // 0x04 unrelated, unreachable code. rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_CDNA3); ASSERT_FALSE(decoded.empty()); EXPECT_EQ(decoded[0]->mnemonic(), "s_trap"); EXPECT_TRUE(std::none_of(decoded.begin(), decoded.end(), [](const auto &inst) { return inst->mnemonic() == "s_setpc_b64"; })); } TEST(BinaryTranslatorE2E, RocrAbortDeadEdgeDoesNotPoisonRecoveredCall) { constexpr uint16_t kPcSreg = 8; constexpr uint16_t kReturnSreg = 30; constexpr uint32_t kLiteralOperand = 255; constexpr uint32_t kInlineInt0 = 128; // Exercise BinaryTranslator's ExplicitOnly entry policy rather than calling // BasicBlock::build with the policy directly: // // builder --conditional--------------------------> call // | // +--> s_trap 2 -X-> dead branch --^ // // Treating the dead post-trap block as an inferred entry would add an // unconstrained path to s[8:9] and make translation fail closed. const std::vector words = { build_s_getpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_CDNA4), // 0x00. build_s_add_u32(kPcSreg, kPcSreg, kLiteralOperand), // 0x04. 40, // 0x08: 0x04 + 40 = helper 0x2c. build_s_addc_u32(kPcSreg + 1, kPcSreg + 1, kInlineInt0), // 0x0c. cdna4::build_sopp(cdna4::kSCbranchScc0Sopp, {.simm16 = 3})[0], // 0x10 -> 0x20. build_s_trap(2), // 0x14. rocjitsu::build_s_branch(1, ROCJITSU_CODE_ARCH_CDNA4), // 0x18 -> 0x20. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x1c. build_s_swappc_b64(kReturnSreg, kPcSreg, ROCJITSU_CODE_ARCH_CDNA4), // 0x20. rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4), // 0x24. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), // 0x28. build_s_setpc_b64(kReturnSreg, ROCJITSU_CODE_ARCH_CDNA4), // 0x2c. }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << result.diagnostics.front().message; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_CDNA3); EXPECT_TRUE(std::ranges::any_of( decoded, [](const auto &inst) { return inst->mnemonic() == "s_call_b64"; })); } TEST(BinaryTranslatorE2E, RejectsUnrecoveredIndirectBranchInstructions) { struct Case { const char *name; std::vector words; const char *mnemonic; }; const std::array cases = {{ {"SetpcS0", {0xBE801D00u, 0x00000000u}, "s_setpc_b64"}, {"SetpcS30WithoutCall", {build_s_setpc_b64(30, ROCJITSU_CODE_ARCH_CDNA4), rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_CDNA4)}, "s_setpc_b64"}, {"Swappc", {0xBE801E00u, 0x00000000u}, "s_swappc_b64"}, }}; for (const auto &test_case : cases) { SCOPED_TRACE(test_case.name); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(test_case.words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(rocjitsu::has_error_containing( result, rocjitsu::DiagnosticKind::Legalization, "indirect branch or call target recovery is not implemented")); const auto diagnostic = std::find_if(result.diagnostics.begin(), result.diagnostics.end(), [&](const auto &d) { return d.mnemonic == test_case.mnemonic; }); EXPECT_NE(diagnostic, result.diagnostics.end()); } } TEST(BinaryTranslatorE2E, RejectsDirectBranchTargetBeforeText) { const std::vector words = { rocjitsu::build_s_branch(-2, ROCJITSU_CODE_ARCH_CDNA4), // 0x00 -> -0x04. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE( rocjitsu::has_error_containing(result, rocjitsu::DiagnosticKind::Legalization, "direct branch target is outside the source .text range")); } TEST(BinaryTranslatorE2E, RejectsDirectBranchTargetAbsentFromRelocatedBody) { const std::vector words = { rocjitsu::build_s_branch(1, ROCJITSU_CODE_ARCH_CDNA4), // 0x00 -> .text end. rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_CDNA4), }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(rocjitsu::has_error_containing( result, rocjitsu::DiagnosticKind::Legalization, "direct branch target is not present in the kernel-local relocated body")); } TEST(BinaryTranslatorE2E, PlacesDescriptorPrologueBeforeLargeRelocatedBody) { constexpr size_t kBodyWordsPastBranchRange = 32769; std::vector words(kBodyWordsPastBranchRange, 0xBF800000u); words.push_back(0xBF810000u); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::enable_workgroup_id_x_sgpr(image); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_RDNA4); auto result = translator.translate(source); EXPECT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); EXPECT_FALSE(rocjitsu::has_error_containing( result, rocjitsu::DiagnosticKind::ResourceLimit, "kernel descriptor prologue branch range exceeds s_branch simm16")); } TEST(BinaryTranslatorE2E, ExpandLegalizationWithoutSemanticRuleFails) { auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(); rocjitsu::AmdGpuCodeObject source_layout(image.data(), image.size()); ASSERT_TRUE(source_layout.is_valid()); ASSERT_FALSE(source_layout.text_sections().empty()); const auto words = make_cdna4_dot2c_unimplemented_expand_words(); const auto *source_text = source_layout.text_sections()[0]; ASSERT_EQ(source_text->size(), words.size() * sizeof(uint32_t)); std::memcpy(image.data() + source_text->sectionOffset(), words.data(), words.size() * sizeof(uint32_t)); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); EXPECT_FALSE(result.ok()); ASSERT_FALSE(result.diagnostics.empty()); const auto diagnostic = std::ranges::find_if(result.diagnostics, [](const auto &d) { return d.kind == rocjitsu::DiagnosticKind::ExpandMissing; }); ASSERT_NE(diagnostic, result.diagnostics.end()); EXPECT_EQ(diagnostic->severity, rocjitsu::DiagnosticSeverity::Error); EXPECT_EQ(diagnostic->guest_offset, std::optional(0)); EXPECT_FALSE(diagnostic->required_work.empty()); } TEST(BinaryTranslatorE2E, DebugContinueAfterFailureCollectsMultipleExpandDiagnostics) { const auto first = make_cdna4_dot2c_unimplemented_expand_words(); const auto second = make_cdna4_dot2c_unimplemented_expand_words(); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {first[0], first[1], second[0], second[1]}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslatorOptions options; options.debug_continue_after_failure = true; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3, 0, options); auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image) << "continued-failure diagnostics must not emit partially translated code"; std::vector expand_offsets; for (const auto &diagnostic : result.diagnostics) { if (diagnostic.kind == rocjitsu::DiagnosticKind::ExpandMissing && diagnostic.guest_offset.has_value()) expand_offsets.push_back(*diagnostic.guest_offset); } EXPECT_EQ(expand_offsets, (std::vector{0, 8})); } TEST(BinaryTranslatorE2E, Gfx1250CopiesUnaffectedInstructionsForB0ToA0) { constexpr uint32_t kGfx1250SNop = 0xBF800000u; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({kGfx1250SNop, kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); EXPECT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); EXPECT_EQ(result.host_arch, ROCJITSU_CODE_ARCH_GFX1250); } TEST(BinaryTranslatorE2E, Gfx1250ReplacesSClauseWithNopForB0ToA0) { constexpr auto source_clause = gfx1250::build_sopp(gfx1250::kSClauseSopp, {.simm16 = 4}); constexpr auto source_end = gfx1250::build_sopp(gfx1250::kSEndpgmSopp, {.simm16 = 0}); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_clause[0], source_end[0]}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[0], gfx1250::build_sopp(gfx1250::kSNopSopp, {.simm16 = 0})[0]); EXPECT_EQ(target_words[1], source_end[0]); } TEST(BinaryTranslatorE2E, Gfx1250PreservesSClauseOutsideB0ToA0) { constexpr auto source_clause = gfx1250::build_sopp(gfx1250::kSClauseSopp, {.simm16 = 4}); constexpr auto source_end = gfx1250::build_sopp(gfx1250::kSEndpgmSopp, {.simm16 = 0}); for (const rocjitsu::ProcessorRevision revision : { rocjitsu::ProcessorRevision::Gfx1250A0, rocjitsu::ProcessorRevision::Gfx1250B0, }) { auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_clause[0], source_end[0]}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(revision, revision)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[0], source_clause[0]); EXPECT_EQ(target_words[1], source_end[0]); } } TEST(BinaryTranslatorE2E, Gfx1250RelocatesDirectCallForB0ToA0) { constexpr uint16_t kReturnSreg = 30; const std::vector words = { rocjitsu::build_s_branch(2, ROCJITSU_CODE_ARCH_GFX1250), rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_GFX1250), rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_GFX1250), rocjitsu::build_s_call_b64(kReturnSreg, 2, ROCJITSU_CODE_ARCH_GFX1250), rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_GFX1250), rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_GFX1250), rocjitsu::build_s_setpc_b64(kReturnSreg, ROCJITSU_CODE_ARCH_GFX1250), }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[0], rocjitsu::build_s_branch(0, ROCJITSU_CODE_ARCH_GFX1250)); EXPECT_EQ(target_words[1], rocjitsu::build_s_call_b64(kReturnSreg, 1, ROCJITSU_CODE_ARCH_GFX1250)); EXPECT_EQ(target_words[2], rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_GFX1250)); EXPECT_EQ(target_words[3], rocjitsu::build_s_setpc_b64(kReturnSreg, ROCJITSU_CODE_ARCH_GFX1250)); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); ASSERT_GE(decoded.size(), 4u); EXPECT_EQ(decoded[1]->mnemonic(), "s_call_i64"); ASSERT_TRUE(decoded[1]->branch_offset_bytes().has_value()); EXPECT_EQ(*decoded[1]->branch_offset_bytes(), 4); } TEST(BinaryTranslatorE2E, Gfx1250DirectTailTransferCompactionIsIdempotent) { constexpr uint16_t kReturnSreg = 30; const std::vector words = { rocjitsu::build_s_call_b64(kReturnSreg, 1, ROCJITSU_CODE_ARCH_GFX1250), rocjitsu::build_s_nop(0, ROCJITSU_CODE_ARCH_GFX1250), rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_GFX1250), }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[0], rocjitsu::build_s_call_b64(kReturnSreg, 0, ROCJITSU_CODE_ARCH_GFX1250)); EXPECT_EQ(target_words[1], rocjitsu::build_s_endpgm(ROCJITSU_CODE_ARCH_GFX1250)); rocjitsu::BinaryTranslator verifier( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto second = verifier.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250KernargPreloadUsesSingleDescriptorEntry) { constexpr uint32_t kGfx1250SNop = 0xBF800000u; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({kGfx1250SNop, kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source_layout(image.data(), image.size()); ASSERT_TRUE(source_layout.is_valid()); const auto *source_rodata = rocjitsu::find_section(source_layout, ".rodata"); ASSERT_NE(source_rodata, nullptr); auto source_kd = rocjitsu::read_kernel_descriptor_for_test(image.data() + source_rodata->sectionOffset()); AMDHSA_BITS_SET(source_kd.kernarg_preload, rocr::llvm::amdhsa::KERNARG_PRELOAD_SPEC_LENGTH, 1); AMDHSA_BITS_SET(source_kd.kernarg_preload, rocr::llvm::amdhsa::KERNARG_PRELOAD_SPEC_OFFSET, 2); rocjitsu::write_kernel_descriptor_for_test(image.data() + source_rodata->sectionOffset(), source_kd); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto *target_rodata = rocjitsu::find_section(translated, ".rodata"); ASSERT_NE(target_rodata, nullptr); const auto target_kd = rocjitsu::read_kernel_descriptor_for_test(translated.image_data() + target_rodata->sectionOffset()); EXPECT_EQ(target_kd.kernarg_preload, source_kd.kernarg_preload) << "gfx1250 preload fields remain active while DBT uses the descriptor's single entry"; } TEST(BinaryTranslatorE2E, Gfx1250CopiesUnaffectedLiteralOperandsForB0ToA0) { constexpr uint32_t kVMovB32Literal = 0x7E0202FFu; constexpr uint32_t kLiteral = 0x11223344u; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {kVMovB32Literal, kLiteral, kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[0], kVMovB32Literal); EXPECT_EQ(target_words[1], kLiteral); EXPECT_EQ(target_words[2], kGfx1250SEndpgm); } TEST(BinaryTranslatorE2E, Gfx1250CopiesFp8ConversionsWhenClampIsClear) { // CLAMP selects the E5M3 encoding these rules exist for. With it clear the // instruction already runs on A0, so both rules must decline rather than // rewrite it. constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; for (const uint16_t opcode : {gfx1250::kVCvtPkFp8F32Vop3, gfx1250::kVCvtSrFp8F32Vop3}) { const auto source_cvt = gfx1250::build_vop3(opcode, {.vdst = 30, .clamp = 0, .src0 = 256 + 22, .src1 = 256 + 2}); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_cvt[0], source_cvt[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[0], source_cvt[0]); EXPECT_EQ(target_words[1], source_cvt[1]); EXPECT_EQ(target_words[2], kGfx1250SEndpgm); } } TEST(BinaryTranslatorE2E, Gfx1250EmulatesCvtPkFp8F32ClampSetForA0) { constexpr auto source_cvt = gfx1250::build_vop3( gfx1250::kVCvtPkFp8F32Vop3, {.vdst = 30, .clamp = 1, .src0 = 256 + 22, .src1 = 256 + 2}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_cvt[0], source_cvt[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &item) { return item->mnemonic() == "v_cvt_pk_fp8_f32"; }), 0); EXPECT_GE(std::ranges::count_if( decoded, [](const auto &item) { return item->mnemonic() == "v_cndmask_b32"; }), 6); } TEST(BinaryTranslatorE2E, Gfx1250EmulatesLiteralCvtPkFp8F32ClampSetForA0) { constexpr auto source_cvt = gfx1250::build_vop3( gfx1250::kVCvtPkFp8F32Vop3, {.vdst = 30, .clamp = 1, .src0 = 255, .src1 = 256 + 2}); constexpr uint32_t kLiteral = 0x3F800000u; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_cvt[0], source_cvt[1], kLiteral, kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &item) { return item->mnemonic() == "v_cvt_pk_fp8_f32"; }), 0); } TEST(BinaryTranslatorE2E, Gfx1250MaterializedE5m3LiteralIgnoresTheGuestSourceBank) { // Src0 is a literal and its guest bank is 1; src1, src2 and the destination // stay in bank 0. The literal is materialized into a low-bank scratch VGPR, // so no helper may run under a src-bank-1 mode (0x04) -- doing so would read // a different physical register than the one the v_mov just wrote. constexpr auto set_vgpr_msb = gfx1250::build_sopp(gfx1250::kSSetVgprMsbSopp, {.simm16 = 0x01}); constexpr auto source_cvt = gfx1250::build_vop3( gfx1250::kVCvtPkFp8F32Vop3, {.vdst = 30, .clamp = 1, .src0 = 255, .src1 = 256 + 2}); constexpr uint32_t kLiteral = 0x3F800000u; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {set_vgpr_msb[0], source_cvt[0], source_cvt[1], kLiteral, kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "v_cvt_pk_fp8_f32"; }), 0); std::vector modes; for (const auto &inst : decoded) { if (inst->mnemonic() == "s_set_vgpr_msb") { ASSERT_NE(inst->raw_encoding(), nullptr); modes.push_back(static_cast(inst->raw_encoding()[0] & 0xffu)); } } ASSERT_FALSE(modes.empty()); EXPECT_EQ(std::ranges::find(modes, 0x04u), modes.end()) << "materialized literal must be read from the low bank"; EXPECT_EQ(modes.back(), 0x01u) << "expansion must restore the incoming VGPR-MSB mode"; } TEST(BinaryTranslatorE2E, Gfx1250EmulatesCvtSrFp8F32ClampSetForA0) { constexpr auto source_cvt = gfx1250::build_vop3(gfx1250::kVCvtSrFp8F32Vop3, {.vdst = 30, .opsel = 12, .clamp = 1, .src0 = 256 + 22, .src1 = 256 + 2}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_cvt[0], source_cvt[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &item) { return item->mnemonic() == "v_cvt_sr_fp8_f32"; }), 0); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &item) { return item->mnemonic() == "v_cvt_f16_f32"; }), 0); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &item) { return item->mnemonic() == "s_getreg_b32"; }), 1); } /// @brief What executing one translated E5M3 replacement left behind. struct Gfx1250E5m3Execution { uint32_t vdst = 0; ///< Destination VGPR of the operand's own bank. uint8_t final_msb_mode = 0; ///< VGPR-MSB mode the replacement left in effect. }; /// @brief One source of a conversion under test, and where its value lives. /// /// @details The expansions materialize a literal into scratch and read a scalar /// through a different operand path than a VGPR, so each kind reaches different /// code. VOP3 carries at most one literal, so at most one source may be one. struct Gfx1250E5m3Operand { enum class Kind : uint8_t { Vgpr, Sgpr, Literal }; Kind kind = Kind::Vgpr; uint32_t value = 0; }; constexpr Gfx1250E5m3Operand e5m3_vgpr(uint32_t value) { return {Gfx1250E5m3Operand::Kind::Vgpr, value}; } constexpr Gfx1250E5m3Operand e5m3_sgpr(uint32_t value) { return {Gfx1250E5m3Operand::Kind::Sgpr, value}; } constexpr Gfx1250E5m3Operand e5m3_literal(uint32_t value) { return {Gfx1250E5m3Operand::Kind::Literal, value}; } /// @brief Run one translated gfx1250 E5M3 replacement on the emulated CU. /// /// @details The B0 conversion is translated on its own and every instruction of /// the A0 replacement then executes in order on a single-lane wavefront. That /// makes the emitted sequence comparable against the execution model's own /// conversion helpers, which mnemonic-counting tests cannot check. /// /// @param vgpr_msb_mode Incoming S_SET_VGPR_MSB layout. Each operand is read and /// written through its own role's bank, so a lowering that forwards the /// wrong role's bank addresses a register the test never wrote. std::optional run_gfx1250_e5m3_replacement(uint16_t opcode, uint8_t opsel, Gfx1250E5m3Operand src0, Gfx1250E5m3Operand src1, uint32_t vdst_initial, bool fp16_ovfl, uint8_t vgpr_msb_mode = 0, uint16_t live_sgprs = 0) { constexpr uint16_t kSrc0Vgpr = 22; constexpr uint16_t kSrc1Vgpr = 2; constexpr uint16_t kDstVgpr = 30; constexpr uint16_t kSrc0Sgpr = 8; constexpr uint16_t kSrc1Sgpr = 9; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; constexpr uint16_t kM0Operand = 125; using Kind = Gfx1250E5m3Operand::Kind; if (src0.kind == Kind::Literal && src1.kind == Kind::Literal) { ADD_FAILURE() << "VOP3 carries at most one literal"; return std::nullopt; } const uint32_t src0_bank = vgpr_msb_mode & 0x3u; const uint32_t src1_bank = (vgpr_msb_mode >> 2) & 0x3u; const uint32_t dst_bank = (vgpr_msb_mode >> 6) & 0x3u; const auto selector = [](Gfx1250E5m3Operand operand, uint16_t vgpr, uint16_t sgpr) -> uint16_t { switch (operand.kind) { case Kind::Vgpr: return static_cast(256 + vgpr); case Kind::Sgpr: return sgpr; case Kind::Literal: break; } return 255; }; std::vector source_words; if (vgpr_msb_mode != 0) { source_words.push_back( gfx1250::build_sopp(gfx1250::kSSetVgprMsbSopp, {.simm16 = vgpr_msb_mode})[0]); } const auto conversion = gfx1250::build_vop3(opcode, {.vdst = kDstVgpr, .opsel = opsel, .clamp = 1, .src0 = selector(src0, kSrc0Vgpr, kSrc0Sgpr), .src1 = selector(src1, kSrc1Vgpr, kSrc1Sgpr)}); source_words.push_back(conversion[0]); source_words.push_back(conversion[1]); if (src0.kind == Kind::Literal) source_words.push_back(src0.value); else if (src1.kind == Kind::Literal) source_words.push_back(src1.value); // Readers that keep every ordinary SGPR live, so the expansion has to borrow // VGPR carriers and guard them instead of taking scratch SGPRs outright. for (uint16_t sgpr = 0; sgpr < live_sgprs; ++sgpr) { source_words.push_back(gfx1250::build_sop1( gfx1250::kSMovB32Sop1, {.ssrc0 = static_cast(sgpr), .sdst = kM0Operand})[0]); } source_words.push_back(kGfx1250SEndpgm); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(source_words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); if (!result.ok()) { ADD_FAILURE() << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); return std::nullopt; } rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); if (translated.text_sections().empty()) { ADD_FAILURE() << "translated code object has no .text"; return std::nullopt; } const auto *text = reinterpret_cast(translated.text_sections()[0]->data()); const size_t word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); rocjitsu::amdgpu::GpuMemory gpu_mem("gfx1250_e5m3_mem"); rocjitsu::amdgpu::L2Cache l2("gfx1250_e5m3_l2"); rocjitsu::amdgpu::ComputeUnitCore::Config cfg{}; cfg.arch = ROCJITSU_CODE_ARCH_GFX1250; cfg.num_wf_slots = 1; cfg.sgprs_per_wf = 106; // Every bank must be addressable: a role's two-bit selector resolves an // operand to reg + 256 * selector, so the file has to span all four. cfg.vgprs_per_wf = 1024; cfg.lds_size_kb = 64; auto cu = rocjitsu::amdgpu::ComputeUnitCore::create("gfx1250_e5m3", cfg, &gpu_mem, &l2); if (cu == nullptr) { ADD_FAILURE() << "could not create a gfx1250 compute unit"; return std::nullopt; } auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf); if (wf == nullptr) { ADD_FAILURE() << "could not dispatch a wavefront"; return std::nullopt; } wf->set_exec(0x1u); wf->set_mode_raw(fp16_ovfl ? rocjitsu::amdgpu::Wavefront::FP16_OVFL_BIT : 0u); const uint32_t vb = wf->vgpr_alloc().base; const uint32_t sb = wf->sgpr_alloc().base; const auto place = [&](Gfx1250E5m3Operand operand, uint32_t bank, uint16_t vgpr, uint16_t sgpr) { if (operand.kind == Kind::Vgpr) cu->write_vgpr(vb + (bank << 8) + vgpr, 0, operand.value); else if (operand.kind == Kind::Sgpr) cu->write_sgpr(sb + sgpr, operand.value); }; place(src0, src0_bank, kSrc0Vgpr, kSrc0Sgpr); place(src1, src1_bank, kSrc1Vgpr, kSrc1Sgpr); cu->write_vgpr(vb + (dst_bank << 8) + kDstVgpr, 0, vdst_initial); auto decoder = rocjitsu::Decoder::create(ROCJITSU_CODE_ARCH_GFX1250); if (!decoder) { ADD_FAILURE() << "no gfx1250 decoder"; return std::nullopt; } const std::string_view source_mnemonic = opcode == gfx1250::kVCvtPkFp8F32Vop3 ? "v_cvt_pk_fp8_f32" : "v_cvt_sr_fp8_f32"; bool borrowed_a_carrier = false; for (size_t index = 0; index < word_count;) { std::unique_ptr inst(decoder->decode(text + index)); if (inst == nullptr) { ADD_FAILURE() << "undecodable replacement word at " << index; return std::nullopt; } if (std::string_view(inst->mnemonic()) == "s_endpgm") break; // Executing the untranslated conversion would compare the model against // itself and pass no matter what the expansion emits. if (std::string_view(inst->mnemonic()) == source_mnemonic) { ADD_FAILURE() << "the source conversion survived translation"; return std::nullopt; } borrowed_a_carrier = borrowed_a_carrier || std::string_view(inst->mnemonic()) == "v_readfirstlane_b32_e32"; cu->execute_instruction(inst.get(), *wf); index += static_cast(inst->size()) / sizeof(uint32_t); } // A caller that walled off the SGPRs asked for the carrier path. Without this // the shape could quietly degenerate into the ordinary one and still compare // equal, testing nothing it was written for. if (live_sgprs != 0 && !borrowed_a_carrier) { ADD_FAILURE() << "the SGPR-pressure shape did not borrow a VGPR carrier"; return std::nullopt; } return Gfx1250E5m3Execution{cu->read_vgpr(vb + (dst_bank << 8) + kDstVgpr, 0), wf->vgpr_msb_mode()}; } TEST(BinaryTranslatorE2E, Gfx1250E5m3PackReplacementMatchesReferenceConversion) { // Deliberately spans the encoding's discontinuities: signed zeros, the tiny // and subnormal quanta, the normal boundary, a rounding tie, max finite, the // rounded and exponent overflow cases, infinity and NaN. static constexpr std::array kValues = { 0x00000000u, 0x80000000u, 0x00000001u, 0x36800000u, 0x37000000u, 0x37800000u, 0x38800000u, 0x3F800000u, 0x3FC00000u, 0x3F900000u, 0x47C00000u, 0x47D00000u, 0x47E00000u, 0x47E80000u, 0x47F00000u, 0x47F80000u, 0x7F000000u, 0x7F800000u, 0x7FC00000u}; constexpr uint32_t kDstInitial = 0xa5a5a5a5u; for (const bool fp16_ovfl : {false, true}) { for (const bool write_high : {false, true}) { for (size_t index = 0; index < kValues.size(); ++index) { const uint32_t src0 = kValues[index]; const uint32_t src1 = kValues[(index + 5) % kValues.size()]; const auto produced = run_gfx1250_e5m3_replacement( gfx1250::kVCvtPkFp8F32Vop3, static_cast(write_high ? 8 : 0), e5m3_vgpr(src0), e5m3_vgpr(src1), kDstInitial, fp16_ovfl); ASSERT_TRUE(produced.has_value()); const uint32_t packed = util::f32_to_fp8_e5m3_rne_mode(std::bit_cast(src0), fp16_ovfl) | (static_cast( util::f32_to_fp8_e5m3_rne_mode(std::bit_cast(src1), fp16_ovfl)) << 8); const uint32_t expected = write_high ? ((kDstInitial & 0x0000ffffu) | (packed << 16)) : ((kDstInitial & 0xffff0000u) | packed); EXPECT_EQ(produced->vdst, expected) << "src0=" << std::hex << src0 << " src1=" << src1 << " fp16_ovfl=" << std::dec << fp16_ovfl << " write_high=" << write_high; } } } } TEST(BinaryTranslatorE2E, Gfx1250E5m3StochasticReplacementMatchesReferenceConversion) { static constexpr std::array kValues = { 0x00000000u, 0x80000000u, 0x00000001u, 0x36800000u, 0x37000000u, 0x38800000u, 0x3F800000u, 0x3FC00000u, 0x47D00000u, 0x47E00000u, 0x47F00000u, 0x47F80000u, 0x7F800000u, 0x7FC00000u}; // Both seed extremes plus one interior value: stochastic rounding must round // down at seed 0 and up at UINT32_MAX for every representable magnitude. static constexpr std::array kSeeds = {0u, 0x0f0f0f0fu, 0xffffffffu}; constexpr uint32_t kDstInitial = 0xa5a5a5a5u; for (const bool fp16_ovfl : {false, true}) { for (uint8_t dst_byte = 0; dst_byte < 4; ++dst_byte) { for (const uint32_t seed : kSeeds) { for (const uint32_t value : kValues) { const auto produced = run_gfx1250_e5m3_replacement( gfx1250::kVCvtSrFp8F32Vop3, static_cast(dst_byte << 2), e5m3_vgpr(value), e5m3_vgpr(seed), kDstInitial, fp16_ovfl); ASSERT_TRUE(produced.has_value()); const uint32_t byte = util::f32_to_fp8_e5m3_sr_mode(std::bit_cast(value), seed, fp16_ovfl); const uint32_t shift = static_cast(dst_byte) * 8u; const uint32_t expected = (kDstInitial & ~(0xffu << shift)) | (byte << shift); EXPECT_EQ(produced->vdst, expected) << "value=" << std::hex << value << " seed=" << seed << std::dec << " fp16_ovfl=" << fp16_ovfl << " dst_byte=" << static_cast(dst_byte); } } } } } TEST(BinaryTranslatorE2E, Gfx1250E5m3ReplacementsHonorPerRoleVgprBanks) { // Every mode gives src0, src1 and the destination three different banks, so a // lowering that forwards one role's selector in place of another's resolves to // a register nothing wrote rather than one that happens to hold the answer. // Rotating through them puts each role on each of selectors 1, 2 and 3, which // is what catches a selector whose high bit is dropped. static constexpr std::array modes = { 0xC9u, // src0 bank 1, src1 bank 2, destination bank 3. 0x87u, // src0 bank 3, src1 bank 1, destination bank 2. 0x4Eu, // src0 bank 2, src1 bank 3, destination bank 1. }; static constexpr std::array kValues = {0x00000000u, 0x37000000u, 0x3F800000u, 0x47E00000u, 0x47F80000u, 0x7FC00000u}; constexpr uint32_t kDstInitial = 0x5a5a5a5au; for (const uint8_t mode : modes) { for (const bool fp16_ovfl : {false, true}) { for (size_t index = 0; index < kValues.size(); ++index) { const uint32_t src0 = kValues[index]; const uint32_t src1 = kValues[(index + 2) % kValues.size()]; const auto packed_run = run_gfx1250_e5m3_replacement(gfx1250::kVCvtPkFp8F32Vop3, 0, e5m3_vgpr(src0), e5m3_vgpr(src1), kDstInitial, fp16_ovfl, mode); ASSERT_TRUE(packed_run.has_value()); const uint32_t packed = util::f32_to_fp8_e5m3_rne_mode(std::bit_cast(src0), fp16_ovfl) | (static_cast( util::f32_to_fp8_e5m3_rne_mode(std::bit_cast(src1), fp16_ovfl)) << 8); EXPECT_EQ(packed_run->vdst, (kDstInitial & 0xffff0000u) | packed) << "src0=" << std::hex << src0 << " src1=" << src1 << std::dec << " fp16_ovfl=" << fp16_ovfl << " mode=" << static_cast(mode); EXPECT_EQ(packed_run->final_msb_mode, mode) << "the incoming mode must be restored"; const auto stochastic_run = run_gfx1250_e5m3_replacement(gfx1250::kVCvtSrFp8F32Vop3, 2u << 2, e5m3_vgpr(src0), e5m3_vgpr(src1), kDstInitial, fp16_ovfl, mode); ASSERT_TRUE(stochastic_run.has_value()); const uint32_t byte = util::f32_to_fp8_e5m3_sr_mode(std::bit_cast(src0), src1, fp16_ovfl); EXPECT_EQ(stochastic_run->vdst, (kDstInitial & ~(0xffu << 16)) | (byte << 16)) << "value=" << std::hex << src0 << " seed=" << src1 << std::dec << " fp16_ovfl=" << fp16_ovfl << " mode=" << static_cast(mode); EXPECT_EQ(stochastic_run->final_msb_mode, mode) << "the incoming mode must be restored"; } } } } TEST(BinaryTranslatorE2E, Gfx1250E5m3ReplacementsMatchForNonVgprSources) { // Each source kind reaches different code: a literal is materialized into // scratch first, a scalar is read through the operand's scalar path, and the // SGPR-pressure shape forces VGPR carriers behind an execz guard. All three // must still produce what the reference conversion does. static constexpr std::array kValues = {0x00000000u, 0x37000000u, 0x3F800000u, 0x47E00000u, 0x7FC00000u}; constexpr uint32_t kDstInitial = 0xa5a5a5a5u; constexpr uint32_t kSeed = 0x0f0f0f0fu; const auto expect_packed = [&](const std::optional &run, uint32_t src0, uint32_t src1, bool fp16_ovfl, const char *shape) { ASSERT_TRUE(run.has_value()) << shape; const uint32_t packed = util::f32_to_fp8_e5m3_rne_mode(std::bit_cast(src0), fp16_ovfl) | (static_cast(util::f32_to_fp8_e5m3_rne_mode( std::bit_cast(src1), fp16_ovfl)) << 8); EXPECT_EQ(run->vdst, (kDstInitial & 0xffff0000u) | packed) << shape << " src0=" << std::hex << src0 << " src1=" << src1 << std::dec << " fp16_ovfl=" << fp16_ovfl; }; for (const bool fp16_ovfl : {false, true}) { for (size_t index = 0; index < kValues.size(); ++index) { const uint32_t src0 = kValues[index]; const uint32_t src1 = kValues[(index + 3) % kValues.size()]; expect_packed(run_gfx1250_e5m3_replacement(gfx1250::kVCvtPkFp8F32Vop3, 0, e5m3_literal(src0), e5m3_vgpr(src1), kDstInitial, fp16_ovfl), src0, src1, fp16_ovfl, "literal src0"); expect_packed(run_gfx1250_e5m3_replacement(gfx1250::kVCvtPkFp8F32Vop3, 0, e5m3_vgpr(src0), e5m3_literal(src1), kDstInitial, fp16_ovfl), src0, src1, fp16_ovfl, "literal src1"); expect_packed(run_gfx1250_e5m3_replacement(gfx1250::kVCvtPkFp8F32Vop3, 0, e5m3_sgpr(src0), e5m3_vgpr(src1), kDstInitial, fp16_ovfl), src0, src1, fp16_ovfl, "scalar src0"); expect_packed(run_gfx1250_e5m3_replacement(gfx1250::kVCvtPkFp8F32Vop3, 0, e5m3_vgpr(src0), e5m3_sgpr(src1), kDstInitial, fp16_ovfl), src0, src1, fp16_ovfl, "scalar src1"); expect_packed(run_gfx1250_e5m3_replacement(gfx1250::kVCvtPkFp8F32Vop3, 0, e5m3_vgpr(src0), e5m3_vgpr(src1), kDstInitial, fp16_ovfl, 0, rocjitsu::REGISTER_SET_MAX_SGPRS), src0, src1, fp16_ovfl, "sgpr pressure"); // The stochastic lowering reads the seed through the same operand paths. for (const auto &[seed, shape] : std::initializer_list>{ {e5m3_literal(kSeed), "literal seed"}, {e5m3_sgpr(kSeed), "scalar seed"}, {e5m3_vgpr(kSeed), "sgpr pressure seed"}}) { const uint16_t live_sgprs = std::string_view(shape) == "sgpr pressure seed" ? rocjitsu::REGISTER_SET_MAX_SGPRS : 0; const auto run = run_gfx1250_e5m3_replacement(gfx1250::kVCvtSrFp8F32Vop3, 0, e5m3_vgpr(src0), seed, kDstInitial, fp16_ovfl, 0, live_sgprs); ASSERT_TRUE(run.has_value()) << shape; const uint32_t byte = util::f32_to_fp8_e5m3_sr_mode(std::bit_cast(src0), kSeed, fp16_ovfl); EXPECT_EQ(run->vdst, (kDstInitial & 0xffffff00u) | byte) << shape << " value=" << std::hex << src0 << std::dec << " fp16_ovfl=" << fp16_ovfl; } } } } TEST(BinaryTranslatorE2E, Gfx1250E5m3PackRejectsDpp) { constexpr uint32_t kEndpgm = 0xBFB00000u; for (const uint16_t opcode : {gfx1250::kVCvtPkFp8F32Vop3, gfx1250::kVCvtSrFp8F32Vop3}) { gfx1250::Vop3VopDpp16MachineInst dpp{}; dpp.vdst = 30; dpp.clamp = 1; dpp.op = opcode; dpp.encoding = 0x35; dpp.src0 = 250; dpp.src1 = 256 + 2; dpp.vsrc0 = 22; dpp.fi = 1; dpp.bank_mask = 0xf; dpp.row_mask = 0xf; std::array words{}; std::memcpy(words.data(), &dpp, sizeof(dpp)); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {words[0], words[1], words[2], kEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_FALSE(result.ok()); ASSERT_FALSE(result.diagnostics.empty()); EXPECT_NE(result.diagnostics.front().message.find("does not support DPP"), std::string::npos); } } TEST(BinaryTranslatorE2E, Gfx1250E5m3PackRejectsLiteral64) { constexpr uint32_t kEndpgm = 0xBFB00000u; constexpr uint32_t kLiteralLow = 0x3F800000u; // The gfx1250 VOP3 decoder reports only the first literal word, so the second // one is decoded on its own. Keep it an s_nop rather than undecodable bytes, // which would fail the kernel before the expansion rule is consulted. constexpr uint32_t kLiteralHigh = 0xBF800000u; for (const uint16_t opcode : {gfx1250::kVCvtPkFp8F32Vop3, gfx1250::kVCvtSrFp8F32Vop3}) { for (const bool literal_in_src0 : {true, false}) { const auto conversion = gfx1250::build_vop3( opcode, {.vdst = 30, .clamp = 1, .src0 = static_cast(literal_in_src0 ? 254 : 256 + 22), .src1 = static_cast(literal_in_src0 ? 256 + 2 : 254)}); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {conversion[0], conversion[1], kLiteralLow, kLiteralHigh, kEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_FALSE(result.ok()); ASSERT_FALSE(result.diagnostics.empty()); EXPECT_NE(result.diagnostics.front().message.find("does not support SRC_LITERAL64"), std::string::npos) << result.diagnostics.front().message; } } } TEST(BinaryTranslatorE2E, Gfx1250E5m3PackUsesCarriersUnderSgprPressure) { for (const uint16_t opcode : {gfx1250::kVCvtPkFp8F32Vop3, gfx1250::kVCvtSrFp8F32Vop3}) { const auto conversion = gfx1250::build_vop3(opcode, {.vdst = 30, .clamp = 1, .src0 = 256 + 22, .src1 = 256 + 23}); auto image = rocjitsu::make_gfx1250_image_with_live_sgprs(conversion, rocjitsu::REGISTER_SET_MAX_SGPRS); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "s_cbranch_execz"; }), 1); EXPECT_GE(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "v_readfirstlane_b32_e32"; }), 4); } } TEST(BinaryTranslatorE2E, Gfx1250CopiesLiteralCvtF32Fp8E32WithoutClampEmulation) { constexpr auto conversion = gfx1250::build_vop1(gfx1250::kVCvtF32Fp8Vop1, {.src0 = 255, .vdst = 30}); constexpr uint32_t kLiteral = 0x3F800000u; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {conversion[0], kLiteral, kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[0], conversion[0]); EXPECT_EQ(target_words[1], kLiteral); EXPECT_EQ(target_words[2], kGfx1250SEndpgm); } TEST(BinaryTranslatorE2E, Gfx1250EmulatesCvtF32Fp8E5m3ForA0) { constexpr auto conversion = gfx1250::build_vop3( gfx1250::kVCvtF32Fp8Vop3, {.vdst = 30, .opsel = 2, .clamp = 1, .src0 = 256 + 22}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {conversion[0], conversion[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "v_cvt_f32_fp8"; }), 0); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "v_cndmask_b32"; }), 2); // Assert the value-defining literal constants of the emitted sequence, not just // mnemonic counts: an off-by-one in the NaN threshold, exponent clamp, or the // F16/F32 reconstruction constants would otherwise pass. These constants are // distinctive, so scan the raw translated text words for each. (Reading them from // the raw text is robust; a decoded VOP3's raw_encoding() only spans its 2-dword // base and does not expose the trailing literal.) const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); const auto contains_word = [&](uint32_t value) { return std::any_of(target_words, target_words + target_word_count, [&](uint32_t w) { return w == value; }); }; EXPECT_TRUE(contains_word(0xffu)) << "missing E5M3 NaN threshold literal 0xff"; EXPECT_TRUE(contains_word(0xf7u)) << "missing E5M3 exponent-clamp literal 0xf7"; EXPECT_TRUE(contains_word(0x47800000u)) << "missing F16 reconstruction constant"; EXPECT_TRUE(contains_word(0x7fa3d000u)) << "missing F32 max-finite constant"; // Pin the byte-select. The source opsel=2 must map through the VOP3 // byte-select bit swap to byte 1, so the single v_bfe_u32 that isolates the // packed byte reads bit offset 1*8=8 (inline-encoded as 128+8=136). Without // the swap it would read byte 2 (offset 16, encoded 144) and silently unpack // the wrong lane's fp8 datum. Inspect the decoded operand so a byte-select // regression cannot pass. int bfe_count = 0; for (const auto &inst : decoded) { if (inst->mnemonic() != "v_bfe_u32") continue; ++bfe_count; const rocjitsu::Operand *offset = inst->src_operand(1); ASSERT_NE(offset, nullptr); EXPECT_EQ(offset->encoding_value(), 136) << "E5M3 byte-select must be byte 1 (offset 8)"; } EXPECT_EQ(bfe_count, 1) << "E5M3 unpack must isolate the byte with exactly one v_bfe_u32"; } TEST(BinaryTranslatorE2E, Gfx1250E5m3CompareMasksTargetDeadSgprs) { constexpr auto conversion = gfx1250::build_vop3( gfx1250::kVCvtF32Fp8Vop3, {.vdst = 30, .opsel = 2, .clamp = 1, .src0 = 256 + 22}); // s103 is the first dead SGPR but cannot be borrowed safely because it backs // src_flat_scratch_base. The two independent Wave32 masks use s104:s105. constexpr uint16_t kLiveSgprs = 103; constexpr uint16_t kExpectedMaskBase = 104; auto image = rocjitsu::make_gfx1250_image_with_live_sgprs(conversion, kLiveSgprs); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); std::optional nan_mask; std::optional exp31_mask; std::vector cndmask_predicates; for (const auto &inst : decoded) { if (inst->mnemonic() == "v_cmp_eq_u32" || inst->mnemonic() == "v_cmp_lt_u32") { ASSERT_NE(inst->raw_encoding(), nullptr); gfx1250::Vop3MachineInst encoded{}; std::memcpy(&encoded, inst->raw_encoding(), sizeof(encoded)); if (inst->mnemonic() == "v_cmp_eq_u32") nan_mask = static_cast(encoded.vdst); else exp31_mask = static_cast(encoded.vdst); } else if (inst->mnemonic() == "v_cndmask_b32") { const rocjitsu::Operand *predicate = inst->src_operand(2); ASSERT_NE(predicate, nullptr); cndmask_predicates.push_back(predicate->encoding_value()); } } ASSERT_TRUE(nan_mask.has_value()); ASSERT_TRUE(exp31_mask.has_value()); ASSERT_EQ(cndmask_predicates.size(), 2u); EXPECT_EQ(*nan_mask, kExpectedMaskBase); EXPECT_EQ(*exp31_mask, kExpectedMaskBase + 1u); EXPECT_EQ(cndmask_predicates[0], *exp31_mask); EXPECT_EQ(cndmask_predicates[1], *nan_mask); } TEST(BinaryTranslatorE2E, Gfx1250E5m3UnpackUsesVgprCarriersUnderSgprPressure) { constexpr auto conversion = gfx1250::build_vop3( gfx1250::kVCvtF32Fp8Vop3, {.vdst = 30, .opsel = 2, .clamp = 1, .src0 = 256 + 22}); for (const uint16_t live_sgprs : {static_cast(rocjitsu::REGISTER_SET_MAX_SGPRS - 1u), static_cast(rocjitsu::REGISTER_SET_MAX_SGPRS)}) { SCOPED_TRACE(live_sgprs); auto image = rocjitsu::make_gfx1250_image_with_live_sgprs(conversion, live_sgprs); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "s_cbranch_execz"; }), 1); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "v_readfirstlane_b32_e32"; }), 2); const auto guard = std::ranges::find_if( decoded, [](const auto &inst) { return inst->mnemonic() == "s_cbranch_execz"; }); ASSERT_NE(guard, decoded.end()); ASSERT_TRUE((*guard)->branch_offset_bytes().has_value()); const uint64_t target = (*guard)->src_loc() + (*guard)->size() + *(*guard)->branch_offset_bytes(); const auto following = std::ranges::find_if(decoded, [&](const auto &inst) { return inst->src_loc() == target; }); ASSERT_NE(following, decoded.end()); EXPECT_EQ((*following)->mnemonic(), "s_mov_b32") << "the EXEC-zero guard must land on the first following guest instruction"; } } TEST(BinaryTranslatorE2E, Gfx1250E5m3SeparatesVectorAndScalarSpillSlots) { constexpr auto conversion = gfx1250::build_vop3( gfx1250::kVCvtF32Fp8Vop3, {.vdst = 30, .opsel = 2, .clamp = 1, .src0 = 256 + 22}); auto image = rocjitsu::make_gfx1250_image_with_live_sgprs(conversion, rocjitsu::REGISTER_SET_MAX_SGPRS); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); auto options = gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0); options.debug_min_free_vgpr = 256; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, options); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); std::vector store_offsets; std::vector load_offsets; for (const auto &inst : decoded) { if (inst->mnemonic() != "scratch_store_b32" && inst->mnemonic() != "scratch_load_b32") continue; ASSERT_NE(inst->raw_encoding(), nullptr); gfx1250::VscratchMachineInst scratch{}; std::memcpy(&scratch, inst->raw_encoding(), sizeof(scratch)); (inst->mnemonic() == "scratch_load_b32" ? load_offsets : store_offsets) .push_back(scratch.ioffset); } EXPECT_EQ(store_offsets, (std::vector{0, 4, 8, 12})); EXPECT_EQ(load_offsets, (std::vector{8, 12, 0, 4})); const auto *rodata = rocjitsu::find_section(translated, ".rodata"); ASSERT_NE(rodata, nullptr); const auto descriptor = rocjitsu::read_kernel_descriptor_for_test(rodata->data()); EXPECT_GE(descriptor.private_segment_fixed_size, 16u); const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250CopiesWmmaOutsideB0ToA0Profile) { constexpr auto source_wmma = gfx1250::build_vop3p( gfx1250::kVWmmaF3216x16x32Bf16Vop3p, {.vdst = 8, .src0 = 256, .src1 = 264, .src2 = 272}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_wmma[0], source_wmma[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[0], source_wmma[0]); EXPECT_EQ(target_words[1], source_wmma[1]); EXPECT_EQ(target_words[2], kGfx1250SEndpgm); } TEST(BinaryTranslatorE2E, Gfx1250EqualRevisionsUseIdentityProfile) { constexpr auto source_ds2 = gfx1250::build_vds( gfx1250::kDsLoad2addrB32Vds, {.offset0 = 1, .offset1 = 3, .addr = 7, .vdst = 9}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_ds2[0], source_ds2[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); for (const rocjitsu::ProcessorRevision revision : {rocjitsu::ProcessorRevision::Gfx1250A0, rocjitsu::ProcessorRevision::Gfx1250B0}) { SCOPED_TRACE(revision == rocjitsu::ProcessorRevision::Gfx1250A0 ? "A0-to-A0" : "B0-to-B0"); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(revision, revision)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[0], source_ds2[0]); EXPECT_EQ(target_words[1], source_ds2[1]); EXPECT_EQ(target_words[2], kGfx1250SEndpgm); } } TEST(BinaryTranslatorE2E, Gfx1250RejectsA0ToB0Translation) { constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250A0, rocjitsu::ProcessorRevision::Gfx1250B0)); auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(rocjitsu::has_error_containing(result, rocjitsu::DiagnosticKind::Legalization, "A0-to-B0 translation is not supported")); } TEST(BinaryTranslatorE2E, Gfx1250ExpandsEveryDs2VariantForA0) { struct Ds2Case { const char *name; uint16_t source_opcode; uint16_t target_opcode; uint8_t element_dwords; bool stride64; }; constexpr std::array cases = { Ds2Case{"store_b32", gfx1250::kDsStore2addrB32Vds, gfx1250::kDsStoreB32Vds, 1, false}, Ds2Case{"store_stride64_b32", gfx1250::kDsStore2addrStride64B32Vds, gfx1250::kDsStoreB32Vds, 1, true}, Ds2Case{"storexchg_b32", gfx1250::kDsStorexchg2addrRtnB32Vds, gfx1250::kDsStorexchgRtnB32Vds, 1, false}, Ds2Case{"storexchg_stride64_b32", gfx1250::kDsStorexchg2addrStride64RtnB32Vds, gfx1250::kDsStorexchgRtnB32Vds, 1, true}, Ds2Case{"load_b32", gfx1250::kDsLoad2addrB32Vds, gfx1250::kDsLoadB32Vds, 1, false}, Ds2Case{"load_stride64_b32", gfx1250::kDsLoad2addrStride64B32Vds, gfx1250::kDsLoadB32Vds, 1, true}, Ds2Case{"store_b64", gfx1250::kDsStore2addrB64Vds, gfx1250::kDsStoreB64Vds, 2, false}, Ds2Case{"store_stride64_b64", gfx1250::kDsStore2addrStride64B64Vds, gfx1250::kDsStoreB64Vds, 2, true}, Ds2Case{"storexchg_b64", gfx1250::kDsStorexchg2addrRtnB64Vds, gfx1250::kDsStorexchgRtnB64Vds, 2, false}, Ds2Case{"storexchg_stride64_b64", gfx1250::kDsStorexchg2addrStride64RtnB64Vds, gfx1250::kDsStorexchgRtnB64Vds, 2, true}, Ds2Case{"load_b64", gfx1250::kDsLoad2addrB64Vds, gfx1250::kDsLoadB64Vds, 2, false}, Ds2Case{"load_stride64_b64", gfx1250::kDsLoad2addrStride64B64Vds, gfx1250::kDsLoadB64Vds, 2, true}, }; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; constexpr gfx1250::VdsBuilderFields source_fields{ .offset0 = 3, .offset1 = 5, .addr = 20, .data0 = 30, .data1 = 40, .vdst = 50}; for (const Ds2Case &test_case : cases) { SCOPED_TRACE(test_case.name); const auto source_ds2 = gfx1250::build_vds(test_case.source_opcode, source_fields); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_ds2[0], source_ds2[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); ASSERT_GE(translated.text_sections()[0]->size(), 6 * sizeof(uint32_t)); const uint16_t scale = static_cast(test_case.element_dwords * sizeof(uint32_t) * (test_case.stride64 ? 64 : 1)); const bool plain_store = test_case.target_opcode == gfx1250::kDsStoreB32Vds || test_case.target_opcode == gfx1250::kDsStoreB64Vds; const auto first = gfx1250::build_vds(test_case.target_opcode, {.offset0 = static_cast((source_fields.offset0 * scale) & 0xff), .offset1 = static_cast((source_fields.offset0 * scale) >> 8), .addr = source_fields.addr, .data0 = source_fields.data0, .vdst = static_cast(plain_store ? 0 : source_fields.vdst)}); const auto second = gfx1250::build_vds(test_case.target_opcode, {.offset0 = static_cast((source_fields.offset1 * scale) & 0xff), .offset1 = static_cast((source_fields.offset1 * scale) >> 8), .addr = source_fields.addr, .data0 = source_fields.data1, .vdst = static_cast( plain_store ? 0 : source_fields.vdst + test_case.element_dwords)}); EXPECT_EQ(target_words[0], first[0]); EXPECT_EQ(target_words[1], first[1]); EXPECT_EQ(target_words[2], second[0]); EXPECT_EQ(target_words[3], second[1]); EXPECT_EQ(target_words[4], gfx1250::build_sopp(gfx1250::kSWaitDscntSopp, {.simm16 = 0})[0]); EXPECT_EQ(target_words[5], kGfx1250SEndpgm); } } TEST(BinaryTranslatorE2E, Gfx1250Ds2LoadPreservesAliasedAddressUntilBothLoadsIssue) { constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; // ADDR=v7 aliases the first destination half v7. The second load must issue // first, exactly matching the executable far-DS2 alias oracle. constexpr gfx1250::VdsBuilderFields fields{.offset0 = 1, .offset1 = 3, .addr = 7, .vdst = 7}; constexpr auto source_ds2 = gfx1250::build_vds(gfx1250::kDsLoad2addrB32Vds, fields); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_ds2[0], source_ds2[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); constexpr auto second = gfx1250::build_vds(gfx1250::kDsLoadB32Vds, {.offset0 = 12, .addr = fields.addr, .vdst = 8}); constexpr auto first = gfx1250::build_vds(gfx1250::kDsLoadB32Vds, {.offset0 = 4, .addr = fields.addr, .vdst = 7}); EXPECT_EQ(target_words[0], second[0]); EXPECT_EQ(target_words[1], second[1]); EXPECT_EQ(target_words[2], first[0]); EXPECT_EQ(target_words[3], first[1]); } TEST(BinaryTranslatorE2E, Gfx1250Ds2ExchangeCyclicOverlapFailsClosed) { constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; // First destination v10 aliases ADDR; second destination v11 aliases DATA0. // Neither issue order preserves every compound source operand. constexpr auto source_ds2 = gfx1250::build_vds( gfx1250::kDsStorexchg2addrRtnB32Vds, {.addr = 10, .data0 = 11, .data1 = 20, .vdst = 10}); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_ds2[0], source_ds2[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); EXPECT_FALSE(result.ok()); const auto diagnostic = std::ranges::find_if(result.diagnostics, [](const auto &item) { return item.kind == rocjitsu::DiagnosticKind::ExpandFailed; }); ASSERT_NE(diagnostic, result.diagnostics.end()); EXPECT_NE(diagnostic->message.find("cyclic"), std::string::npos); } TEST(BinaryTranslatorE2E, Gfx1250Ds2OversizedStride64OffsetFailsClosed) { constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; constexpr auto source_ds2 = gfx1250::build_vds(gfx1250::kDsLoad2addrStride64B64Vds, {.offset1 = 255, .addr = 4, .vdst = 8}); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_ds2[0], source_ds2[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); EXPECT_FALSE(result.ok()); const auto diagnostic = std::ranges::find_if(result.diagnostics, [](const auto &item) { return item.kind == rocjitsu::DiagnosticKind::ExpandFailed; }); ASSERT_NE(diagnostic, result.diagnostics.end()); EXPECT_NE(diagnostic->message.find("16-bit"), std::string::npos); } TEST(BinaryTranslatorE2E, Gfx1250Ds2MovesData1VgprBankToSecondData0Role) { constexpr auto set_vgpr_msb = gfx1250::build_sopp(gfx1250::kSSetVgprMsbSopp, {.simm16 = 0x10}); constexpr auto source_ds2 = gfx1250::build_vds(gfx1250::kDsStore2addrB32Vds, {.offset0 = 1, .offset1 = 2, .addr = 4, .data0 = 8, .data1 = 12}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {set_vgpr_msb[0], source_ds2[0], source_ds2[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); // The generated bank transitions carry these immediates, and each generated // s_set_vgpr_msb must be preceded by an s_wait_xcnt drain. Collect the generated // transitions dynamically so the assertion does not depend on absolute offsets. std::vector generated_modes; for (size_t index = 0; index < decoded.size(); ++index) { if (decoded[index]->mnemonic() != "s_set_vgpr_msb") continue; ASSERT_NE(decoded[index]->raw_encoding(), nullptr); const uint16_t imm = static_cast(decoded[index]->raw_encoding()[0] & 0xffffu); // The 0x0010 transition is the copied source s_set_vgpr_msb; the two generated // transitions (0x1014, 0x1410) must each be guarded by s_wait_xcnt. if (imm != 0x0010) { ASSERT_GT(index, 0u); EXPECT_EQ(decoded[index - 1]->mnemonic(), "s_wait_xcnt"); } generated_modes.push_back(imm); } EXPECT_EQ(generated_modes, (std::vector{0x0010, 0x1014, 0x1410})); } TEST(BinaryTranslatorE2E, Gfx1250Ds2AdjustsDestinationBankForSecondLoad) { constexpr auto source_ds2 = gfx1250::build_vds( gfx1250::kDsLoad2addrB64Vds, {.offset0 = 1, .offset1 = 2, .addr = 4, .vdst = 254}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_ds2[0], source_ds2[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); // Two generated bank transitions (0x0040, 0x4000), each guarded by s_wait_xcnt. std::vector generated_modes; for (size_t index = 0; index < decoded.size(); ++index) { if (decoded[index]->mnemonic() != "s_set_vgpr_msb") continue; ASSERT_NE(decoded[index]->raw_encoding(), nullptr); ASSERT_GT(index, 0u); EXPECT_EQ(decoded[index - 1]->mnemonic(), "s_wait_xcnt"); generated_modes.push_back(static_cast(decoded[index]->raw_encoding()[0] & 0xffffu)); } EXPECT_EQ(generated_modes, (std::vector{0x0040, 0x4000})); } TEST(SemanticTranslator, Gfx1250ClassifiesLivenessFreeExpandRules) { std::vector liveness_free_rules; for (const rocjitsu::TranslationRule &rule : rocjitsu::semantic_expand_rules_gfx1250_b0_to_a0()) { if (!rule.requires_liveness) { liveness_free_rules.push_back((static_cast(rule.src_encoding_id) << 16) | rule.src_opcode); } } const std::vector expected = { (static_cast(gfx1250::encoding::kSop1) << 16) | gfx1250::kSBarrierSignalIsfirstSop1, (static_cast(gfx1250::encoding::kSopp) << 16) | gfx1250::kSClauseSopp, (static_cast(gfx1250::encoding::kVop3p) << 16) | gfx1250::kVWmmaF3216x16x128F8f6f4Vop3p, (static_cast(gfx1250::encoding::kVop3p) << 16) | gfx1250::kVWmmaI3216x16x64Iu8Vop3p, (static_cast(gfx1250::encoding::kVop3p) << 16) | gfx1250::kVSwmmacI3216x16x128Iu8Vop3p, (static_cast(gfx1250::encoding::kVop3pOpHi1) << 16) | gfx1250::kVWmmaF3216x16x128Fp8Fp8Vop3p, (static_cast(gfx1250::encoding::kVop3pOpHi1) << 16) | gfx1250::kVWmmaF3216x16x128Fp8Bf8Vop3p, (static_cast(gfx1250::encoding::kVop3pOpHi1) << 16) | gfx1250::kVWmmaF3216x16x128Bf8Fp8Vop3p, (static_cast(gfx1250::encoding::kVop3pOpHi1) << 16) | gfx1250::kVWmmaF3216x16x128Bf8Bf8Vop3p, }; // The standalone 32x16 FP4 and packed-f16 K=128 rules are deliberately // absent because they query operand-bank liveness. EXPECT_EQ(liveness_free_rules, expected); rocjitsu::SemanticTranslator translator(ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0); constexpr auto tensor_mask_clear = gfx1250::build_sop2(gfx1250::kSPackHhB32B16Sop2, {.ssrc0 = 128, .ssrc1 = 0, .sdst = 0}); constexpr auto cluster_m0_clear = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = 128, .sdst = 125}); constexpr auto v_nop = gfx1250::build_vop1(gfx1250::kVNopVop1); const auto tensor_clear_inst = rocjitsu::decode_one(tensor_mask_clear[0], ROCJITSU_CODE_ARCH_GFX1250); const auto cluster_clear_inst = rocjitsu::decode_one(cluster_m0_clear[0], ROCJITSU_CODE_ARCH_GFX1250); const auto v_nop_inst = rocjitsu::decode_one(v_nop[0], ROCJITSU_CODE_ARCH_GFX1250); ASSERT_NE(tensor_clear_inst, nullptr); ASSERT_NE(cluster_clear_inst, nullptr); ASSERT_NE(v_nop_inst, nullptr); EXPECT_FALSE(translator.has_expand_rule(*tensor_clear_inst)); EXPECT_FALSE(translator.has_expand_rule(*cluster_clear_inst)); EXPECT_FALSE(translator.has_expand_rule(*v_nop_inst)); } TEST(BinaryTranslatorE2E, Gfx1250UsesNeutralScaledK128Fp8Bf8Wmma) { struct WmmaCase { const char *name; uint16_t source_opcode; uint8_t matrix_a_fmt; uint8_t matrix_b_fmt; }; constexpr std::array cases = { WmmaCase{"fp8_fp8", gfx1250::kVWmmaF3216x16x128Fp8Fp8Vop3p, 0, 0}, WmmaCase{"fp8_bf8", gfx1250::kVWmmaF3216x16x128Fp8Bf8Vop3p, 0, 1}, WmmaCase{"bf8_fp8", gfx1250::kVWmmaF3216x16x128Bf8Fp8Vop3p, 1, 0}, WmmaCase{"bf8_bf8", gfx1250::kVWmmaF3216x16x128Bf8Bf8Vop3p, 1, 1}, }; constexpr gfx1250::Vop3pBuilderFields fields{ .vdst = 54, .neg_hi = 7, .opsel = 7, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 256 + 48, .opsel_hi = 3, .neg = 7, }; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; constexpr auto completion_wait = kGfx1250WmmaCompletionWait; EXPECT_EQ(rocjitsu::semantic_expand_rules_gfx1250_b0_to_a0().size(), 41u); for (const WmmaCase &test_case : cases) { SCOPED_TRACE(test_case.name); auto source_wmma = gfx1250::build_vop3p(test_case.source_opcode, fields); source_wmma[0] |= uint32_t{1} << 14; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_wmma[0], source_wmma[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); ASSERT_GE(translated.text_sections()[0]->size(), 6 * sizeof(uint32_t)); const auto *words = reinterpret_cast(translated.text_sections()[0]->data()); gfx1250::Vop3pMachineInst prefix{}; gfx1250::Vop3pMachineInst matrix{}; std::memcpy(&prefix, words, sizeof(prefix)); std::memcpy(&matrix, words + 2, sizeof(matrix)); EXPECT_EQ(prefix.op, 0x35u); EXPECT_EQ(prefix.src0, 128u); EXPECT_EQ(prefix.src1, 128u); EXPECT_EQ(prefix.src2, 256u); EXPECT_EQ(prefix.opsel, 0u) << "clear source matrix-A reuse in the new encoding"; EXPECT_EQ(prefix.pad_14, 0u) << "clear source matrix-B reuse in the new encoding"; EXPECT_EQ(prefix.opsel_hi, 0u); EXPECT_EQ(matrix.op, gfx1250::kVWmmaF3216x16x128F8f6f4Vop3p); EXPECT_EQ(matrix.vdst, fields.vdst); EXPECT_EQ(matrix.src0, fields.src0); EXPECT_EQ(matrix.src1, fields.src1); EXPECT_EQ(matrix.src2, fields.src2); EXPECT_EQ(matrix.opsel, test_case.matrix_a_fmt); EXPECT_EQ((matrix.pad_14 << 2) | matrix.opsel_hi, test_case.matrix_b_fmt); EXPECT_EQ(matrix.neg_hi, 4u) << "preserve only C absolute"; EXPECT_EQ(matrix.neg, 4u) << "preserve only C negate"; EXPECT_EQ(words[4], completion_wait[0]); EXPECT_EQ(words[5], kGfx1250SEndpgm); } } TEST(BinaryTranslatorE2E, Gfx1250K128WmmaTranslatesCapturedEncodingWithoutK64Fallback) { // Captured compiler encoding. OPSEL_HI is three, but these source fields do // not select matrix formats for this opcode and must not gate translation. constexpr std::array source_wmma = {0xCC800088u, 0x1A02E542u}; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_wmma[0], source_wmma[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); EXPECT_EQ(translated.text_sections()[0]->size(), 6 * sizeof(uint32_t)); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ(std::ranges::count_if(decoded, [](const auto &candidate) { return candidate->mnemonic() == "v_wmma_scale_f32_16x16x128_f8f6f4"; }), 1); EXPECT_EQ(std::ranges::count_if(decoded, [](const auto &candidate) { return candidate->mnemonic().find("16x16x64_fp8") != std::string_view::npos || candidate->mnemonic().find("16x16x64_bf8") != std::string_view::npos; }), 0); } TEST(BinaryTranslatorE2E, Gfx1250F32K128WmmaRejectsNonVgprMatrixInputs) { constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; for (const gfx1250::Vop3pBuilderFields fields : { gfx1250::Vop3pBuilderFields{ .vdst = 54, .src0 = 128, .src1 = 256 + 32, .src2 = 256 + 48, .opsel_hi = 3}, gfx1250::Vop3pBuilderFields{ .vdst = 54, .src0 = 256 + 16, .src1 = 128, .src2 = 256 + 48, .opsel_hi = 3}, }) { const auto source_wmma = gfx1250::build_vop3p(gfx1250::kVWmmaF3216x16x128Fp8Fp8Vop3p, fields); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_wmma[0], source_wmma[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE( rocjitsu::has_error_containing(result, rocjitsu::DiagnosticKind::ExpandFailed, "K=128 WMMA matrix operands are not ordinary VGPR ranges")); } } TEST(BinaryTranslatorE2E, Gfx1250F32K128WmmaRejectsClamp) { constexpr auto source_wmma = gfx1250::build_vop3p(gfx1250::kVWmmaF3216x16x128Fp8Fp8Vop3p, {.vdst = 54, .clamp = 1, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 256 + 48, .opsel_hi = 3}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_wmma[0], source_wmma[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(rocjitsu::has_error_containing( result, rocjitsu::DiagnosticKind::ExpandFailed, "Input is malformed, CLAMP \"must be set to zero\" for WMMA/SWMMAC producing " "floating-point results")); } // The f32 lowering re-encodes VDST, SRC0, SRC1, and SRC2 unchanged, so the // operand shapes that the packed-f16 lowering cannot address must still // translate here. Narrowing this path would reject code objects that the // hardware accepts. TEST(BinaryTranslatorE2E, Gfx1250F32K128WmmaAcceptsOperandsThePackedF16PathRejects) { struct OperandCase { const char *name; gfx1250::Vop3pBuilderFields fields; }; const std::array cases = { OperandCase{"odd_matrix_and_destination", {.vdst = 55, .src0 = 256 + 17, .src1 = 256 + 33, .src2 = 256 + 49}}, OperandCase{"sgpr_accumulator", {.vdst = 54, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 8}}, OperandCase{"nonzero_inline_accumulator", {.vdst = 54, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 129}}, }; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; for (const OperandCase &test_case : cases) { SCOPED_TRACE(test_case.name); const auto source_wmma = gfx1250::build_vop3p(gfx1250::kVWmmaF3216x16x128Fp8Fp8Vop3p, test_case.fields); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_wmma[0], source_wmma[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const auto *words = reinterpret_cast(translated.text_sections()[0]->data()); gfx1250::Vop3pMachineInst matrix{}; std::memcpy(&matrix, words + 2, sizeof(matrix)); EXPECT_EQ(matrix.op, gfx1250::kVWmmaF3216x16x128F8f6f4Vop3p); EXPECT_EQ(matrix.vdst, test_case.fields.vdst); EXPECT_EQ(matrix.src0, test_case.fields.src0); EXPECT_EQ(matrix.src1, test_case.fields.src1); EXPECT_EQ(matrix.src2, test_case.fields.src2); } } TEST(BinaryTranslatorE2E, Gfx1250LowersF16K128Fp8Bf8WmmaThroughF32Scratch) { constexpr std::array source_opcodes = { gfx1250::kVWmmaF1616x16x128Fp8Fp8Vop3p, gfx1250::kVWmmaF1616x16x128Fp8Bf8Vop3p, gfx1250::kVWmmaF1616x16x128Bf8Fp8Vop3p, gfx1250::kVWmmaF1616x16x128Bf8Bf8Vop3p, }; constexpr gfx1250::Vop3pBuilderFields fields{ .vdst = 54, .neg_hi = 4, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 256 + 48, .neg = 4, }; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; for (const uint16_t source_opcode : source_opcodes) { const auto source_wmma = gfx1250::build_vop3p(source_opcode, fields); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_wmma[0], source_wmma[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); const auto matrix = std::ranges::find_if(decoded, [](const auto &candidate) { return candidate->mnemonic() == "v_wmma_scale_f32_16x16x128_f8f6f4"; }); ASSERT_NE(matrix, decoded.end()); ASSERT_NE(matrix, decoded.begin()); EXPECT_EQ((*std::prev(matrix))->mnemonic(), "s_wait_alu"); ASSERT_NE(std::next(matrix), decoded.end()); EXPECT_EQ((*std::next(matrix))->mnemonic(), "s_wait_alu"); ASSERT_NE((*matrix)->raw_encoding(), nullptr); gfx1250::Vop3pMachineInst lowered_matrix{}; std::memcpy(&lowered_matrix, (*matrix)->raw_encoding() + 2, sizeof(lowered_matrix)); EXPECT_EQ(lowered_matrix.vdst % 2u, 0u) << "WMMA scratch tuples must start at an even VGPR"; EXPECT_EQ(lowered_matrix.opsel, source_opcode == gfx1250::kVWmmaF1616x16x128Bf8Fp8Vop3p || source_opcode == gfx1250::kVWmmaF1616x16x128Bf8Bf8Vop3p ? 1u : 0u); EXPECT_EQ(lowered_matrix.opsel_hi, source_opcode == gfx1250::kVWmmaF1616x16x128Fp8Bf8Vop3p || source_opcode == gfx1250::kVWmmaF1616x16x128Bf8Bf8Vop3p ? 1u : 0u); EXPECT_EQ(lowered_matrix.neg_hi, 4u); EXPECT_EQ(lowered_matrix.neg, 4u); auto after_wait = std::next(matrix, 2); ASSERT_LE(std::distance(decoded.begin(), after_wait) + 16, std::distance(decoded.begin(), decoded.end())); for (uint16_t slot = 0; slot < 16; ++slot, ++after_wait) EXPECT_EQ((*after_wait)->mnemonic(), "v_nop_e32"); EXPECT_EQ(std::ranges::count_if(decoded, [](const auto &candidate) { return candidate->mnemonic() == "v_wmma_scale_f32_16x16x128_f8f6f4"; }), 1); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &candidate) { return candidate->mnemonic() == "v_cvt_f32_f16"; }), 8); EXPECT_EQ(std::ranges::count_if(decoded, [](const auto &candidate) { return candidate->mnemonic() == "v_cvt_pk_f16_f32"; }), 4); EXPECT_EQ( std::ranges::count_if( decoded, [](const auto &candidate) { return candidate->mnemonic() == "s_getreg_b32"; }), 1); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &candidate) { return candidate->mnemonic() == "v_maximum_f32"; }), 8); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &candidate) { return candidate->mnemonic() == "v_minimum_f32"; }), 8); EXPECT_EQ(std::ranges::count_if(decoded, [](const auto &candidate) { return candidate->mnemonic().find("16x16x64") != std::string_view::npos; }), 0); } } TEST(BinaryTranslatorE2E, Gfx1250F16K128WmmaAcceptsInlineZeroAccumulator) { constexpr auto source_wmma = gfx1250::build_vop3p(gfx1250::kVWmmaF1616x16x128Fp8Fp8Vop3p, {.vdst = 54, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 128}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_wmma[0], source_wmma[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ( std::ranges::count_if( decoded, [](const auto &candidate) { return candidate->mnemonic() == "v_cvt_f32_f16"; }), 0); const auto matrix = std::ranges::find_if(decoded, [](const auto &candidate) { return candidate->mnemonic() == "v_wmma_scale_f32_16x16x128_f8f6f4"; }); ASSERT_NE(matrix, decoded.end()); gfx1250::Vop3pMachineInst lowered_matrix{}; std::memcpy(&lowered_matrix, (*matrix)->raw_encoding() + 2, sizeof(lowered_matrix)); EXPECT_EQ(lowered_matrix.src2, 128u); } TEST(BinaryTranslatorE2E, Gfx1250F16K128WmmaRejectsNonVgprNonzeroAccumulator) { for (const uint16_t accumulator : {uint16_t{0}, uint16_t{129}}) { const auto source_wmma = gfx1250::build_vop3p(gfx1250::kVWmmaF1616x16x128Fp8Fp8Vop3p, {.vdst = 54, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = accumulator}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_wmma[0], source_wmma[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(rocjitsu::has_error_containing( result, rocjitsu::DiagnosticKind::ExpandFailed, "K=128 WMMA accumulator is not a VGPR range or inline zero")); } } TEST(BinaryTranslatorE2E, Gfx1250F16K128WmmaUsesAccumulatorBankAsGeneratedSrc0) { constexpr auto set_vgpr_msb = gfx1250::build_sopp(gfx1250::kSSetVgprMsbSopp, {.simm16 = 0x10}); constexpr auto source_wmma = gfx1250::build_vop3p(gfx1250::kVWmmaF1616x16x128Fp8Fp8Vop3p, {.vdst = 54, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 256 + 48}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {set_vgpr_msb[0], source_wmma[0], source_wmma[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); std::vector modes; for (const auto &candidate : decoded) { if (candidate->mnemonic() == "s_set_vgpr_msb") modes.push_back(static_cast(candidate->raw_encoding()[0] & 0xffu)); } ASSERT_GE(modes.size(), 2u); EXPECT_EQ(modes[0], 0x10u); EXPECT_EQ(modes[1], 0x01u) << "the generated conversion reads the accumulator through SRC0"; } TEST(BinaryTranslatorE2E, Gfx1250F16K128WmmaAllowsSequentialTuplesToCrossVgprBanks) { constexpr auto source_wmma = gfx1250::build_vop3p(gfx1250::kVWmmaF1616x16x128Fp8Fp8Vop3p, {.vdst = 254, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 256 + 254}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_wmma[0], source_wmma[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); std::vector unpack_sources; std::vector packed_destinations; std::vector modes; for (const auto &candidate : decoded) { if (candidate->mnemonic() == "s_set_vgpr_msb") { modes.push_back(static_cast(candidate->raw_encoding()[0] & 0xffu)); } else if (candidate->mnemonic() == "v_cvt_f32_f16") { gfx1250::Vop3MachineInst conversion{}; std::memcpy(&conversion, candidate->raw_encoding(), sizeof(conversion)); unpack_sources.push_back(conversion.src0); } else if (candidate->mnemonic() == "v_cvt_pk_f16_f32") { gfx1250::Vop3MachineInst conversion{}; std::memcpy(&conversion, candidate->raw_encoding(), sizeof(conversion)); packed_destinations.push_back(conversion.vdst); } } EXPECT_EQ(unpack_sources, (std::vector{510, 510, 511, 511, 256, 256, 257, 257})); EXPECT_EQ(packed_destinations, (std::vector{254, 255, 0, 1})); EXPECT_NE(std::ranges::find(modes, 0x01u), modes.end()) << "the high accumulator dwords must select SRC0 bank 1"; EXPECT_NE(std::ranges::find(modes, 0x40u), modes.end()) << "the high destination dwords must select DST bank 1"; } TEST(BinaryTranslatorE2E, Gfx1250F16K128WmmaRejectsOddVgprTuples) { constexpr auto source_wmma = gfx1250::build_vop3p(gfx1250::kVWmmaF1616x16x128Fp8Fp8Vop3p, {.vdst = 54, .src0 = 256 + 17, .src1 = 256 + 32, .src2 = 256 + 48}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_wmma[0], source_wmma[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(rocjitsu::has_error_containing( result, rocjitsu::DiagnosticKind::ExpandFailed, "K=128 WMMA matrix operands and destination are not even VGPR ranges")); } TEST(BinaryTranslatorE2E, Gfx1250F16K128WmmaUsesCarrierUnderSgprPressure) { constexpr auto source_wmma = gfx1250::build_vop3p(gfx1250::kVWmmaF1616x16x128Fp8Fp8Vop3p, {.vdst = 54, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 256 + 48}); auto image = rocjitsu::make_gfx1250_image_with_live_sgprs(source_wmma, rocjitsu::REGISTER_SET_MAX_SGPRS); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ( std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "v_readfirstlane_b32_e32"; }), 1); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "s_cbranch_execz"; }), 1); } TEST(BinaryTranslatorE2E, Gfx1250F16K128WmmaUsesSpillBackedScratchWhenVgprsAreFull) { using namespace rocr::llvm::amdhsa; constexpr auto source_wmma = gfx1250::build_vop3p(gfx1250::kVWmmaF1616x16x128Fp8Fp8Vop3p, {.vdst = 54, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 256 + 48}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_wmma[0], source_wmma[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); auto descriptor = rocjitsu::read_kernel_descriptor_for_test(rodata->data()); AMDHSA_BITS_SET(descriptor.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, 63); rocjitsu::write_kernel_descriptor_for_test(image.data() + rodata->sectionOffset(), descriptor); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); auto options = gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0); options.debug_min_free_vgpr = 256; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, options); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "scratch_store_b32"; }), 9); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "scratch_load_b32"; }), 9); EXPECT_EQ(std::ranges::count_if(decoded, [](const auto &inst) { return inst->mnemonic() == "v_wmma_scale_f32_16x16x128_f8f6f4"; }), 1); } // Numerical evidence that the packed-f16 lowering preserves values, including // the FP16 overflow contract. MI400 Shader Programming 4.6.12 makes a WMMA // result of 16 bits or fewer saturate to +/-MAX when MODE.FP16_OVFL is set, // infinities included, while V_CVT_PK_F16_F32 preserves infinities in the same // mode. The lowering compensates with an explicit clamp, so both mode settings // have to be exercised, and the accumulator has to carry both infinity signs // plus values that overflow FP16 only after the matrix product. TEST(BinaryTranslatorE2E, Gfx1250F16K128WmmaLoweringMatchesUnloweredExecution) { constexpr uint16_t kMatrixA = 16; // A occupies 16 dwords of packed FP8. constexpr uint16_t kMatrixB = 32; // B occupies 16 dwords of packed FP8. constexpr uint16_t kMatrixC = 48; // C occupies 4 dwords of packed FP16. constexpr uint8_t kDestination = 52; constexpr uint32_t kMatrixARegs = 16; constexpr uint32_t kMatrixBRegs = 16; constexpr uint32_t kMatrixCRegs = 4; constexpr uint32_t kDestinationRegs = 4; constexpr auto matrix = gfx1250::build_vop3p(gfx1250::kVWmmaF1616x16x128Fp8Fp8Vop3p, {.vdst = kDestination, .src0 = 256 + kMatrixA, .src1 = 256 + kMatrixB, .src2 = 256 + kMatrixC}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const auto *text_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t text_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); auto decoder = rocjitsu::Decoder::create(ROCJITSU_CODE_ARCH_GFX1250); ASSERT_NE(decoder, nullptr); std::vector body_words; for (size_t offset = 0; offset < text_word_count;) { std::unique_ptr inst(decoder->decode(text_words + offset)); ASSERT_NE(inst, nullptr) << "translated word " << offset << " failed to decode"; const std::string_view mnemonic(inst->mnemonic()); if (mnemonic == "s_endpgm") break; EXPECT_NE(mnemonic, "s_set_vgpr_msb") << "bank-0 operands must not need a mode transition"; EXPECT_NE(mnemonic, "scratch_store_b32") << "the scratch lease must not spill here"; const size_t words = static_cast(inst->size()) / sizeof(uint32_t); ASSERT_GT(words, 0u); body_words.insert(body_words.end(), text_words + offset, text_words + offset + words); offset += words; } rocjitsu::amdgpu::GpuMemory gpu_mem("gfx1250_f16_k128_diff_mem"); rocjitsu::amdgpu::L2Cache l2("gfx1250_f16_k128_diff_l2"); rocjitsu::amdgpu::ComputeUnitCore::Config cfg{}; cfg.arch = ROCJITSU_CODE_ARCH_GFX1250; cfg.num_wf_slots = 1; cfg.sgprs_per_wf = 106; cfg.vgprs_per_wf = 256; cfg.lds_size_kb = 64; auto cu = rocjitsu::amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2); ASSERT_NE(cu, nullptr); auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf); ASSERT_NE(wf, nullptr); ASSERT_EQ(wf->wf_size(), 32u); wf->set_exec((1ULL << wf->wf_size()) - 1ULL); const uint32_t vb = wf->vgpr_alloc().base; const uint32_t lanes = wf->wf_size(); // FP8 E4M3 magnitudes reach 448, and K=128 terms accumulate, so ordinary // pseudo-random matrices already overflow FP16 in most output elements. The // accumulator adds both infinities, which only the clamp handles. NaN is // excluded here and covered separately below. auto fill_inputs = [&](uint32_t accumulator_override) { uint32_t state = 0x0f16ac21u; auto next = [&state] { state = state * 1664525u + 1013904223u; return state; }; auto without_fp8_nan = [](uint32_t word) { uint32_t out = 0; for (uint32_t byte = 0; byte < 4; ++byte) { uint32_t value = (word >> (byte * 8)) & 0xffu; if ((value & 0x7fu) == 0x7fu) value ^= 1u; out |= value << (byte * 8); } return out; }; auto without_f16_nan = [](uint32_t word) { uint32_t out = 0; for (uint32_t half = 0; half < 2; ++half) { uint32_t value = (word >> (half * 16)) & 0xffffu; if ((value & 0x7c00u) == 0x7c00u && (value & 0x03ffu) != 0) value &= 0xfbffu; out |= value << (half * 16); } return out; }; for (uint32_t reg = 0; reg < kMatrixARegs; ++reg) for (uint32_t lane = 0; lane < lanes; ++lane) cu->write_vgpr(vb + kMatrixA + reg, lane, without_fp8_nan(next())); for (uint32_t reg = 0; reg < kMatrixBRegs; ++reg) for (uint32_t lane = 0; lane < lanes; ++lane) cu->write_vgpr(vb + kMatrixB + reg, lane, without_fp8_nan(next())); for (uint32_t reg = 0; reg < kMatrixCRegs; ++reg) for (uint32_t lane = 0; lane < lanes; ++lane) cu->write_vgpr(vb + kMatrixC + reg, lane, accumulator_override != 0 ? accumulator_override : without_f16_nan(next())); if (accumulator_override != 0) return; // Packed FP16 pairs: +Inf/-Inf, +MAX/-MAX, +0/-0, denormal/one. static constexpr std::array kSpecials = {0xFC007C00u, 0xFBFF7BFFu, 0x80000000u, 0x3C000001u}; for (uint32_t reg = 0; reg < kMatrixCRegs; ++reg) for (uint32_t i = 0; i < kSpecials.size(); ++i) cu->write_vgpr(vb + kMatrixC + reg, i * 7u, kSpecials[i]); }; auto zero_destination = [&] { for (uint32_t reg = 0; reg < kDestinationRegs; ++reg) for (uint32_t lane = 0; lane < lanes; ++lane) cu->write_vgpr(vb + kDestination + reg, lane, 0u); }; auto snapshot_destination = [&] { std::vector out; out.reserve(kDestinationRegs * lanes); for (uint32_t reg = 0; reg < kDestinationRegs; ++reg) for (uint32_t lane = 0; lane < lanes; ++lane) out.push_back(cu->read_vgpr(vb + kDestination + reg, lane)); return out; }; auto run_source = [&] { const std::array words = {matrix[0], matrix[1]}; std::unique_ptr inst(decoder->decode(words.data())); EXPECT_NE(inst, nullptr); EXPECT_EQ(std::string_view(inst->mnemonic()), "v_wmma_f16_16x16x128_fp8_fp8"); cu->execute_instruction(inst.get(), *wf); }; auto run_lowered = [&] { for (size_t offset = 0; offset < body_words.size();) { std::unique_ptr inst(decoder->decode(body_words.data() + offset)); ASSERT_NE(inst, nullptr); cu->execute_instruction(inst.get(), *wf); offset += static_cast(inst->size()) / sizeof(uint32_t); } }; std::array, 2> source_by_mode; for (const bool fp16_ovfl : {false, true}) { SCOPED_TRACE(fp16_ovfl ? "FP16_OVFL=1" : "FP16_OVFL=0"); wf->set_mode_raw(fp16_ovfl ? rocjitsu::amdgpu::Wavefront::FP16_OVFL_BIT : 0u); fill_inputs(0); zero_destination(); run_source(); const std::vector run_a = snapshot_destination(); fill_inputs(0); zero_destination(); run_lowered(); const std::vector run_b = snapshot_destination(); EXPECT_EQ(run_a, run_b); for (size_t i = 0; i < run_a.size() && run_a != run_b; ++i) { if (run_a[i] == run_b[i]) continue; ADD_FAILURE() << "first divergence at D reg " << (i / lanes) << " lane " << (i % lanes) << ": source=0x" << std::hex << run_a[i] << " lowered=0x" << run_b[i] << std::dec; break; } source_by_mode[fp16_ovfl ? 1 : 0] = run_a; } // The comparison above only proves the two paths agree. This proves they // agree on something: every element that reaches infinity with the mode clear // must saturate to the same-signed MAX with it set, and nothing else moves. ASSERT_EQ(source_by_mode[0].size(), source_by_mode[1].size()); size_t saturated_positive = 0; size_t saturated_negative = 0; for (size_t i = 0; i < source_by_mode[0].size(); ++i) { for (uint32_t half = 0; half < 2; ++half) { const auto clear = static_cast(source_by_mode[0][i] >> (half * 16)); const auto set = static_cast(source_by_mode[1][i] >> (half * 16)); if (clear == 0x7C00u) { EXPECT_EQ(set, 0x7BFFu) << "+infinity must saturate to +MAX"; ++saturated_positive; } else if (clear == 0xFC00u) { EXPECT_EQ(set, 0xFBFFu) << "-infinity must saturate to -MAX"; ++saturated_negative; } else { EXPECT_EQ(set, clear) << "FP16_OVFL must not disturb representable results"; } } } EXPECT_GT(saturated_positive, 0u) << "no output element reached +infinity"; EXPECT_GT(saturated_negative, 0u) << "no output element reached -infinity"; // A NaN accumulator produces a NaN result on both paths, but not the same // encoding: V_MAXIMUM_F32 and V_MINIMUM_F32 return the canonical quiet NaN, // so the clamp drops the sign and payload the source instruction carries // through. IEEE 754 leaves both uninterpreted and the ISA promises only that // a NaN input yields a NaN output, so compare NaN-ness instead of bits. wf->set_mode_raw(0u); fill_inputs(0xFE00FE00u); zero_destination(); run_source(); const std::vector nan_source = snapshot_destination(); fill_inputs(0xFE00FE00u); zero_destination(); run_lowered(); const std::vector nan_lowered = snapshot_destination(); auto all_halves_are_nan = [](const std::vector &values) { return std::ranges::all_of(values, [](uint32_t word) { for (uint32_t half = 0; half < 2; ++half) { const auto value = static_cast(word >> (half * 16)); if ((value & 0x7C00u) != 0x7C00u || (value & 0x03FFu) == 0) return false; } return true; }); }; EXPECT_TRUE(all_halves_are_nan(nan_source)); EXPECT_TRUE(all_halves_are_nan(nan_lowered)); if (!wf->is_halted()) wf->halt(); } TEST(BinaryTranslatorE2E, Gfx1250MasksM0AroundOffFormClusterLoadForA0) { // An off-form (NULL-saddr) cluster load is NOT demoted to a global load. Like // every cluster-load form, it stays a cluster load and is wrapped so it runs // with M0 = 0; the opcode is unchanged. constexpr auto cluster = gfx1250::build_vglobal(gfx1250::kClusterLoadB32Vglobal, {.saddr = 124, .vdst = 8, .vaddr = 12}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {cluster[0], cluster[1], cluster[2], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); // The load stays a cluster load (no demotion to global_load). EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "cluster_load_b32"; }), 1); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "global_load_b32"; }), 0); // And it is framed by an M0=0 save/clear/restore. M0 encodes as 125 on gfx1250, // inline 0 as 128; SOP1 s_mov is SSRC0[7:0], SDST[22:16]. constexpr uint16_t kGfx1250M0Operand = 125; constexpr uint16_t kInlineZeroOperand = 128; bool cleared_m0 = false; for (const auto &inst : decoded) { if (inst->mnemonic() != "s_mov_b32" || inst->raw_encoding() == nullptr) continue; const uint32_t word = inst->raw_encoding()[0]; if (((word >> 16) & 0x7fu) == kGfx1250M0Operand && (word & 0xffu) == kInlineZeroOperand) cleared_m0 = true; } EXPECT_TRUE(cleared_m0) << "off-form cluster load must still set M0 (125) = inline 0 (128)"; rocjitsu::BinaryTranslator verifier( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto second_result = verifier.translate(translated); ASSERT_TRUE(second_result.ok()) << (second_result.diagnostics.empty() ? "" : second_result.diagnostics.front().message); EXPECT_EQ(second_result.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250MasksM0AroundSaddrClusterLoadForA0) { constexpr auto cluster = gfx1250::build_vglobal(gfx1250::kClusterLoadB64Vglobal, {.saddr = 4, .vdst = 8, .vaddr = 12}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {cluster[0], cluster[1], cluster[2], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "cluster_load_b64"; }), 1); // The load is wrapped to run with M0 = 0, saving/restoring the original M0 // through a dead SGPR. Assert operands, not just mnemonics: M0 encodes as 125 on // gfx1250 and NULL as 124, so a write to 124 would be a discarded NULL write. // SOP1 (s_mov): SSRC0[7:0], SDST[22:16]; inline 0 encodes as 128. The sequence // is: s_mov scratch, M0 (save) / s_mov M0, 0 (clear) / cluster_load / // s_mov M0, scratch (restore). constexpr uint16_t kGfx1250M0Operand = 125; constexpr uint16_t kInlineZeroOperand = 128; bool saved_m0 = false; // some s_mov reads M0 as source bool cleared_m0 = false; // some s_mov writes M0 from inline 0 bool restored_m0 = false; // some s_mov writes M0 from a non-inline (scratch) source for (const auto &inst : decoded) { if (inst->mnemonic() != "s_mov_b32" || inst->raw_encoding() == nullptr) continue; const uint32_t word = inst->raw_encoding()[0]; const uint16_t ssrc0 = word & 0xffu; const uint16_t sdst = (word >> 16) & 0x7fu; if (ssrc0 == kGfx1250M0Operand) saved_m0 = true; if (sdst == kGfx1250M0Operand && ssrc0 == kInlineZeroOperand) cleared_m0 = true; if (sdst == kGfx1250M0Operand && ssrc0 != kInlineZeroOperand) restored_m0 = true; } EXPECT_TRUE(saved_m0) << "no s_mov_b32 reads M0 (125) to save it"; EXPECT_TRUE(cleared_m0) << "no s_mov_b32 sets M0 (125) = inline 0 (128)"; EXPECT_TRUE(restored_m0) << "no s_mov_b32 restores M0 (125) from the saved scratch"; rocjitsu::BinaryTranslator verifier( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto second_result = verifier.translate(translated); ASSERT_TRUE(second_result.ok()) << (second_result.diagnostics.empty() ? "" : second_result.diagnostics.front().message); EXPECT_EQ(second_result.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250ClusterLoadsUseVgprCarrierUnderSgprPressure) { for (const uint16_t opcode : {gfx1250::kClusterLoadB32Vglobal, gfx1250::kClusterLoadAsyncToLdsB32Vglobal}) { SCOPED_TRACE(opcode == gfx1250::kClusterLoadB32Vglobal ? "cluster_load_b32" : "cluster_load_async_to_lds_b32"); const auto cluster = gfx1250::build_vglobal(opcode, {.saddr = 124, .vdst = 8, .vaddr = 12}); auto image = rocjitsu::make_gfx1250_image_with_live_sgprs(cluster, rocjitsu::REGISTER_SET_MAX_SGPRS); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "s_cbranch_execz"; }), 0); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "v_readfirstlane_b32_e32"; }), 0); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "v_writelane_b32"; }), 1); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "v_readlane_b32"; }), 1); EXPECT_EQ( std::ranges::count_if(decoded, [&](const auto &inst) { return inst->opcode() == opcode; }), 1); ASSERT_GE(decoded.size(), 8u); ASSERT_EQ(decoded[0]->mnemonic(), "s_wait_idle"); ASSERT_EQ(decoded[1]->mnemonic(), "v_writelane_b32"); ASSERT_EQ(decoded[6]->mnemonic(), "v_readlane_b32"); ASSERT_NE(decoded[1]->raw_encoding(), nullptr); ASSERT_NE(decoded[6]->raw_encoding(), nullptr); gfx1250::Vop3MachineInst save{}; gfx1250::Vop3MachineInst restore{}; std::memcpy(&save, decoded[1]->raw_encoding(), sizeof(save)); std::memcpy(&restore, decoded[6]->raw_encoding(), sizeof(restore)); EXPECT_EQ(save.src0, restore.vdst) << "the carrier must restore the borrowed SGPR"; EXPECT_EQ(static_cast(256u + save.vdst), restore.src0) << "the carrier save and restore must use the same VGPR"; constexpr uint16_t kInlineZero = 128; EXPECT_EQ(save.src1, kInlineZero) << "the EXEC-independent carrier must save through lane zero"; EXPECT_EQ(restore.src1, kInlineZero) << "the EXEC-independent carrier must restore through lane zero"; const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } } TEST(BinaryTranslatorE2E, Gfx1250ClusterLoadFailsClosedWithoutExecIndependentCarrier) { constexpr auto cluster = gfx1250::build_vglobal(gfx1250::kClusterLoadB32Vglobal, {.saddr = 124, .vdst = 8, .vaddr = 12}); auto image = rocjitsu::make_gfx1250_image_with_live_sgprs(cluster, rocjitsu::REGISTER_SET_MAX_SGPRS); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); auto options = gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0); options.debug_min_free_vgpr = 256; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, options); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(rocjitsu::has_error_containing( result, rocjitsu::DiagnosticKind::ExpandFailed, "gfx1250 cluster load could not allocate scalar scratch for M0 preservation")); } TEST(BinaryTranslatorE2E, Gfx1250ClusterLoadPreservesGuestM0ClearWithoutRestore) { constexpr auto clear_m0 = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = 128, .sdst = 125}); constexpr auto cluster = gfx1250::build_vglobal(gfx1250::kClusterLoadB32Vglobal, {.saddr = 124, .vdst = 8, .vaddr = 12}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {clear_m0[0], cluster[0], cluster[1], cluster[2], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); ASSERT_EQ(translated.text_sections()[0]->size(), 5 * sizeof(uint32_t)); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[0], clear_m0[0]); EXPECT_EQ(target_words[1], cluster[0]); EXPECT_EQ(target_words[2], cluster[1]); EXPECT_EQ(target_words[3], cluster[2]); EXPECT_EQ(target_words[4], kGfx1250SEndpgm); rocjitsu::BinaryTranslator verifier( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto second_result = verifier.translate(translated); ASSERT_TRUE(second_result.ok()) << (second_result.diagnostics.empty() ? "" : second_result.diagnostics.front().message); EXPECT_EQ(second_result.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250ClusterLoadDoesNotReuseNonzeroM0Write) { constexpr auto nonzero_m0 = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = 1, .sdst = 125}); constexpr auto clear_m0 = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = 128, .sdst = 125}); constexpr auto cluster = gfx1250::build_vglobal(gfx1250::kClusterLoadB32Vglobal, {.saddr = 124, .vdst = 8, .vaddr = 12}); constexpr auto generated_save = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = 125, .sdst = 0}); constexpr auto generated_restore = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = 0, .sdst = 125}); constexpr auto dependency_barrier = gfx1250::build_sopp(gfx1250::kSWaitIdleSopp); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {nonzero_m0[0], cluster[0], cluster[1], cluster[2], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); ASSERT_EQ(translated.text_sections()[0]->size(), 9 * sizeof(uint32_t)); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[0], nonzero_m0[0]); EXPECT_EQ(target_words[1], dependency_barrier[0]); EXPECT_EQ(target_words[2], generated_save[0]); EXPECT_EQ(target_words[3], clear_m0[0]); EXPECT_EQ(target_words[4], cluster[0]); EXPECT_EQ(target_words[5], cluster[1]); EXPECT_EQ(target_words[6], cluster[2]); EXPECT_EQ(target_words[7], generated_restore[0]); EXPECT_EQ(target_words[8], kGfx1250SEndpgm); rocjitsu::BinaryTranslator verifier( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto second_result = verifier.translate(translated); ASSERT_TRUE(second_result.ok()) << (second_result.diagnostics.empty() ? "" : second_result.diagnostics.front().message); EXPECT_EQ(second_result.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250ClusterLoadDoesNotReuseZeroWriteToOtherSgpr) { constexpr auto clear_s7 = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = 128, .sdst = 7}); constexpr auto clear_m0 = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = 128, .sdst = 125}); constexpr auto cluster = gfx1250::build_vglobal(gfx1250::kClusterLoadB32Vglobal, {.saddr = 124, .vdst = 8, .vaddr = 12}); constexpr auto dependency_barrier = gfx1250::build_sopp(gfx1250::kSWaitIdleSopp); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {clear_s7[0], cluster[0], cluster[1], cluster[2], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); ASSERT_EQ(translated.text_sections()[0]->size(), 9 * sizeof(uint32_t)); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[0], clear_s7[0]); EXPECT_EQ(target_words[1], dependency_barrier[0]); EXPECT_EQ(target_words[3], clear_m0[0]); const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250ClusterLoadDoesNotReuseM0ClearBypassedByBranch) { constexpr auto branch_to_cluster = gfx1250::build_sopp(gfx1250::kSCbranchScc0Sopp, {.simm16 = 2}); constexpr auto source_save = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = 125, .sdst = 12}); constexpr auto clear_m0 = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = 128, .sdst = 125}); constexpr auto cluster = gfx1250::build_vglobal(gfx1250::kClusterLoadB32Vglobal, {.saddr = 124, .vdst = 8, .vaddr = 12}); constexpr auto source_restore = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = 12, .sdst = 125}); constexpr auto dependency_barrier = gfx1250::build_sopp(gfx1250::kSWaitIdleSopp); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {branch_to_cluster[0], source_save[0], clear_m0[0], cluster[0], cluster[1], cluster[2], source_restore[0], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); ASSERT_EQ(translated.text_sections()[0]->size(), 12 * sizeof(uint32_t)); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); constexpr auto generated_save = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = 125, .sdst = 0}); constexpr auto generated_restore = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = 0, .sdst = 125}); EXPECT_EQ(target_words[0], branch_to_cluster[0]); EXPECT_EQ(target_words[1], source_save[0]); EXPECT_EQ(target_words[2], clear_m0[0]); EXPECT_EQ(target_words[3], dependency_barrier[0]); EXPECT_EQ(target_words[4], generated_save[0]); EXPECT_EQ(target_words[5], clear_m0[0]); EXPECT_EQ(target_words[6], cluster[0]); EXPECT_EQ(target_words[7], cluster[1]); EXPECT_EQ(target_words[8], cluster[2]); EXPECT_EQ(target_words[9], generated_restore[0]); EXPECT_EQ(target_words[10], source_restore[0]); EXPECT_EQ(target_words[11], kGfx1250SEndpgm); rocjitsu::BinaryTranslator verifier( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto second_result = verifier.translate(translated); ASSERT_TRUE(second_result.ok()) << (second_result.diagnostics.empty() ? "" : second_result.diagnostics.front().message); EXPECT_EQ(second_result.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250MaterializesDsAddtidAddressForA0) { constexpr auto addtid = gfx1250::build_vds(gfx1250::kDsLoadAddtidB32Vds, {.offset0 = 0x34, .offset1 = 0x12, .vdst = 8}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {addtid[0], addtid[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "ds_load_addtid_b32"; }), 0); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "ds_load_b32"; }), 1); EXPECT_EQ(std::ranges::count_if(decoded, [](const auto &inst) { return inst->mnemonic() == "v_mbcnt_lo_u32_b32" || inst->mnemonic() == "v_mbcnt_hi_u32_b32"; }), 2); // Pin the A0 ADDTID address formula (M0 + tid*4) & 0xfffff, not just mnemonics: // this is the stepping-specific contract, so a miscompiled shift/mask/base would // silently target the wrong LDS offset. The emitted sequence is // tid = v_mbcnt_lo/hi (checked by count above) // tid*4 = v_lshlrev_b32 by 2 // +M0 = v_add_nc_u32 SRC0 = M0 (gfx1250 M0 = 125; NULL is 124 — the inverse // of CDNA. Reading NULL would use base 0.) // &mask = v_bfe_u32 offset 0 width 20 (0xfffff = 20-bit LDS byte address) // Inline integer N encodes as 128+N in a VOP3 source; VOP3 SRC0 is the low 9 // bits of word[1], SRC1 the next 9 (bits [17:9]), SRC2 the next 9 (word[1] // bits... read from the raw words the builders emit). constexpr uint16_t kGfx1250M0Operand = 125; constexpr uint16_t kInline = 128; // inline integer 0 encodes as 128. auto v_lshl = std::ranges::find_if( decoded, [](const auto &inst) { return inst->mnemonic() == "v_lshlrev_b32"; }); ASSERT_NE(v_lshl, decoded.end()); ASSERT_NE((*v_lshl)->raw_encoding(), nullptr); // SRC0 is the shift amount: inline 2 -> encoding 130 (shift left by 2 == *4). EXPECT_EQ((*v_lshl)->raw_encoding()[1] & 0x1ffu, static_cast(kInline + 2)); auto v_add = std::ranges::find_if( decoded, [](const auto &inst) { return inst->mnemonic() == "v_add_nc_u32"; }); ASSERT_NE(v_add, decoded.end()); ASSERT_NE((*v_add)->raw_encoding(), nullptr); EXPECT_EQ((*v_add)->raw_encoding()[1] & 0x1ffu, kGfx1250M0Operand); auto v_bfe = std::ranges::find_if( decoded, [](const auto &inst) { return inst->mnemonic() == "v_bfe_u32"; }); ASSERT_NE(v_bfe, decoded.end()); ASSERT_NE((*v_bfe)->raw_encoding(), nullptr); // SRC1 (bits [17:9]) is the field offset (inline 0 -> 128); SRC2 (bits [26:18]) // is the field width (inline 20 -> 148). Together: (value >> 0) & ((1<<20)-1). EXPECT_EQ(((*v_bfe)->raw_encoding()[1] >> 9) & 0x1ffu, kInline); EXPECT_EQ(((*v_bfe)->raw_encoding()[1] >> 18) & 0x1ffu, static_cast(kInline + 20)); } // A getpc-plus-literal names its target by distance from the getpc, so relocating the body that // contains it silently retargets it. A callee packed up against its caller moves whenever the // source left padding between them, and its instructions are copied verbatim -- so nothing else in // the pipeline notices that the pair now reaches a different address. The literal must therefore be // recomputed from the getpc's final placement. // Read back the single `R_AMDGPU_RELATIVE64` addend and resolve it to `.text` words, so a test can // assert where a function pointer ends up pointing rather than only that translation succeeded. [[nodiscard]] std::vector text_words_at_relative64_addend(std::span image, size_t count) { rocjitsu::Elf64_Ehdr ehdr{}; std::memcpy(&ehdr, image.data(), sizeof(ehdr)); std::vector shdrs(ehdr.e_shnum); std::memcpy(shdrs.data(), image.data() + ehdr.e_shoff, ehdr.e_shnum * sizeof(rocjitsu::Elf64_Shdr)); std::optional addend; for (const rocjitsu::Elf64_Shdr §ion : shdrs) { if (section.sh_type != rocjitsu::SHT_RELA) continue; for (size_t i = 0; i < section.sh_size / sizeof(rocjitsu::Elf64_Rela); ++i) { rocjitsu::Elf64_Rela rela{}; std::memcpy(&rela, image.data() + section.sh_offset + i * sizeof(rocjitsu::Elf64_Rela), sizeof(rela)); if ((rela.r_info & 0xffffffffu) == rocjitsu::R_AMDGPU_RELATIVE64) addend = static_cast(rela.r_addend); } } if (!addend) return {}; for (const rocjitsu::Elf64_Shdr §ion : shdrs) { if ((section.sh_flags & rocjitsu::SHF_EXECINSTR) == 0 || *addend < section.sh_addr || *addend - section.sh_addr >= section.sh_size) { continue; } const uint64_t offset = section.sh_offset + (*addend - section.sh_addr); std::vector words(count); std::memcpy(words.data(), image.data() + offset, count * sizeof(uint32_t)); return words; } return {}; } // Translated text replaces .text wholesale, so a device function body that reaches no kernel scope // would not be preserved: its address range is reoccupied by relocated code. A body a function // pointer names is therefore adopted as a translation root instead, which both carries it into the // relocated text and gives the addend a placement to be rewritten to. Kernel bodies need no such // adoption because their descriptors already make them roots. TEST(BinaryTranslatorE2E, Gfx1250AdoptsDeviceFunctionBodyReachedByNoScope) { constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; constexpr uint32_t kGfx1250SNop = 0xBF800000u; // The kernel ends at word 0, so nothing below it lands in a scope. A function-pointer table names // the body at word 3, so an address the decoded graph never accounts for can still arrive there. // That is what separates this from a dead local a linker merely retained: the identical object // without the table is accepted by the test below, so the table is the only thing being pinned. const std::vector words = { kGfx1250SEndpgm, kGfx1250SNop, kGfx1250SNop, kGfx1250SNop, kGfx1250SEndpgm, }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( words, /*text_function_words=*/2, /*text_function_offset_words=*/3, /*function_pointer_table_target_words=*/3); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); ASSERT_TRUE(result.dispatchable()); EXPECT_NE(result.elf_bytes, image); // The addend must land on the adopted body, not on whatever now occupies its original range. EXPECT_EQ(text_words_at_relative64_addend(result.elf_bytes, 2), (std::vector{kGfx1250SNop, kGfx1250SEndpgm})); } // The same shape with the pointer array's own symbol stripped. Adoption is driven by the // discovered function tables, and discovery needs a qualifying `STT_OBJECT` around the slot, so a // compiler-anonymous pointer array names no adoptable body. That boundary is pinned here rather // than left unstated: the body is still dropped, its addend still has nowhere to point, and // replace_text() still refuses the object -- the behavior that predates adoption. Widening it // needs reachability asked of the raw relocations rather than of the tables. TEST(BinaryTranslatorE2E, Gfx1250RefusesDeviceFunctionBodyHeldByAnUnnamedPointerSlot) { constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; constexpr uint32_t kGfx1250SNop = 0xBF800000u; const std::vector words = { kGfx1250SEndpgm, kGfx1250SNop, kGfx1250SNop, kGfx1250SNop, kGfx1250SEndpgm, }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( words, /*text_function_words=*/2, /*text_function_offset_words=*/3, /*function_pointer_table_target_words=*/3, /*name_function_pointer_table_with_symbol=*/false); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); ASSERT_TRUE(rocjitsu::discover_relocation_function_tables(source).empty()) << "the slot must not qualify as a table, or this covers the same path as the test above"; rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); } TEST(BinaryTranslatorE2E, Gfx1250RepointsPcRelativeDataRangeBoundsAfterRelocation) { constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; constexpr uint32_t kGfx1250SNop = 0xBF800000u; constexpr uint32_t kGfx1250GetPcS0 = 0xBE804700u; // s_get_pc_i64 s[0:1] constexpr uint32_t kGfx1250AddNcU64S0 = 0xA980FE00u; // s_add_nc_u64 s[0:1], s[0:1], lit64 constexpr uint32_t kGfx1250SetPcS30 = 0xBE80481Eu; // s_set_pc_i64 s[30:31] // s_call_i64 s[30:31], simm16 jumps to (this instruction + 4) + simm16 * 4. The callee starts at // word 8, so simm16 is 7; words 2..7 are the padding that disappears when it is packed against // the caller, which is what moves the getpc. constexpr uint32_t kGfx1250CallToWord8 = 0xBA1E0007u; constexpr size_t kCalleeWord = 8; std::vector words = { kGfx1250CallToWord8, kGfx1250SEndpgm, kGfx1250SNop, kGfx1250SNop, kGfx1250SNop, kGfx1250SNop, kGfx1250SNop, kGfx1250SNop, kGfx1250GetPcS0, kGfx1250AddNcU64S0, 0u /*lit lo*/, 0u /*lit hi*/, kGfx1250SetPcS30, }; const auto sections = [](std::span img) { rocjitsu::Elf64_Ehdr ehdr{}; std::memcpy(&ehdr, img.data(), sizeof(ehdr)); std::vector shdrs(ehdr.e_shnum); std::memcpy(shdrs.data(), img.data() + ehdr.e_shoff, ehdr.e_shnum * sizeof(rocjitsu::Elf64_Shdr)); return shdrs; }; const auto verify_target = [&](bool points_one_past_end) { SCOPED_TRACE(points_one_past_end ? "one-past-end pointer" : "section-start pointer"); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); // Locate .text and a real non-executable allocated target, then point the literal at either // bound of that section. Both bounds are valid values in a compiler-generated [begin, end) // pair, and both must follow the section when text growth shifts it. auto shdrs = sections(image); const auto text = std::ranges::find_if(shdrs, [](const rocjitsu::Elf64_Shdr &s) { return (s.sh_flags & rocjitsu::SHF_EXECINSTR) != 0; }); const auto data = std::ranges::find_if(shdrs, [](const rocjitsu::Elf64_Shdr &s) { return (s.sh_flags & rocjitsu::SHF_ALLOC) != 0 && (s.sh_flags & rocjitsu::SHF_EXECINSTR) == 0 && s.sh_size != 0; }); ASSERT_NE(text, shdrs.end()); ASSERT_NE(data, shdrs.end()); const uint64_t source_getpc_result = text->sh_addr + kCalleeWord * sizeof(uint32_t) + sizeof(uint32_t); const uint64_t target_section_offset = points_one_past_end ? data->sh_size : 0; const uint64_t target_vaddr = data->sh_addr + target_section_offset; const uint64_t source_literal = target_vaddr - source_getpc_result; std::memcpy(image.data() + text->sh_offset + (kCalleeWord + 2) * sizeof(uint32_t), &source_literal, sizeof(source_literal)); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); // Find the relocated getpc by its encoding; the callee moved, so its offset is not known up // front. auto out_shdrs = sections(result.elf_bytes); const auto out_text = std::ranges::find_if(out_shdrs, [](const rocjitsu::Elf64_Shdr &s) { return (s.sh_flags & rocjitsu::SHF_EXECINSTR) != 0; }); const auto out_data = std::ranges::find_if(out_shdrs, [](const rocjitsu::Elf64_Shdr &s) { return (s.sh_flags & rocjitsu::SHF_ALLOC) != 0 && (s.sh_flags & rocjitsu::SHF_EXECINSTR) == 0 && s.sh_size != 0; }); ASSERT_NE(out_text, out_shdrs.end()); ASSERT_NE(out_data, out_shdrs.end()); std::optional getpc_word; for (size_t i = 0; i + 3 < out_text->sh_size / sizeof(uint32_t); ++i) { uint32_t word = 0; std::memcpy(&word, result.elf_bytes.data() + out_text->sh_offset + i * sizeof(uint32_t), sizeof(word)); if (word == kGfx1250GetPcS0) { getpc_word = i; break; } } ASSERT_TRUE(getpc_word.has_value()) << "the callee body must survive into translated text"; ASSERT_NE(*getpc_word, kCalleeWord) << "the callee must have moved, or this proves nothing"; uint64_t relocated_literal = 0; std::memcpy(&relocated_literal, result.elf_bytes.data() + out_text->sh_offset + (*getpc_word + 2) * sizeof(uint32_t), sizeof(relocated_literal)); const uint64_t relocated_getpc_result = out_text->sh_addr + *getpc_word * sizeof(uint32_t) + sizeof(uint32_t); EXPECT_EQ(relocated_getpc_result + relocated_literal, out_data->sh_addr + target_section_offset) << "the relocated body must still reach the same data address"; EXPECT_NE(relocated_literal, source_literal) << "the body moved, so the literal had to change"; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); }; ASSERT_NO_FATAL_FAILURE(verify_target(false)); ASSERT_NO_FATAL_FAILURE(verify_target(true)); } TEST(BinaryTranslatorE2E, Gfx1250FailsClosedOnExcludedBarrierSignalIsfirst) { // Inline constants encode -1 at 193, so the excluded id is 195. constexpr uint8_t kExcludedBarrierIdInline = 195; constexpr auto barrier = gfx1250::build_sop1(gfx1250::kSBarrierSignalIsfirstSop1, {.ssrc0 = kExcludedBarrierIdInline}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({barrier[0], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image) << "a fail-closed translation must leave the object unchanged"; EXPECT_TRUE(rocjitsu::has_error_containing( result, rocjitsu::DiagnosticKind::ExpandFailed, "s_barrier_signal_isfirst cannot name barrier id -3 (inline selector 195)")); } // Every other barrier id must survive translation unchanged. This operand // admits inline constants and M0; the neighbouring inline ids and an M0 source // therefore cover the supported spellings. // // M0 is the one case here that copies a run-time value. It is copied rather // than refused because that form draws the id from a zero-extended low field // and so cannot reach the excluded negative id. If that ever stopped holding, // this case would belong in the fail-closed test above instead. TEST(BinaryTranslatorE2E, Gfx1250CopiesRemainingBarrierSignalIsfirstFormsForA0) { constexpr uint8_t kInlineMinusOne = 193; constexpr uint8_t kInlineMinusTwo = 194; constexpr uint8_t kInlineMinusFour = 196; constexpr uint8_t kInlinePlusOne = 129; constexpr uint8_t kM0Selector = 125; for (const uint8_t barrier_id : {kInlineMinusOne, kInlineMinusTwo, kInlineMinusFour, kInlinePlusOne, kM0Selector}) { const auto barrier = gfx1250::build_sop1(gfx1250::kSBarrierSignalIsfirstSop1, {.ssrc0 = barrier_id}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({barrier[0], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << "barrier id " << static_cast(barrier_id) << ": " << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[0], barrier[0]) << "barrier id " << static_cast(barrier_id) << " must be copied verbatim"; } } TEST(BinaryTranslatorE2E, Gfx1250WrapsBareF8f6f4WmmaInNeutralScaleForA0) { constexpr gfx1250::Vop3pBuilderFields fields{ .vdst = 32, .neg_hi = 4, .opsel = 4, .src0 = 256 + 64, .src1 = 256 + 128, .src2 = 256 + 192, .opsel_hi = 1, .neg = 4, }; constexpr auto wmma = gfx1250::build_vop3p(gfx1250::kVWmmaF3216x16x128F8f6f4Vop3p, fields); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; constexpr auto completion_wait = kGfx1250WmmaCompletionWait; constexpr size_t kPrefixAndMatrixWords = 4; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {wmma[0], wmma[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); ASSERT_GE(decoded.size(), 3u); EXPECT_EQ(decoded[0]->mnemonic(), "v_wmma_scale_f32_16x16x128_f8f6f4"); EXPECT_EQ(decoded[0]->size(), 4 * static_cast(sizeof(uint32_t))); EXPECT_EQ(decoded[1]->mnemonic(), "s_wait_alu"); EXPECT_EQ(decoded[2]->mnemonic(), "s_endpgm"); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); ASSERT_GE(translated.text_sections()[0]->size() / sizeof(uint32_t), kPrefixAndMatrixWords + 2u); gfx1250::Vop3pMachineInst prefix{}; std::memcpy(&prefix, target_words, sizeof(prefix)); EXPECT_EQ(prefix.op, 0x35u); EXPECT_EQ(prefix.src0, 128u); EXPECT_EQ(prefix.src1, 128u); EXPECT_EQ(prefix.src2, 256u); EXPECT_EQ(prefix.neg, 0u); EXPECT_EQ(prefix.neg_hi, 0u); EXPECT_EQ(prefix.opsel, 0u); EXPECT_EQ(prefix.opsel_hi, 0u); EXPECT_EQ(prefix.pad_14, 0u); EXPECT_EQ(target_words[2], wmma[0]); EXPECT_EQ(target_words[3], wmma[1]); EXPECT_EQ(target_words[kPrefixAndMatrixWords], completion_wait[0]); EXPECT_EQ(target_words[kPrefixAndMatrixWords + 1], kGfx1250SEndpgm); } TEST(BinaryTranslatorE2E, Gfx1250PreservesSupportedK64LowPrecisionWmmaAndAppendsWaitForA0) { struct Case { const char *name; uint16_t opcode; }; const std::array cases = {{ {"v_wmma_f32_16x16x64_fp8_fp8", gfx1250::kVWmmaF3216x16x64Fp8Fp8Vop3p}, {"v_wmma_f32_16x16x64_fp8_bf8", gfx1250::kVWmmaF3216x16x64Fp8Bf8Vop3p}, {"v_wmma_f32_16x16x64_bf8_fp8", gfx1250::kVWmmaF3216x16x64Bf8Fp8Vop3p}, {"v_wmma_f32_16x16x64_bf8_bf8", gfx1250::kVWmmaF3216x16x64Bf8Bf8Vop3p}, {"v_wmma_f16_16x16x64_fp8_fp8", gfx1250::kVWmmaF1616x16x64Fp8Fp8Vop3p}, {"v_wmma_f16_16x16x64_fp8_bf8", gfx1250::kVWmmaF1616x16x64Fp8Bf8Vop3p}, {"v_wmma_f16_16x16x64_bf8_fp8", gfx1250::kVWmmaF1616x16x64Bf8Fp8Vop3p}, {"v_wmma_f16_16x16x64_bf8_bf8", gfx1250::kVWmmaF1616x16x64Bf8Bf8Vop3p}, }}; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; constexpr auto completion_wait = kGfx1250WmmaCompletionWait; for (const auto &c : cases) { const auto wmma = gfx1250::build_vop3p( c.opcode, {.vdst = 32, .src0 = 256 + 64, .src1 = 256 + 128, .src2 = 256 + 192}); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {wmma[0], wmma[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()) << c.name; rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); struct TraceClassification { bool copied_original; bool semantic_lowering; bool changed; size_t target_word_count; }; std::optional wmma_trace; translator.set_trace_callback([&](const rocjitsu::TranslationTraceEvent &event) { if (event.source_words.size() != wmma.size() || event.source_words[0] != wmma[0] || event.source_words[1] != wmma[1]) { return; } wmma_trace = {.copied_original = event.copied_original, .semantic_lowering = event.semantic_lowering, .changed = event.changed, .target_word_count = event.target_words.size()}; }); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << c.name << ": " << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()) << c.name; ASSERT_GE(translated.text_sections()[0]->size(), 4 * sizeof(uint32_t)) << c.name; const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[0], wmma[0]) << c.name; EXPECT_EQ(target_words[1], wmma[1]) << c.name; EXPECT_EQ(target_words[2], completion_wait[0]) << c.name; EXPECT_EQ(target_words[3], kGfx1250SEndpgm) << c.name; ASSERT_TRUE(wmma_trace.has_value()) << c.name; EXPECT_TRUE(wmma_trace->copied_original) << c.name; EXPECT_FALSE(wmma_trace->semantic_lowering) << c.name; EXPECT_TRUE(wmma_trace->changed) << c.name; EXPECT_EQ(wmma_trace->target_word_count, 3u) << c.name; const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << c.name << ": " << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes) << c.name; } } TEST(BinaryTranslatorE2E, Gfx1250DoesNotCreditTrailingWaitAcrossDependentWmma) { constexpr auto producer = gfx1250::build_vop3p(gfx1250::kVWmmaF3216x16x64Fp8Fp8Vop3p, {.vdst = 32, .src0 = 256 + 64, .src1 = 256 + 128, .src2 = 256 + 192}); constexpr auto consumer = gfx1250::build_vop3p(gfx1250::kVWmmaF3216x16x64Fp8Fp8Vop3p, {.vdst = 96, .src0 = 256 + 8, .src1 = 256 + 16, .src2 = 256 + 32}); constexpr auto completion_wait = kGfx1250WmmaCompletionWait; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {producer[0], producer[1], consumer[0], consumer[1], completion_wait[0], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); ASSERT_GE(translated.text_sections()[0]->size(), 7 * sizeof(uint32_t)); EXPECT_EQ(target_words[0], producer[0]); EXPECT_EQ(target_words[1], producer[1]); EXPECT_EQ(target_words[2], completion_wait[0]); EXPECT_EQ(target_words[3], consumer[0]); EXPECT_EQ(target_words[4], consumer[1]); EXPECT_EQ(target_words[5], completion_wait[0]); EXPECT_EQ(target_words[6], kGfx1250SEndpgm); const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250CompletionWaitCreditChecksOnlyVaVdst) { struct Case { const char *name; uint16_t simm16; bool expect_inserted_wait; }; const std::array cases = {{ {"nonzero VA_VDST", 0x1f9f, true}, {"zero VA_VDST with other waits", 0x0000, false}, }}; constexpr auto wmma = gfx1250::build_vop3p(gfx1250::kVWmmaF3216x16x64Fp8Fp8Vop3p, {.vdst = 32, .src0 = 256 + 64, .src1 = 256 + 128, .src2 = 256 + 192}); constexpr auto completion_wait = kGfx1250WmmaCompletionWait; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; for (const auto &c : cases) { const auto source_wait = gfx1250::build_sopp(gfx1250::kSWaitAluSopp, {.simm16 = c.simm16}); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {wmma[0], wmma[1], source_wait[0], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << c.name << ": " << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()) << c.name; ASSERT_GE(translated.text_sections()[0]->size(), (c.expect_inserted_wait ? 5u : 4u) * sizeof(uint32_t)) << c.name; const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t source_wait_index = c.expect_inserted_wait ? 3u : 2u; if (c.expect_inserted_wait) { EXPECT_EQ(target_words[2], completion_wait[0]) << c.name; } EXPECT_EQ(target_words[source_wait_index], source_wait[0]) << c.name; EXPECT_EQ(target_words[source_wait_index + 1], kGfx1250SEndpgm) << c.name; const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << c.name << ": " << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes) << c.name; } } TEST(BinaryTranslatorE2E, Gfx1250CreditsWaitAcrossCanonicalBankTransitionScaffolding) { constexpr auto lower_bank_wmma = gfx1250::build_vop3p(gfx1250::kVWmmaF3216x16x64Fp8Fp8Vop3p, {.vdst = 32, .src0 = 256 + 64, .src1 = 256 + 128, .src2 = 256 + 192}); constexpr auto wait_xcnt = gfx1250::build_sopp(gfx1250::kSWaitXcntSopp, {.simm16 = 0}); constexpr auto select_bank_one = gfx1250::build_sopp(gfx1250::kSSetVgprMsbSopp, {.simm16 = 0x0055}); constexpr auto upper_bank_wmma = gfx1250::build_vop3p(gfx1250::kVWmmaF3216x16x64Fp8Fp8Vop3p, {.vdst = 96, .src0 = 256 + 8, .src1 = 256 + 16, .src2 = 256 + 24}); constexpr auto completion_wait = kGfx1250WmmaCompletionWait; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {lower_bank_wmma[0], lower_bank_wmma[1], wait_xcnt[0], select_bank_one[0], upper_bank_wmma[0], upper_bank_wmma[1], completion_wait[0], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "s_wait_alu"; }), 1); const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250CreditsPhysicallyAdjacentWaitAcrossBasicBlockBoundary) { // The conditional branch makes the wait a block leader while the physically // preceding WMMA remains on the fallthrough path. constexpr auto branch_to_wait = gfx1250::build_sopp(gfx1250::kSCbranchScc0Sopp, {.simm16 = 2}); constexpr auto wmma = gfx1250::build_vop3p(gfx1250::kVWmmaF3216x16x64Fp8Fp8Vop3p, {.vdst = 32, .src0 = 256 + 64, .src1 = 256 + 128, .src2 = 256 + 192}); constexpr auto completion_wait = kGfx1250WmmaCompletionWait; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {branch_to_wait[0], wmma[0], wmma[1], completion_wait[0], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "s_wait_alu"; }), 1); const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250CopiesA0CompatibleLowPrecisionSwmmac) { // FP8/BF8 sparse K=128 instructions exist on both steppings. They are not in // the gfx1250 B0-additions table and their opcodes fit the A0 VOP3P opcode // field, so B0-to-A0 translation must preserve their authoritative bytes. constexpr auto swmmac = gfx1250::build_vop3p(gfx1250::kVSwmmacF3216x16x128Fp8Fp8Vop3p, {.vdst = 32, .src0 = 256 + 64, .src1 = 256 + 128, .src2 = 256 + 192}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {swmmac[0], swmmac[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto text = translated.text_sections(); ASSERT_EQ(text.size(), 1u); ASSERT_GE(text[0]->size(), 3 * sizeof(uint32_t)); EXPECT_EQ(std::memcmp(text[0]->data(), swmmac.data(), 2 * sizeof(uint32_t)), 0); const auto *target_words = reinterpret_cast(text[0]->data()); EXPECT_EQ(target_words[2], kGfx1250SEndpgm); } TEST(BinaryTranslatorE2E, Gfx1250Allows32BitFlatScratchBaseHiSource) { // The same special value in a 32-bit source position is unaffected; s_mov_b32 // must translate cleanly (identity, same-arch stepping copy). constexpr uint16_t kFlatScratchBaseHi = 231; constexpr auto mov32 = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = kFlatScratchBaseHi, .sdst = 0}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({mov32[0], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); EXPECT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); } namespace { // Populate a C-API code object handle owning a copy of @p image, for // rj_code_translate. rj_code_object_t is non-copyable (RefCounted), so fill a // caller-provided handle in place. void init_c_code_object(rj_code_object_t &handle, const std::vector &image) { handle.owned_co = std::make_unique(image.data(), image.size()); handle.co = handle.owned_co.get(); } } // namespace TEST(RjCodeTranslateCApi, RejectsSameArchGfx1250) { // The C ABI carries no silicon revision, and gfx1250 A0 and B0 share an ELF // machine ID, so a same-architecture translation is direction-ambiguous. // Revision-aware B0-to-A0 translation uses the C++ BinaryTranslator instead. constexpr auto mov = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = 128, .sdst = 0}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({mov[0], kGfx1250SEndpgm}); rj_code_object_t source{}; init_c_code_object(source, image); ASSERT_TRUE(source.co->is_valid()); rj_code_dbt_options_t options{}; options.guest_arch = ROCJITSU_CODE_ARCH_GFX1250; options.host_arch = ROCJITSU_CODE_ARCH_GFX1250; rj_code_object_t *translated = nullptr; EXPECT_NE(rj_code_translate(&source, &options, &translated), ROCJITSU_STATUS_SUCCESS); EXPECT_EQ(translated, nullptr); } TEST(BinaryTranslatorEnforcesGfx1250Revisions, SameArchUnspecifiedRevisionFailsClosed) { // Direct BinaryTranslator use with same-arch gfx1250 and no revisions fails // closed because no translation profile was selected. constexpr auto cluster = gfx1250::build_vglobal(gfx1250::kClusterLoadB64Vglobal, {.saddr = 4, .vdst = 8, .vaddr = 12}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {cluster[0], cluster[1], cluster[2], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250); auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image) << "fail-closed must leave the object unchanged"; } TEST(BinaryTranslatorEnforcesGfx1250Revisions, TranslatesGfx1250B0ToA0WithRevisions) { // With both revisions provided, B0-to-A0 translation selects its profile. The // cluster load remains a cluster load framed by an M0=0 sequence. constexpr auto cluster = gfx1250::build_vglobal(gfx1250::kClusterLoadB64Vglobal, {.saddr = 4, .vdst = 8, .vaddr = 12}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {cluster[0], cluster[1], cluster[2], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "cluster_load_b64"; }), 1); } TEST(BinaryTranslatorE2E, Gfx1250GeneratedVgprMsbTransitionsCarryPreviousState) { constexpr uint16_t kAllVgprMsbFieldsHwreg = 1u | (12u << 6) | (7u << 11); constexpr auto original_mode = gfx1250::build_sopk(gfx1250::kSSetregImm32B32Sopk, {.simm16 = kAllVgprMsbFieldsHwreg}); constexpr auto addtid = gfx1250::build_vds(gfx1250::kDsStoreAddtidB32Vds, {.offset0 = 4, .data0 = 8}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {original_mode[0], 1u << 14, addtid[0], addtid[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); ASSERT_GE(decoded.size(), 5u); EXPECT_EQ(decoded[0]->mnemonic(), "s_setreg_imm32_b32"); // The dependency barrier also separates the guest MODE write from the // generated transition. The transition retains its dedicated memory drain. EXPECT_EQ(decoded[1]->mnemonic(), "s_wait_idle"); EXPECT_EQ(decoded[2]->mnemonic(), "s_wait_xcnt"); EXPECT_EQ(decoded[3]->mnemonic(), "s_set_vgpr_msb"); ASSERT_NE(decoded[3]->raw_encoding(), nullptr); EXPECT_EQ(decoded[3]->raw_encoding()[0] & 0xffffu, 0x0100u); // Every generated s_set_vgpr_msb must be immediately preceded by an s_wait_xcnt. std::vector generated_modes; for (size_t index = 1; index < decoded.size(); ++index) { if (decoded[index]->mnemonic() == "s_set_vgpr_msb") { ASSERT_NE(decoded[index]->raw_encoding(), nullptr); ASSERT_GT(index, 0u); EXPECT_EQ(decoded[index - 1]->mnemonic(), "s_wait_xcnt") << "s_set_vgpr_msb at index " << index << " is not guarded by s_wait_xcnt"; generated_modes.push_back(static_cast(decoded[index]->raw_encoding()[0] & 0xffffu)); } } // Input mode has SRC0 bank 1, SRC1 bank 0. The ds_store_addtid_b32 store-data // operand is a SRC1-role VGPR, so the store mode carries src1_bank (0) in the // SRC1 field, not src0_bank. With src1_bank 0 the store transition is a no-op // and is elided, leaving only the compute transition and the final restore. EXPECT_EQ(generated_modes, (std::vector{0x0100, 0x0001})); } TEST(BinaryTranslatorE2E, Gfx1250AddtidStoreUsesSpillBackedScratchWhenVgprsAreFull) { using namespace rocr::llvm::amdhsa; constexpr auto addtid = gfx1250::build_vds(gfx1250::kDsStoreAddtidB32Vds, {.offset0 = 4, .data0 = 8}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {addtid[0], addtid[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); auto descriptor = rocjitsu::read_kernel_descriptor_for_test(rodata->data()); AMDHSA_BITS_SET(descriptor.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, 63); rocjitsu::write_kernel_descriptor_for_test(image.data() + rodata->sectionOffset(), descriptor); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); auto options = gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0); options.debug_min_free_vgpr = 256; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, options); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "scratch_store_b32"; }), 1); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "scratch_load_b32"; }), 1); EXPECT_EQ( std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "ds_store_addtid_b32"; }), 0); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "ds_store_b32"; }), 1); std::optional ds_store_index; std::optional ds_wait_index; std::optional scratch_load_index; std::optional scratch_store; std::optional scratch_load; std::optional ds_store; for (size_t index = 0; index < decoded.size(); ++index) { if (decoded[index]->mnemonic() == "scratch_store_b32") { scratch_store.emplace(); std::memcpy(&*scratch_store, decoded[index]->raw_encoding(), sizeof(*scratch_store)); } else if (decoded[index]->mnemonic() == "ds_store_b32") { ds_store_index = index; ds_store.emplace(); std::memcpy(&*ds_store, decoded[index]->raw_encoding(), sizeof(*ds_store)); } else if (decoded[index]->mnemonic() == "s_wait_dscnt") { ds_wait_index = index; } else if (decoded[index]->mnemonic() == "scratch_load_b32") { scratch_load_index = index; scratch_load.emplace(); std::memcpy(&*scratch_load, decoded[index]->raw_encoding(), sizeof(*scratch_load)); } } ASSERT_TRUE(ds_store_index.has_value()); ASSERT_TRUE(ds_wait_index.has_value()); ASSERT_TRUE(scratch_load_index.has_value()); EXPECT_EQ(*ds_wait_index, *ds_store_index + 1u) << "the DS store must complete before its address VGPR is restored"; EXPECT_LT(*ds_wait_index, *scratch_load_index); ASSERT_TRUE(scratch_store.has_value()); ASSERT_TRUE(scratch_load.has_value()); ASSERT_TRUE(ds_store.has_value()); EXPECT_EQ(scratch_store->ioffset, scratch_load->ioffset); EXPECT_EQ(scratch_store->vsrc, scratch_load->vdst); EXPECT_EQ(ds_store->addr, scratch_load->vdst); const auto *translated_rodata = rocjitsu::find_section(translated, ".rodata"); ASSERT_NE(translated_rodata, nullptr); const auto translated_descriptor = rocjitsu::read_kernel_descriptor_for_test(translated_rodata->data()); EXPECT_GE(translated_descriptor.private_segment_fixed_size, sizeof(uint32_t)); const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250SpillBackedRulesFailClosedForDynamicStackKernel) { using namespace rocr::llvm::amdhsa; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; const auto expect_failure = [&](std::vector words, std::string_view diagnostic) { SCOPED_TRACE(diagnostic); words.push_back(kGfx1250SEndpgm); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); auto descriptor = rocjitsu::read_kernel_descriptor_for_test(rodata->data()); descriptor.private_segment_fixed_size = 32; AMDHSA_BITS_SET(descriptor.kernel_code_properties, KERNEL_CODE_PROPERTY_USES_DYNAMIC_STACK, 1); AMDHSA_BITS_SET(descriptor.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, 63); rocjitsu::write_kernel_descriptor_for_test(image.data() + rodata->sectionOffset(), descriptor); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); auto options = gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0); options.debug_min_free_vgpr = 256; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, options); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE( rocjitsu::has_error_containing(result, rocjitsu::DiagnosticKind::ExpandFailed, diagnostic)); }; constexpr auto addtid = gfx1250::build_vds(gfx1250::kDsStoreAddtidB32Vds, {.offset0 = 4, .data0 = 8}); expect_failure( {addtid[0], addtid[1]}, "gfx1250 DS store ADDTID cannot use private-memory spills in a dynamic-stack kernel"); constexpr auto e5m3 = gfx1250::build_vop3(gfx1250::kVCvtF32Fp8Vop3, {.vdst = 30, .opsel = 2, .clamp = 1, .src0 = 256 + 22}); expect_failure({e5m3[0], e5m3[1]}, "gfx1250 E5M3 unpack cannot use private-memory spills in a dynamic-stack kernel"); constexpr auto pack = gfx1250::build_vop3( gfx1250::kVCvtPkFp8F32Vop3, {.vdst = 30, .clamp = 1, .src0 = 256 + 22, .src1 = 256 + 2}); expect_failure({pack[0], pack[1]}, "gfx1250 E5M3 pack cannot use private-memory spills in a dynamic-stack kernel"); constexpr auto stochastic = gfx1250::build_vop3( gfx1250::kVCvtSrFp8F32Vop3, {.vdst = 30, .clamp = 1, .src0 = 256 + 22, .src1 = 256 + 2}); expect_failure( {stochastic[0], stochastic[1]}, "gfx1250 stochastic E5M3 cannot use private-memory spills in a dynamic-stack kernel"); } TEST(BinaryTranslatorE2E, Gfx1250SpillWaitsForOutstandingVgprProducer) { using namespace rocr::llvm::amdhsa; constexpr auto pending_load = gfx1250::build_vds(gfx1250::kDsLoadB32Vds, {.addr = 4, .vdst = 0}); constexpr auto addtid = gfx1250::build_vds(gfx1250::kDsStoreAddtidB32Vds, {.offset0 = 4, .data0 = 8}); constexpr auto consume_v0 = gfx1250::build_vop1(gfx1250::kVMovB32Vop1, {.src0 = 256, .vdst = 9}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {pending_load[0], pending_load[1], addtid[0], addtid[1], consume_v0[0], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); auto descriptor = rocjitsu::read_kernel_descriptor_for_test(rodata->data()); AMDHSA_BITS_SET(descriptor.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, 63); rocjitsu::write_kernel_descriptor_for_test(image.data() + rodata->sectionOffset(), descriptor); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); auto options = gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0); options.debug_min_free_vgpr = 256; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, options); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); std::optional pending_load_index; std::optional wait_idle_index; std::optional scratch_store_index; for (size_t index = 0; index < decoded.size(); ++index) { if (decoded[index]->mnemonic() == "ds_load_b32") pending_load_index = index; else if (decoded[index]->mnemonic() == "s_wait_idle") wait_idle_index = index; else if (decoded[index]->mnemonic() == "scratch_store_b32") { scratch_store_index = index; ASSERT_NE(decoded[index]->raw_encoding(), nullptr); gfx1250::VscratchMachineInst scratch{}; std::memcpy(&scratch, decoded[index]->raw_encoding(), sizeof(scratch)); EXPECT_EQ(scratch.vsrc, 0u); } } ASSERT_TRUE(pending_load_index.has_value()); ASSERT_TRUE(wait_idle_index.has_value()); ASSERT_TRUE(scratch_store_index.has_value()); EXPECT_LT(*pending_load_index, *wait_idle_index); EXPECT_EQ(*scratch_store_index, *wait_idle_index + 1u) << "the borrowed live VGPR must not be read before its producer completes"; const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250DeadScratchWaitsForOutstandingVgprProducer) { constexpr auto pending_load = gfx1250::build_vds(gfx1250::kDsLoadB32Vds, {.addr = 4, .vdst = 0}); constexpr auto addtid = gfx1250::build_vds(gfx1250::kDsStoreAddtidB32Vds, {.offset0 = 4, .data0 = 8}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {pending_load[0], pending_load[1], addtid[0], addtid[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); std::optional pending_load_index; std::optional wait_idle_index; std::optional scratch_write_index; for (size_t index = 0; index < decoded.size(); ++index) { if (decoded[index]->mnemonic() == "ds_load_b32") pending_load_index = index; else if (decoded[index]->mnemonic() == "s_wait_idle") wait_idle_index = index; else if (decoded[index]->mnemonic() == "v_mbcnt_lo_u32_b32") { scratch_write_index = index; const rocjitsu::Operand *destination = decoded[index]->dst_operand(0); ASSERT_NE(destination, nullptr); EXPECT_EQ(destination->encoding_value(), 1u) << "globally unused v1 is preferred over site-dead v0"; } } ASSERT_TRUE(pending_load_index.has_value()); ASSERT_TRUE(wait_idle_index.has_value()); ASSERT_TRUE(scratch_write_index.has_value()); EXPECT_LT(*pending_load_index, *wait_idle_index); EXPECT_LT(*wait_idle_index, *scratch_write_index) << "a dead scratch VGPR may still have an outstanding producer"; EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "scratch_store_b32"; }), 0); const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250AddtidLoadWaitsForOutstandingVgprProducer) { constexpr auto pending_load = gfx1250::build_vds(gfx1250::kDsLoadB32Vds, {.addr = 4, .vdst = 0}); constexpr auto addtid = gfx1250::build_vds(gfx1250::kDsLoadAddtidB32Vds, {.offset0 = 4, .vdst = 0}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {pending_load[0], pending_load[1], addtid[0], addtid[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); std::optional wait_idle_index; std::optional scratch_write_index; for (size_t index = 0; index < decoded.size(); ++index) { if (decoded[index]->mnemonic() == "s_wait_idle") wait_idle_index = index; else if (decoded[index]->mnemonic() == "v_mbcnt_lo_u32_b32") scratch_write_index = index; } ASSERT_TRUE(wait_idle_index.has_value()); ASSERT_TRUE(scratch_write_index.has_value()); EXPECT_LT(*wait_idle_index, *scratch_write_index) << "the ADDTID load destination may still have an outstanding producer"; const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250AddtidStoreModeUsesSrc1BankNotSrc0) { // Regression for the ADDTID store-data bank: the emitted ds_store_b32 keeps // the original store-data VGPR in data0, a SRC1-role operand, so its high bank // must come from src1_bank. Choose differing nonzero banks (SRC0 bank 1, SRC1 // bank 2) so a src0_bank/src1_bank swap is observable: the store mode must be // 0x08 (bank 2 in the SRC1 field), not 0x04 (bank 1). constexpr uint16_t kAllVgprMsbFieldsHwreg = 1u | (12u << 6) | (7u << 11); constexpr auto original_mode = gfx1250::build_sopk(gfx1250::kSSetregImm32B32Sopk, {.simm16 = kAllVgprMsbFieldsHwreg}); constexpr uint32_t kModeLiteral = (1u << 14) | (2u << 16); // SRC0 bank 1, SRC1 bank 2. constexpr auto addtid = gfx1250::build_vds(gfx1250::kDsStoreAddtidB32Vds, {.offset0 = 4, .data0 = 8}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {original_mode[0], kModeLiteral, addtid[0], addtid[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); // Assert the full generated s_set_vgpr_msb sequence, not just the store mode's // low byte, so this also catches a regression that drops the previous-mode // carry (SIMM16[15:8]) or restores the wrong original mode. The set_vgpr_msb // byte is {DST[7:6], SRC2[5:4], SRC1[3:2], SRC0[1:0]}; the high byte carries // the immediately-preceding mode. The live mode is SRC0 bank 1, SRC1 bank 2 = // 0x09. Transitions: compute (new 0x00, prev 0x09) = 0x0900; store (new // src1_bank 2 in the SRC1 field 0x08, prev 0x00) = 0x0008; restore (new 0x09, // prev 0x08) = 0x0809. A src0_bank/src1_bank swap would make the store mode // 0x04 instead of 0x08. std::vector generated_modes; for (size_t index = 0; index < decoded.size(); ++index) { if (decoded[index]->mnemonic() == "s_set_vgpr_msb") { ASSERT_NE(decoded[index]->raw_encoding(), nullptr); generated_modes.push_back(static_cast(decoded[index]->raw_encoding()[0] & 0xffffu)); } } EXPECT_EQ(generated_modes, (std::vector{0x0900, 0x0008, 0x0809})) << "ADDTID store mode must carry src1_bank (2) in the SRC1 field, and every transition " "must record the previous mode in SIMM16[15:8]"; } TEST(BinaryTranslatorE2E, Gfx1250Iu8WmmaSpacingCreditsOnlySameBlockCanonicalVNops) { enum class Consumer { EndProgram, ValuThenVNops, DenseWmma, ScalarNopThenDenseWmma, }; struct Case { const char *name; uint16_t opcode; size_t required_slots; size_t existing_slots; Consumer consumer; size_t trailing_slots; }; constexpr std::array cases = { Case{.name = "dense_no_existing_padding", .opcode = gfx1250::kVWmmaI3216x16x64Iu8Vop3p, .required_slots = 9, .existing_slots = 0, .consumer = Consumer::EndProgram, .trailing_slots = 0}, Case{.name = "sparse_no_existing_padding", .opcode = gfx1250::kVSwmmacI3216x16x128Iu8Vop3p, .required_slots = 5, .existing_slots = 0, .consumer = Consumer::EndProgram, .trailing_slots = 0}, Case{.name = "dense_partial_credit", .opcode = gfx1250::kVWmmaI3216x16x64Iu8Vop3p, .required_slots = 9, .existing_slots = 4, .consumer = Consumer::EndProgram, .trailing_slots = 0}, Case{.name = "sparse_partial_credit", .opcode = gfx1250::kVSwmmacI3216x16x128Iu8Vop3p, .required_slots = 5, .existing_slots = 3, .consumer = Consumer::EndProgram, .trailing_slots = 0}, Case{.name = "sparse_exact_credit", .opcode = gfx1250::kVSwmmacI3216x16x128Iu8Vop3p, .required_slots = 5, .existing_slots = 5, .consumer = Consumer::EndProgram, .trailing_slots = 0}, Case{.name = "dense_over_padded", .opcode = gfx1250::kVWmmaI3216x16x64Iu8Vop3p, .required_slots = 9, .existing_slots = 12, .consumer = Consumer::EndProgram, .trailing_slots = 0}, Case{.name = "dense_credit_stops_at_valu", .opcode = gfx1250::kVWmmaI3216x16x64Iu8Vop3p, .required_slots = 9, .existing_slots = 0, .consumer = Consumer::ValuThenVNops, .trailing_slots = 8}, Case{.name = "back_to_back_dense", .opcode = gfx1250::kVWmmaI3216x16x64Iu8Vop3p, .required_slots = 9, .existing_slots = 0, .consumer = Consumer::DenseWmma, .trailing_slots = 0}, Case{.name = "dense_partial_credit_before_matrix_consumer", .opcode = gfx1250::kVWmmaI3216x16x64Iu8Vop3p, .required_slots = 9, .existing_slots = 4, .consumer = Consumer::DenseWmma, .trailing_slots = 0}, Case{.name = "scalar_nop_receives_no_credit", .opcode = gfx1250::kVWmmaI3216x16x64Iu8Vop3p, .required_slots = 9, .existing_slots = 0, .consumer = Consumer::ScalarNopThenDenseWmma, .trailing_slots = 0}, }; const uint32_t v_nop = gfx1250::build_vop1(gfx1250::kVNopVop1)[0]; const uint32_t v_mov = gfx1250::build_vop1(gfx1250::kVMovB32Vop1, {.src0 = 128, .vdst = 0})[0]; const uint32_t s_nop = gfx1250::build_sopp(gfx1250::kSNopSopp, {.simm16 = 8})[0]; const auto consumer_wmma = gfx1250::build_vop3p(gfx1250::kVWmmaI3216x16x64Iu8Vop3p, {.vdst = 64, .src0 = 256 + 72, .src1 = 256 + 136, .src2 = 256 + 200}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; for (const Case &test_case : cases) { SCOPED_TRACE(test_case.name); const auto source_wmma = gfx1250::build_vop3p( test_case.opcode, {.vdst = 32, .src0 = 256 + 64, .src1 = 256 + 128, .src2 = 256 + 192}); std::vector source_words = {source_wmma[0], source_wmma[1]}; source_words.insert(source_words.end(), test_case.existing_slots, v_nop); if (test_case.consumer == Consumer::ValuThenVNops) { source_words.push_back(v_mov); source_words.insert(source_words.end(), test_case.trailing_slots, v_nop); } else if (test_case.consumer == Consumer::DenseWmma) { source_words.insert(source_words.end(), consumer_wmma.begin(), consumer_wmma.end()); } else if (test_case.consumer == Consumer::ScalarNopThenDenseWmma) { source_words.push_back(s_nop); source_words.insert(source_words.end(), consumer_wmma.begin(), consumer_wmma.end()); } source_words.push_back(kGfx1250SEndpgm); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(source_words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const size_t expected_prefix_slots = std::max(test_case.required_slots, test_case.existing_slots); std::vector expected_words = {source_wmma[0], source_wmma[1]}; expected_words.insert(expected_words.end(), expected_prefix_slots, v_nop); if (test_case.consumer == Consumer::ValuThenVNops) { expected_words.push_back(v_mov); expected_words.insert(expected_words.end(), test_case.trailing_slots, v_nop); } else if (test_case.consumer == Consumer::DenseWmma) { expected_words.insert(expected_words.end(), consumer_wmma.begin(), consumer_wmma.end()); expected_words.insert(expected_words.end(), 9, v_nop); } else if (test_case.consumer == Consumer::ScalarNopThenDenseWmma) { expected_words.push_back(s_nop); expected_words.insert(expected_words.end(), consumer_wmma.begin(), consumer_wmma.end()); expected_words.insert(expected_words.end(), 9, v_nop); } expected_words.push_back(kGfx1250SEndpgm); ASSERT_EQ(translated.text_sections()[0]->size(), expected_words.size() * sizeof(uint32_t)); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(std::vector(target_words, target_words + expected_words.size()), expected_words); rocjitsu::BinaryTranslator verifier( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto second_result = verifier.translate(translated); ASSERT_TRUE(second_result.ok()) << (second_result.diagnostics.empty() ? "" : second_result.diagnostics.front().message); EXPECT_EQ(second_result.elf_bytes, result.elf_bytes); } } TEST(BinaryTranslatorE2E, Gfx1250Iu8WmmaDoesNotCreditVNopsAcrossBlockBoundary) { constexpr auto branch_to_padding = gfx1250::build_sopp(gfx1250::kSCbranchScc1Sopp, {.simm16 = 2}); constexpr auto source_wmma = gfx1250::build_vop3p(gfx1250::kVWmmaI3216x16x64Iu8Vop3p, {.vdst = 32, .src0 = 256 + 64, .src1 = 256 + 128, .src2 = 256 + 192}); const uint32_t v_nop = gfx1250::build_vop1(gfx1250::kVNopVop1)[0]; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; std::vector source_words = {branch_to_padding[0], source_wmma[0], source_wmma[1]}; source_words.insert(source_words.end(), 9, v_nop); source_words.push_back(kGfx1250SEndpgm); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(source_words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); ASSERT_EQ(translated.text_sections()[0]->size(), 22 * sizeof(uint32_t)); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[1], source_wmma[0]); EXPECT_EQ(target_words[2], source_wmma[1]); for (size_t slot = 0; slot < 18; ++slot) EXPECT_EQ(target_words[3 + slot], v_nop); EXPECT_EQ(target_words[21], kGfx1250SEndpgm); rocjitsu::BinaryTranslator verifier( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto second_result = verifier.translate(translated); ASSERT_TRUE(second_result.ok()) << (second_result.diagnostics.empty() ? "" : second_result.diagnostics.front().message); EXPECT_EQ(second_result.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250TensorLoadAlwaysClearsDynamicMulticastMask) { // Multicast is selected dynamically by D# group-1 bits [15:0], not by an // instruction bit. A0 must therefore clear the runtime mask for every tensor // load and restore the descriptor word after the instruction. constexpr auto source_tensor = gfx1250::build_vimage( gfx1250::kTensorLoadToLdsVimage, {.vaddr4 = 124, .vaddr0 = 8, .vaddr1 = 0, .vaddr2 = 124, .vaddr3 = 124}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_tensor[0], source_tensor[1], source_tensor[2], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); ASSERT_GE(translated.text_sections()[0]->size(), 8 * sizeof(uint32_t)); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); // D0 occupies s[8:11] and D1 occupies s[0:7], so the first dead ordinary // scalar register is s12. constexpr auto save_d1 = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = 0, .sdst = 12}); constexpr auto clear_mask = gfx1250::build_sop2(gfx1250::kSPackHhB32B16Sop2, {.ssrc0 = 128, .ssrc1 = 0, .sdst = 0}); constexpr auto restore_d1 = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = 12, .sdst = 0}); constexpr auto dependency_barrier = gfx1250::build_sopp(gfx1250::kSWaitIdleSopp); EXPECT_EQ(target_words[0], dependency_barrier[0]); EXPECT_EQ(target_words[1], save_d1[0]); EXPECT_EQ(target_words[2], clear_mask[0]); EXPECT_EQ(target_words[3], source_tensor[0]); EXPECT_EQ(target_words[4], source_tensor[1]); EXPECT_EQ(target_words[5], source_tensor[2]); EXPECT_EQ(target_words[6], restore_d1[0]); EXPECT_EQ(target_words[7], kGfx1250SEndpgm); auto second_result = translator.translate(translated); ASSERT_TRUE(second_result.ok()) << (second_result.diagnostics.empty() ? "" : second_result.diagnostics.front().message); EXPECT_EQ(second_result.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250TensorLoadUsesHighDeadSgprUnderCompilerPressure) { constexpr auto source_tensor = gfx1250::build_vimage( gfx1250::kTensorLoadToLdsVimage, {.vaddr4 = 124, .vaddr0 = 8, .vaddr1 = 0, .vaddr2 = 124, .vaddr3 = 124}); auto image = rocjitsu::make_gfx1250_image_with_live_sgprs( source_tensor, rocjitsu::REGISTER_SET_ALLOCATABLE_SGPRS); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "s_cbranch_execz"; }), 0); EXPECT_EQ( std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "v_readfirstlane_b32_e32"; }), 0); EXPECT_EQ(std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "tensor_load_to_lds"; }), 1); ASSERT_GE(translated.text_sections()[0]->size(), 7 * sizeof(uint32_t)); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); // The generic 102-SGPR ceiling leaves s102/s103 dead, but gfx1250 scratch // allocation skips those scratch-base registers and uses s104. constexpr uint8_t kScratch = 104; constexpr auto save_d1 = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = 0, .sdst = kScratch}); constexpr auto clear_mask = gfx1250::build_sop2(gfx1250::kSPackHhB32B16Sop2, {.ssrc0 = 128, .ssrc1 = 0, .sdst = 0}); constexpr auto restore_d1 = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = kScratch, .sdst = 0}); constexpr auto dependency_barrier = gfx1250::build_sopp(gfx1250::kSWaitIdleSopp); EXPECT_EQ(target_words[0], dependency_barrier[0]); EXPECT_EQ(target_words[1], save_d1[0]); EXPECT_EQ(target_words[2], clear_mask[0]); EXPECT_EQ(target_words[3], source_tensor[0]); EXPECT_EQ(target_words[4], source_tensor[1]); EXPECT_EQ(target_words[5], source_tensor[2]); EXPECT_EQ(target_words[6], restore_d1[0]); const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250TensorScratchWaitsForOutstandingSgprProducer) { constexpr auto pending_load = gfx1250::build_smem(gfx1250::kSLoadB32Smem, {.sbase = 8, .sdata = 12, .soffset = 128}); constexpr auto source_tensor = gfx1250::build_vimage( gfx1250::kTensorLoadToLdsVimage, {.vaddr4 = 124, .vaddr0 = 8, .vaddr1 = 0, .vaddr2 = 124, .vaddr3 = 124}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {pending_load[0], pending_load[1], source_tensor[0], source_tensor[1], source_tensor[2], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); std::optional pending_load_index; std::optional wait_idle_index; std::optional scratch_write_index; for (size_t index = 0; index < decoded.size(); ++index) { if (decoded[index]->mnemonic() == "s_load_b32") pending_load_index = index; else if (decoded[index]->mnemonic() == "s_wait_idle") wait_idle_index = index; else if (decoded[index]->mnemonic() == "s_mov_b32") { ASSERT_NE(decoded[index]->raw_encoding(), nullptr); gfx1250::Sop1MachineInst move{}; std::memcpy(&move, decoded[index]->raw_encoding(), sizeof(move)); if (move.ssrc0 == 0 && move.sdst == 12) scratch_write_index = index; } } ASSERT_TRUE(pending_load_index.has_value()); ASSERT_TRUE(wait_idle_index.has_value()); ASSERT_TRUE(scratch_write_index.has_value()); EXPECT_LT(*pending_load_index, *wait_idle_index); EXPECT_EQ(*scratch_write_index, *wait_idle_index + 1u) << "the dead SGPR scratch may still have an outstanding producer"; const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250TensorLoadFailsClosedWhenEverySgprIsLive) { constexpr auto source_tensor = gfx1250::build_vimage( gfx1250::kTensorLoadToLdsVimage, {.vaddr4 = 124, .vaddr0 = 8, .vaddr1 = 0, .vaddr2 = 124, .vaddr3 = 124}); auto image = rocjitsu::make_gfx1250_image_with_live_sgprs(source_tensor, rocjitsu::REGISTER_SET_MAX_SGPRS); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(rocjitsu::has_error_containing( result, rocjitsu::DiagnosticKind::ExpandFailed, "gfx1250 tensor-load mask rule could not allocate scalar scratch")); } TEST(BinaryTranslatorE2E, Gfx1250TensorLoadPreservesGuestMaskClearWithoutRestore) { // An immediately preceding clear in the same block already guarantees the // A0 mask invariant. Preserve the guest's choice not to restore the mask. constexpr auto source_clear = gfx1250::build_sop2(gfx1250::kSPackHhB32B16Sop2, {.ssrc0 = 128, .ssrc1 = 0, .sdst = 0}); constexpr auto source_tensor = gfx1250::build_vimage( gfx1250::kTensorLoadToLdsVimage, {.vaddr4 = 124, .vaddr0 = 8, .vaddr1 = 0, .vaddr2 = 124, .vaddr3 = 124}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_clear[0], source_tensor[0], source_tensor[1], source_tensor[2], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); ASSERT_EQ(translated.text_sections()[0]->size(), 5 * sizeof(uint32_t)); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[0], source_clear[0]); EXPECT_EQ(target_words[1], source_tensor[0]); EXPECT_EQ(target_words[2], source_tensor[1]); EXPECT_EQ(target_words[3], source_tensor[2]); EXPECT_EQ(target_words[4], kGfx1250SEndpgm); auto second_result = translator.translate(translated); ASSERT_TRUE(second_result.ok()) << (second_result.diagnostics.empty() ? "" : second_result.diagnostics.front().message); EXPECT_EQ(second_result.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250TensorLoadDoesNotReuseNonzeroMaskWrite) { constexpr auto source_nonzero = gfx1250::build_sop2(gfx1250::kSPackHhB32B16Sop2, {.ssrc0 = 1, .ssrc1 = 0, .sdst = 0}); constexpr auto source_tensor = gfx1250::build_vimage( gfx1250::kTensorLoadToLdsVimage, {.vaddr4 = 124, .vaddr0 = 8, .vaddr1 = 0, .vaddr2 = 124, .vaddr3 = 124}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_nonzero[0], source_tensor[0], source_tensor[1], source_tensor[2], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto first = translator.translate(source); ASSERT_TRUE(first.ok()) << (first.diagnostics.empty() ? "" : first.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(first.elf_bytes.data(), first.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); constexpr auto save_d1 = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = 0, .sdst = 12}); constexpr auto clear_mask = gfx1250::build_sop2(gfx1250::kSPackHhB32B16Sop2, {.ssrc0 = 128, .ssrc1 = 0, .sdst = 0}); constexpr auto restore_d1 = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = 12, .sdst = 0}); constexpr auto dependency_barrier = gfx1250::build_sopp(gfx1250::kSWaitIdleSopp); EXPECT_EQ(target_words[0], source_nonzero[0]); EXPECT_EQ(target_words[1], dependency_barrier[0]); EXPECT_EQ(target_words[2], save_d1[0]); EXPECT_EQ(target_words[3], clear_mask[0]); EXPECT_EQ(target_words[4], source_tensor[0]); EXPECT_EQ(target_words[5], source_tensor[1]); EXPECT_EQ(target_words[6], source_tensor[2]); EXPECT_EQ(target_words[7], restore_d1[0]); EXPECT_EQ(target_words[8], kGfx1250SEndpgm); const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, first.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250TensorLoadDoesNotReuseClearOfDifferentDescriptor) { constexpr auto clear_s4 = gfx1250::build_sop2(gfx1250::kSPackHhB32B16Sop2, {.ssrc0 = 128, .ssrc1 = 4, .sdst = 4}); constexpr auto clear_s0 = gfx1250::build_sop2(gfx1250::kSPackHhB32B16Sop2, {.ssrc0 = 128, .ssrc1 = 0, .sdst = 0}); constexpr auto source_tensor = gfx1250::build_vimage( gfx1250::kTensorLoadToLdsVimage, {.vaddr4 = 124, .vaddr0 = 8, .vaddr1 = 0, .vaddr2 = 124, .vaddr3 = 124}); constexpr auto dependency_barrier = gfx1250::build_sopp(gfx1250::kSWaitIdleSopp); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {clear_s4[0], source_tensor[0], source_tensor[1], source_tensor[2], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); ASSERT_EQ(translated.text_sections()[0]->size(), 9 * sizeof(uint32_t)); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[0], clear_s4[0]); EXPECT_EQ(target_words[1], dependency_barrier[0]); EXPECT_EQ(target_words[3], clear_s0[0]); const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250TensorLoadDoesNotReuseMaskPrefixBypassedByBranch) { // The branch makes the tensor load a block leader, so it can bypass the // guest-authored clear and must receive a new wrapper. The guest save keeps // s12 live across the load, making s13 the first free scratch register. constexpr auto branch_to_tensor = gfx1250::build_sopp(gfx1250::kSBranchSopp, {.simm16 = 2}); constexpr auto source_save = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = 0, .sdst = 12}); constexpr auto source_clear = gfx1250::build_sop2(gfx1250::kSPackHhB32B16Sop2, {.ssrc0 = 128, .ssrc1 = 0, .sdst = 0}); constexpr auto source_tensor = gfx1250::build_vimage( gfx1250::kTensorLoadToLdsVimage, {.vaddr4 = 124, .vaddr0 = 8, .vaddr1 = 0, .vaddr2 = 124, .vaddr3 = 124}); constexpr auto source_restore = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = 12, .sdst = 0}); constexpr auto dependency_barrier = gfx1250::build_sopp(gfx1250::kSWaitIdleSopp); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {branch_to_tensor[0], source_save[0], source_clear[0], source_tensor[0], source_tensor[1], source_tensor[2], source_restore[0], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); ASSERT_GE(translated.text_sections()[0]->size(), 10 * sizeof(uint32_t)); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); constexpr auto relocated_branch = gfx1250::build_sopp(gfx1250::kSBranchSopp, {.simm16 = 0}); constexpr auto generated_save = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = 0, .sdst = 13}); constexpr auto generated_restore = gfx1250::build_sop1(gfx1250::kSMovB32Sop1, {.ssrc0 = 13, .sdst = 0}); EXPECT_EQ(target_words[0], relocated_branch[0]); EXPECT_EQ(target_words[1], dependency_barrier[0]); EXPECT_EQ(target_words[2], generated_save[0]); EXPECT_EQ(target_words[3], source_clear[0]); EXPECT_EQ(target_words[4], source_tensor[0]); EXPECT_EQ(target_words[5], source_tensor[1]); EXPECT_EQ(target_words[6], source_tensor[2]); EXPECT_EQ(target_words[7], generated_restore[0]); EXPECT_EQ(target_words[8], source_restore[0]); EXPECT_EQ(target_words[9], kGfx1250SEndpgm); auto second_result = translator.translate(translated); ASSERT_TRUE(second_result.ok()) << (second_result.diagnostics.empty() ? "" : second_result.diagnostics.front().message); EXPECT_EQ(second_result.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250RegularWmmaScaleEncodesSrc2AndWaitsForCompletion) { // Real VOP3PX2 encoding from the mxscale offline oracle. Bits [58:50] are // zero in the B0 object even though SQ decodes that unused field as a scalar // dependency. The A0 output must encode VGPR0 without changing other bits. constexpr std::array source_scale = {0xCC350000u, 0x0202954Eu, 0xCC332042u, 0x050A01CAu}; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_scale[0], source_scale[1], source_scale[2], source_scale[3], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); ASSERT_EQ(target_word_count, 6u); constexpr uint32_t kScaleSrc2Mask = 0x1ffu << 18; constexpr auto completion_wait = kGfx1250WmmaCompletionWait; EXPECT_EQ(target_words[0], source_scale[0]); EXPECT_EQ(target_words[1], (source_scale[1] & ~kScaleSrc2Mask) | (0x100u << 18)); EXPECT_EQ(target_words[2], source_scale[2]); EXPECT_EQ(target_words[3], source_scale[3]); EXPECT_EQ(target_words[4], completion_wait[0]); EXPECT_EQ(target_words[5], kGfx1250SEndpgm); } TEST(BinaryTranslatorE2E, Gfx1250FloatingScaledWmmaRejectsNonzeroCmFields) { struct CmCase { const char *name; uint16_t prefix_opcode; uint8_t prefix_cm; uint8_t matrix_cm; const char *diagnostic; }; constexpr std::array cases = { CmCase{"regular_prefix", 0x35, 1, 0, "SCL_CM must be set to zero"}, CmCase{"regular_matrix", 0x35, 0, 1, "CLAMP \"must be set to zero\""}, CmCase{"scale16_prefix", 0x3a, 1, 0, "SCL_CM must be set to zero"}, CmCase{"scale16_matrix", 0x3a, 0, 1, "CLAMP \"must be set to zero\""}, }; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; for (const CmCase &test_case : cases) { SCOPED_TRACE(test_case.name); const auto prefix = gfx1250::build_vop3p( test_case.prefix_opcode, {.clamp = test_case.prefix_cm, .src0 = 4, .src1 = 6, .src2 = 0}); const auto matrix = gfx1250::build_vop3p(gfx1250::kVWmmaF3216x16x128F8f6f4Vop3p, {.vdst = 96, .clamp = test_case.matrix_cm, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 128}); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {prefix[0], prefix[1], matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(rocjitsu::has_error_containing(result, rocjitsu::DiagnosticKind::ExpandFailed, test_case.diagnostic)); } } TEST(BinaryTranslatorE2E, Gfx1250BareFloatingWmmaRejectsClamp) { constexpr auto matrix = gfx1250::build_vop3p( gfx1250::kVWmmaF3216x16x128F8f6f4Vop3p, {.vdst = 96, .clamp = 1, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 128}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(rocjitsu::has_error_containing( result, rocjitsu::DiagnosticKind::ExpandFailed, "CLAMP \"must be set to zero\" for WMMA/SWMMAC producing floating-point results")); } TEST(BinaryTranslatorE2E, Gfx1250PreservesVop3DppExtensionWord) { // Real rocSPARSE encoding: v_fma_f32_e64_dpp is a 64-bit VOP3 followed by // one DPP control DWORD. The decoder must consume and preserve all 12 bytes, // rather than treating the control word as a new instruction. constexpr std::array source_vop3_dpp = {0xD6130000u, 0x020200FAu, 0xFF08E403u}; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {source_vop3_dpp[0], source_vop3_dpp[1], source_vop3_dpp[2], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); EXPECT_EQ(target_words[0], source_vop3_dpp[0]); EXPECT_EQ(target_words[1], source_vop3_dpp[1]); EXPECT_EQ(target_words[2], source_vop3_dpp[2]); EXPECT_EQ(target_words[3], kGfx1250SEndpgm); } TEST(BinaryTranslatorE2E, Gfx1250SplitsRegularScaleFp4AcrossMForA0) { constexpr auto scale = gfx1250::build_vop3p(0x35, {.src0 = 256 + 64, .src1 = 256 + 66, .src2 = 0, .opsel_hi = 1}); constexpr auto matrix = gfx1250::build_vop3p( gfx1250::kVWmmaF3232x16x128F4Vop3p, {.vdst = 96, .neg_hi = 4, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 256 + 48, .neg = 4}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {scale[0], scale[1], matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto *words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); std::vector pass_offsets; for (size_t i = 0; i < word_count; ++i) { if (((words[i] >> 16) & 0xffu) == 0x35u) pass_offsets.push_back(i); } ASSERT_EQ(pass_offsets.size(), 2u); for (uint16_t half = 0; half < 2; ++half) { const size_t offset = pass_offsets[half]; ASSERT_LE(offset + 4, word_count); EXPECT_EQ((words[offset] >> 11) & 0x1u, half); EXPECT_EQ(words[offset] & ((1u << 13) | (1u << 14)), 0u); EXPECT_EQ((words[offset + 1] >> 18) & 0x1ffu, 0x100u); EXPECT_EQ((words[offset + 1] >> 27) & 0x1u, 1u) << "B scale selection uses SCL_OPSEL_HI[0]"; gfx1250::Vop3pMachineInst replacement{}; std::memcpy(&replacement, words + offset + 2, sizeof(replacement)); EXPECT_EQ(replacement.op, gfx1250::kVWmmaF3216x16x128F8f6f4Vop3p); EXPECT_EQ(replacement.vdst, 96u + half * 8u); EXPECT_EQ(replacement.src0, 256u + 16u + half * 8u); EXPECT_EQ(replacement.src1, 256u + 32u); EXPECT_EQ(replacement.src2, 256u + 48u + half * 8u); EXPECT_EQ(replacement.opsel, 4u); EXPECT_EQ(replacement.pad_14, 1u); EXPECT_EQ(replacement.opsel_hi, 0u); EXPECT_EQ(replacement.neg_hi, 4u); EXPECT_EQ(replacement.clamp, 0u); EXPECT_EQ(replacement.neg, 4u); } const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250RegularScaleFp4RejectsEveryDestructiveUpperInputOverlap) { struct OverlapCase { const char *name; gfx1250::Vop3pBuilderFields matrix; }; constexpr std::array cases = { OverlapCase{"upper_a", {.vdst = 96, .src0 = 256 + 88, .src1 = 256 + 32, .src2 = 256 + 48}}, OverlapCase{"shared_b", {.vdst = 96, .src0 = 256 + 16, .src1 = 256 + 96, .src2 = 256 + 48}}, OverlapCase{"upper_c", {.vdst = 96, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 256 + 88}}, }; constexpr auto scale = gfx1250::build_vop3p(0x35, {.src0 = 256 + 64, .src1 = 256 + 66, .src2 = 0}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; for (const OverlapCase &test_case : cases) { SCOPED_TRACE(test_case.name); const auto matrix = gfx1250::build_vop3p(gfx1250::kVWmmaF3232x16x128F4Vop3p, test_case.matrix); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {scale[0], scale[1], matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(rocjitsu::has_error_containing( result, rocjitsu::DiagnosticKind::ExpandFailed, "regular-Scale 32x16 lower destination overlaps an input needed by the upper half")); } } TEST(BinaryTranslatorE2E, Gfx1250RegularScaleFp4RejectsScaleSourceOverlap) { struct ScaleCase { const char *name; uint16_t scale_src0; uint16_t scale_src1; }; constexpr std::array cases = { ScaleCase{"scale_src0", 256 + 96, 256 + 66}, ScaleCase{"scale_src1", 256 + 64, 256 + 96}, }; // Scale operands ignore VGPR-MSB and always name bank-zero registers. Keep // matrix A/B in nonzero banks while D remains in bank zero so a lowering that // incorrectly applies the matrix SRC0/SRC1 bank to the scale operands misses // this destructive overlap. constexpr auto set_vgpr_msb = gfx1250::build_sopp(gfx1250::kSSetVgprMsbSopp, {.simm16 = 0x09}); constexpr auto matrix = gfx1250::build_vop3p(gfx1250::kVWmmaF3232x16x128F4Vop3p, {.vdst = 96, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 256 + 48}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; for (const ScaleCase &test_case : cases) { SCOPED_TRACE(test_case.name); const auto scale = gfx1250::build_vop3p( 0x35, {.src0 = test_case.scale_src0, .src1 = test_case.scale_src1, .src2 = 0}); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {set_vgpr_msb[0], scale[0], scale[1], matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(rocjitsu::has_error_containing( result, rocjitsu::DiagnosticKind::ExpandFailed, "regular-Scale 32x16 lower destination overlaps an input needed by the upper half")); } } TEST(BinaryTranslatorE2E, Gfx1250RegularScaleFp4AcceptsScalarAndInlineScaleSources) { constexpr auto scale = gfx1250::build_vop3p(0x35, {.src0 = 4, .src1 = 128, .src2 = 0}); constexpr auto matrix = gfx1250::build_vop3p(gfx1250::kVWmmaF3232x16x128F4Vop3p, {.vdst = 96, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 256 + 48}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {scale[0], scale[1], matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto *words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); size_t prefix_count = 0; for (size_t i = 0; i + 1 < word_count; ++i) { if (((words[i] >> 16) & 0xffu) != 0x35u) continue; gfx1250::Vop3pMachineInst prefix{}; std::memcpy(&prefix, words + i, sizeof(prefix)); EXPECT_EQ(prefix.src0, 4u); EXPECT_EQ(prefix.src1, 128u); EXPECT_EQ(prefix.src2, 256u); ++prefix_count; } EXPECT_EQ(prefix_count, 2u); } TEST(BinaryTranslatorE2E, Gfx1250RegularScaleFp4AdjustsUpperCAndDestinationBanks) { constexpr auto set_vgpr_msb = gfx1250::build_sopp(gfx1250::kSSetVgprMsbSopp, {.simm16 = 0x90}); constexpr auto scale = gfx1250::build_vop3p(0x35, {.src0 = 256 + 64, .src1 = 256 + 66, .src2 = 0}); constexpr auto matrix = gfx1250::build_vop3p(gfx1250::kVWmmaF3232x16x128F4Vop3p, {.vdst = 252, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 256 + 252}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {set_vgpr_msb[0], scale[0], scale[1], matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto *words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); std::vector pass_offsets; for (size_t i = 0; i < word_count; ++i) { if (((words[i] >> 16) & 0xffu) == 0x35u) pass_offsets.push_back(i); } ASSERT_EQ(pass_offsets.size(), 2u); gfx1250::Vop3pMachineInst upper{}; std::memcpy(&upper, words + pass_offsets[1] + 2, sizeof(upper)); EXPECT_EQ(upper.vdst, 4u); EXPECT_EQ(upper.src0, 256u + 24u); EXPECT_EQ(upper.src2, 256u + 4u); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); std::vector modes; for (const auto &inst : decoded) { if (inst->mnemonic() == "s_set_vgpr_msb") { ASSERT_NE(inst->raw_encoding(), nullptr); modes.push_back(static_cast(inst->raw_encoding()[0] & 0xffu)); } } EXPECT_NE(std::ranges::find(modes, 0xe0u), modes.end()); EXPECT_EQ(modes.back(), 0x90u); } TEST(BinaryTranslatorE2E, Gfx1250RegularScaleFp4AdjustsUpperABank) { constexpr auto scale = gfx1250::build_vop3p(0x35, {.src0 = 256 + 64, .src1 = 256 + 66, .src2 = 0}); constexpr auto matrix = gfx1250::build_vop3p(gfx1250::kVWmmaF3232x16x128F4Vop3p, {.vdst = 96, .src0 = 256 + 252, .src1 = 256 + 32, .src2 = 256 + 48}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {scale[0], scale[1], matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto *words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); std::vector pass_offsets; for (size_t i = 0; i < word_count; ++i) { if (((words[i] >> 16) & 0xffu) == 0x35u) pass_offsets.push_back(i); } ASSERT_EQ(pass_offsets.size(), 2u); gfx1250::Vop3pMachineInst upper{}; std::memcpy(&upper, words + pass_offsets[1] + 2, sizeof(upper)); EXPECT_EQ(upper.src0, 256u + 4u); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); std::vector modes; for (const auto &inst : decoded) { if (inst->mnemonic() == "s_set_vgpr_msb") { ASSERT_NE(inst->raw_encoding(), nullptr); modes.push_back(static_cast(inst->raw_encoding()[0] & 0xffu)); } } EXPECT_NE(std::ranges::find(modes, 0x01u), modes.end()); EXPECT_EQ(modes.back(), 0u); } TEST(BinaryTranslatorE2E, Gfx1250RegularScaleFp4RejectsBank3UpperHalfOverflow) { constexpr auto set_vgpr_msb = gfx1250::build_sopp(gfx1250::kSSetVgprMsbSopp, {.simm16 = 0xf0}); constexpr auto scale = gfx1250::build_vop3p(0x35, {.src0 = 256 + 64, .src1 = 256 + 66, .src2 = 0}); constexpr auto matrix = gfx1250::build_vop3p(gfx1250::kVWmmaF3232x16x128F4Vop3p, {.vdst = 252, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 256 + 252}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {set_vgpr_msb[0], scale[0], scale[1], matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(rocjitsu::has_error_containing( result, rocjitsu::DiagnosticKind::ExpandFailed, "regular-Scale 32x16 FP4 split exceeds the VGPR address space")); } TEST(BinaryTranslatorE2E, Gfx1250SplitsScale16Fp4AcrossMOnlyForA0) { auto scale16 = gfx1250::build_vop3p(0x3a, {.neg_hi = 2, .opsel = 4, .src0 = 256 + 64, .src1 = 256 + 66, .src2 = 0, .opsel_hi = 1, .neg = 2}); scale16[0] |= uint32_t{1} << 14; constexpr auto matrix = gfx1250::build_vop3p( gfx1250::kVWmmaF3232x16x128F4Vop3p, {.vdst = 96, .neg_hi = 4, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 256 + 48, .neg = 4}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; constexpr auto completion_wait = kGfx1250WmmaCompletionWait; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {scale16[0], scale16[1], matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto *words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); std::vector pass_offsets; for (size_t i = 0; i < word_count; ++i) { if (((words[i] >> 16) & 0xffu) == 0x3au) pass_offsets.push_back(i); } ASSERT_EQ(pass_offsets.size(), 2u); for (uint16_t half = 0; half < 2; ++half) { const size_t offset = pass_offsets[half]; ASSERT_LE(offset + 4, word_count); gfx1250::Vop3pMachineInst prefix{}; std::memcpy(&prefix, words + offset, sizeof(prefix)); EXPECT_EQ(prefix.op, 0x3au); EXPECT_EQ(prefix.src0, 256u + 64u); EXPECT_EQ(prefix.src1, 256u + 66u); EXPECT_EQ(prefix.src2, 256u); EXPECT_EQ(prefix.opsel, half); EXPECT_EQ(prefix.opsel_hi, 1u); EXPECT_EQ(prefix.pad_14, 0u); EXPECT_EQ(prefix.neg_hi, 2u); EXPECT_EQ(prefix.neg, 2u); EXPECT_EQ(words[offset] & ((1u << 13) | (1u << 14)), 0u); gfx1250::Vop3pMachineInst replacement{}; std::memcpy(&replacement, words + offset + 2, sizeof(replacement)); const uint16_t destination = static_cast(96u + half * 8u); EXPECT_EQ(replacement.op, gfx1250::kVWmmaF3216x16x128F8f6f4Vop3p); EXPECT_EQ(replacement.vdst, destination); EXPECT_EQ(replacement.src0, 256u + 16u + half * 8u); EXPECT_EQ(replacement.src1, 256u + 32u); EXPECT_EQ(replacement.src2, 256u + 48u + half * 8u); EXPECT_EQ(replacement.opsel, 4u); EXPECT_EQ(replacement.pad_14, 1u); EXPECT_EQ(replacement.opsel_hi, 0u); EXPECT_EQ(replacement.neg_hi, 4u); EXPECT_EQ(replacement.neg, 4u); } EXPECT_EQ(word_count, 10u); EXPECT_EQ(words[8], completion_wait[0]); EXPECT_EQ(words[9], kGfx1250SEndpgm); const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } TEST(BinaryTranslatorE2E, Gfx1250Scale16Fp4AcceptsScalarAndInlineScaleSources) { constexpr auto scale16 = gfx1250::build_vop3p(0x3a, {.src0 = 4, .src1 = 128, .src2 = 0}); constexpr auto matrix = gfx1250::build_vop3p(gfx1250::kVWmmaF3232x16x128F4Vop3p, {.vdst = 96, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 256 + 48}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {scale16[0], scale16[1], matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto *words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); size_t prefix_count = 0; for (size_t i = 0; i + 1 < word_count; ++i) { if (((words[i] >> 16) & 0xffu) != 0x3au) continue; gfx1250::Vop3pMachineInst prefix{}; std::memcpy(&prefix, words + i, sizeof(prefix)); EXPECT_EQ(prefix.src0, 4u); EXPECT_EQ(prefix.src1, 128u); EXPECT_EQ(prefix.src2, 256u); ++prefix_count; } EXPECT_EQ(prefix_count, 2u); } TEST(BinaryTranslatorE2E, Gfx1250Scale16RejectsMisalignedOrOutOfRangeVgprPairs) { struct MatrixCase { const char *name; uint16_t opcode; }; constexpr std::array matrix_cases = { MatrixCase{"m16", gfx1250::kVWmmaF3216x16x128F8f6f4Vop3p}, MatrixCase{"m32", gfx1250::kVWmmaF3232x16x128F4Vop3p}, }; struct ScaleCase { const char *name; bool source1; uint16_t base; }; constexpr std::array scale_cases = { ScaleCase{"src0_odd", false, 65}, ScaleCase{"src1_odd", true, 67}, ScaleCase{"src0_v255", false, 255}, ScaleCase{"src1_v255", true, 255}, }; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; for (const MatrixCase &matrix_case : matrix_cases) { for (const ScaleCase &scale_case : scale_cases) { SCOPED_TRACE(std::string(matrix_case.name) + "_" + scale_case.name); const auto scale16 = gfx1250::build_vop3p( 0x3a, {.src0 = static_cast(256 + (scale_case.source1 ? 64 : scale_case.base)), .src1 = static_cast(256 + (scale_case.source1 ? scale_case.base : 66)), .src2 = 0}); const auto matrix = gfx1250::build_vop3p( matrix_case.opcode, {.vdst = 96, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 256 + 48}); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {scale16[0], scale16[1], matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(rocjitsu::has_error_containing( result, rocjitsu::DiagnosticKind::ExpandFailed, "Scale16 VGPR scale sources must be even-aligned pairs in v0:v255")); } } } TEST(BinaryTranslatorE2E, Gfx1250Scale16Fp4RejectsEveryDestructiveUpperInputOverlap) { struct OverlapCase { const char *name; gfx1250::Vop3pBuilderFields scale; gfx1250::Vop3pBuilderFields matrix; }; constexpr std::array cases = { OverlapCase{"scale_src0_second_register", {.src0 = 256 + 96, .src1 = 256 + 64, .src2 = 0}, {.vdst = 97, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 256 + 48}}, OverlapCase{"scale_src1_second_register", {.src0 = 256 + 64, .src1 = 256 + 96, .src2 = 0}, {.vdst = 97, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 256 + 48}}, OverlapCase{"upper_a", {.src0 = 256 + 64, .src1 = 256 + 66, .src2 = 0}, {.vdst = 96, .src0 = 256 + 88, .src1 = 256 + 32, .src2 = 256 + 48}}, OverlapCase{"shared_b", {.src0 = 256 + 64, .src1 = 256 + 66, .src2 = 0}, {.vdst = 96, .src0 = 256 + 16, .src1 = 256 + 96, .src2 = 256 + 48}}, OverlapCase{"upper_c", {.src0 = 256 + 64, .src1 = 256 + 66, .src2 = 0}, {.vdst = 96, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 256 + 88}}, }; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; for (const OverlapCase &test_case : cases) { SCOPED_TRACE(test_case.name); const auto scale16 = gfx1250::build_vop3p(0x3a, test_case.scale); const auto matrix = gfx1250::build_vop3p(gfx1250::kVWmmaF3232x16x128F4Vop3p, test_case.matrix); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {scale16[0], scale16[1], matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(rocjitsu::has_error_containing( result, rocjitsu::DiagnosticKind::ExpandFailed, "Scale16 32x16 lower destination overlaps an input needed by the upper half")); } } TEST(BinaryTranslatorE2E, Gfx1250Scale16Fp4AdjustsUpperCAndDestinationBanks) { constexpr auto set_vgpr_msb = gfx1250::build_sopp(gfx1250::kSSetVgprMsbSopp, {.simm16 = 0x90}); constexpr auto scale16 = gfx1250::build_vop3p(0x3a, {.src0 = 256 + 64, .src1 = 256 + 66, .src2 = 256}); constexpr auto matrix = gfx1250::build_vop3p(gfx1250::kVWmmaF3232x16x128F4Vop3p, {.vdst = 252, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 256 + 252}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {set_vgpr_msb[0], scale16[0], scale16[1], matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto *words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); std::vector pass_offsets; for (size_t i = 0; i < word_count; ++i) { if (((words[i] >> 16) & 0xffu) == 0x3au) pass_offsets.push_back(i); } ASSERT_EQ(pass_offsets.size(), 2u); gfx1250::Vop3pMachineInst upper{}; std::memcpy(&upper, words + pass_offsets[1] + 2, sizeof(upper)); EXPECT_EQ(upper.vdst, 4u); EXPECT_EQ(upper.src0, 256u + 24u); EXPECT_EQ(upper.src2, 256u + 4u); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); std::vector modes; for (const auto &inst : decoded) { if (inst->mnemonic() == "s_set_vgpr_msb") { ASSERT_NE(inst->raw_encoding(), nullptr); modes.push_back(static_cast(inst->raw_encoding()[0] & 0xffu)); } } EXPECT_NE(std::ranges::find(modes, 0xe0u), modes.end()); EXPECT_EQ(modes.back(), 0x90u); } TEST(BinaryTranslatorE2E, Gfx1250Scale16Fp4RejectsBank3UpperHalfOverflow) { constexpr auto set_vgpr_msb = gfx1250::build_sopp(gfx1250::kSSetVgprMsbSopp, {.simm16 = 0xf0}); constexpr auto scale16 = gfx1250::build_vop3p(0x3a, {.src0 = 256 + 64, .src1 = 256 + 66, .src2 = 256}); constexpr auto matrix = gfx1250::build_vop3p(gfx1250::kVWmmaF3232x16x128F4Vop3p, {.vdst = 252, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 256 + 252}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {set_vgpr_msb[0], scale16[0], scale16[1], matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE( rocjitsu::has_error_containing(result, rocjitsu::DiagnosticKind::ExpandFailed, "Scale16 32x16 FP4 split exceeds the VGPR address space")); } TEST(BinaryTranslatorE2E, Gfx1250PreservesNativeM16Scale16ForA0) { constexpr auto scale16 = gfx1250::build_vop3p( 0x3a, {.neg_hi = 2, .opsel = 1, .src0 = 4, .src1 = 6, .src2 = 0, .opsel_hi = 1}); auto matrix = gfx1250::build_vop3p(gfx1250::kVWmmaF3216x16x128F8f6f4Vop3p, {.vdst = 96, .neg_hi = 4, .opsel = 3, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 128, .neg = 4}); matrix[0] |= uint32_t{1} << 14; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; constexpr auto completion_wait = kGfx1250WmmaCompletionWait; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {scale16[0], scale16[1], matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); ASSERT_EQ(target_word_count, 6u); constexpr uint32_t kUnusedScaleSrc2Mask = 0x1ffu << 18; EXPECT_EQ(target_words[0], scale16[0]); EXPECT_EQ(target_words[1], (scale16[1] & ~kUnusedScaleSrc2Mask) | (static_cast(256u) << 18)); EXPECT_EQ(target_words[2], matrix[0]); EXPECT_EQ(target_words[3], matrix[1]); EXPECT_EQ(target_words[4], completion_wait[0]); EXPECT_EQ(target_words[5], kGfx1250SEndpgm); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); EXPECT_EQ(std::ranges::count_if(decoded, [](const auto &candidate) { return candidate->mnemonic() == "v_wmma_scale16_f32_16x16x128_f8f6f4"; }), 1); } TEST(BinaryTranslatorE2E, Gfx1250Scale16M32DoesNotNeedScratchWhenVgprsAreFull) { using namespace rocr::llvm::amdhsa; constexpr auto scale16 = gfx1250::build_vop3p(0x3a, {.src0 = 256 + 64, .src1 = 256 + 66, .src2 = 256}); constexpr auto matrix = gfx1250::build_vop3p(gfx1250::kVWmmaF3232x16x128F4Vop3p, {.vdst = 96, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 256 + 48}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {scale16[0], scale16[1], matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); auto descriptor = rocjitsu::read_kernel_descriptor_for_test(rodata->data()); AMDHSA_BITS_SET(descriptor.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, 63); rocjitsu::write_kernel_descriptor_for_test(image.data() + rodata->sectionOffset(), descriptor); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); auto options = gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0); options.debug_min_free_vgpr = 256; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, options); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); struct ScratchAccess { size_t word_offset; uint8_t vgpr; uint32_t byte_offset; }; std::vector stores; std::vector loads; size_t wait_store_count = 0; size_t wait_load_count = 0; for (const auto &inst : decoded) { if (inst->mnemonic() == "s_wait_storecnt") { ++wait_store_count; continue; } if (inst->mnemonic() == "s_wait_loadcnt") { ++wait_load_count; continue; } if (inst->mnemonic() != "scratch_store_b32" && inst->mnemonic() != "scratch_load_b32") continue; ASSERT_NE(inst->raw_encoding(), nullptr); gfx1250::VscratchMachineInst scratch{}; std::memcpy(&scratch, inst->raw_encoding(), sizeof(scratch)); const bool is_load = inst->mnemonic() == "scratch_load_b32"; (is_load ? loads : stores) .push_back( ScratchAccess{.word_offset = inst->src_loc() / sizeof(uint32_t), .vgpr = static_cast(is_load ? scratch.vdst : scratch.vsrc), .byte_offset = scratch.ioffset}); } EXPECT_TRUE(stores.empty()); EXPECT_TRUE(loads.empty()); EXPECT_EQ(wait_store_count, 0u); EXPECT_EQ(wait_load_count, 0u); EXPECT_EQ(std::ranges::count_if(decoded, [](const auto &candidate) { return candidate->mnemonic() == "v_wmma_scale16_f32_16x16x128_f8f6f4"; }), 2); ASSERT_GT(target_word_count, 0u); EXPECT_EQ(target_words[target_word_count - 1], kGfx1250SEndpgm); } TEST(BinaryTranslatorE2E, Gfx1250Scale16M32IgnoresUnavailableScratchSpillSpace) { using namespace rocr::llvm::amdhsa; constexpr auto scale16 = gfx1250::build_vop3p(0x3a, {.src0 = 256 + 64, .src1 = 256 + 66, .src2 = 256}); constexpr auto matrix = gfx1250::build_vop3p(gfx1250::kVWmmaF3232x16x128F4Vop3p, {.vdst = 96, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = 256 + 48}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {scale16[0], scale16[1], matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); auto descriptor = rocjitsu::read_kernel_descriptor_for_test(rodata->data()); AMDHSA_BITS_SET(descriptor.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, 63); descriptor.private_segment_fixed_size = 0x7ffffffdu; rocjitsu::write_kernel_descriptor_for_test(image.data() + rodata->sectionOffset(), descriptor); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); auto options = gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0); options.debug_min_free_vgpr = 256; rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, options); const auto result = translator.translate(source); EXPECT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); } // The standalone form produces floating-point results, so it carries the same // control requirement as the scaled entry points. A malformed encoding must be // refused rather than split. TEST(BinaryTranslatorE2E, Gfx1250Standalone32x16Fp4RejectsNonZeroClampForA0) { constexpr auto matrix = gfx1250::build_vop3p( gfx1250::kVWmmaF3232x16x128F4Vop3p, {.vdst = 96, .clamp = 1, .src0 = 256 + 16, .src1 = 256 + 40, .src2 = 256 + 64}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image) << "fail-closed must leave the object unchanged"; EXPECT_TRUE(rocjitsu::has_error_containing( result, rocjitsu::DiagnosticKind::ExpandFailed, "Input is malformed, CLAMP \"must be set to zero\" for WMMA/SWMMAC producing " "floating-point results")); } TEST(BinaryTranslatorE2E, Gfx1250SplitsStandalone32x16Fp4IntoScaledHalvesForA0) { // A0 has no M=32 FP4 matrix opcode, and it cannot resume an unprefixed // low-precision matrix body after a trap. The lowering is therefore two M=16 // halves, each carrying a neutral inline-zero scale prefix. constexpr uint16_t kMatrixA = 16; constexpr uint16_t kMatrixB = 40; constexpr uint16_t kMatrixC = 64; constexpr uint8_t kDestination = 96; constexpr uint16_t kHalfDwords = 8; constexpr auto matrix = gfx1250::build_vop3p(gfx1250::kVWmmaF3232x16x128F4Vop3p, {.vdst = kDestination, .src0 = 256 + kMatrixA, .src1 = 256 + kMatrixB, .src2 = 256 + kMatrixC}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); // The B0-only M=32 opcode must be gone, replaced by two scaled M=16 forms. EXPECT_EQ( std::ranges::count_if( decoded, [](const auto &inst) { return inst->mnemonic() == "v_wmma_f32_32x16x128_f4"; }), 0); EXPECT_EQ(std::ranges::count_if(decoded, [](const auto &inst) { return inst->mnemonic() == "v_wmma_scale_f32_16x16x128_f8f6f4"; }), 2); const auto *target_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t target_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); constexpr uint32_t kScalePrefixOp = 0x35; std::vector pass_offsets; for (size_t i = 0; i < target_word_count; ++i) { if (((target_words[i] >> 16) & 0xffu) == kScalePrefixOp) pass_offsets.push_back(i); } ASSERT_EQ(pass_offsets.size(), 2u) << "each half must carry its own scale prefix"; constexpr uint16_t kInlineZero = 128; constexpr uint16_t kVgpr0 = 256; for (size_t half = 0; half < pass_offsets.size(); ++half) { const size_t prefix = pass_offsets[half]; ASSERT_LT(prefix + 3, target_word_count); // Inline integer zero in a scale source reads as the E8M0 value for 1.0. EXPECT_EQ(target_words[prefix + 1] & 0x1ffu, kInlineZero) << "matrix A scale must be neutral"; EXPECT_EQ((target_words[prefix + 1] >> 9) & 0x1ffu, kInlineZero) << "matrix B scale must be neutral"; EXPECT_EQ((target_words[prefix + 1] >> 18) & 0x1ffu, kVgpr0) << "the unused scale SRC2 must encode VGPR0"; // The matrix body is the A0 M=16 operation with both formats forced to FP4 // and A, C, and D sliced by eight dwords while B is shared. Read it back as // the encoding struct so the whole matrix-B format is checked: the high bit // alone leaves opsel_hi free to select formats 5 through 7. gfx1250::Vop3pMachineInst body{}; std::memcpy(&body, &target_words[prefix + 2], sizeof(body)); EXPECT_EQ(body.op, gfx1250::kVWmmaF3216x16x128F8f6f4Vop3p); EXPECT_EQ(body.opsel, 4u) << "matrix A format must be FP4"; EXPECT_EQ(body.pad_14, 1u) << "matrix B format must be FP4"; EXPECT_EQ(body.opsel_hi, 0u) << "matrix B format must be exactly FP4"; const uint16_t delta = static_cast(half * kHalfDwords); EXPECT_EQ(body.vdst, kDestination + delta); EXPECT_EQ(body.src0, 256u + kMatrixA + delta); EXPECT_EQ(body.src1, 256u + kMatrixB) << "matrix B is shared by both halves"; EXPECT_EQ(body.src2, 256u + kMatrixC + delta); } const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } // The upper half slices eight dwords off matrix A, so a base near the top of a // bank makes the split cross into the next one. That must produce a bank // transition and restore the entry mode afterwards. TEST(BinaryTranslatorE2E, Gfx1250Standalone32x16Fp4AdjustsUpperABank) { constexpr auto matrix = gfx1250::build_vop3p(gfx1250::kVWmmaF3232x16x128F4Vop3p, {.vdst = 96, .src0 = 256 + 252, .src1 = 256 + 32, .src2 = 256 + 48}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto *words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); std::vector pass_offsets; for (size_t i = 0; i < word_count; ++i) { if (((words[i] >> 16) & 0xffu) == 0x35u) pass_offsets.push_back(i); } ASSERT_EQ(pass_offsets.size(), 2u); // 252 + 8 wraps to 4 in the next bank. gfx1250::Vop3pMachineInst upper{}; std::memcpy(&upper, words + pass_offsets[1] + 2, sizeof(upper)); EXPECT_EQ(upper.src0, 256u + 4u); EXPECT_EQ(upper.src1, 256u + 32u) << "matrix B is not sliced and keeps its bank"; const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); std::vector modes; for (size_t index = 0; index < decoded.size(); ++index) { if (decoded[index]->mnemonic() != "s_set_vgpr_msb") continue; ASSERT_NE(decoded[index]->raw_encoding(), nullptr); modes.push_back(static_cast(decoded[index]->raw_encoding()[0] & 0xffu)); ASSERT_GT(index, 0u); EXPECT_EQ(decoded[index - 1]->mnemonic(), "s_wait_xcnt") << "each bank change must be preceded by a drain"; } ASSERT_GE(modes.size(), 2u) << "the crossing must emit a transition and a restore"; EXPECT_EQ(modes.front(), 0x01u) << "the upper half selects the next Src0 bank"; EXPECT_EQ(modes.back(), 0x00u) << "the entry mode must be restored"; const auto second = translator.translate(translated); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, result.elf_bytes); } // Matrix A and B must be register ranges; the standalone form has no lowering // for anything else and must leave the object untouched. TEST(BinaryTranslatorE2E, Gfx1250FailsClosedOnStandalone32x16Fp4NonVgprMatrixForA0) { constexpr uint16_t kInlineZero = 128; const std::vector>> cases = { {"matrix A", gfx1250::build_vop3p(gfx1250::kVWmmaF3232x16x128F4Vop3p, {.vdst = 96, .src0 = kInlineZero, .src1 = 256 + 32, .src2 = 256 + 48})}, {"matrix B", gfx1250::build_vop3p(gfx1250::kVWmmaF3232x16x128F4Vop3p, {.vdst = 96, .src0 = 256 + 16, .src1 = kInlineZero, .src2 = 256 + 48})}, }; for (const auto &[name, matrix] : cases) { SCOPED_TRACE(name); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image) << "a fail-closed translation must leave the object unchanged"; // Matched in full rather than by its tail: the scaled caller reports a // message of the same shape, so a suffix would not notice this rule // adopting the other caller's wording. EXPECT_TRUE(rocjitsu::has_error_containing(result, rocjitsu::DiagnosticKind::ExpandFailed, "gfx1250 32x16 FP4 operands are not VGPR ranges")); } } // An inline C operand names no register range, so neither half may slice it. TEST(BinaryTranslatorE2E, Gfx1250Standalone32x16Fp4PreservesInlineMatrixCForA0) { constexpr uint16_t kInlineZero = 128; constexpr auto matrix = gfx1250::build_vop3p(gfx1250::kVWmmaF3232x16x128F4Vop3p, {.vdst = 96, .src0 = 256 + 16, .src1 = 256 + 32, .src2 = kInlineZero}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); const auto *words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); std::vector pass_offsets; for (size_t i = 0; i < word_count; ++i) { if (((words[i] >> 16) & 0xffu) == 0x35u) pass_offsets.push_back(i); } ASSERT_EQ(pass_offsets.size(), 2u); for (const size_t prefix : pass_offsets) { gfx1250::Vop3pMachineInst body{}; std::memcpy(&body, words + prefix + 2, sizeof(body)); EXPECT_EQ(body.src2, kInlineZero) << "an inline C operand must be copied to both halves"; } } // Numerical evidence that the M=32 FP4 split preserves values. The encoding // tests above only prove the right bits are emitted; this one runs both the // original B0-only M=32 instruction and the translated pair of scaled M=16 // halves on the instruction simulator over identical inputs and requires // bit-identical destination registers. The two paths dispatch through // different execution kernels (exec_wmma_f32 versus // exec_wmma_f32_scaled_mixed, the latter substituting the neutral E8M0 word // for the inline-zero scale operands), so the comparison is a real // differential rather than a tautology. TEST(BinaryTranslatorE2E, Gfx1250Standalone32x16Fp4SplitMatchesUnsplitExecution) { // Every operand range stays inside VGPR bank 0 so the lowering needs no // s_set_vgpr_msb transitions and the translated body is exactly two pairs. constexpr uint16_t kMatrixA = 16; // A occupies 16 dwords, halves at +0/+8. constexpr uint16_t kMatrixB = 32; // B occupies 8 dwords and is shared. constexpr uint16_t kMatrixC = 48; // C occupies 16 dwords, halves at +0/+8. constexpr uint8_t kDestination = 64; constexpr uint32_t kMatrixARegs = 16; constexpr uint32_t kMatrixBRegs = 8; constexpr uint32_t kMatrixCRegs = 16; constexpr uint32_t kDestinationRegs = 16; constexpr auto matrix = gfx1250::build_vop3p(gfx1250::kVWmmaF3232x16x128F4Vop3p, {.vdst = kDestination, .src0 = 256 + kMatrixA, .src1 = 256 + kMatrixB, .src2 = 256 + kMatrixC}); constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; constexpr auto completion_wait = kGfx1250WmmaCompletionWait; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {matrix[0], matrix[1], kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const auto *text_words = reinterpret_cast(translated.text_sections()[0]->data()); const size_t text_word_count = translated.text_sections()[0]->size() / sizeof(uint32_t); auto decoder = rocjitsu::Decoder::create(ROCJITSU_CODE_ARCH_GFX1250); ASSERT_NE(decoder, nullptr); // Collect the translated body, stopping at the terminator. std::vector body_words; for (size_t offset = 0; offset < text_word_count;) { std::unique_ptr inst(decoder->decode(text_words + offset)); ASSERT_NE(inst, nullptr) << "translated word " << offset << " failed to decode"; if (std::string_view(inst->mnemonic()) == "s_endpgm") break; EXPECT_NE(std::string_view(inst->mnemonic()), "s_set_vgpr_msb") << "bank-0 operands must not need a mode transition"; const size_t words = static_cast(inst->size()) / sizeof(uint32_t); ASSERT_GT(words, 0u); body_words.insert(body_words.end(), text_words + offset, text_words + offset + words); offset += words; } ASSERT_EQ(body_words.size(), 9u) << "expected two four-word VOP3PX2 pairs and one completion wait"; EXPECT_EQ(body_words.back(), completion_wait[0]); body_words.pop_back(); rocjitsu::amdgpu::GpuMemory gpu_mem("gfx1250_fp4_split_diff_mem"); rocjitsu::amdgpu::L2Cache l2("gfx1250_fp4_split_diff_l2"); rocjitsu::amdgpu::ComputeUnitCore::Config cfg{}; cfg.arch = ROCJITSU_CODE_ARCH_GFX1250; cfg.num_wf_slots = 1; cfg.sgprs_per_wf = 106; cfg.vgprs_per_wf = 256; cfg.lds_size_kb = 64; auto cu = rocjitsu::amdgpu::ComputeUnitCore::create("gfx1250", cfg, &gpu_mem, &l2); ASSERT_NE(cu, nullptr); auto *wf = cu->dispatch_wf(0, 0, cfg.sgprs_per_wf, cfg.vgprs_per_wf); ASSERT_NE(wf, nullptr); ASSERT_EQ(wf->wf_size(), 32u); wf->set_exec((1ULL << wf->wf_size()) - 1ULL); const uint32_t vb = wf->vgpr_alloc().base; const uint32_t lanes = wf->wf_size(); // Deterministic inputs. A and B are packed FP4, so raw pseudo-random dwords // are all valid. C holds FP32 accumulators, so the exponent is confined to a // wide but finite range and the interesting encodings are injected by hand. auto fill_inputs = [&] { uint32_t state = 0x1234abcdu; auto next = [&state] { state = state * 1664525u + 1013904223u; return state; }; for (uint32_t reg = 0; reg < kMatrixARegs; ++reg) for (uint32_t lane = 0; lane < lanes; ++lane) cu->write_vgpr(vb + kMatrixA + reg, lane, next()); for (uint32_t reg = 0; reg < kMatrixBRegs; ++reg) for (uint32_t lane = 0; lane < lanes; ++lane) cu->write_vgpr(vb + kMatrixB + reg, lane, next()); for (uint32_t reg = 0; reg < kMatrixCRegs; ++reg) { for (uint32_t lane = 0; lane < lanes; ++lane) { const uint32_t bits = next(); const uint32_t exponent = ((bits >> 23u) % 60u) + 100u; cu->write_vgpr(vb + kMatrixC + reg, lane, (bits & 0x807fffffu) | (exponent << 23u)); } } // qNaN, -Inf, +Inf, smallest denormal, -0.0, +0.0. static constexpr std::array kSpecials = {0x7fc00000u, 0xff800000u, 0x7f800000u, 0x00000001u, 0x80000000u, 0x00000000u}; // Placed in both halves of C. Each half is accumulated by a different // instruction in the translated form, so confining these to one half would // leave the other never seeing them. for (uint32_t i = 0; i < kSpecials.size(); ++i) { cu->write_vgpr(vb + kMatrixC + i, i * 5u, kSpecials[i]); cu->write_vgpr(vb + kMatrixC + 8u + i, i * 5u, kSpecials[i]); } }; auto zero_destination = [&] { for (uint32_t reg = 0; reg < kDestinationRegs; ++reg) for (uint32_t lane = 0; lane < lanes; ++lane) cu->write_vgpr(vb + kDestination + reg, lane, 0u); }; auto snapshot_destination = [&] { std::vector out; out.reserve(kDestinationRegs * lanes); for (uint32_t reg = 0; reg < kDestinationRegs; ++reg) for (uint32_t lane = 0; lane < lanes; ++lane) out.push_back(cu->read_vgpr(vb + kDestination + reg, lane)); return out; }; // Run A: the untranslated B0 M=32 instruction. fill_inputs(); zero_destination(); { const std::array words = {matrix[0], matrix[1]}; std::unique_ptr inst(decoder->decode(words.data())); ASSERT_NE(inst, nullptr); ASSERT_EQ(std::string_view(inst->mnemonic()), "v_wmma_f32_32x16x128_f4"); cu->execute_instruction(inst.get(), *wf); } const std::vector run_a = snapshot_destination(); // Both halves of the destination must have been written, or a split that // silently dropped one of them would still compare equal. const size_t half_size = 8u * lanes; ASSERT_TRUE(std::ranges::any_of(run_a.begin(), run_a.begin() + half_size, [](uint32_t v) { return v != 0u; })) << "the unsplit run left the lower destination half empty"; ASSERT_TRUE(std::ranges::any_of(run_a.begin() + half_size, run_a.end(), [](uint32_t v) { return v != 0u; })) << "the unsplit run left the upper destination half empty"; // Run B: the translated pair of scaled M=16 halves over the same inputs. fill_inputs(); zero_destination(); size_t executed = 0; for (size_t offset = 0; offset < body_words.size();) { std::unique_ptr inst(decoder->decode(body_words.data() + offset)); ASSERT_NE(inst, nullptr); cu->execute_instruction(inst.get(), *wf); offset += static_cast(inst->size()) / sizeof(uint32_t); ++executed; } EXPECT_EQ(executed, 2u); const std::vector run_b = snapshot_destination(); EXPECT_EQ(run_a, run_b); if (run_a != run_b) { for (size_t i = 0; i < run_a.size(); ++i) { if (run_a[i] == run_b[i]) continue; ADD_FAILURE() << "first divergence at D reg " << (i / lanes) << " lane " << (i % lanes) << ": unsplit=0x" << std::hex << run_a[i] << " split=0x" << run_b[i] << std::dec; break; } } if (!wf->is_halted()) wf->halt(); } TEST(BinaryTranslatorE2E, Gfx1250RecoversAddNcU64PcBuilder) { constexpr uint16_t kPcSreg = 4; constexpr uint16_t kGfx1250SAddNcU64Opcode = 83; constexpr uint16_t kLiteralOperand = 255; constexpr uint32_t kTargetOffset = 20; constexpr uint32_t kGfx1250SNop = 0xBF800000u; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; const std::vector words = { rocjitsu::build_s_getpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_GFX1250), rocjitsu::build_sop2_encoding(ROCJITSU_CODE_ARCH_GFX1250, kGfx1250SAddNcU64Opcode, kPcSreg, kPcSreg, kLiteralOperand), kTargetOffset - sizeof(uint32_t), rocjitsu::build_s_setpc_b64(kPcSreg, ROCJITSU_CODE_ARCH_GFX1250), kGfx1250SNop, kGfx1250SEndpgm, }; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); EXPECT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); } TEST(BinaryTranslatorE2E, Gfx1250RecoveredBuilderRewriteKeepsTheXcntDrain) { // A recovered gfx1250 signed-delta range holds the s_wait_xcnt the compiler // put ahead of the pair writes. The canonical builder overwrites the whole // range and writes that same pair, so the drain has to be re-emitted in front // of it instead of being NOP-filled away with the rest of the range. constexpr uint64_t kWord = sizeof(uint32_t); constexpr uint16_t kPcSreg = 8; constexpr uint16_t kLiteralOperand = 255; std::vector text(8 * kWord, 0); rocjitsu::KernelTextLayout layout; layout.body_end = text.size(); layout.blocks.push_back( {.source_start = 0, .source_end = 4 * kWord, .target_start = 0, .target_end = 4 * kWord}); layout.blocks.push_back({.source_start = 4 * kWord, .source_end = 8 * kWord, .target_start = 4 * kWord, .target_end = 8 * kWord}); layout.recovered_builder_fixups.push_back({.source_target_offset = 4 * kWord, .source_call_sreg = kPcSreg, .source_requires_xcnt_drain = true, .target_getpc_offset = 0, .target_recovery_begin_offset = kWord, .target_recovery_end_offset = 4 * kWord}); const auto patched = rocjitsu::patch_recovered_builder_fixups(text, layout, ROCJITSU_CODE_ARCH_GFX1250); ASSERT_TRUE(patched.ok) << patched.message; std::array replacement{}; std::memcpy(replacement.data(), text.data() + kWord, replacement.size() * sizeof(uint32_t)); EXPECT_EQ(replacement[0], rocjitsu::build_s_wait_xcnt(ROCJITSU_CODE_ARCH_GFX1250).value()); EXPECT_EQ(replacement[1], rocjitsu::build_sop2_encoding(ROCJITSU_CODE_ARCH_GFX1250, gfx1250::kSAddNcU64Sop2, kPcSreg, kPcSreg, kLiteralOperand)); // s_get_pc_i64 leaves the address of the next instruction, and the target is // the block at 4 * kWord. EXPECT_EQ(replacement[2], 3 * kWord); } TEST(BinaryTranslatorE2E, Gfx1250RecoveredBuilderRewriteOmitsAnUnneededXcntDrain) { constexpr uint64_t kWord = sizeof(uint32_t); constexpr uint16_t kPcSreg = 8; constexpr uint16_t kLiteralOperand = 255; std::vector text(8 * kWord, 0); rocjitsu::KernelTextLayout layout; layout.body_end = text.size(); layout.blocks.push_back( {.source_start = 0, .source_end = 4 * kWord, .target_start = 0, .target_end = 4 * kWord}); layout.blocks.push_back({.source_start = 4 * kWord, .source_end = 8 * kWord, .target_start = 4 * kWord, .target_end = 8 * kWord}); layout.recovered_builder_fixups.push_back({.source_target_offset = 4 * kWord, .source_call_sreg = kPcSreg, .target_getpc_offset = 0, .target_recovery_begin_offset = kWord, .target_recovery_end_offset = 4 * kWord}); const auto patched = rocjitsu::patch_recovered_builder_fixups(text, layout, ROCJITSU_CODE_ARCH_GFX1250); ASSERT_TRUE(patched.ok) << patched.message; uint32_t first = 0; std::memcpy(&first, text.data() + kWord, sizeof(first)); EXPECT_EQ(first, rocjitsu::build_sop2_encoding(ROCJITSU_CODE_ARCH_GFX1250, gfx1250::kSAddNcU64Sop2, kPcSreg, kPcSreg, kLiteralOperand)); } TEST(BinaryTranslatorE2E, RecoveredBuilderReuseRejectsDisagreeingXcntDrain) { // Only the first consumer of a shared range writes the replacement, so a // second consumer that wants different words must not be silently dropped. constexpr uint64_t kWord = sizeof(uint32_t); constexpr uint16_t kPcSreg = 8; std::vector text(8 * kWord, 0); rocjitsu::KernelTextLayout layout; layout.body_end = text.size(); layout.blocks.push_back( {.source_start = 0, .source_end = 4 * kWord, .target_start = 0, .target_end = 4 * kWord}); layout.blocks.push_back({.source_start = 4 * kWord, .source_end = 8 * kWord, .target_start = 4 * kWord, .target_end = 8 * kWord}); layout.recovered_builder_fixups.push_back({.source_target_offset = 4 * kWord, .source_call_sreg = kPcSreg, .target_getpc_offset = 0, .target_recovery_begin_offset = kWord, .target_recovery_end_offset = 4 * kWord}); layout.recovered_builder_fixups.push_back({.source_target_offset = 4 * kWord, .source_call_sreg = kPcSreg, .source_requires_xcnt_drain = true, .target_getpc_offset = 0, .target_recovery_begin_offset = kWord, .target_recovery_end_offset = 4 * kWord}); const auto patched = rocjitsu::patch_recovered_builder_fixups(text, layout, ROCJITSU_CODE_ARCH_GFX1250); EXPECT_FALSE(patched.ok); EXPECT_NE(patched.message.find("incompatible replacements"), std::string::npos) << patched.message; } TEST(BinaryTranslatorE2E, Gfx1250IgnoresUndecodablePaddingAfterTerminator) { constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {kGfx1250SEndpgm, 0, kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); EXPECT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); } TEST(BinaryTranslatorE2E, Gfx1250RejectsReachableZeroInKernelBody) { constexpr uint32_t kGfx1250SNop = 0xBF800000u; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {kGfx1250SNop, 0, kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_EQ(result.elf_bytes, image); EXPECT_TRUE(rocjitsu::has_error_containing( result, rocjitsu::DiagnosticKind::Legalization, "reachable kernel code falls through into undecodable .text bytes")); } TEST(BinaryTranslatorE2E, Gfx1250AcceptsClangUnreachableKernelStub) { // clang emits this one-instruction body for rocPRIM target-specialized // trampolines whose source ends in __builtin_unreachable(). The following // zero is alignment padding, not a second instruction. constexpr uint32_t kSetReplayMode = 0xb9800641u; constexpr uint32_t kLiteralOne = 1u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {kSetReplayMode, kLiteralOne, 0}, 2); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); ASSERT_EQ(decoded.size(), 2u) << "the synthesized endpgm must be the only instruction after the stub body"; ASSERT_NE(decoded[0]->raw_encoding(), nullptr); EXPECT_EQ(decoded[0]->mnemonic(), "s_setreg_imm32_b32"); EXPECT_EQ(decoded[0]->raw_encoding()[0], kSetReplayMode); EXPECT_EQ(decoded[0]->raw_encoding()[1], kLiteralOne); EXPECT_EQ(decoded[1]->mnemonic(), "s_endpgm"); const auto kernel_symbol = rocjitsu::find_elf_symbol_for_test(result.elf_bytes, "kernel"); ASSERT_TRUE(kernel_symbol.has_value()); EXPECT_EQ(kernel_symbol->st_size, 3 * sizeof(uint32_t)) << "the translated function extent must include its synthesized terminator"; } TEST(BinaryTranslatorE2E, Gfx1250AcceptsClangUnreachableKernelStubWithPrefetchPrologue) { // Newer clang adds the exact gfx1250 prefetch prologue in front of the same // unreachable rocPRIM target-specialization body. constexpr uint32_t kGlobalPrefetchB8[] = {0xee174000u, 0x00040000u, 0u}; constexpr uint32_t kVNop = 0x7e000000u; constexpr uint32_t kSetReplayMode[] = {0xb9800641u, 1u}; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text( {kGlobalPrefetchB8[0], kGlobalPrefetchB8[1], kGlobalPrefetchB8[2], kVNop, kSetReplayMode[0], kSetReplayMode[1], 0}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_FALSE(translated.text_sections().empty()); const auto decoded = decode_text_instructions(*translated.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); ASSERT_EQ(decoded.size(), 4u) << "the synthesized endpgm must be the only instruction after the stub body"; EXPECT_EQ(decoded[0]->mnemonic(), "global_prefetch_b8"); EXPECT_EQ(decoded[1]->mnemonic(), "v_nop_e32"); EXPECT_EQ(decoded[2]->mnemonic(), "s_setreg_imm32_b32"); ASSERT_NE(decoded[2]->raw_encoding(), nullptr); EXPECT_EQ(decoded[2]->raw_encoding()[0], kSetReplayMode[0]); EXPECT_EQ(decoded[2]->raw_encoding()[1], kSetReplayMode[1]); EXPECT_EQ(decoded[3]->mnemonic(), "s_endpgm"); } TEST(BinaryTranslatorE2E, Gfx1250MaterializesSectionFinalClangUnreachableKernelStub) { constexpr uint32_t kSetReplayMode = 0xb9800641u; constexpr uint32_t kLiteralOne = 1u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({kSetReplayMode, kLiteralOne}); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto first = translator.translate(source); ASSERT_TRUE(first.ok()) << (first.diagnostics.empty() ? "" : first.diagnostics.front().message); rocjitsu::AmdGpuCodeObject first_output(first.elf_bytes.data(), first.elf_bytes.size()); const auto decoded = decode_text_instructions(*first_output.text_sections()[0], ROCJITSU_CODE_ARCH_GFX1250); ASSERT_EQ(decoded.size(), 2u); EXPECT_EQ(decoded[0]->mnemonic(), "s_setreg_imm32_b32"); EXPECT_EQ(decoded[1]->mnemonic(), "s_endpgm"); auto second = translator.translate(first_output); ASSERT_TRUE(second.ok()) << (second.diagnostics.empty() ? "" : second.diagnostics.front().message); EXPECT_EQ(second.elf_bytes, first.elf_bytes); } TEST(BinaryTranslatorE2E, MatchedSemanticExpandRuleFailureIsDiagnostic) { auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(); rocjitsu::AmdGpuCodeObject source_layout(image.data(), image.size()); ASSERT_TRUE(source_layout.is_valid()); ASSERT_FALSE(source_layout.text_sections().empty()); const auto words = make_cdna4_bitop3_b16_unsupported_op_sel_words(); const auto *source_text = source_layout.text_sections()[0]; ASSERT_EQ(source_text->size(), words.size() * sizeof(uint32_t)); std::memcpy(image.data() + source_text->sectionOffset(), words.data(), words.size() * sizeof(uint32_t)); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); ASSERT_TRUE(source.is_valid()); rocjitsu::BinaryTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); auto result = translator.translate(source); EXPECT_FALSE(result.ok()); ASSERT_FALSE(result.diagnostics.empty()); const auto diagnostic = std::ranges::find_if(result.diagnostics, [](const auto &d) { return d.kind == rocjitsu::DiagnosticKind::ExpandFailed; }); ASSERT_NE(diagnostic, result.diagnostics.end()); EXPECT_EQ(diagnostic->severity, rocjitsu::DiagnosticSeverity::Error); EXPECT_EQ(diagnostic->guest_offset, std::optional(0)); EXPECT_FALSE(diagnostic->message.empty()); } // Regression: the gfx1250 B0-to-A0 patcher used to zero COMPUTE_PGM_RSRC3 and then write the // small architectural default into INST_PREF_SIZE, discarding the value the producing compiler // chose to cover its own kernel body. Shrinking the instruction preload that way turned // TDM-pipelined MXFP GEMM kernels into intermittent MEMORY_APERTURE_VIOLATION aborts on real // gfx1250 parts, so a nonzero source request must survive translation untouched. TEST(BinaryTranslatorE2E, Gfx1250PreservesSourceInstPrefSize) { using namespace rocr::llvm::amdhsa; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; // 107 exceeds the 6-bit GFX11 field: GFX12+ widened INST_PREF_SIZE to 8 bits, and reading it // through the narrow definition truncates 107 to 43. 25% of the kernels in the gfx1250 Gluon // suites request more than 63 units, so this width matters in practice. constexpr uint32_t kSourceInstPrefSize = 107u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); auto descriptor = rocjitsu::read_kernel_descriptor_for_test(rodata->data()); AMDHSA_BITS_SET(descriptor.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE, kSourceInstPrefSize); // NAMED_BAR_CNT is a GFX125-only control with no GFX10/GFX11 equivalent. Rebuilding RSRC3 // used to drop it, which silently removes a kernel's named-barrier allocation. AMDHSA_BITS_SET(descriptor.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT, 5); rocjitsu::write_kernel_descriptor_for_test(image.data() + rodata->sectionOffset(), descriptor); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); const auto *target_rodata = rocjitsu::find_section(translated, ".rodata"); ASSERT_NE(target_rodata, nullptr); const auto target = rocjitsu::read_kernel_descriptor_for_test(target_rodata->data()); EXPECT_EQ(AMDHSA_BITS_GET(target.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX12_PLUS_INST_PREF_SIZE), kSourceInstPrefSize); EXPECT_EQ(AMDHSA_BITS_GET(target.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX125_NAMED_BAR_CNT), 5u); } // Zero is a legitimate request: INST_PREF_SIZE == 0 disables the instruction preload for the // wave. A same-architecture translation must carry that through rather than substitute the // architectural default, which would silently re-enable prefetch the producer turned off. The // default is reserved for sources whose RSRC3 is not in the GFX10+ layout at all (GFX90A-family, // where those bits are ACCUM_OFFSET) and therefore has to be rebuilt. TEST(BinaryTranslatorE2E, Gfx1250PreservesDisabledInstPrefSizeFromGfx10PlusSource) { using namespace rocr::llvm::amdhsa; constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text({kGfx1250SEndpgm}); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); auto descriptor = rocjitsu::read_kernel_descriptor_for_test(rodata->data()); AMDHSA_BITS_SET(descriptor.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX10_PLUS_INST_PREF_SIZE, 0); rocjitsu::write_kernel_descriptor_for_test(image.data() + rodata->sectionOffset(), descriptor); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); ASSERT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); ASSERT_TRUE(translated.is_valid()); const auto *target_rodata = rocjitsu::find_section(translated, ".rodata"); ASSERT_NE(target_rodata, nullptr); const auto target = rocjitsu::read_kernel_descriptor_for_test(target_rodata->data()); EXPECT_EQ(AMDHSA_BITS_GET(target.compute_pgm_rsrc3, COMPUTE_PGM_RSRC3_GFX10_PLUS_INST_PREF_SIZE), 0u); } TEST(KernelDescriptorTranslator, Gfx1250UsesWave32SixteenVgprGranularity) { using namespace rocr::llvm::amdhsa; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); const auto *text = rocjitsu::find_section(layout, ".text"); ASSERT_NE(rodata, nullptr); ASSERT_NE(text, nullptr); auto descriptor = rocjitsu::read_kernel_descriptor_for_test(rodata->data()); AMDHSA_BITS_SET(descriptor.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_GRANULATED_WORKITEM_VGPR_COUNT, 63); // Leave ENABLE_WAVEFRONT_SIZE32 clear to verify that the Wave32-only gfx1250 // architecture does not misinterpret a legacy producer's missing bit. rocjitsu::write_kernel_descriptor_for_test(image.data() + rodata->sectionOffset(), descriptor); rocjitsu::KernelDescriptorTranslator translator(ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250); const auto translations = translator.translate_image( image, text->sectionOffset(), text->size(), rocjitsu::KernelDescriptorTranslationOptions{}); ASSERT_EQ(translations.size(), 1u); EXPECT_TRUE(translations[0].supported); EXPECT_EQ(translations[0].guest_wavefront_size, 32); EXPECT_EQ(translations[0].target_wave_size, 32); EXPECT_EQ(translations[0].guest_vgpr_count, 1024u); EXPECT_EQ(translations[0].target_vgpr_count, 1024u); EXPECT_EQ(translations[0].target_vgpr_granulated, 63u); } TEST(KernelDescriptorTranslator, Gfx1250AcceptsLdsAboveTcpPartitionSize) { auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); const auto *text = rocjitsu::find_section(layout, ".text"); ASSERT_NE(rodata, nullptr); ASSERT_NE(text, nullptr); // gfx1250's descriptor limit is 320 KiB per workgroup. A TCP may be // configured with a smaller LDS/vector-cache partition, but that partition // size is not the architectural descriptor validation limit. constexpr uint32_t kStaticLdsBytes = 176128u; rocjitsu::write_value_for_test( image, rodata->sectionOffset() + offsetof(rocjitsu::TestKernelDescriptor, group_segment_fixed_size), kStaticLdsBytes); rocjitsu::KernelDescriptorTranslator translator(ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250); const auto translations = translator.translate_image( image, text->sectionOffset(), text->size(), rocjitsu::KernelDescriptorTranslationOptions{}); ASSERT_EQ(translations.size(), 1u); EXPECT_TRUE(translations[0].supported); EXPECT_EQ(translations[0].target_lds_size, kStaticLdsBytes); } TEST(KernelDescriptorTranslator, CdnaToCdnaRejectsOversizedLdsWithoutVirtualization) { auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); const auto *text = rocjitsu::find_section(layout, ".text"); ASSERT_NE(text, nullptr); rocjitsu::write_value_for_test( image, rodata->sectionOffset() + offsetof(rocjitsu::TestKernelDescriptor, group_segment_fixed_size), 105600u); rocjitsu::KernelDescriptorTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); const auto translations = translator.translate_image( image, text->sectionOffset(), text->size(), rocjitsu::KernelDescriptorTranslationOptions{}); ASSERT_EQ(translations.size(), 1u); EXPECT_FALSE(translations[0].supported); EXPECT_EQ(translations[0].target_lds_size, 105600u); EXPECT_FALSE(translations[0].needs_lds_overflow_buf); EXPECT_EQ(translations[0].lds_overflow_size, 0u); const bool reported_lds_limit = std::ranges::any_of(translations[0].diagnostics, [](const auto &diagnostic) { return diagnostic.message.find("target LDS size exceeds host per-workgroup limit") != std::string::npos; }); EXPECT_TRUE(reported_lds_limit); } TEST(KernelDescriptorTranslator, CdnaToCdnaVirtualizesOversizedStaticLdsDescriptor) { auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(); rocjitsu::enable_kernarg_segment_ptr_sgpr(image); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); const auto *text = rocjitsu::find_section(layout, ".text"); ASSERT_NE(text, nullptr); rocjitsu::write_value_for_test( image, rodata->sectionOffset() + offsetof(rocjitsu::TestKernelDescriptor, group_segment_fixed_size), 105600u); auto source_descriptor = rocjitsu::read_elf_struct_for_test( image, rodata->sectionOffset()); uint32_t source_rsrc2 = source_descriptor.compute_pgm_rsrc2; AMDHSA_BITS_SET(source_rsrc2, rocr::llvm::amdhsa::COMPUTE_PGM_RSRC2_GRANULATED_LDS_SIZE, 22); rocjitsu::write_value_for_test( image, rodata->sectionOffset() + offsetof(rocjitsu::TestKernelDescriptor, compute_pgm_rsrc2), source_rsrc2); rocjitsu::KernelDescriptorTranslationOptions options; options.virtualize_lds = true; rocjitsu::KernelDescriptorTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); const auto translations = translator.translate_image(image, text->sectionOffset(), text->size(), options); ASSERT_EQ(translations.size(), 1u); EXPECT_TRUE(translations[0].supported); EXPECT_EQ(translations[0].target_lds_size, 0u); EXPECT_TRUE(translations[0].needs_lds_overflow_buf); EXPECT_EQ(translations[0].lds_overflow_size, 105600u); EXPECT_EQ(translations[0].kernarg_size, 16u); EXPECT_EQ(translations[0].kernarg_wrapper_original_pointer_offset, 16u); EXPECT_EQ(translations[0].lds_overflow_kernarg_pointer_offset, 24u); EXPECT_EQ(translations[0].target_kernarg_size, 48u); rocjitsu::AmdGpuCodeObject mutated(image.data(), image.size()); ASSERT_TRUE(mutated.is_valid()); rocjitsu::CodeObjectPatcher patcher(mutated); auto patch_plan = translations[0]; patch_plan.target_entry_text_offset = patch_plan.entry_text_offset; patch_plan.target_body_entry_text_offset = patch_plan.entry_text_offset; ASSERT_TRUE(patcher.apply_kernel_descriptor_translation(patch_plan, ROCJITSU_CODE_ARCH_CDNA3)); const auto patched_image = patcher.emit(); const auto patched_kd = rocjitsu::read_elf_struct_for_test( patched_image, rodata->sectionOffset()); EXPECT_EQ(patched_kd.group_segment_fixed_size, 0u); EXPECT_EQ(patched_kd.kernarg_size, 48u); EXPECT_EQ(AMDHSA_BITS_GET(patched_kd.compute_pgm_rsrc2, rocr::llvm::amdhsa::COMPUTE_PGM_RSRC2_GRANULATED_LDS_SIZE), 0u); } // SHARED_VGPR_COUNT is carried rather than recomputed, so the registers it reserves have to come // out of the target budget before the allocation is chosen. Planning that ignores them picks an // allocation the shared blocks no longer fit alongside, and the only way to encode that is to // take blocks away from a body that still uses them. TEST(KernelDescriptorTranslator, SharedVgprBlocksAreReservedFromTheTargetVgprBudget) { auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); const auto *text = rocjitsu::find_section(layout, ".text"); ASSERT_NE(text, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); // Wave64 is what gives shared VGPR blocks any meaning. AMDHSA_BITS_SET(source_kd->kernel_code_properties, rocr::llvm::amdhsa::KERNEL_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32, 0); AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc3, rocr::llvm::amdhsa::COMPUTE_PGM_RSRC3_GFX10_PLUS_SHARED_VGPR_COUNT, 9); auto plan = [&](uint32_t minimum_vgprs) { rocjitsu::KernelDescriptorTranslationOptions options; options.minimum_vgprs = minimum_vgprs; rocjitsu::KernelDescriptorTranslator translator(ROCJITSU_CODE_ARCH_RDNA3, ROCJITSU_CODE_ARCH_RDNA3); return translator.translate_image(image, text->sectionOffset(), text->size(), options); }; // Nine blocks reserve 72 of the 256 registers, leaving 184 for the translated allocation. const auto fits = plan(/*minimum_vgprs=*/180); ASSERT_EQ(fits.size(), 1u); EXPECT_TRUE(fits[0].supported); const auto overcommits = plan(/*minimum_vgprs=*/200); ASSERT_EQ(overcommits.size(), 1u); EXPECT_FALSE(overcommits[0].supported) << "the body still needs the shared blocks its descriptor declared"; } TEST(KernelDescriptorTranslator, VirtualLdsPreservesKernargPreloadRangeWhenSizeIsZero) { auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(); rocjitsu::enable_kernarg_segment_ptr_sgpr(image, /*kernarg_size=*/0); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); const auto *text = rocjitsu::find_section(layout, ".text"); ASSERT_NE(text, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); AMDHSA_BITS_SET(source_kd->kernarg_preload, rocr::llvm::amdhsa::KERNARG_PRELOAD_SPEC_OFFSET, 2); AMDHSA_BITS_SET(source_kd->kernarg_preload, rocr::llvm::amdhsa::KERNARG_PRELOAD_SPEC_LENGTH, 6); rocjitsu::KernelDescriptorTranslationOptions options; options.virtualize_lds = true; rocjitsu::KernelDescriptorTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); const auto translations = translator.translate_image(image, text->sectionOffset(), text->size(), options); ASSERT_EQ(translations.size(), 1u); EXPECT_TRUE(translations[0].supported); EXPECT_EQ(translations[0].kernarg_size, 32u); EXPECT_EQ(translations[0].kernarg_wrapper_original_pointer_offset, 32u); EXPECT_EQ(translations[0].lds_overflow_kernarg_pointer_offset, 40u); EXPECT_EQ(translations[0].target_kernarg_size, 64u); } TEST(KernelDescriptorTranslator, VirtualLdsKeepsOddKernargPreloadCopyExtentExact) { auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(); rocjitsu::enable_kernarg_segment_ptr_sgpr(image, /*kernarg_size=*/0); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); const auto *text = rocjitsu::find_section(layout, ".text"); ASSERT_NE(text, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); AMDHSA_BITS_SET(source_kd->kernarg_preload, rocr::llvm::amdhsa::KERNARG_PRELOAD_SPEC_OFFSET, 0); AMDHSA_BITS_SET(source_kd->kernarg_preload, rocr::llvm::amdhsa::KERNARG_PRELOAD_SPEC_LENGTH, 11); rocjitsu::KernelDescriptorTranslationOptions options; options.virtualize_lds = true; rocjitsu::KernelDescriptorTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); const auto translations = translator.translate_image(image, text->sectionOffset(), text->size(), options); ASSERT_EQ(translations.size(), 1u); EXPECT_TRUE(translations[0].supported); EXPECT_EQ(translations[0].kernarg_size, 44u); // Wrapper fields remain aligned independently of the exact source-copy size. EXPECT_EQ(translations[0].kernarg_wrapper_original_pointer_offset, 48u); EXPECT_EQ(translations[0].lds_overflow_kernarg_pointer_offset, 56u); EXPECT_EQ(translations[0].target_kernarg_size, 80u); } TEST(KernelDescriptorTranslator, VirtualLdsAcceptsZeroKernargSizeWithWrapper) { auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(); rocjitsu::enable_kernarg_segment_ptr_sgpr(image, /*kernarg_size=*/0); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); const auto *text = rocjitsu::find_section(layout, ".text"); ASSERT_NE(text, nullptr); rocjitsu::write_value_for_test( image, rodata->sectionOffset() + offsetof(rocjitsu::TestKernelDescriptor, group_segment_fixed_size), 105600u); rocjitsu::KernelDescriptorTranslationOptions options; options.virtualize_lds = true; rocjitsu::KernelDescriptorTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); const auto translations = translator.translate_image(image, text->sectionOffset(), text->size(), options); ASSERT_EQ(translations.size(), 1u); EXPECT_TRUE(translations[0].supported); EXPECT_TRUE(translations[0].needs_lds_overflow_buf); EXPECT_EQ(translations[0].kernarg_size, 0u); EXPECT_EQ(translations[0].kernarg_wrapper_original_pointer_offset, 0u); EXPECT_EQ(translations[0].lds_overflow_kernarg_pointer_offset, 8u); EXPECT_EQ(translations[0].target_kernarg_size, 32u); } TEST(KernelDescriptorTranslator, VirtualLdsAddsKernargSegmentPointerWhenMissing) { using namespace rocr::llvm::amdhsa; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); const auto *text = rocjitsu::find_section(layout, ".text"); ASSERT_NE(text, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); source_kd->group_segment_fixed_size = 105600u; source_kd->kernarg_size = 0; AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, 2); AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1); AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_ENABLE_PRIVATE_SEGMENT, 1); AMDHSA_BITS_SET(source_kd->kernel_code_properties, KERNEL_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR, 1); rocjitsu::KernelDescriptorTranslationOptions options; options.virtualize_lds = true; rocjitsu::KernelDescriptorTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); const auto translations = translator.translate_image(image, text->sectionOffset(), text->size(), options); ASSERT_EQ(translations.size(), 1u); EXPECT_TRUE(translations[0].supported); EXPECT_TRUE(translations[0].needs_lds_overflow_buf); EXPECT_EQ(translations[0].kernarg_size, 0u); EXPECT_EQ(translations[0].kernarg_wrapper_original_pointer_offset, 0u); EXPECT_EQ(translations[0].lds_overflow_kernarg_pointer_offset, 8u); EXPECT_EQ(translations[0].target_kernarg_size, 32u); EXPECT_FALSE(translations[0].source_has_kernarg_segment_ptr); EXPECT_TRUE(translations[0].has_kernarg_segment_ptr); EXPECT_EQ(translations[0].kernarg_segment_ptr_sgpr, 2u); EXPECT_EQ(translations[0].source_user_sgpr_count, 2u); EXPECT_EQ(translations[0].target_user_sgpr_count, 4u); EXPECT_EQ(translations[0].workgroup_id_sgpr_x, 2); EXPECT_EQ(translations[0].lds_overflow_workgroup_id_sgpr_x, 4); EXPECT_EQ(translations[0].user_sgpr_repair_start, 2u); // gfx950 initializes architected FLAT_SCRATCH rather than an additional // ordinary SGPR, so only the enabled workgroup-id SGPR needs repair. EXPECT_EQ(translations[0].user_sgpr_repair_count, 1u); EXPECT_TRUE(translations[0].has_dispatch_ptr); EXPECT_EQ(translations[0].dispatch_ptr_sgpr, 0u); } TEST(KernelDescriptorTranslator, VirtualLdsRejectsMissingKernargSegmentPointerWithFullUserSgprs) { using namespace rocr::llvm::amdhsa; auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(); rocjitsu::AmdGpuCodeObject layout(image.data(), image.size()); ASSERT_TRUE(layout.is_valid()); const auto *rodata = rocjitsu::find_section(layout, ".rodata"); ASSERT_NE(rodata, nullptr); const auto *text = rocjitsu::find_section(layout, ".text"); ASSERT_NE(text, nullptr); ASSERT_GE(rodata->size(), sizeof(rocjitsu::TestKernelDescriptor)); auto *source_kd = reinterpret_cast(image.data() + rodata->sectionOffset()); source_kd->group_segment_fixed_size = 105600u; source_kd->kernarg_size = 0; AMDHSA_BITS_SET(source_kd->compute_pgm_rsrc2, COMPUTE_PGM_RSRC2_USER_SGPR_COUNT, 15); rocjitsu::KernelDescriptorTranslationOptions options; options.virtualize_lds = true; rocjitsu::KernelDescriptorTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_CDNA3); const auto translations = translator.translate_image(image, text->sectionOffset(), text->size(), options); ASSERT_EQ(translations.size(), 1u); EXPECT_FALSE(translations[0].supported); EXPECT_TRUE(translations[0].needs_lds_overflow_buf); const bool reported_user_sgpr_limit = std::ranges::any_of(translations[0].diagnostics, [](const auto &diagnostic) { return diagnostic.message.find("USER_SGPR_COUNT") != std::string::npos && diagnostic.message.find("16 SGPR") != std::string::npos; }); EXPECT_TRUE(reported_user_sgpr_limit); } TEST(KernelDescriptorTranslator, IgnoresNonAllocExecutableSectionsForEntryRange) { auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(); const auto ehdr = rocjitsu::read_elf_struct_for_test(image, 0); auto shdrs = rocjitsu::read_elf_array_for_test(image, ehdr.e_shoff, ehdr.e_shnum); constexpr uint64_t fake_exec_vaddr = 0x9000; shdrs[5].sh_flags = rocjitsu::SHF_EXECINSTR; shdrs[5].sh_addr = fake_exec_vaddr; shdrs[5].sh_size = sizeof(uint32_t); for (size_t i = 0; i < shdrs.size(); ++i) rocjitsu::write_elf_struct_for_test(image, ehdr.e_shoff + i * sizeof(rocjitsu::Elf64_Shdr), shdrs[i]); rocjitsu::write_kernel_descriptor_entry_offset(image.data() + shdrs[2].sh_offset, static_cast(fake_exec_vaddr) - static_cast(shdrs[2].sh_addr)); rocjitsu::KernelDescriptorTranslator translator(ROCJITSU_CODE_ARCH_CDNA4, ROCJITSU_CODE_ARCH_RDNA4); const auto translations = translator.translate_image( image, shdrs[1].sh_offset, shdrs[1].sh_size, rocjitsu::KernelDescriptorTranslationOptions{}); EXPECT_TRUE(translations.empty()) << "non-loadable executable sections must not extend valid kernel entry range"; } namespace { /// @brief Scalar selector naming the low half of the flat-scratch base. constexpr uint16_t kFlatScratchBaseLoSelector = 230; /// @brief Scalar selector naming the high half of the flat-scratch base. constexpr uint16_t kFlatScratchBaseHiSelector = 231; /// @brief Translate a single-instruction gfx1250 kernel with the B0-to-A0 profile. [[nodiscard]] std::vector translate_gfx1250_b0_to_a0_words(std::vector words) { constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; words.push_back(kGfx1250SEndpgm); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); auto result = translator.translate(source); EXPECT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); if (!result.ok()) return {}; rocjitsu::AmdGpuCodeObject translated(result.elf_bytes.data(), result.elf_bytes.size()); EXPECT_FALSE(translated.text_sections().empty()); if (translated.text_sections().empty()) return {}; const auto *section = translated.text_sections()[0]; const auto *target_words = reinterpret_cast(section->data()); return std::vector(target_words, target_words + section->size() / sizeof(uint32_t)); } } // namespace // A scalar 64-bit read reaches the whole flat-scratch base through either // selector. The A0 profile spells it with the low selector, so the high // selector is rewritten in place and the instruction keeps its size. TEST(BinaryTranslatorE2E, Gfx1250RewritesScalarFlatScratchBase64BitSourceForA0) { const auto source = gfx1250::build_sop1(gfx1250::kSMovB64Sop1, {.ssrc0 = static_cast(kFlatScratchBaseHiSelector), .sdst = 10}); const auto out = translate_gfx1250_b0_to_a0_words({source[0]}); ASSERT_GE(out.size(), 2u); EXPECT_EQ(out[0] & 0xffu, kFlatScratchBaseLoSelector) << "the scalar read must use the low selector on A0"; // Only the source selector changes: destination, opcode, and encoding stay. EXPECT_EQ(out[0] & ~0xffu, source[0] & ~0xffu); EXPECT_EQ(out.size(), 2u) << "a scalar rewrite must not add instructions"; } // A 64-bit read in the first scalar source position of a two-source encoding. TEST(BinaryTranslatorE2E, Gfx1250RewritesScalarFlatScratchBaseInFirstSourceOfSop2ForA0) { const auto source = gfx1250::build_sop2( gfx1250::kSLshlB64Sop2, {.ssrc0 = static_cast(kFlatScratchBaseHiSelector), .ssrc1 = 130, .sdst = 12}); const auto out = translate_gfx1250_b0_to_a0_words({source[0]}); ASSERT_GE(out.size(), 2u); EXPECT_EQ(out[0] & 0xffu, kFlatScratchBaseLoSelector); EXPECT_EQ((out[0] >> 8) & 0xffu, 130u) << "the 32-bit shift source is unaffected"; EXPECT_EQ(out.size(), 2u); } // The second scalar source position is a separate encoding field, so it needs // its own coverage: a rewrite that only ever touched the first would pass every // test above. TEST(BinaryTranslatorE2E, Gfx1250RewritesScalarFlatScratchBaseInSecondSourceOfSop2ForA0) { constexpr uint8_t kOrdinaryPair = 20; const auto source = gfx1250::build_sop2( gfx1250::kSAndB64Sop2, {.ssrc0 = kOrdinaryPair, .ssrc1 = static_cast(kFlatScratchBaseHiSelector), .sdst = 12}); const auto out = translate_gfx1250_b0_to_a0_words({source[0]}); ASSERT_GE(out.size(), 2u); EXPECT_EQ(out[0] & 0xffu, kOrdinaryPair) << "the first source is untouched"; EXPECT_EQ((out[0] >> 8) & 0xffu, kFlatScratchBaseLoSelector) << "the second source must use the low selector on A0"; EXPECT_EQ(out.size(), 2u) << "a scalar rewrite must not add instructions"; } // The two-source scalar compare is the remaining modelled scalar encoding, and // each of its fields is separate from the ones covered above. TEST(BinaryTranslatorE2E, Gfx1250RewritesScalarFlatScratchBaseInBothSourcesOfSopcForA0) { constexpr uint8_t kOrdinaryPair = 20; struct SopcCase { const char *name; uint8_t ssrc0; uint8_t ssrc1; uint32_t expected_ssrc0; uint32_t expected_ssrc1; }; const SopcCase cases[] = { {"first source", static_cast(kFlatScratchBaseHiSelector), kOrdinaryPair, kFlatScratchBaseLoSelector, kOrdinaryPair}, {"second source", kOrdinaryPair, static_cast(kFlatScratchBaseHiSelector), kOrdinaryPair, kFlatScratchBaseLoSelector}, {"both sources", static_cast(kFlatScratchBaseHiSelector), static_cast(kFlatScratchBaseHiSelector), kFlatScratchBaseLoSelector, kFlatScratchBaseLoSelector}, }; for (const SopcCase &test_case : cases) { SCOPED_TRACE(test_case.name); const auto source = gfx1250::build_sopc(gfx1250::kSCmpEqU64Sopc, {.ssrc0 = test_case.ssrc0, .ssrc1 = test_case.ssrc1}); const auto out = translate_gfx1250_b0_to_a0_words({source[0]}); ASSERT_GE(out.size(), 2u); EXPECT_EQ(out[0] & 0xffu, test_case.expected_ssrc0); EXPECT_EQ((out[0] >> 8) & 0xffu, test_case.expected_ssrc1); EXPECT_EQ(out.size(), 2u) << "a scalar rewrite must not add instructions"; } } // The single-source vector encoding is the last modelled layout without // coverage. Its source is nine bits wide, so it reaches the selector and takes // the staged pair like the other vector forms. TEST(BinaryTranslatorE2E, Gfx1250MovesFlatScratchBaseInSingleSourceVectorFormToSgprPairForA0) { const auto source = gfx1250::build_vop1(gfx1250::kVMovB64Vop1, {.src0 = kFlatScratchBaseHiSelector, .vdst = 4}); const auto out = translate_gfx1250_b0_to_a0_words({source[0]}); ASSERT_GE(out.size(), 3u) << "the rewrite prepends one scalar move"; ASSERT_EQ((out[0] >> 23) & 0x1ffu, 381u) << "prologue must be a SOP1 instruction"; EXPECT_EQ(out[0] & 0xffu, kFlatScratchBaseLoSelector); const uint32_t pair_base = (out[0] >> 16) & 0x7fu; EXPECT_EQ(pair_base % 2u, 0u) << "a 64-bit scalar operand must be even-aligned"; EXPECT_EQ(out[1] & 0x1ffu, pair_base) << "the vector source must name the borrowed pair"; } // Two affected positions in one instruction share a single borrowed pair. This // is the only direct check that the prologue is emitted once rather than once // per rewritten source. TEST(BinaryTranslatorE2E, Gfx1250SharesOneBorrowedPairAcrossTwoVectorSourcesForA0) { const auto source = gfx1250::build_vop3( gfx1250::kVAddNcU64Vop3, {.vdst = 0, .src0 = kFlatScratchBaseHiSelector, .src1 = kFlatScratchBaseHiSelector}); const auto out = translate_gfx1250_b0_to_a0_words({source[0], source[1]}); ASSERT_GE(out.size(), 4u); ASSERT_EQ((out[0] >> 23) & 0x1ffu, 381u) << "prologue must be a SOP1 instruction"; EXPECT_EQ(out[0] & 0xffu, kFlatScratchBaseLoSelector); const uint32_t pair_base = (out[0] >> 16) & 0x7fu; EXPECT_EQ(out[2] & 0x1ffu, pair_base) << "src0 must name the borrowed pair"; EXPECT_EQ((out[2] >> 9) & 0x1ffu, pair_base) << "src1 must name the same pair"; EXPECT_EQ(out.size(), 4u) << "one move serves both positions, so only one is emitted"; } // The third source is the last modelled field and the only one no other case // reaches, so a rewrite that walked just the first two would pass every test // above. TEST(BinaryTranslatorE2E, Gfx1250MovesFlatScratchBaseInThirdVectorSourceToSgprPairForA0) { const auto source = gfx1250::build_vop3( gfx1250::kVFmaF64Vop3, {.vdst = 0, .src0 = 256 + 2, .src1 = 256 + 4, .src2 = kFlatScratchBaseHiSelector}); const auto out = translate_gfx1250_b0_to_a0_words({source[0], source[1]}); ASSERT_GE(out.size(), 4u) << "the rewrite prepends one scalar move"; ASSERT_EQ((out[0] >> 23) & 0x1ffu, 381u) << "prologue must be a SOP1 instruction"; EXPECT_EQ(out[0] & 0xffu, kFlatScratchBaseLoSelector); const uint32_t pair_base = (out[0] >> 16) & 0x7fu; EXPECT_EQ(pair_base % 2u, 0u) << "a 64-bit scalar operand must be even-aligned"; EXPECT_EQ(out[2] & 0x1ffu, 256u + 2u) << "src0 is unchanged"; EXPECT_EQ((out[2] >> 9) & 0x1ffu, 256u + 4u) << "src1 is unchanged"; EXPECT_EQ((out[2] >> 18) & 0x1ffu, pair_base) << "src2 must name the borrowed pair"; } // VOP3P carries packed 64-bit sources through nine-bit fields, so it reaches // the selector exactly as VOP3 does. Leaving it unmodelled would send it to the // conservative scan, which skips vector-typed operands and would copy the // instruction through unchanged. TEST(BinaryTranslatorE2E, Gfx1250MovesVop3pFlatScratchBaseSourceToSgprPairForA0) { const auto source = gfx1250::build_vop3p( gfx1250::kVPkMulF32Vop3p, {.vdst = 0, .src0 = kFlatScratchBaseHiSelector, .src1 = 258}); const auto out = translate_gfx1250_b0_to_a0_words({source[0], source[1]}); ASSERT_GE(out.size(), 4u) << "the rewrite prepends one scalar move"; ASSERT_EQ((out[0] >> 23) & 0x1ffu, 381u) << "prologue must be a SOP1 instruction"; EXPECT_EQ(out[0] & 0xffu, kFlatScratchBaseLoSelector); const uint32_t pair_base = (out[0] >> 16) & 0x7fu; EXPECT_EQ(pair_base % 2u, 0u) << "a 64-bit scalar operand must be even-aligned"; EXPECT_EQ(out[1], source[0]) << "the first word is unchanged"; EXPECT_EQ(out[2] & 0x1ffu, pair_base) << "src0 must name the borrowed pair"; EXPECT_EQ((out[2] >> 9) & 0x1ffu, 258u) << "src1 is unchanged"; } // A decoder may report more sources than the encoding has source fields: an // accumulate form repeats its destination and a literal takes a position of its // own. Neither can carry the selector, so their presence must not by itself // refuse the rewrite. TEST(BinaryTranslatorE2E, Gfx1250RewritesCompactFormsWithExtraDecodedOperandsForA0) { struct ExtraOperandCase { const char *name; std::vector words; }; const std::vector cases = { {"destination alias", {gfx1250::build_vop2(gfx1250::kVFmacF64Vop2, {.src0 = kFlatScratchBaseHiSelector, .vsrc1 = 2})[0]}}, }; for (const ExtraOperandCase &test_case : cases) { SCOPED_TRACE(test_case.name); const auto out = translate_gfx1250_b0_to_a0_words(test_case.words); ASSERT_GE(out.size(), 3u) << "the rewrite prepends one scalar move"; ASSERT_EQ((out[0] >> 23) & 0x1ffu, 381u) << "prologue must be a SOP1 instruction"; EXPECT_EQ(out[0] & 0xffu, kFlatScratchBaseLoSelector); const uint32_t pair_base = (out[0] >> 16) & 0x7fu; EXPECT_EQ(out[1] & 0x1ffu, pair_base) << "src0 must name the borrowed pair"; EXPECT_EQ((out[1] >> 9) & 0xffu, 2u) << "the VGPR source is unchanged"; } } // With every ordinary scalar register live there is no pair to borrow. The // rewrite must fail rather than clobber one, and the object must come back // unchanged. TEST(BinaryTranslatorE2E, Gfx1250FailsClosedWhenNoDeadSgprPairIsAvailableForA0) { constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; const auto vector_read = gfx1250::build_vop3( gfx1250::kVAddNcU64Vop3, {.vdst = 0, .src0 = kFlatScratchBaseHiSelector, .src1 = 256 + 2}); std::vector words = {vector_read[0], vector_read[1]}; // Reading every ordinary scalar register after the rewrite keeps them all // live across it, so the allocator has nothing to take. for (uint16_t base = 0; base + 1 <= 101; base += 2) { words.push_back( gfx1250::build_sop2(gfx1250::kSAndB32Sop2, {.ssrc0 = static_cast(base), .ssrc1 = static_cast(base + 1), .sdst = 102})[0]); } words.push_back(kGfx1250SEndpgm); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); rocjitsu::BinaryTranslator translator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); const auto result = translator.translate(source); EXPECT_FALSE(result.ok()); EXPECT_FALSE(result.dispatchable()); EXPECT_EQ(result.elf_bytes, image) << "fail-closed must leave the object unchanged"; EXPECT_TRUE(rocjitsu::has_error_containing( result, rocjitsu::DiagnosticKind::ExpandFailed, "flat-scratch-base rewrite could not allocate a dead SGPR pair")); } // An unmodelled encoding carrying a wide vector register whose number equals // the selector must still be copied; only genuine scalar reads are refused. TEST(BinaryTranslatorE2E, Gfx1250LeavesWideVgprMatchingSelectorNumberUnchangedForA0) { constexpr uint8_t kVgprMatchingSelector = 231; // A NULL scalar base makes the address a 64-bit vector pair, which is a // source operand of exactly the affected width. constexpr uint8_t kNullScalarBase = 124; const auto load = gfx1250::build_vglobal(gfx1250::kGlobalLoadB64Vglobal, {.saddr = kNullScalarBase, .vdst = 0, .vaddr = kVgprMatchingSelector}); const auto out = translate_gfx1250_b0_to_a0_words({load[0], load[1], load[2]}); ASSERT_GE(out.size(), 4u); EXPECT_EQ(out[0], load[0]); EXPECT_EQ(out[1], load[1]); EXPECT_EQ(out[2], load[2]); } // With the low registers live the rewrite must borrow from above them. The // scalar file is fixed on this target rather than descriptor-allocated, so the // requirement is that the borrowed pair stays inside the architectural file. TEST(BinaryTranslatorE2E, Gfx1250BorrowsFlatScratchBasePairAboveLiveRegistersForA0) { constexpr uint16_t kHighestOrdinarySgpr = 101; const auto vector_read = gfx1250::build_vop3( gfx1250::kVAddNcU64Vop3, {.vdst = 0, .src0 = kFlatScratchBaseHiSelector, .src1 = 256 + 2}); std::vector words = {vector_read[0], vector_read[1]}; // Reading s0..s7 after the rewrite keeps them live across it, so the // allocator cannot choose any of them. for (uint8_t base = 0; base < 8; base += 2) { words.push_back(gfx1250::build_sop2( gfx1250::kSAndB32Sop2, {.ssrc0 = base, .ssrc1 = static_cast(base + 1), .sdst = 20})[0]); } const auto out = translate_gfx1250_b0_to_a0_words(words); ASSERT_GE(out.size(), 3u); ASSERT_EQ((out[0] >> 23) & 0x1ffu, 381u) << "prologue must be a SOP1 instruction"; EXPECT_EQ(out[0] & 0xffu, kFlatScratchBaseLoSelector); const uint32_t pair_base = (out[0] >> 16) & 0x7fu; EXPECT_GE(pair_base, 8u) << "s0..s7 are live, so the pair must sit above them"; EXPECT_EQ(pair_base % 2u, 0u) << "a 64-bit scalar operand must be even-aligned"; EXPECT_LE(pair_base + 1u, kHighestOrdinarySgpr) << "the borrowed pair must stay inside the ordinary scalar file"; EXPECT_EQ(out[2] & 0x1ffu, pair_base) << "the vector read must name the borrowed pair"; } // The compact vector formats reach the same rewrite as their VOP3 forms. TEST(BinaryTranslatorE2E, Gfx1250MovesCompactVectorFlatScratchBaseSourceToSgprPairForA0) { struct CompactCase { const char *name; std::vector words; }; const std::vector cases = { {"vop2", {gfx1250::build_vop2(gfx1250::kVAddNcU64Vop2, {.src0 = kFlatScratchBaseHiSelector, .vsrc1 = 2})[0]}}, {"vopc", {gfx1250::build_vopc(gfx1250::kVCmpEqU64Vopc, {.src0 = kFlatScratchBaseHiSelector, .vsrc1 = 2})[0]}}, }; for (const CompactCase &test_case : cases) { SCOPED_TRACE(test_case.name); const auto out = translate_gfx1250_b0_to_a0_words(test_case.words); ASSERT_GE(out.size(), 3u) << "the rewrite prepends one scalar move"; EXPECT_EQ((out[0] >> 23) & 0x1ffu, 381u) << "prologue must be a SOP1 instruction"; EXPECT_EQ(out[0] & 0xffu, kFlatScratchBaseLoSelector); const uint32_t pair_base = (out[0] >> 16) & 0x7fu; EXPECT_EQ(pair_base % 2u, 0u); EXPECT_EQ(out[1] & 0x1ffu, pair_base) << "src0 must name the borrowed pair"; EXPECT_EQ((out[1] >> 9) & 0xffu, 2u) << "the VGPR source is unchanged"; } } // The compact formats decode their second source as a plain VGPR index, so a // VGPR whose number matches the selector must not be mistaken for it. TEST(BinaryTranslatorE2E, Gfx1250LeavesCompactVgprMatchingSelectorNumberUnchangedForA0) { constexpr uint8_t kVgprMatchingSelector = 231; const std::vector sources = { gfx1250::build_vop2(gfx1250::kVAddNcU64Vop2, {.src0 = 256 + 4, .vsrc1 = kVgprMatchingSelector})[0], gfx1250::build_vopc(gfx1250::kVCmpEqU64Vopc, {.src0 = 256 + 4, .vsrc1 = kVgprMatchingSelector})[0], }; for (const uint32_t source_word : sources) { SCOPED_TRACE(source_word); const auto out = translate_gfx1250_b0_to_a0_words({source_word}); ASSERT_GE(out.size(), 2u); EXPECT_EQ(out[0], source_word) << "an ordinary VGPR must be copied verbatim"; } } // A literal source reports the literal's own value as its encoding value, so // the constants 230 and 231 collide numerically with the two selectors. Reading // the operand instead of the encoding field mistakes such a constant for the // base, and rewriting the field then erases the marker that says a literal // follows -- stranding its dword as a standalone illegal instruction. // // The first VOP2 case is the RCCL all_reduce_perf kernel that first exposed // this: `v_mul_u64_e32 v[10:11], 0xe7, v[0:1]`, words 0x541400ff 0x000000e7. TEST(BinaryTranslatorE2E, Gfx1250LeavesLiteralMatchingSelectorNumberUnchangedForA0) { struct LiteralCase { const char *name; std::vector words; }; constexpr uint32_t kLiteralSelector = 255; constexpr uint32_t kLiteral64Selector = 254; const std::vector cases = { // The reported reproducer, and its twin on the other colliding constant. {"vop2 literal 0xe7", {gfx1250::build_vop2(gfx1250::kVMulU64Vop2, {.src0 = kLiteralSelector, .vsrc1 = 0, .vdst = 10})[0], kFlatScratchBaseHiSelector}}, {"vop2 literal 0xe6", {gfx1250::build_vop2(gfx1250::kVMulU64Vop2, {.src0 = kLiteralSelector, .vsrc1 = 0, .vdst = 10})[0], kFlatScratchBaseLoSelector}}, // The scalar path rewrites the selector in place, so a stale literal there // keeps the instruction's size and changes only what it reads: a silent // wrong answer alongside the orphaned dword. {"sop1 literal 0xe7", {gfx1250::build_sop1(gfx1250::kSMovB64Sop1, {.ssrc0 = static_cast(kLiteralSelector), .sdst = 0})[0], kFlatScratchBaseHiSelector}}, // A 64-bit literal reports only its low dword as the encoding value, so it // collides the same way while stranding two dwords rather than one. {"vop2 literal64 low dword 0xe7", {gfx1250::build_vop2(gfx1250::kVMulU64Vop2, {.src0 = kLiteral64Selector, .vsrc1 = 0, .vdst = 10})[0], kFlatScratchBaseHiSelector, 1u}}, }; for (const LiteralCase &test_case : cases) { SCOPED_TRACE(test_case.name); const auto out = translate_gfx1250_b0_to_a0_words(test_case.words); ASSERT_GE(out.size(), test_case.words.size() + 1u); for (size_t i = 0; i < test_case.words.size(); ++i) { EXPECT_EQ(out[i], test_case.words[i]) << "word " << i << " of a literal-source instruction must be copied verbatim"; } EXPECT_EQ(out.size(), test_case.words.size() + 1u) << "no prologue may be prepended for a literal that is not the selector"; } } // The cases above all put their source fields in word 0. VOP3 puts them in word // 1 and its literal in word 2, so it is the only layout where the field being // rewritten and the literal that must survive live in different words. // // Both sources here read 231: src0 genuinely names the selector, while src1 is // an ordinary literal that merely collides with its number. The two must be // treated differently in a single instruction -- src0 repointed at the borrowed // pair, src1 and its trailing dword untouched. TEST(BinaryTranslatorE2E, Gfx1250RewritesVop3SelectorButKeepsCollidingLiteralForA0) { constexpr uint32_t kLiteralSelector = 255; const auto source = gfx1250::build_vop3( gfx1250::kVMulU64Vop3, {.vdst = 10, .src0 = kFlatScratchBaseHiSelector, .src1 = kLiteralSelector}); const auto out = translate_gfx1250_b0_to_a0_words({source[0], source[1], kFlatScratchBaseHiSelector}); ASSERT_GE(out.size(), 5u) << "the rewrite prepends one scalar move and keeps four words"; ASSERT_EQ((out[0] >> 23) & 0x1ffu, 381u) << "prologue must be a SOP1 instruction"; EXPECT_EQ(out[0] & 0xffu, kFlatScratchBaseLoSelector); const uint32_t pair_base = (out[0] >> 16) & 0x7fu; EXPECT_EQ(pair_base % 2u, 0u) << "a 64-bit scalar operand must be even-aligned"; EXPECT_EQ(out[1], source[0]) << "vdst, opcode, and modifiers are unchanged"; EXPECT_EQ(out[2] & 0x1ffu, pair_base) << "the real selector must name the borrowed pair"; EXPECT_EQ((out[2] >> 9) & 0x1ffu, kLiteralSelector) << "the literal marker must survive, or its dword becomes an instruction"; EXPECT_EQ(out[3], kFlatScratchBaseHiSelector) << "the literal dword itself is unchanged"; EXPECT_EQ(out.size(), 5u) << "only the prologue is added"; } namespace { /// @brief Mnemonics reported once per translation for a deferred family, in the /// order the translation reported them. /// /// The report is an ordinary translation diagnostic, so it reaches the library /// callback and the hotswap renderer alongside every other one. Reading it back /// off the result needs no stream redirection. [[nodiscard]] std::vector deferred_family_reports(const rocjitsu::TranslatedCodeObject &result) { std::vector reported; for (const auto &diagnostic : result.diagnostics) { if (diagnostic.severity == rocjitsu::DiagnosticSeverity::Warning && diagnostic.kind == rocjitsu::DiagnosticKind::Legalization && diagnostic.message.find("target-specific handling is not yet implemented") != std::string::npos) { reported.push_back(diagnostic.mnemonic); } } return reported; } /// @brief Translate a single-kernel gfx1250 object with the B0-to-A0 profile. [[nodiscard]] rocjitsu::TranslatedCodeObject translate_gfx1250_b0_to_a0_result(rocjitsu::BinaryTranslator &translator, std::vector words) { constexpr uint32_t kGfx1250SEndpgm = 0xBFB00000u; words.push_back(kGfx1250SEndpgm); auto image = rocjitsu::make_minimal_amdgpu_elf_with_descriptor_after_text(words); rocjitsu::AmdGpuCodeObject source(image.data(), image.size()); auto result = translator.translate(source); EXPECT_TRUE(result.ok()) << (result.diagnostics.empty() ? "" : result.diagnostics.front().message); return result; } [[nodiscard]] rocjitsu::BinaryTranslator make_gfx1250_b0_to_a0_translator() { return rocjitsu::BinaryTranslator( ROCJITSU_CODE_ARCH_GFX1250, ROCJITSU_CODE_ARCH_GFX1250, 0, gfx1250_revision_options(rocjitsu::ProcessorRevision::Gfx1250B0, rocjitsu::ProcessorRevision::Gfx1250A0)); } /// @brief An s_monitor_sleep, whose A0 handling is deferred (DEGFXMI400-12268). [[nodiscard]] uint32_t gfx1250_deferred_word() { return gfx1250::build_sopp(gfx1250::kSMonitorSleepSopp, {.simm16 = 1})[0]; } } // namespace // A deferred family passes through unchanged and is reported so the gap stays // visible. The report describes the mnemonic, not the site, so repeating it per // instruction adds nothing: one RCCL all_reduce run emitted 104,831 copies. TEST(BinaryTranslatorE2E, Gfx1250ReportsOncePerDeferredMnemonicWithinOneTranslation) { const uint32_t deferred = gfx1250_deferred_word(); auto translator = make_gfx1250_b0_to_a0_translator(); const auto result = translate_gfx1250_b0_to_a0_result( translator, {deferred, deferred, deferred, deferred, deferred}); EXPECT_EQ(deferred_family_reports(result), (std::vector{"s_monitor_sleep"})) << "five deferred instructions must produce one diagnostic, not five"; } // Distinct deferred mnemonics are distinct gaps, so suppressing one must not // suppress another. TEST(BinaryTranslatorE2E, Gfx1250ReportsSeparatelyForEachDeferredMnemonic) { const uint32_t deferred = gfx1250_deferred_word(); const uint32_t barrier_state = gfx1250::build_sop1(gfx1250::kSGetBarrierStateSop1, {.ssrc0 = 0, .sdst = 0})[0]; auto translator = make_gfx1250_b0_to_a0_translator(); const auto result = translate_gfx1250_b0_to_a0_result( translator, {deferred, barrier_state, deferred, barrier_state}); auto reported = deferred_family_reports(result); std::ranges::sort(reported); EXPECT_EQ(reported, (std::vector{"s_get_barrier_state", "s_monitor_sleep"})); } // The suppression is scoped to one translation. Process-wide state would let // the first code object that uses a deferred mnemonic hide the same gap in // every object loaded afterwards, which is exactly how HotSwap runs: many code // objects translated in one process. TEST(BinaryTranslatorE2E, Gfx1250ReportsAgainForEachSeparateTranslation) { const uint32_t deferred = gfx1250_deferred_word(); auto first_translator = make_gfx1250_b0_to_a0_translator(); auto second_translator = make_gfx1250_b0_to_a0_translator(); const auto first = translate_gfx1250_b0_to_a0_result(first_translator, {deferred, deferred}); const auto second = translate_gfx1250_b0_to_a0_result(second_translator, {deferred, deferred}); EXPECT_EQ(deferred_family_reports(first), (std::vector{"s_monitor_sleep"})); EXPECT_EQ(deferred_family_reports(second), (std::vector{"s_monitor_sleep"})) << "a second code object must report the gap independently"; } // A translator instance reused across code objects must behave the same way, // since the state lives on the translator rather than in a local. TEST(BinaryTranslatorE2E, Gfx1250ReportsAgainWhenOneTranslatorIsReused) { const uint32_t deferred = gfx1250_deferred_word(); auto translator = make_gfx1250_b0_to_a0_translator(); const auto first = translate_gfx1250_b0_to_a0_result(translator, {deferred, deferred}); const auto second = translate_gfx1250_b0_to_a0_result(translator, {deferred, deferred}); EXPECT_EQ(deferred_family_reports(first), (std::vector{"s_monitor_sleep"})); EXPECT_EQ(deferred_family_reports(second), (std::vector{"s_monitor_sleep"})) << "translate() must reset the per-translation suppression state"; } // The report points at the first instruction of the family so a reader can find // one, and carries the mnemonic so callback consumers can group by gap. TEST(BinaryTranslatorE2E, Gfx1250DeferredFamilyReportCarriesOffsetAndMnemonic) { constexpr uint32_t kGfx1250SNop = 0xBF800000u; const uint32_t deferred = gfx1250_deferred_word(); auto translator = make_gfx1250_b0_to_a0_translator(); const auto result = translate_gfx1250_b0_to_a0_result(translator, {kGfx1250SNop, deferred, deferred}); const auto reported = std::ranges::find_if(result.diagnostics, [](const auto &diagnostic) { return diagnostic.mnemonic == "s_monitor_sleep"; }); ASSERT_NE(reported, result.diagnostics.end()); EXPECT_EQ(reported->severity, rocjitsu::DiagnosticSeverity::Warning); EXPECT_EQ(reported->guest_offset, std::optional(sizeof(uint32_t))) << "the offset must name the first deferred instruction, not the s_nop before it"; } // s_sleep and s_sleep_var behave identically on A0 and B0. The only sleep-family // A0 erratum, DEGFXMI400-12268, is specific to s_monitor_sleep('forever') with // MWAIT=0. Copying a plain sleep through is therefore the correct translation, // not an unimplemented one, and reporting it drowned the reports that do name a // real gap: one RCCL all_reduce run emitted 104,831 of them. TEST(BinaryTranslatorE2E, Gfx1250CopiesPlainSleepWithoutReportingAGap) { const std::vector sleeps = { gfx1250::build_sopp(gfx1250::kSSleepSopp, {.simm16 = 1})[0], gfx1250::build_sop1(gfx1250::kSSleepVarSop1, {.ssrc0 = 0})[0], }; auto translator = make_gfx1250_b0_to_a0_translator(); const auto result = translate_gfx1250_b0_to_a0_result(translator, sleeps); EXPECT_TRUE(deferred_family_reports(result).empty()) << "a plain sleep translates correctly and must not be reported as a gap"; for (const auto &diagnostic : result.diagnostics) { EXPECT_EQ(diagnostic.severity, rocjitsu::DiagnosticSeverity::Warning) << diagnostic.message; } } // A vector 64-bit read cannot name the selector on A0, so the base is moved // into a dead SGPR pair and the source position is repointed at that pair. TEST(BinaryTranslatorE2E, Gfx1250MovesVectorFlatScratchBase64BitSourceToSgprPairForA0) { const auto source = gfx1250::build_vop3( gfx1250::kVAddNcU64Vop3, {.vdst = 0, .src0 = kFlatScratchBaseHiSelector, .src1 = 256 + 2}); const auto out = translate_gfx1250_b0_to_a0_words({source[0], source[1]}); ASSERT_GE(out.size(), 4u) << "the rewrite prepends one scalar move"; // The prologue is a 64-bit scalar read of the base through the low selector. const auto expected_prologue_op = static_cast(gfx1250::kSMovB64Sop1); EXPECT_EQ((out[0] >> 23) & 0x1ffu, 381u) << "prologue must be a SOP1 instruction"; EXPECT_EQ((out[0] >> 8) & 0xffu, expected_prologue_op); EXPECT_EQ(out[0] & 0xffu, kFlatScratchBaseLoSelector); const uint32_t pair_base = (out[0] >> 16) & 0x7fu; EXPECT_EQ(pair_base % 2u, 0u) << "a 64-bit scalar operand must be even-aligned"; EXPECT_LE(pair_base + 1u, 105u) << "the borrowed pair must be an ordinary SGPR pair"; // The vector instruction now reads that pair; its other operands are intact. EXPECT_EQ(out[1] & 0xffffu, source[0] & 0xffffu) << "vdst and modifiers are unchanged"; EXPECT_EQ(out[2] & 0x1ffu, pair_base) << "src0 must name the borrowed pair"; EXPECT_EQ((out[2] >> 9) & 0x1ffu, 256u + 2u) << "src1 is unchanged"; }