/* * Copyright (c) Advanced Micro Devices, Inc., or its affiliates. * * SPDX-License-Identifier: MIT */ // clang-format off namespace amd::pal { #define RUNTIME_KERNEL(...) #__VA_ARGS__ const char* SchedulerSourceCode = RUNTIME_KERNEL( \n extern void __amd_scheduler(__global void*, __global void*, uint); \n __kernel void __amd_rocclr_scheduler(__global void* queue, __global void* params, uint paramIdx) { __amd_scheduler(queue, params, paramIdx); } \n); const char* SchedulerSourceCode20 = RUNTIME_KERNEL( \n extern void __amd_scheduler_pal(__global void*, __global void*, uint); \n __kernel void __amd_rocclr_scheduler(__global void* queue, __global void* params, uint paramIdx) { __amd_scheduler_pal(queue, params, paramIdx); } \n); /* Trap handler for compute. * * The reference version for this are maintained at * https://github.com/ROCm/ROCR-Runtime/tree/amd-staging/runtime/hsa-runtime/core/runtime/trap_handler * * The trap handler source is copied from the above URL, with the following * modifications: * - Add the following directive to declare the trap_entry symbol (this is * later used by pal::Program::GetTrapHandlerAddress to locate the load * address of the trap handler): * * .globl trap_entry * .type trap_entry,@function * .align 256 * - Remove code related to architectures or functionalities not supported * by CLR. * * Make sure to update the TrapHandlerABIVersion definition when appropriate. */ const char* TrapHandlerCode = RUNTIME_KERNEL( \n.if .amdgcn.gfx_generation_number == 11 \n.set SQ_WAVE_PC_HI_ADDRESS_MASK , 0xFFFF \n.set SQ_WAVE_PC_HI_HT_SHIFT , 24 \n.set SQ_WAVE_PC_HI_TRAP_ID_SHIFT , 16 \n.set SQ_WAVE_PC_HI_TRAP_ID_SIZE , 8 \n.set SQ_WAVE_PC_HI_TRAP_ID_BFE , (SQ_WAVE_PC_HI_TRAP_ID_SHIFT | (SQ_WAVE_PC_HI_TRAP_ID_SIZE << 16)) \n.set SQ_WAVE_STATUS_HALT_SHIFT , 13 \n.set SQ_WAVE_STATUS_TRAP_SKIP_EXPORT_SHIFT , 18 \n.set SQ_WAVE_STATUS_HALT_BFE , (SQ_WAVE_STATUS_HALT_SHIFT | (1 << 16)) \n.set SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT , 8 \n.set SQ_WAVE_TRAPSTS_ILLEGAL_INST_SHIFT , 11 \n.set SQ_WAVE_TRAPSTS_XNACK_ERROR_SHIFT , 28 \n.set SQ_WAVE_TRAPSTS_MATH_EXCP , 0x7F \n.set SQ_WAVE_TRAPSTS_PERF_SNAPSHOT_SHIFT , 26 \n.set SQ_WAVE_TRAPSTS_HOST_TRAP_SHIFT , 22 \n.set SQ_WAVE_MODE_EXCP_EN_SHIFT , 12 \n.set SQ_WAVE_MODE_EXCP_EN_SIZE , 8 \n.set TRAP_ID_ABORT , 2 \n.set TRAP_ID_DEBUGTRAP , 3 \n.set DOORBELL_ID_SIZE , 10 \n.set DOORBELL_ID_MASK , ((1 << DOORBELL_ID_SIZE) - 1) \n.set EC_QUEUE_WAVE_ABORT_M0 , (1 << (DOORBELL_ID_SIZE + 0)) \n.set EC_QUEUE_WAVE_TRAP_M0 , (1 << (DOORBELL_ID_SIZE + 1)) \n.set EC_QUEUE_WAVE_MATH_ERROR_M0 , (1 << (DOORBELL_ID_SIZE + 2)) \n.set EC_QUEUE_WAVE_ILLEGAL_INSTRUCTION_M0 , (1 << (DOORBELL_ID_SIZE + 3)) \n.set EC_QUEUE_WAVE_MEMORY_VIOLATION_M0 , (1 << (DOORBELL_ID_SIZE + 4)) \n.set EC_QUEUE_WAVE_APERTURE_VIOLATION_M0 , (1 << (DOORBELL_ID_SIZE + 5)) \n \n.set TTMP6_SPI_TTMPS_SETUP_DISABLED_SHIFT , 31 \n.set TTMP6_WAVE_STOPPED_SHIFT , 30 \n.set TTMP6_SAVED_STATUS_HALT_SHIFT , 29 \n.set TTMP6_SAVED_STATUS_HALT_MASK , (1 << TTMP6_SAVED_STATUS_HALT_SHIFT) \n.set TTMP6_SAVED_TRAP_ID_SHIFT , 25 \n.set TTMP6_SAVED_TRAP_ID_SIZE , 4 \n.set TTMP6_SAVED_TRAP_ID_MASK , (((1 << TTMP6_SAVED_TRAP_ID_SIZE) - 1) << TTMP6_SAVED_TRAP_ID_SHIFT) \n.set TTMP6_SAVED_TRAP_ID_BFE , (TTMP6_SAVED_TRAP_ID_SHIFT | (TTMP6_SAVED_TRAP_ID_SIZE << 16)) \n \n.set TTMP_PC_HI_SHIFT , 7 \n.set TTMP_DEBUG_ENABLED_SHIFT , 23 \n \n// ABI between first and second level trap handler: \n// ttmp0 = PC[31:0] \n// ttmp6 = 0[6:0], DispatchPktIndx[24:0] \n// ttmp8 = WorkgroupIdX \n// ttmp9 = WorkgroupIdY \n// ttmp10 = WorkgroupIdZ \n// ttmp11 = 0[7:0], DebugEnabled[0], 0[15:0], NoScratch[0], WaveIdInWG[5:0] \n// ttmp12 = SQ_WAVE_STATUS \n// ttmp14 = TMA[31:0] \n// ttmp15 = TMA[63:32] \n \n .globl trap_entry \n .type trap_entry,@function \n .align 256 \n \ntrap_entry: \n // Extract trap_id from ttmp2 \n s_bfe_u32 ttmp2, ttmp1, SQ_WAVE_PC_HI_TRAP_ID_BFE \n s_cbranch_scc0 .not_s_trap // If trap_id == 0, it's not an s_trap nor host trap \n \n // Check if the it was an host trap. \n s_bitcmp1_b32 ttmp1, SQ_WAVE_PC_HI_HT_SHIFT \n s_cbranch_scc1 .not_s_trap \n \n // It's an s_trap; advance the PC \n s_add_u32 ttmp0, ttmp0, 0x4 \n s_addc_u32 ttmp1, ttmp1, 0x0 \n \n // If llvm.debugtrap and debugger is not attached. \n s_cmp_eq_u32 ttmp2, TRAP_ID_DEBUGTRAP \n s_cbranch_scc0 .no_skip_debugtrap \n s_bitcmp0_b32 ttmp11, TTMP_DEBUG_ENABLED_SHIFT \n s_cbranch_scc0 .no_skip_debugtrap \n \n // Ignore llvm.debugtrap. \n s_branch .exit_trap \n \n.not_s_trap: \n.no_skip_debugtrap: \n // Save trap id and halt status in ttmp6. \n s_andn2_b32 ttmp6, ttmp6, (TTMP6_SAVED_TRAP_ID_MASK | TTMP6_SAVED_STATUS_HALT_MASK) \n s_bfe_u32 ttmp2, ttmp1, SQ_WAVE_PC_HI_TRAP_ID_BFE \n s_min_u32 ttmp2, ttmp2, 0xF \n s_lshl_b32 ttmp2, ttmp2, TTMP6_SAVED_TRAP_ID_SHIFT \n s_or_b32 ttmp6, ttmp6, ttmp2 \n s_bfe_u32 ttmp2, ttmp12, SQ_WAVE_STATUS_HALT_BFE \n s_lshl_b32 ttmp2, ttmp2, TTMP6_SAVED_STATUS_HALT_SHIFT \n s_or_b32 ttmp6, ttmp6, ttmp2 \n \n // Fetch doorbell id for our queue. \n s_sendmsg_rtn_b32 ttmp3, sendmsg(MSG_RTN_GET_DOORBELL) \n s_waitcnt lgkmcnt(0) \n s_and_b32 ttmp3, ttmp3, DOORBELL_ID_MASK \n \n // Map trap reason to an exception code. \n s_getreg_b32 ttmp2, hwreg(HW_REG_TRAPSTS) \n \n s_bitcmp1_b32 ttmp2, SQ_WAVE_TRAPSTS_XNACK_ERROR_SHIFT \n s_cbranch_scc0 .not_memory_violation \n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_MEMORY_VIOLATION_M0 \n \n // Aperture violation requires XNACK_ERROR == 0. \n s_branch .not_aperture_violation \n \n.not_memory_violation: \n s_bitcmp1_b32 ttmp2, SQ_WAVE_TRAPSTS_MEM_VIOL_SHIFT \n s_cbranch_scc0 .not_aperture_violation \n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_APERTURE_VIOLATION_M0 \n \n.not_aperture_violation: \n s_bitcmp1_b32 ttmp2, SQ_WAVE_TRAPSTS_ILLEGAL_INST_SHIFT \n s_cbranch_scc0 .not_illegal_instruction \n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_ILLEGAL_INSTRUCTION_M0 \n \n.not_illegal_instruction: \n s_and_b32 ttmp2, ttmp2, SQ_WAVE_TRAPSTS_MATH_EXCP \n s_cbranch_scc0 .not_math_exception \n s_getreg_b32 ttmp7, hwreg(HW_REG_MODE) \n s_lshl_b32 ttmp2, ttmp2, SQ_WAVE_MODE_EXCP_EN_SHIFT \n s_and_b32 ttmp2, ttmp2, ttmp7 \n s_cbranch_scc0 .not_math_exception \n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_MATH_ERROR_M0 \n \n.not_math_exception: \n s_bfe_u32 ttmp2, ttmp6, TTMP6_SAVED_TRAP_ID_BFE \n s_cmp_eq_u32 ttmp2, TRAP_ID_ABORT \n s_cbranch_scc0 .not_abort_trap \n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_ABORT_M0 \n \n.not_abort_trap: \n // If no other exception was flagged then report a generic error. \n s_andn2_b32 ttmp2, ttmp3, DOORBELL_ID_MASK \n s_cbranch_scc1 .send_interrupt \n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0 \n \n.send_interrupt: \n // m0 = interrupt data = (exception_code << DOORBELL_ID_SIZE) | doorbell_id \n s_mov_b32 ttmp2, m0 \n s_mov_b32 m0, ttmp3 \n s_nop 0x0 // Manually inserted wait states \n s_sendmsg sendmsg(MSG_INTERRUPT) \n s_waitcnt lgkmcnt(0) // Wait for the message to go out. \n s_mov_b32 m0, ttmp2 \n \n // Parking the wave requires saving the original pc in the preserved ttmps. \n // Register layout before parking the wave: \n // \n // ttmp7: 0[31:0] \n // ttmp11: 1st_level_ttmp11[31:23] 0[15:0] 1st_level_ttmp11[6:0] \n // \n // After parking the wave: \n // \n // ttmp7: pc_lo[31:0] \n // ttmp11: 1st_level_ttmp11[31:23] pc_hi[15:0] 1st_level_ttmp11[6:0] \n // Save the PC \n s_mov_b32 ttmp7, ttmp0 \n s_and_b32 ttmp1, ttmp1, SQ_WAVE_PC_HI_ADDRESS_MASK \n s_lshl_b32 ttmp1, ttmp1, TTMP_PC_HI_SHIFT \n s_andn2_b32 ttmp11, ttmp11, (SQ_WAVE_PC_HI_ADDRESS_MASK << TTMP_PC_HI_SHIFT) \n s_or_b32 ttmp11, ttmp11, ttmp1 \n \n // Park the wave \n s_getpc_b64 [ttmp0, ttmp1] \n s_add_u32 ttmp0, ttmp0, .parked - . \n s_addc_u32 ttmp1, ttmp1, 0x0 \n \n.halt_wave: \n // Halt the wavefront upon restoring STATUS below. \n s_bitset1_b32 ttmp6, TTMP6_WAVE_STOPPED_SHIFT \n s_bitset1_b32 ttmp12, SQ_WAVE_STATUS_HALT_SHIFT \n // Set WAVE.SKIP_EXPORT as a maker so the debugger knows the trap handler was \n // entered and has decided to halt the wavee. \n s_bitset1_b32 ttmp12, SQ_WAVE_STATUS_TRAP_SKIP_EXPORT_SHIFT \n \n.exit_trap: \n // Restore SQ_WAVE_STATUS. \n s_and_b64 exec, exec, exec // restore STATUS.EXECZ, not writable by s_setreg_b32 \n s_and_b64 vcc, vcc, vcc // restore STATUS.VCCZ, not writable by s_setreg_b32 \n s_setreg_b32 hwreg(HW_REG_STATUS), ttmp12 \n \n // Return to original (possibly modified) PC. \n s_rfe_b64 [ttmp0, ttmp1] \n \n.parked: \n s_trap 0x2 \n s_branch .parked \n \n// For gfx11, add padding instructions so we can ensure instruction cache \n// prefetch always has something to load. \n.rept (256 - ((. - trap_entry) % 64)) / 4 \n s_code_end \n.endr \n.elseif .amdgcn.gfx_generation_number == 12 \n.set DOORBELL_ID_SIZE , 10 \n.set DOORBELL_ID_MASK , ((1 << DOORBELL_ID_SIZE) - 1) \n.set EC_QUEUE_WAVE_ABORT_M0 , (1 << (DOORBELL_ID_SIZE + 0)) \n.set EC_QUEUE_WAVE_TRAP_M0 , (1 << (DOORBELL_ID_SIZE + 1)) \n.set EC_QUEUE_WAVE_MATH_ERROR_M0 , (1 << (DOORBELL_ID_SIZE + 2)) \n.set EC_QUEUE_WAVE_ILLEGAL_INSTRUCTION_M0 , (1 << (DOORBELL_ID_SIZE + 3)) \n.set EC_QUEUE_WAVE_MEMORY_VIOLATION_M0 , (1 << (DOORBELL_ID_SIZE + 4)) \n.set EC_QUEUE_WAVE_APERTURE_VIOLATION_M0 , (1 << (DOORBELL_ID_SIZE + 5)) \n \n.set SQ_WAVE_EXCP_FLAG_PRIV_ADDR_WATCH_MASK , (1 << 4) - 1 \n.set SQ_WAVE_EXCP_FLAG_PRIV_MEMVIOL_SHIFT , 4 \n.set SQ_WAVE_EXCP_FLAG_PRIV_ILLEGAL_INST_SHIFT , 6 \n.set SQ_WAVE_EXCP_FLAG_PRIV_HT_SHIFT , 7 \n.set SQ_WAVE_EXCP_FLAG_PRIV_WAVE_START_SHIFT , 8 \n.set SQ_WAVE_EXCP_FLAG_PRIV_WAVE_END_SHIFT , 9 \n.set SQ_WAVE_EXCP_FLAG_PRIV_TRAP_AFTER_INST_SHIFT , 11 \n.set SQ_WAVE_EXCP_FLAG_PRIV_XNACK_ERROR_SHIFT , 12 \n \n.set SQ_WAVE_EXCP_FLAG_USER_MATH_EXCP_SHIFT , 0 \n.set SQ_WAVE_EXCP_FLAG_USER_MATH_EXCP_SIZE , 7 \n \n.set SQ_WAVE_TRAP_CTRL_MATH_EXCP_MASK , ((1 << 7) - 1) \n.set SQ_WAVE_TRAP_CTRL_ADDR_WATCH_SHIFT , 7 \n.set SQ_WAVE_TRAP_CTRL_WAVE_END_SHIFT , 8 \n.set SQ_WAVE_TRAP_CTRL_TRAP_AFTER_INST , 9 \n \n.set SQ_WAVE_PC_HI_ADDRESS_MASK , 0xFFFF \n.set SQ_WAVE_PC_HI_TRAP_ID_BFE , (SQ_WAVE_PC_HI_TRAP_ID_SHIFT | (SQ_WAVE_PC_HI_TRAP_ID_SIZE << 16)) \n.set SQ_WAVE_PC_HI_TRAP_ID_SHIFT , 28 \n.set SQ_WAVE_PC_HI_TRAP_ID_SIZE , 4 \n.set SQ_WAVE_STATE_PRIV_HALT_BFE , (SQ_WAVE_STATE_PRIV_HALT_SHIFT | (1 << 16)) \n.set SQ_WAVE_STATE_PRIV_HALT_SHIFT , 14 \n.set SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_SHIFT , 2 \n.set TRAP_ID_ABORT , 2 \n.set TRAP_ID_DEBUGTRAP , 3 \n.set TTMP6_SAVED_STATUS_HALT_MASK , (1 << TTMP6_SAVED_STATUS_HALT_SHIFT) \n.set TTMP6_SAVED_STATUS_HALT_SHIFT , 29 \n.set TTMP6_SAVED_TRAP_ID_BFE , (TTMP6_SAVED_TRAP_ID_SHIFT | (TTMP6_SAVED_TRAP_ID_SIZE << 16)) \n.set TTMP6_SAVED_TRAP_ID_MASK , (((1 << TTMP6_SAVED_TRAP_ID_SIZE) - 1) << TTMP6_SAVED_TRAP_ID_SHIFT) \n.set TTMP6_SAVED_TRAP_ID_SHIFT , 25 \n.set TTMP6_SAVED_TRAP_ID_SIZE , 4 \n.set TTMP6_WAVE_STOPPED_SHIFT , 30 \n.set TTMP8_DEBUG_FLAG_SHIFT , 31 \n.set TTMP11_DEBUG_ENABLED_SHIFT , 23 \n.set TTMP_PC_HI_SHIFT , 7 \n \n// ABI between first and second level trap handler: \n// { ttmp1, ttmp0 } = TrapID[3:0], zeros, PC[47:0] \n// ttmp11 = 0[7:0], DebugEnabled[0], 0[15:0], NoScratch[0], 0[5:0] \n// ttmp12 = SQ_WAVE_STATE_PRIV \n// ttmp14 = TMA[31:0] \n// ttmp15 = TMA[63:32] \n \n .globl trap_entry \n .type trap_entry,@function \n .align 256 \n \ntrap_entry: \n // Clear ttmp3 as it will contain the exception code. \n s_mov_b32 ttmp3, 0 \n \n // Branch if not a trap (an exception instead). \n s_bfe_u32 ttmp2, ttmp1, SQ_WAVE_PC_HI_TRAP_ID_BFE \n s_cbranch_scc0 .check_exceptions \n \n // If caused by s_trap then advance PC, then figure out the trap ID: \n // - if trapID is DEBUGTRAP and debugger is attach, report WAVE_TRAP, \n // - if trapID is ABORTTRAP, report WAVE_ABORT, \n // - report WAVE_TRAP for any other trap ID. \n s_add_u32 ttmp0, ttmp0, 0x4 \n s_addc_u32 ttmp1, ttmp1, 0x0 \n \n // If llvm.debugtrap and debugger is not attached. \n s_cmp_eq_u32 ttmp2, TRAP_ID_DEBUGTRAP \n s_cbranch_scc0 .not_debug_trap \n \n s_bitcmp1_b32 ttmp11, TTMP11_DEBUG_ENABLED_SHIFT \n s_cbranch_scc0 .check_exceptions \n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0 \n \n.not_debug_trap: \n s_cmp_eq_u32 ttmp2, TRAP_ID_ABORT \n s_cbranch_scc0 .not_abort_trap \n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_ABORT_M0 \n s_branch .check_exceptions \n \n.not_abort_trap: \n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0 \n \n // We need to explititly look for all exceptions we want to report to the \n // host: \n // - EXCP_FLAG_PRIV.XNACK_ERROR (&& EXCP_FLAG_PRIV.MEMVIOL) \n // -> WAVE_MEMORY_VIOLATION \n // - EXCP_FLAG_PRIV.MEMVIOL (and !EXCP_FLAG_PRIV.XNACK_ERROR) \n // -> WAVE_APERTURE_VIOLATION \n // - EXCP_FLAG_PRIV.ILLEGAL_INST -> WAVE_ILLEGAL_INSTRUCTION \n // - EXCP_FLAG_PRIV.WAVE_START -> WAVE_TRAP \n // - EXCP_FLAG_PRIV.WAVE_END && TRAP_CTRL.WAVE_END -> WAVE_TRAP \n // - TRAP_CTRL.TRAP_AFTER_INST -> WAVE_TRAP \n // - EXCP_FLAG_PRIV.ADDR_WATCH && TRAP_CTL.WATCH -> WAVE_TRAP \n // - (EXCP_FLAG_USER[ALU] & TRAP_CTRL[ALU]) != 0 -> WAVE_MATH_ERROR \n.check_exceptions: \n s_getreg_b32 ttmp2, hwreg(HW_REG_EXCP_FLAG_PRIV) \n s_getreg_b32 ttmp13, hwreg(HW_REG_TRAP_CTRL) \n \n s_bitcmp1_b32 ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_XNACK_ERROR_SHIFT \n s_cbranch_scc0 .not_memory_violation \n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_MEMORY_VIOLATION_M0 \n \n // Aperture violation requires XNACK_ERROR == 0. \n s_branch .not_aperture_violation \n \n.not_memory_violation: \n s_bitcmp1_b32 ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_MEMVIOL_SHIFT \n s_cbranch_scc0 .not_aperture_violation \n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_APERTURE_VIOLATION_M0 \n \n.not_aperture_violation: \n s_bitcmp1_b32 ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_ILLEGAL_INST_SHIFT \n s_cbranch_scc0 .not_illegal_instruction \n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_ILLEGAL_INSTRUCTION_M0 \n \n.not_illegal_instruction: \n s_bitcmp1_b32 ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_WAVE_START_SHIFT \n s_cbranch_scc0 .not_wave_end \n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0 \n \n.not_wave_start: \n s_bitcmp1_b32 ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_WAVE_END_SHIFT \n s_cbranch_scc0 .not_wave_end \n s_bitcmp1_b32 ttmp13, SQ_WAVE_TRAP_CTRL_WAVE_END_SHIFT \n s_cbranch_scc0 .not_wave_end \n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0 \n \n.not_wave_end: \n s_bitcmp1_b32 ttmp13, SQ_WAVE_TRAP_CTRL_TRAP_AFTER_INST \n s_cbranch_scc0 .not_trap_after_inst \n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0 \n \n.not_trap_after_inst: \n s_and_b32 ttmp2, ttmp2, SQ_WAVE_EXCP_FLAG_PRIV_ADDR_WATCH_MASK \n s_cbranch_scc0 .not_addr_watch \n s_bitcmp1_b32 ttmp13, SQ_WAVE_TRAP_CTRL_ADDR_WATCH_SHIFT \n s_cbranch_scc0 .not_addr_watch \n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_TRAP_M0 \n \n.not_addr_watch: \n s_getreg_b32 ttmp2, hwreg(HW_REG_EXCP_FLAG_USER, SQ_WAVE_EXCP_FLAG_USER_MATH_EXCP_SHIFT, SQ_WAVE_EXCP_FLAG_USER_MATH_EXCP_SIZE) \n s_and_b32 ttmp13, ttmp13, SQ_WAVE_TRAP_CTRL_MATH_EXCP_MASK \n s_and_b32 ttmp2, ttmp2, ttmp13 \n s_cbranch_scc0 .not_math_exception \n s_or_b32 ttmp3, ttmp3, EC_QUEUE_WAVE_MATH_ERROR_M0 \n \n.not_math_exception: \n s_cmp_eq_u32 ttmp3, 0 \n // This was not a s_trap we are interested in or an exception, return to \n // the user code. \n s_cbranch_scc1 .exit_trap \n \n.send_interrupt: \n // Fetch doorbell id for our queue. \n s_sendmsg_rtn_b32 ttmp2, sendmsg(MSG_RTN_GET_DOORBELL) \n s_wait_kmcnt 0 \n s_and_b32 ttmp2, ttmp2, DOORBELL_ID_MASK \n s_or_b32 ttmp3, ttmp2, ttmp3 \n \n // Save trap id and halt status in ttmp6. \n s_andn2_b32 ttmp6, ttmp6, (TTMP6_SAVED_TRAP_ID_MASK | TTMP6_SAVED_STATUS_HALT_MASK) \n s_bfe_u32 ttmp2, ttmp1, SQ_WAVE_PC_HI_TRAP_ID_BFE \n s_min_u32 ttmp2, ttmp2, 0xF \n s_lshl_b32 ttmp2, ttmp2, TTMP6_SAVED_TRAP_ID_SHIFT \n s_or_b32 ttmp6, ttmp6, ttmp2 \n s_bfe_u32 ttmp2, ttmp12, SQ_WAVE_STATE_PRIV_HALT_BFE \n s_lshl_b32 ttmp2, ttmp2, TTMP6_SAVED_STATUS_HALT_SHIFT \n s_or_b32 ttmp6, ttmp6, ttmp2 \n \n // m0 = interrupt data = (exception_code << DOORBELL_ID_SIZE) | doorbell_id \n s_mov_b32 ttmp2, m0 \n s_mov_b32 m0, ttmp3 \n s_nop 0x0 // Manually inserted wait states \n s_sendmsg sendmsg(MSG_INTERRUPT) \n // Wait for the message to go out. \n s_wait_kmcnt 0 \n s_mov_b32 m0, ttmp2 \n \n // Parking the wave requires saving the original pc in the preserved ttmps. \n // Register layout before parking the wave: \n // \n // ttmp10: ?[31:0] \n // ttmp11: 1st_level_ttmp11[31:23] 0[15:0] 1st_level_ttmp11[6:0] \n // \n // After parking the wave: \n // \n // ttmp10: pc_lo[31:0] \n // ttmp11: 1st_level_ttmp11[31:23] pc_hi[15:0] 1st_level_ttmp11[6:0] \n // \n // Save the PC \n s_mov_b32 ttmp10, ttmp0 \n s_and_b32 ttmp1, ttmp1, SQ_WAVE_PC_HI_ADDRESS_MASK \n s_lshl_b32 ttmp1, ttmp1, TTMP_PC_HI_SHIFT \n s_andn2_b32 ttmp11, ttmp11, (SQ_WAVE_PC_HI_ADDRESS_MASK << TTMP_PC_HI_SHIFT) \n s_or_b32 ttmp11, ttmp11, ttmp1 \n \n // Park the wave \n s_getpc_b64 [ttmp0, ttmp1] \n s_add_u32 ttmp0, ttmp0, .parked - . \n s_addc_u32 ttmp1, ttmp1, 0x0 \n \n.halt_wave: \n // Halt the wavefront upon restoring STATUS below. \n s_bitset1_b32 ttmp6, TTMP6_WAVE_STOPPED_SHIFT \n s_bitset1_b32 ttmp12, SQ_WAVE_STATE_PRIV_HALT_SHIFT \n \n // Initialize TTMP registers \n s_bitcmp1_b32 ttmp8, TTMP8_DEBUG_FLAG_SHIFT \n s_cbranch_scc1 .ttmps_initialized \n s_mov_b32 ttmp4, 0 \n s_mov_b32 ttmp5, 0 \n s_bitset1_b32 ttmp8, TTMP8_DEBUG_FLAG_SHIFT \n.ttmps_initialized: \n \n.exit_trap: \n // Restore SQ_WAVE_STATUS. \n s_and_b64 exec, exec, exec // Restore STATUS.EXECZ, not writable by s_setreg_b32 \n s_and_b64 vcc, vcc, vcc // Restore STATUS.VCCZ, not writable by s_setreg_b32 \n s_setreg_b32 hwreg(HW_REG_STATE_PRIV, 0, SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_SHIFT), ttmp12 \n s_lshr_b32 ttmp12, ttmp12, (SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_SHIFT + 1) \n s_setreg_b32 hwreg(HW_REG_STATE_PRIV, SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_SHIFT + 1, 32 - SQ_WAVE_STATE_PRIV_BARRIER_COMPLETE_SHIFT - 1), ttmp12 \n \n // Return to original (possibly modified) PC. \n s_rfe_b64 [ttmp0, ttmp1] \n \n.parked: \n s_trap 0x2 \n s_branch .parked \n \n// Add s_code_end padding so instruction prefetch always has something to read. \n.rept (256 - ((. - trap_entry) % 64)) / 4 \n s_code_end \n.endr \n.endif \n); } // namespace amd::pal // clang-format on