Files
TheRock/compiler/amd-llvm/amd/comgr/test-lit/hotswap/rewriter/hotswap-reusable-pc-call-targets.s
T
derek.holloway 7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
Flatten subprojects into a single base monorepo
2026-09-02 16:34:47 -07:00

384 lines
13 KiB
ArmAsm

// COM: Production activation kernels select one of several local callees with
// COM: get-PC/carry materialization, merge the selected address, and reuse it
// COM: across many register calls. Resolve the finite reaching-target set so
// COM: an unrelated required far rewrite may safely use external gateway
// COM: padding. A selector bypass would leave the target unknown and must
// COM: continue to fail closed.
// RUN: %clang -target amdgcn-amd-amdhsa -mcpu=gfx1250 -nostdlib %s -o %t.elf
// RUN: env AMD_COMGR_EMIT_VERBOSE_LOGS=1 hotswap-rewrite %t.elf \
// RUN: amdgcn-amd-amdhsa--gfx1250 amdgcn-amd-amdhsa--gfx1250 \
// RUN: --output %t.out.elf 2>&1 | %FileCheck --check-prefix=LOG %s
// LOG: hotswap: resolved PC-materialized call
// LOG: hotswap: resolved reusable PC-materialized call
// LOG-SAME: to 1 target(s)
// LOG: hotswap: resolved reusable PC-materialized call
// LOG-SAME: to 2 target(s)
// LOG-NOT: hotswap: unresolved call target
// LOG: hotswap: planned 1 shared far-dispatch gateway group(s) for 8 source site(s)
// LOG: RESULT: SUCCESS
// RUN: sed 's/^\.set unsafe_selector, 0$/.set unsafe_selector, 1/' \
// RUN: %s > %t.bypass.s
// RUN: %clang -target amdgcn-amd-amdhsa -mcpu=gfx1250 -nostdlib \
// RUN: %t.bypass.s -o %t.bypass.elf
// RUN: env AMD_COMGR_EMIT_VERBOSE_LOGS=1 hotswap-rewrite %t.bypass.elf \
// RUN: amdgcn-amd-amdhsa--gfx1250 amdgcn-amd-amdhsa--gfx1250 \
// RUN: --expect-status ERROR 2>&1 \
// RUN: | %FileCheck --check-prefixes=BYPASS,FAIL %s
// BYPASS: hotswap: unresolved call target
// OVERLAP: hotswap: unresolved call target
// CLOBBER: hotswap: resolved reusable PC-materialized call
// CLOBBER: hotswap: unresolved call target
// BOOTSTRAP-CLOBBER: hotswap: resolved PC-materialized call
// BOOTSTRAP-CLOBBER: hotswap: unresolved call target
// TAIL: hotswap: resolved reusable PC-materialized call
// TAIL: hotswap: unresolved call target
// INDIRECT: hotswap: resolved reusable PC-materialized call
// INDIRECT: hotswap: unresolved call target
// EXTERNAL: hotswap: unresolved call target
// UNDECODED: hotswap: resolved reusable PC-materialized call
// UNDECODED: hotswap: unresolved call target
// RECONVERGE: hotswap: resolved reusable PC-materialized call
// RECONVERGE: hotswap: unresolved call target
// FAIL: hotswap: unresolved control-flow target disables NOP-sled emission,
// FAIL-SAME: trampoline coalescing, source relocation, and .text gateways
// FAIL: hotswap: error: no safe short-branch gateway for far site
// FAIL: RESULT: ERROR
// RUN: sed 's/^\.set overlap_delta, 0$/.set overlap_delta, 1/' \
// RUN: %s > %t.overlap.s
// RUN: %clang -target amdgcn-amd-amdhsa -mcpu=gfx1250 -nostdlib \
// RUN: %t.overlap.s -o %t.overlap.elf
// RUN: env AMD_COMGR_EMIT_VERBOSE_LOGS=1 hotswap-rewrite %t.overlap.elf \
// RUN: amdgcn-amd-amdhsa--gfx1250 amdgcn-amd-amdhsa--gfx1250 \
// RUN: --expect-status ERROR 2>&1 \
// RUN: | %FileCheck --check-prefixes=OVERLAP,FAIL %s
// RUN: sed 's/^\.set clobber_target, 0$/.set clobber_target, 1/' \
// RUN: %s > %t.clobber.s
// RUN: %clang -target amdgcn-amd-amdhsa -mcpu=gfx1250 -nostdlib \
// RUN: %t.clobber.s -o %t.clobber.elf
// RUN: env AMD_COMGR_EMIT_VERBOSE_LOGS=1 hotswap-rewrite %t.clobber.elf \
// RUN: amdgcn-amd-amdhsa--gfx1250 amdgcn-amd-amdhsa--gfx1250 \
// RUN: --expect-status ERROR 2>&1 \
// RUN: | %FileCheck --check-prefixes=CLOBBER,FAIL %s
// RUN: sed 's/^\.set clobber_bootstrap, 0$/.set clobber_bootstrap, 1/' \
// RUN: %s > %t.bootstrap-clobber.s
// RUN: %clang -target amdgcn-amd-amdhsa -mcpu=gfx1250 -nostdlib \
// RUN: %t.bootstrap-clobber.s -o %t.bootstrap-clobber.elf
// RUN: env AMD_COMGR_EMIT_VERBOSE_LOGS=1 hotswap-rewrite \
// RUN: %t.bootstrap-clobber.elf amdgcn-amd-amdhsa--gfx1250 \
// RUN: amdgcn-amd-amdhsa--gfx1250 --expect-status ERROR 2>&1 \
// RUN: | %FileCheck --check-prefixes=BOOTSTRAP-CLOBBER,FAIL %s
// RUN: sed 's/^\.set tail_escape, 0$/.set tail_escape, 1/' \
// RUN: %s > %t.tail.s
// RUN: %clang -target amdgcn-amd-amdhsa -mcpu=gfx1250 -nostdlib \
// RUN: %t.tail.s -o %t.tail.elf
// RUN: env AMD_COMGR_EMIT_VERBOSE_LOGS=1 hotswap-rewrite %t.tail.elf \
// RUN: amdgcn-amd-amdhsa--gfx1250 amdgcn-amd-amdhsa--gfx1250 \
// RUN: --expect-status ERROR 2>&1 \
// RUN: | %FileCheck --check-prefixes=TAIL,FAIL %s
// RUN: sed 's/^\.set indirect_escape, 0$/.set indirect_escape, 1/' \
// RUN: %s > %t.indirect.s
// RUN: %clang -target amdgcn-amd-amdhsa -mcpu=gfx1250 -nostdlib \
// RUN: %t.indirect.s -o %t.indirect.elf
// RUN: env AMD_COMGR_EMIT_VERBOSE_LOGS=1 hotswap-rewrite %t.indirect.elf \
// RUN: amdgcn-amd-amdhsa--gfx1250 amdgcn-amd-amdhsa--gfx1250 \
// RUN: --expect-status ERROR 2>&1 \
// RUN: | %FileCheck --check-prefixes=INDIRECT,FAIL %s
// RUN: sed 's/^\.set external_entry, 0$/.set external_entry, 1/' \
// RUN: %s > %t.external.s
// RUN: %clang -target amdgcn-amd-amdhsa -mcpu=gfx1250 -nostdlib \
// RUN: %t.external.s -o %t.external.elf
// RUN: env AMD_COMGR_EMIT_VERBOSE_LOGS=1 hotswap-rewrite %t.external.elf \
// RUN: amdgcn-amd-amdhsa--gfx1250 amdgcn-amd-amdhsa--gfx1250 \
// RUN: --expect-status ERROR 2>&1 \
// RUN: | %FileCheck --check-prefixes=EXTERNAL,FAIL %s
// RUN: sed 's/^\.set outside_selector, 0$/.set outside_selector, 1/' \
// RUN: %s > %t.outside.s
// RUN: %clang -target amdgcn-amd-amdhsa -mcpu=gfx1250 -nostdlib \
// RUN: %t.outside.s -o %t.outside.elf
// RUN: env AMD_COMGR_EMIT_VERBOSE_LOGS=1 hotswap-rewrite %t.outside.elf \
// RUN: amdgcn-amd-amdhsa--gfx1250 amdgcn-amd-amdhsa--gfx1250 \
// RUN: --expect-status ERROR 2>&1 \
// RUN: | %FileCheck --check-prefix=OUTSIDE %s
// OUTSIDE: hotswap: resolved PC-materialized call
// OUTSIDE: hotswap: resolved reusable PC-materialized call
// OUTSIDE-SAME: to 1 target(s)
// OUTSIDE: hotswap: resolved reusable PC-materialized call
// OUTSIDE-SAME: to 3 target(s)
// OUTSIDE: hotswap: unresolved call target
// OUTSIDE-SAME: (s_swap_pc_i64)
// OUTSIDE: hotswap: unresolved control-flow target disables NOP-sled emission,
// OUTSIDE-SAME: trampoline coalescing, source relocation, and .text gateways
// OUTSIDE: hotswap: error: no safe short-branch gateway for far site
// OUTSIDE: RESULT: ERROR
// An undecoded slot between the two reused calls must not let the later call
// resolve: its bytes could clobber the target pair or divert control flow.
// RUN: sed 's/^\.set undecoded_gap, 0$/.set undecoded_gap, 1/' \
// RUN: %s > %t.undecoded.s
// RUN: %clang -target amdgcn-amd-amdhsa -mcpu=gfx1250 -nostdlib \
// RUN: %t.undecoded.s -o %t.undecoded.elf
// RUN: env AMD_COMGR_EMIT_VERBOSE_LOGS=1 hotswap-rewrite %t.undecoded.elf \
// RUN: amdgcn-amd-amdhsa--gfx1250 amdgcn-amd-amdhsa--gfx1250 \
// RUN: --expect-status ERROR 2>&1 \
// RUN: | %FileCheck --check-prefixes=UNDECODED,FAIL %s
// A back-edge that reloads the target pair through an unprovable definition
// makes the reconverged call Unknown; a stale finite result from the first
// visit must not survive.
// RUN: sed 's/^\.set reconverge_reload, 0$/.set reconverge_reload, 1/' \
// RUN: %s > %t.reconverge.s
// RUN: %clang -target amdgcn-amd-amdhsa -mcpu=gfx1250 -nostdlib \
// RUN: %t.reconverge.s -o %t.reconverge.elf
// RUN: env AMD_COMGR_EMIT_VERBOSE_LOGS=1 hotswap-rewrite %t.reconverge.elf \
// RUN: amdgcn-amd-amdhsa--gfx1250 amdgcn-amd-amdhsa--gfx1250 \
// RUN: --expect-status ERROR 2>&1 \
// RUN: | %FileCheck --check-prefixes=RECONVERGE,FAIL %s
// RUN: %llvm-objdump -d %t.out.elf | %FileCheck --check-prefix=DISASM \
// RUN: --implicit-check-not=s_add_pc_i64 %s
// DISASM-LABEL: <reusable_pc_targets>:
// DISASM: s_swap_pc_i64 s[6:7]
// DISASM-NEXT: s_swap_pc_i64 s[6:7]
// DISASM-NEXT: s_branch
// DISASM-NEXT: s_get_pc_i64
// DISASM-NEXT: s_branch
// DISASM-NEXT: s_branch
// DISASM-NEXT: s_get_pc_i64
// DISASM-NEXT: s_branch
// DISASM-LABEL: <gateway_barrier>:
// DISASM-NEXT: s_endpgm
// DISASM-NEXT: s_get_pc_i64
// DISASM-NEXT: s_add_nc_u64
// DISASM-NEXT: s_set_pc_i64
// DISASM: ds_load_b32 v0, v2 offset:256
// DISASM-NEXT: ds_load_b32 v1, v2 offset:768
// RUN: hotswap-rewrite %t.out.elf \
// RUN: amdgcn-amd-amdhsa--gfx1250 amdgcn-amd-amdhsa--gfx1250 \
// RUN: --check-idempotent | %FileCheck --check-prefix=IDEM %s
// IDEM: IDEMPOTENT: YES
.set unsafe_selector, 0
.set outside_selector, 0
.set overlap_delta, 0
.set clobber_target, 0
.set clobber_bootstrap, 0
.set tail_escape, 0
.set indirect_escape, 0
.set external_entry, 0
.set undecoded_gap, 0
.set reconverge_reload, 0
.amdgcn_target "amdgcn-amd-amdhsa--gfx1250"
.text
.if external_entry
.type external_materialization_entry,@function
external_materialization_entry:
// This independent function enters after get-PC and delta creation.
s_branch .Lfirst_add_low
s_endpgm
.size external_materialization_entry, .-external_materialization_entry
.endif
.local callee_bootstrap
.type callee_bootstrap,@function
callee_bootstrap:
s_mov_b32 s8, 3
.if clobber_bootstrap
s_mov_b32 s10, 0
.endif
s_set_pc_i64 s[12:13]
.size callee_bootstrap, .-callee_bootstrap
.globl reusable_pc_targets
.p2align 8
.type reusable_pc_targets,@function
reusable_pc_targets:
// A straight-line first call is resolved by the one-shot matcher. Its
// proven target remains reusable after the audited callee returns.
.Lbootstrap_getpc:
s_get_pc_i64 s[10:11]
s_add_nc_u64 s[10:11], s[10:11], callee_bootstrap-(.Lbootstrap_getpc+4)
s_swap_pc_i64 s[12:13], s[10:11]
s_swap_pc_i64 s[12:13], s[10:11]
.if unsafe_selector
// This edge reaches the call without executing either get-PC sequence.
s_cmp_eq_u32 s0, 2
s_cbranch_scc1 .Lselected
.endif
.if outside_selector
s_cmp_eq_u32 s0, 3
s_cbranch_scc1 .Lselect_outside
.endif
s_cmp_eq_u32 s0, 0
s_cbranch_scc1 .Lselect_second
.Lselect_first:
s_get_pc_i64 s[2:3]
.if overlap_delta
// The temporary aliases the PC pair and destroys its low half.
s_add_co_i32 s2, callee_first-(.Lselect_first+4)-4, 4
.Lfirst_add_low:
s_add_co_u32 s2, s2, s2
.else
s_add_co_i32 s4, callee_first-(.Lselect_first+4)-4, 4
.Lfirst_add_low:
s_add_co_u32 s2, s2, s4
.endif
s_add_co_ci_u32 s3, s3, 0
s_branch .Lselected
.Lselect_second:
s_get_pc_i64 s[2:3]
.if overlap_delta
s_add_co_i32 s2, callee_second-(.Lselect_second+4)-4, 4
s_add_co_u32 s2, s2, s2
.else
s_add_co_i32 s4, callee_second-(.Lselect_second+4)-4, 4
s_add_co_u32 s2, s2, s4
.endif
s_add_co_ci_u32 s3, s3, 0
.if outside_selector
s_branch .Lselected
.Lselect_outside:
s_get_pc_i64 s[2:3]
s_add_co_i32 s4, outside_text_end-(.Lselect_outside+4)-4, 4
s_add_co_u32 s2, s2, s4
s_add_co_ci_u32 s3, s3, 0
.endif
.Lselected:
s_swap_pc_i64 s[6:7], s[2:3]
.if undecoded_gap
// Keep the first call's return continuation decoded, then place an unknown
// word before the second call. The solver must not carry the finite target
// set across it.
s_nop 0
.long 0xffffffff
.endif
.if reconverge_reload
// One path reaches the second reused call with the proven pair; a sibling
// path reloads it through an unprovable definition. They reconverge on the
// second call, which must therefore be Unknown -- not the finite result the
// proven path alone recorded on its earlier visit.
s_cmp_eq_u32 s0, 5
s_cbranch_scc1 .Lreload_join
s_branch .Lreused_second
.Lreload_join:
s_mov_b32 s2, 0
s_mov_b32 s3, 0
.Lreused_second:
.endif
// Production kernels reuse this selected pair for multiple calls.
s_swap_pc_i64 s[6:7], s[2:3]
s_branch .Lpatch0
.Lpatch0:
ds_load_2addr_stride64_b32 v[0:1], v2 offset0:1 offset1:3
s_branch .Lpatch1
.Lpatch1:
ds_load_2addr_stride64_b32 v[0:1], v2 offset0:1 offset1:3
s_branch .Lpatch2
.Lpatch2:
ds_load_2addr_stride64_b32 v[0:1], v2 offset0:1 offset1:3
s_branch .Lpatch3
.Lpatch3:
ds_load_2addr_stride64_b32 v[0:1], v2 offset0:1 offset1:3
s_branch .Lpatch4
.Lpatch4:
ds_load_2addr_stride64_b32 v[0:1], v2 offset0:1 offset1:3
s_branch .Lpatch5
.Lpatch5:
ds_load_2addr_stride64_b32 v[0:1], v2 offset0:1 offset1:3
s_branch .Lpatch6
.Lpatch6:
ds_load_2addr_stride64_b32 v[0:1], v2 offset0:1 offset1:3
s_branch .Lpatch7
.Lpatch7:
ds_load_2addr_stride64_b32 v[0:1], v2 offset0:1 offset1:3
.Lpatch_done:
s_wait_dscnt 0x0
s_endpgm
.size reusable_pc_targets, .-reusable_pc_targets
.local callee_first
.type callee_first,@function
callee_first:
s_mov_b32 s8, 1
.if clobber_target
s_mov_b32 s2, 0
.endif
.if tail_escape
s_branch clobber_helper
.endif
.if indirect_escape
s_set_pc_i64 s[10:11]
.endif
s_set_pc_i64 s[6:7]
.size callee_first, .-callee_first
.local callee_second
.type callee_second,@function
callee_second:
s_mov_b32 s8, 2
.if clobber_target
s_mov_b32 s2, 0
.endif
.if tail_escape
s_branch clobber_helper
.endif
.if indirect_escape
s_set_pc_i64 s[10:11]
.endif
s_set_pc_i64 s[6:7]
.size callee_second, .-callee_second
.if tail_escape
.local clobber_helper
.type clobber_helper,@function
clobber_helper:
s_mov_b32 s2, 0
s_set_pc_i64 s[6:7]
.size clobber_helper, .-clobber_helper
.endif
.type gateway_barrier,@function
gateway_barrier:
s_endpgm
.size gateway_barrier, .-gateway_barrier
.fill 20, 1, 0
.rept 40000
s_mov_b32 s10, s11
.endr
outside_text_end:
.rodata
.p2align 8
.amdhsa_kernel reusable_pc_targets
.amdhsa_next_free_vgpr 3
.amdhsa_next_free_sgpr 14
.end_amdhsa_kernel
.amdgpu_metadata
amdhsa.version:
- 3
- 0
amdhsa.kernels:
- .name: reusable_pc_targets
.symbol: reusable_pc_targets.kd
.sgpr_count: 14
.vgpr_count: 3
.kernarg_segment_size: 0
.group_segment_fixed_size: 0
.private_segment_fixed_size: 0
.kernarg_segment_align: 8
.wavefront_size: 64
.max_flat_workgroup_size: 256
.end_amdgpu_metadata