derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
185 lines
8.2 KiB
YAML
185 lines
8.2 KiB
YAML
# RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sme2 -force-streaming -verify-machineinstrs -enable-subreg-liveness -start-before=greedy %s -o - | FileCheck %s
|
|
|
|
# No available group of four strided x4 registers, fall back on default allocation order
|
|
---
|
|
name: form_4x_tuple_many_live
|
|
tracksRegLiveness: true
|
|
stack:
|
|
- { id: 0, name: '', type: default, offset: 0, size: 32, alignment: 16,
|
|
stack-id: scalable-vector, callee-saved-register: '', callee-saved-restored: true,
|
|
debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
|
|
body: |
|
|
bb.0.entry:
|
|
liveins: $x0, $x1, $z0, $z17
|
|
|
|
; CHECK-LABEL: form_4x_tuple_many_live
|
|
; CHECK: stp d11, d10, [sp, #-48]!
|
|
; CHECK-NEXT: stp d9, d8, [sp, #16]
|
|
; CHECK-NEXT: str x29, [sp, #32]
|
|
; CHECK-NEXT: addvl sp, sp, #-2
|
|
; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x30, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 48 + 16 * VG
|
|
; CHECK-NEXT: .cfi_offset w29, -16
|
|
; CHECK-NEXT: .cfi_offset b8, -24
|
|
; CHECK-NEXT: .cfi_offset b9, -32
|
|
; CHECK-NEXT: .cfi_offset b10, -40
|
|
; CHECK-NEXT: .cfi_offset b11, -48
|
|
; CHECK-NEXT: lsl x9, x1, #1
|
|
; CHECK-NEXT: ptrue pn8.b
|
|
; CHECK-NEXT: mov w8, wzr
|
|
; CHECK-NEXT: ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]
|
|
; CHECK-NEXT: ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x1]
|
|
; CHECK-NEXT: ptrue p0.b
|
|
; CHECK-NEXT: add x10, x9, x1
|
|
; CHECK-NEXT: ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x9]
|
|
; CHECK-NEXT: ld1b { z4.b - z7.b }, pn8/z, [x0, x10]
|
|
; CHECK-NEXT: mov z8.d, z16.d
|
|
; CHECK-NEXT: mov z9.d, z18.d
|
|
; CHECK-NEXT: mov z21.d, z22.d
|
|
; CHECK-NEXT: mov z10.d, z19.d
|
|
; CHECK-NEXT: mov z22.d, z23.d
|
|
; CHECK-NEXT: mov z25.d, z26.d
|
|
; CHECK-NEXT: mov z11.d, z4.d
|
|
; CHECK-NEXT: mov z23.d, z5.d
|
|
; CHECK-NEXT: mov z26.d, z27.d
|
|
; CHECK-NEXT: mov z27.d, z6.d
|
|
; CHECK-NEXT: mov z29.d, z30.d
|
|
; CHECK-NEXT: mov z30.d, z31.d
|
|
; CHECK-NEXT: mov z31.d, z7.d
|
|
; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z8.b - z11.b }, z0.b[0]
|
|
; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b[0]
|
|
; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b[0]
|
|
; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]
|
|
; CHECK-NEXT: st1b { z0.b }, p0, [x0]
|
|
; CHECK-NEXT: st1b { z17.b }, p0, [x0]
|
|
; CHECK-NEXT: addvl sp, sp, #2
|
|
; CHECK-NEXT: ldp d9, d8, [sp, #16]
|
|
; CHECK-NEXT: ldr x29, [sp, #32]
|
|
; CHECK-NEXT: ldp d11, d10, [sp], #48
|
|
; CHECK-NEXT: ret
|
|
|
|
%0:gpr64common = COPY $x0
|
|
%1:gpr64 = COPY $x1
|
|
%2:zpr = COPY $z0
|
|
%3:zpr = COPY $z17
|
|
%5:matrixindexgpr32_8_11 = COPY $wzr
|
|
%6:gpr64 = UBFMXri %1, 63, 62
|
|
%pred:pnr_p8to15 = PTRUE_C_B implicit $vg
|
|
%7:ppr_3b = PTRUE_B 31, implicit $vg
|
|
%8:gpr64 = ADDXrr %6, %1
|
|
%9:zpr4stridedorcontiguous = LD1B_4Z_IMM_PSEUDO %pred, %0, 0
|
|
%10:zpr4stridedorcontiguous = LD1B_4Z_PSEUDO %pred, %0, %1
|
|
%11:zpr4stridedorcontiguous = LD1B_4Z_PSEUDO %pred, %0, %6
|
|
%12:zpr4stridedorcontiguous = LD1B_4Z_PSEUDO %pred, %0, %8
|
|
%13:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %9.zsub0, %10.zsub0, %11.zsub0, %12.zsub0
|
|
%14:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %9.zsub1, %10.zsub1, %11.zsub1, %12.zsub1
|
|
%15:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %9.zsub2, %10.zsub2, %11.zsub2, %12.zsub2
|
|
%16:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %9.zsub3, %10.zsub3, %11.zsub3, %12.zsub3
|
|
$za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %13, undef %28:zpr_4b, 0
|
|
$za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %14, undef %30:zpr_4b, 0
|
|
$za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %15, undef %32:zpr_4b, 0
|
|
$za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %16, undef %34:zpr_4b, 0
|
|
ST1B_IMM %2, %7, %0, 0 :: (store (<vscale x 1 x s128>) into %stack.0)
|
|
ST1B_IMM %3, %7, %0, 0 :: (store (<vscale x 1 x s128>) into %stack.0)
|
|
RET_ReallyLR
|
|
...
|
|
|
|
# First multi-vector load to be allocated is not the first operand of the FORM_TRANSPOSED pseudo
|
|
---
|
|
name: form_4x_tuple_allocation_order
|
|
tracksRegLiveness: true
|
|
stack:
|
|
- { id: 0, name: '', type: default, offset: 0, size: 32, alignment: 16,
|
|
stack-id: scalable-vector, callee-saved-register: '', callee-saved-restored: true,
|
|
debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
|
|
body: |
|
|
bb.0.entry:
|
|
liveins: $x0, $x1, $z0
|
|
|
|
; CHECK: str x29, [sp, #-16]!
|
|
; CHECK-NEXT: addvl sp, sp, #-2
|
|
; CHECK-NEXT: .cfi_escape 0x0f, 0x08, 0x8f, 0x10, 0x92, 0x2e, 0x00, 0x40, 0x1e, 0x22 // sp + 16 + 16 * VG
|
|
; CHECK-NEXT: .cfi_offset w29, -16
|
|
; CHECK-NEXT: lsl x9, x1, #1
|
|
; CHECK-NEXT: ptrue pn8.b
|
|
; CHECK-NEXT: mov w8, wzr
|
|
; CHECK-NEXT: ptrue p0.b
|
|
; CHECK-NEXT: add x10, x9, x1
|
|
; CHECK-NEXT: ld1b { z19.b, z23.b, z27.b, z31.b }, pn8/z, [x0, x10]
|
|
; CHECK-NEXT: ld1b { z18.b, z22.b, z26.b, z30.b }, pn8/z, [x0, x9]
|
|
; CHECK-NEXT: ld1b { z17.b, z21.b, z25.b, z29.b }, pn8/z, [x0, x1]
|
|
; CHECK-NEXT: ld1b { z16.b, z20.b, z24.b, z28.b }, pn8/z, [x0]
|
|
; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z16.b - z19.b }, z0.b[0]
|
|
; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z20.b - z23.b }, z0.b[0]
|
|
; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z24.b - z27.b }, z0.b[0]
|
|
; CHECK-NEXT: udot za.s[w8, 0, vgx4], { z28.b - z31.b }, z0.b[0]
|
|
; CHECK-NEXT: st1b { z0.b }, p0, [x0]
|
|
; CHECK-NEXT: addvl sp, sp, #2
|
|
; CHECK-NEXT: ldr x29, [sp], #16
|
|
; CHECK-NEXT: ret
|
|
|
|
%0:gpr64common = COPY $x0
|
|
%1:gpr64 = COPY $x1
|
|
%2:zpr = COPY $z0
|
|
%5:matrixindexgpr32_8_11 = COPY $wzr
|
|
%6:gpr64 = UBFMXri %1, 63, 62
|
|
%pred:pnr_p8to15 = PTRUE_C_B implicit $vg
|
|
%7:ppr_3b = PTRUE_B 31, implicit $vg
|
|
%8:gpr64 = ADDXrr %6, %1
|
|
%9:zpr4stridedorcontiguous = LD1B_4Z_PSEUDO %pred, %0, %8
|
|
%10:zpr4stridedorcontiguous = LD1B_4Z_PSEUDO %pred, %0, %6
|
|
%11:zpr4stridedorcontiguous = LD1B_4Z_PSEUDO %pred, %0, %1
|
|
%12:zpr4stridedorcontiguous = LD1B_4Z_IMM_PSEUDO %pred, %0, 0
|
|
%13:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %12.zsub0, %11.zsub0, %10.zsub0, %9.zsub0
|
|
%14:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %12.zsub1, %11.zsub1, %10.zsub1, %9.zsub1
|
|
%15:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %12.zsub2, %11.zsub2, %10.zsub2, %9.zsub2
|
|
%16:zpr4mul4 = FORM_TRANSPOSED_REG_TUPLE_X4_PSEUDO %12.zsub3, %11.zsub3, %10.zsub3, %9.zsub3
|
|
$za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %13, undef %28:zpr_4b, 0
|
|
$za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %14, undef %30:zpr_4b, 0
|
|
$za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %15, undef %32:zpr_4b, 0
|
|
$za = UDOT_VG4_M4ZZI_BtoS $za, %5, 0, %16, undef %34:zpr_4b, 0
|
|
ST1B_IMM %2, %7, %0, 0 :: (store (<vscale x 1 x s128>) into %stack.0)
|
|
RET_ReallyLR
|
|
...
|
|
|
|
# Strided order is [ $z16_z24 $z17_z25 $z18_z26 $z19_z27 $z20_z28 $z21_z29 $z22_z30 $z23_z31 $z0_z8 $z1_z9 $z2_z10 $z3_z11 $z4_z12 $z5_z13 $z6_z14 $z7_z15 ]
|
|
# Ensure we don't allocate $z23_z31 & $z0_z8 although they are consecutive
|
|
---
|
|
name: udot_form_2x_tuple_live_reg_order
|
|
tracksRegLiveness: true
|
|
body: |
|
|
bb.0.entry:
|
|
liveins: $x0, $x1, $z16, $z17, $z18, $z19, $z20, $z21, $z22
|
|
|
|
; CHECK: stp d9, d8, [sp, #-16]!
|
|
; CHECK-NEXT: .cfi_def_cfa_offset 16
|
|
; CHECK-NEXT: .cfi_offset b8, -8
|
|
; CHECK-NEXT: .cfi_offset b9, -16
|
|
; CHECK-NEXT: ptrue pn8.b
|
|
; CHECK-NEXT: mov w8, wzr
|
|
; CHECK-NEXT: ld1b { z0.b, z8.b }, pn8/z, [x0]
|
|
; CHECK-NEXT: ld1b { z1.b, z9.b }, pn8/z, [x0, x1]
|
|
; CHECK-NEXT: udot za.s[w8, 0, vgx2], { z0.b, z1.b }, z0.b
|
|
; CHECK-NEXT: udot za.s[w8, 0, vgx2], { z8.b, z9.b }, z0.b
|
|
; CHECK-NEXT: ldp d9, d8, [sp], #16
|
|
; CHECK-NEXT: ret
|
|
|
|
%0:gpr64 = COPY $x1
|
|
%1:gpr64common = COPY $x0
|
|
%2:zpr = COPY $z16
|
|
%3:zpr = COPY $z17
|
|
%4:zpr = COPY $z18
|
|
%5:zpr = COPY $z19
|
|
%6:zpr = COPY $z20
|
|
%7:zpr = COPY $z21
|
|
%8:zpr = COPY $z22
|
|
%9:matrixindexgpr32_8_11 = COPY $wzr
|
|
%10:pnr_p8to15 = PTRUE_C_B implicit $vg
|
|
%11:zpr2stridedorcontiguous = LD1B_2Z_IMM_PSEUDO %10, %1, 0
|
|
%12:zpr2stridedorcontiguous = LD1B_2Z_PSEUDO %10, %1, %0
|
|
%13:zpr2 = FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO %11.zsub0, %12.zsub0
|
|
%14:zpr2 = FORM_TRANSPOSED_REG_TUPLE_X2_PSEUDO %11.zsub1, %12.zsub1
|
|
$za = UDOT_VG2_M2ZZ_BtoS $za, %9, 0, %13, undef %15:zpr_4b
|
|
$za = UDOT_VG2_M2ZZ_BtoS $za, %9, 0, %14, undef %16:zpr_4b
|
|
RET_ReallyLR
|
|
...
|