derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
271 lines
11 KiB
LLVM
271 lines
11 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
|
|
; RUN: llc < %s -mtriple=x86_64-pc-windows-gnu -mattr=egpr,ndd,ccmp,avx512vl -verify-machineinstrs | FileCheck %s
|
|
|
|
define fastcc i64 @foo(i32 %dim1, i32 %dim2, i32 %dim3, i32 %dim4, ptr %p1, ptr %p2, ptr %p3, ptr %p4, ptr %p5, ptr %p6, ptr %p7, ptr %p8, ptr %p9, ptr %p10, ptr %p11, ptr %p12, ptr %p13, ptr %p14, ptr %p15, ptr %p16, ptr %p17, ptr %p18, ptr %p19, i64 %stride, i64 %extent, i64 %arg1, i1 %cond1, i1 %cond2, i64 %idx, <8 x i1> %mask1, i64 %offset, <8 x i1> %mask2) nounwind {
|
|
; CHECK-LABEL: foo:
|
|
; CHECK: # %bb.0: # %entry
|
|
; CHECK-NEXT: pushq %rbp
|
|
; CHECK-NEXT: pushq %r15
|
|
; CHECK-NEXT: pushq %r14
|
|
; CHECK-NEXT: pushq %r13
|
|
; CHECK-NEXT: pushq %r12
|
|
; CHECK-NEXT: pushq %rsi
|
|
; CHECK-NEXT: pushq %rdi
|
|
; CHECK-NEXT: pushq %rbx
|
|
; CHECK-NEXT: subq $56, %rsp
|
|
; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rbp
|
|
; CHECK-NEXT: andq $-32, %rsp
|
|
; CHECK-NEXT: movq %rsp, %rbx
|
|
; CHECK-NEXT: movq 312(%rbp), %rax
|
|
; CHECK-NEXT: vpmovsxwd (%rax), %ymm0
|
|
; CHECK-NEXT: vpslld $31, %ymm0, %ymm0
|
|
; CHECK-NEXT: vptestmd %ymm0, %ymm0, %k2
|
|
; CHECK-NEXT: movq 304(%rbp), %r29
|
|
; CHECK-NEXT: movzbl 296(%rbp), %r26d
|
|
; CHECK-NEXT: movzbl 288(%rbp), %r30d
|
|
; CHECK-NEXT: movq 280(%rbp), %r31
|
|
; CHECK-NEXT: movq 272(%rbp), %r20
|
|
; CHECK-NEXT: movq 264(%rbp), %r21
|
|
; CHECK-NEXT: movq 240(%rbp), %r9
|
|
; CHECK-NEXT: movq 232(%rbp), %r10
|
|
; CHECK-NEXT: movq 224(%rbp), %r11
|
|
; CHECK-NEXT: movq 216(%rbp), %r16
|
|
; CHECK-NEXT: movq 208(%rbp), %r17
|
|
; CHECK-NEXT: movq 192(%rbp), %r18
|
|
; CHECK-NEXT: movq 184(%rbp), %r19
|
|
; CHECK-NEXT: movq 176(%rbp), %r22
|
|
; CHECK-NEXT: movq 168(%rbp), %r23
|
|
; CHECK-NEXT: movq 160(%rbp), %r24
|
|
; CHECK-NEXT: movq 152(%rbp), %r25
|
|
; CHECK-NEXT: movq 144(%rbp), %r14
|
|
; CHECK-NEXT: movq 136(%rbp), %r15
|
|
; CHECK-NEXT: movq 128(%rbp), %r12
|
|
; CHECK-NEXT: movq 120(%rbp), %r13
|
|
; CHECK-NEXT: movq 112(%rbp), %rsi
|
|
; CHECK-NEXT: leaq 15(,%r21,4), %rdx
|
|
; CHECK-NEXT: andq $-16, %rdx
|
|
; CHECK-NEXT: movq %rdx, %rax
|
|
; CHECK-NEXT: callq ___chkstk_ms
|
|
; CHECK-NEXT: subq %rax, %rsp
|
|
; CHECK-NEXT: andq $-32, %rsp
|
|
; CHECK-NEXT: movq %rdx, %rax
|
|
; CHECK-NEXT: callq ___chkstk_ms
|
|
; CHECK-NEXT: subq %rax, %rsp
|
|
; CHECK-NEXT: andq $-32, %rsp
|
|
; CHECK-NEXT: movl %ecx, %r8d
|
|
; CHECK-NEXT: xorl %edi, %edi
|
|
; CHECK-NEXT: xorl %eax, %eax
|
|
; CHECK-NEXT: kmovw %k2, 14(%rbx) # 2-byte Spill
|
|
; CHECK-NEXT: movq %r8, 24(%rbx) # 8-byte Spill
|
|
; CHECK-NEXT: jmp .LBB0_1
|
|
; CHECK-NEXT: .p2align 4
|
|
; CHECK-NEXT: .LBB0_6: # %loop_outer.backedge
|
|
; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1
|
|
; CHECK-NEXT: subq %r21, %rdi
|
|
; CHECK-NEXT: .LBB0_1: # %loop_outer
|
|
; CHECK-NEXT: # =>This Loop Header: Depth=1
|
|
; CHECK-NEXT: # Child Loop BB0_8 Depth 2
|
|
; CHECK-NEXT: testq %r31, %r31
|
|
; CHECK-NEXT: ctestgq {dfv=} %r20, %r20
|
|
; CHECK-NEXT: jns .LBB0_10
|
|
; CHECK-NEXT: # %bb.2: # %then_branch
|
|
; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1
|
|
; CHECK-NEXT: testb $1, %r30b
|
|
; CHECK-NEXT: je .LBB0_3
|
|
; CHECK-NEXT: # %bb.5: # %early_exit
|
|
; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1
|
|
; CHECK-NEXT: decq %rax
|
|
; CHECK-NEXT: jmp .LBB0_6
|
|
; CHECK-NEXT: .p2align 4
|
|
; CHECK-NEXT: .LBB0_10: # %else_branch
|
|
; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1
|
|
; CHECK-NEXT: leaq -1(%rax), %r27
|
|
; CHECK-NEXT: testb $1, %r26b
|
|
; CHECK-NEXT: movq %r27, 16(%rbx) # 8-byte Spill
|
|
; CHECK-NEXT: je .LBB0_11
|
|
; CHECK-NEXT: # %bb.7: # %memset_path
|
|
; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1
|
|
; CHECK-NEXT: subq $32, %rsp
|
|
; CHECK-NEXT: xorl %ecx, %ecx
|
|
; CHECK-NEXT: xorl %edx, %edx
|
|
; CHECK-NEXT: vzeroupper
|
|
; CHECK-NEXT: callq memset
|
|
; CHECK-NEXT: movzbl 288(%rbp), %r30d
|
|
; CHECK-NEXT: addq $32, %rsp
|
|
; CHECK-NEXT: .p2align 4
|
|
; CHECK-NEXT: .LBB0_8: # %inner_loop
|
|
; CHECK-NEXT: # Parent Loop BB0_1 Depth=1
|
|
; CHECK-NEXT: # => This Inner Loop Header: Depth=2
|
|
; CHECK-NEXT: testb $1, %r30b
|
|
; CHECK-NEXT: jne .LBB0_8
|
|
; CHECK-NEXT: # %bb.9: # in Loop: Header=BB0_1 Depth=1
|
|
; CHECK-NEXT: kmovw 14(%rbx), %k2 # 2-byte Reload
|
|
; CHECK-NEXT: movq 304(%rbp), %r29
|
|
; CHECK-NEXT: movq 240(%rbp), %r9
|
|
; CHECK-NEXT: movq 232(%rbp), %r10
|
|
; CHECK-NEXT: movq 224(%rbp), %r11
|
|
; CHECK-NEXT: movq 216(%rbp), %r16
|
|
; CHECK-NEXT: movq 208(%rbp), %r17
|
|
; CHECK-NEXT: movq 192(%rbp), %r18
|
|
; CHECK-NEXT: movq 184(%rbp), %r19
|
|
; CHECK-NEXT: movq 176(%rbp), %r22
|
|
; CHECK-NEXT: movq 168(%rbp), %r23
|
|
; CHECK-NEXT: movq 160(%rbp), %r24
|
|
; CHECK-NEXT: movq 152(%rbp), %r25
|
|
; CHECK-NEXT: movzbl 296(%rbp), %r26d
|
|
; CHECK-NEXT: movq 280(%rbp), %r31
|
|
; CHECK-NEXT: movq 272(%rbp), %r20
|
|
; CHECK-NEXT: movq 264(%rbp), %r21
|
|
; CHECK-NEXT: movq 24(%rbx), %r8 # 8-byte Reload
|
|
; CHECK-NEXT: movq 16(%rbx), %rax # 8-byte Reload
|
|
; CHECK-NEXT: subq %r21, %rdi
|
|
; CHECK-NEXT: jmp .LBB0_1
|
|
; CHECK-NEXT: .p2align 4
|
|
; CHECK-NEXT: .LBB0_11: # %compute_path
|
|
; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1
|
|
; CHECK-NEXT: testb $1, %r30b
|
|
; CHECK-NEXT: jne .LBB0_12
|
|
; CHECK-NEXT: # %bb.14: # %loop_exit
|
|
; CHECK-NEXT: # in Loop: Header=BB0_1 Depth=1
|
|
; CHECK-NEXT: movq 256(%rbp), %rcx
|
|
; CHECK-NEXT: addq %rax, %rcx
|
|
; CHECK-NEXT: andl $1, %r21d, %r27d
|
|
; CHECK-NEXT: andl $1, %r8d, %edx
|
|
; CHECK-NEXT: movq %r20, %rax
|
|
; CHECK-NEXT: movq %r31, %r20
|
|
; CHECK-NEXT: movl %r26d, %r31d
|
|
; CHECK-NEXT: movq 320(%rbp), %r26
|
|
; CHECK-NEXT: movq 200(%rbp), %r28
|
|
; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0
|
|
; CHECK-NEXT: vmovdqu32 %ymm0, (%r28,%r26) {%k2}
|
|
; CHECK-NEXT: movl %r31d, %r26d
|
|
; CHECK-NEXT: movq %r20, %r31
|
|
; CHECK-NEXT: movq %rax, %r20
|
|
; CHECK-NEXT: vmovups %ymm0, (%rcx) {%k2}
|
|
; CHECK-NEXT: cmpl %edx, %r27d
|
|
; CHECK-NEXT: movq 16(%rbx), %rax # 8-byte Reload
|
|
; CHECK-NEXT: je .LBB0_6
|
|
; CHECK-NEXT: # %bb.15:
|
|
; CHECK-NEXT: movl $1, %eax
|
|
; CHECK-NEXT: jmp .LBB0_13
|
|
; CHECK-NEXT: .LBB0_3: # %vector_loop.preheader
|
|
; CHECK-NEXT: xorl %eax, %eax
|
|
; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0
|
|
; CHECK-NEXT: .p2align 4
|
|
; CHECK-NEXT: .LBB0_4: # %vector_loop
|
|
; CHECK-NEXT: # =>This Inner Loop Header: Depth=1
|
|
; CHECK-NEXT: vmovdqu (%r9), %ymm1
|
|
; CHECK-NEXT: vpord (%r18), %ymm1, %ymm1
|
|
; CHECK-NEXT: vmovups %ymm0, (%r23) {%k2}
|
|
; CHECK-NEXT: vmovups %ymm0, (%r24) {%k2}
|
|
; CHECK-NEXT: vmovups (%r25,%rdi), %ymm2
|
|
; CHECK-NEXT: vcmpltps (%r14,%rdi), %ymm2, %k1
|
|
; CHECK-NEXT: vptestnmd %ymm1, %ymm1, %k1 {%k1}
|
|
; CHECK-NEXT: vmovups (%r10), %ymm1 {%k1} {z}
|
|
; CHECK-NEXT: vmovups %ymm1, (%r22) {%k2}
|
|
; CHECK-NEXT: vmovdqu32 %ymm0, (%r13) {%k2}
|
|
; CHECK-NEXT: vmovdqu32 %ymm0, (%r16) {%k2}
|
|
; CHECK-NEXT: vmovups %ymm0, (%r19) {%k2}
|
|
; CHECK-NEXT: vmovups (%r15), %ymm1 {%k2} {z}
|
|
; CHECK-NEXT: vmovups %ymm1, (%r17,%r29,4) {%k2}
|
|
; CHECK-NEXT: vmovups %ymm0, (%r11) {%k2}
|
|
; CHECK-NEXT: vmovups %ymm0, (%r12) {%k2}
|
|
; CHECK-NEXT: vmovups %ymm0, (%rsi) {%k2}
|
|
; CHECK-NEXT: incq %rax
|
|
; CHECK-NEXT: addq $4, %rdi
|
|
; CHECK-NEXT: jmp .LBB0_4
|
|
; CHECK-NEXT: .LBB0_12:
|
|
; CHECK-NEXT: xorl %eax, %eax
|
|
; CHECK-NEXT: .LBB0_13: # %exit
|
|
; CHECK-NEXT: leaq 8(%rbp), %rsp
|
|
; CHECK-NEXT: popq %rbx
|
|
; CHECK-NEXT: popq %rdi
|
|
; CHECK-NEXT: popq %rsi
|
|
; CHECK-NEXT: popq %r12
|
|
; CHECK-NEXT: popq %r13
|
|
; CHECK-NEXT: popq %r14
|
|
; CHECK-NEXT: popq %r15
|
|
; CHECK-NEXT: popq %rbp
|
|
; CHECK-NEXT: vzeroupper
|
|
; CHECK-NEXT: retq
|
|
entry:
|
|
%buf1 = alloca float, i64 %stride, align 32
|
|
%buf2 = alloca float, i64 %stride, align 32
|
|
%dim1_ext = zext i32 %dim1 to i64
|
|
br label %loop_outer
|
|
|
|
loop_outer: ; preds = %loop_exit, %inner_loop, %early_exit, %entry
|
|
%iv = phi i64 [ 0, %entry ], [ %iv_dec, %early_exit ], [ %iv_dec2, %loop_exit ], [ %iv_dec2, %inner_loop ]
|
|
%cmp1 = icmp slt i64 0, %arg1
|
|
%cmp2 = icmp sgt i64 0, %extent
|
|
%both = and i1 %cmp1, %cmp2
|
|
br i1 %both, label %then_branch, label %else_branch
|
|
|
|
then_branch: ; preds = %loop_outer
|
|
br i1 %cond1, label %early_exit, label %vector_loop
|
|
|
|
early_exit: ; preds = %then_branch
|
|
%iv_dec = add i64 %iv, -1
|
|
br label %loop_outer
|
|
|
|
else_branch: ; preds = %loop_outer
|
|
%iv_dec2 = add i64 %iv, -1
|
|
%gep1 = getelementptr i8, ptr %p19, i64 %iv
|
|
br i1 %cond2, label %memset_path, label %compute_path
|
|
|
|
vector_loop: ; preds = %vector_loop, %then_branch
|
|
%inner_iv = phi i64 [ %inner_iv_next, %vector_loop ], [ 0, %then_branch ]
|
|
%gep2 = getelementptr i32, ptr %p10, i64 %inner_iv
|
|
%vec1 = load <8 x i32>, ptr %p17, align 1
|
|
%vec2 = load <8 x i32>, ptr %p11, align 1
|
|
%base_offset = mul i64 %stride, %iv
|
|
%base1 = getelementptr i8, ptr %p5, i64 %base_offset
|
|
%gep3 = getelementptr float, ptr %base1, i64 %inner_iv
|
|
%base2 = getelementptr i8, ptr %p6, i64 %base_offset
|
|
%gep4 = getelementptr float, ptr %base2, i64 %inner_iv
|
|
call void @llvm.masked.store.v8f32.p0(<8 x float> zeroinitializer, ptr %p8, <8 x i1> %mask1)
|
|
call void @llvm.masked.store.v8f32.p0(<8 x float> zeroinitializer, ptr %p7, <8 x i1> %mask1)
|
|
%or_vec = or <8 x i32> %vec1, %vec2
|
|
%cmp_vec1 = icmp eq <8 x i32> %or_vec, zeroinitializer
|
|
%load1 = load <8 x float>, ptr %gep3, align 1
|
|
%load2 = load <8 x float>, ptr %gep4, align 1
|
|
%cmp_vec2 = fcmp ogt <8 x float> %load1, %load2
|
|
%and_mask = and <8 x i1> %cmp_vec1, %cmp_vec2
|
|
%masked_load = call <8 x float> @llvm.masked.load.v8f32.p0(ptr %p16, <8 x i1> %and_mask, <8 x float> zeroinitializer)
|
|
call void @llvm.masked.store.v8f32.p0(<8 x float> %masked_load, ptr %p9, <8 x i1> %mask1)
|
|
call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr %p2, <8 x i1> %mask1)
|
|
call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr %p14, <8 x i1> %mask1)
|
|
call void @llvm.masked.store.v8f32.p0(<8 x float> zeroinitializer, ptr %p10, <8 x i1> %mask1)
|
|
%load3 = call <8 x float> @llvm.masked.load.v8f32.p0(ptr %p4, <8 x i1> %mask1, <8 x float> zeroinitializer)
|
|
%gep5 = getelementptr float, ptr %p13, i64 %idx
|
|
call void @llvm.masked.store.v8f32.p0(<8 x float> %load3, ptr %gep5, <8 x i1> %mask1)
|
|
call void @llvm.masked.store.v8f32.p0(<8 x float> zeroinitializer, ptr %p15, <8 x i1> %mask1)
|
|
call void @llvm.masked.store.v8f32.p0(<8 x float> zeroinitializer, ptr %p3, <8 x i1> %mask1)
|
|
call void @llvm.masked.store.v8f32.p0(<8 x float> zeroinitializer, ptr %p1, <8 x i1> %mask1)
|
|
%inner_iv_next = add i64 %inner_iv, 1
|
|
br label %vector_loop
|
|
|
|
memset_path: ; preds = %else_branch
|
|
call void @llvm.memset.p0.i64(ptr null, i8 0, i64 %dim1_ext, i1 false)
|
|
br label %inner_loop
|
|
|
|
inner_loop: ; preds = %inner_loop, %memset_path
|
|
br i1 %cond1, label %inner_loop, label %loop_outer
|
|
|
|
compute_path: ; preds = %else_branch
|
|
%and1 = and i64 %stride, 1
|
|
%and2 = and i64 %dim1_ext, 1
|
|
br i1 %cond1, label %exit, label %loop_exit
|
|
|
|
exit: ; preds = %loop_exit, %compute_path
|
|
%ret_val = phi i64 [ 1, %loop_exit ], [ 0, %compute_path ]
|
|
ret i64 %ret_val
|
|
|
|
loop_exit: ; preds = %compute_path
|
|
%gep6 = getelementptr i8, ptr %p12, i64 %offset
|
|
call void @llvm.masked.store.v8i32.p0(<8 x i32> zeroinitializer, ptr %gep6, <8 x i1> %mask1)
|
|
call void @llvm.masked.store.v8f32.p0(<8 x float> zeroinitializer, ptr %gep1, <8 x i1> %mask1)
|
|
%cmp_exit = icmp eq i64 %and1, %and2
|
|
br i1 %cmp_exit, label %loop_outer, label %exit
|
|
}
|