; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme,+sme2p1 -stop-before=finalize-isel -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK-BEFORE-ISEL
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+sve,+sme,+sme2p1 -stop-after=finalize-isel -verify-machineinstrs < %s | FileCheck %s --check-prefix=CHECK-AFTER-ISEL

target triple = "aarch64-unknown-linux-gnu"

declare void @bar_enabled(<vscale x 4 x i32>) #0
declare void @bar(<vscale x 4 x i32>)
declare <vscale x 4 x i32> @bar_retv_enabled() #0
declare <vscale x 4 x i32> @bar_retv()

; Non-streaming -> calls streaming callee
define void @foo_non_streaming_pass_arg(ptr %arg) {
  ; CHECK-BEFORE-ISEL-LABEL: name: foo_non_streaming_pass_arg
  ; CHECK-BEFORE-ISEL: bb.0.entry:
  ; CHECK-BEFORE-ISEL-NEXT:   liveins: $x0
  ; CHECK-BEFORE-ISEL-NEXT: {{  $}}
  ; CHECK-BEFORE-ISEL-NEXT:   [[COPY:%[0-9]+]]:gpr64common = COPY $x0
  ; CHECK-BEFORE-ISEL-NEXT:   [[LDR_ZXI:%[0-9]+]]:zpr = LDR_ZXI [[COPY]], 0 :: (load (<vscale x 1 x s128>) from %ir.arg)
  ; CHECK-BEFORE-ISEL-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
  ; CHECK-BEFORE-ISEL-NEXT:   CHECK_MATCHING_VL_PSEUDO
  ; CHECK-BEFORE-ISEL-NEXT:   MSRpstatesvcrImm1 1, 1, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit-def $sp, implicit-def $z0, implicit $vg, implicit-def $vg, implicit-def $fpmr
  ; CHECK-BEFORE-ISEL-NEXT:   $z0 = COPY [[LDR_ZXI]]
  ; CHECK-BEFORE-ISEL-NEXT:   BL @bar_enabled, csr_aarch64_sve_aapcs, implicit-def dead $lr, implicit $sp, implicit $z0, implicit-def $sp
  ; CHECK-BEFORE-ISEL-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
  ; CHECK-BEFORE-ISEL-NEXT:   MSRpstatesvcrImm1 1, 0, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit $vg, implicit-def $vg, implicit-def $fpmr
  ; CHECK-BEFORE-ISEL-NEXT:   RET_ReallyLR
  ;
  ; CHECK-AFTER-ISEL-LABEL: name: foo_non_streaming_pass_arg
  ; CHECK-AFTER-ISEL: bb.0.entry:
  ; CHECK-AFTER-ISEL-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
  ; CHECK-AFTER-ISEL-NEXT:   liveins: $x0
  ; CHECK-AFTER-ISEL-NEXT: {{  $}}
  ; CHECK-AFTER-ISEL-NEXT:   [[COPY:%[0-9]+]]:gpr64common = COPY $x0
  ; CHECK-AFTER-ISEL-NEXT:   [[LDR_ZXI:%[0-9]+]]:zpr = LDR_ZXI [[COPY]], 0 :: (load (<vscale x 1 x s128>) from %ir.arg)
  ; CHECK-AFTER-ISEL-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
  ; CHECK-AFTER-ISEL-NEXT:   [[RDVLI_XI:%[0-9]+]]:gpr64 = RDVLI_XI 1, implicit $vg
  ; CHECK-AFTER-ISEL-NEXT:   [[COPY1:%[0-9]+]]:gpr64sp = COPY [[RDVLI_XI]]
  ; CHECK-AFTER-ISEL-NEXT:   [[ADDSVL_XXI:%[0-9]+]]:gpr64sp = ADDSVL_XXI [[COPY1]], -1, implicit $vg
  ; CHECK-AFTER-ISEL-NEXT:   [[COPY2:%[0-9]+]]:gpr64 = COPY [[ADDSVL_XXI]]
  ; CHECK-AFTER-ISEL-NEXT:   CBZX [[COPY2]], %bb.2
  ; CHECK-AFTER-ISEL-NEXT: {{  $}}
  ; CHECK-AFTER-ISEL-NEXT: bb.1.entry:
  ; CHECK-AFTER-ISEL-NEXT:   successors:
  ; CHECK-AFTER-ISEL-NEXT: {{  $}}
  ; CHECK-AFTER-ISEL-NEXT:   BRK 1
  ; CHECK-AFTER-ISEL-NEXT: {{  $}}
  ; CHECK-AFTER-ISEL-NEXT: bb.2.entry:
  ; CHECK-AFTER-ISEL-NEXT:   MSRpstatesvcrImm1 1, 1, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit-def $sp, implicit-def $z0, implicit $vg, implicit-def $vg, implicit-def $fpmr
  ; CHECK-AFTER-ISEL-NEXT:   $z0 = COPY [[LDR_ZXI]]
  ; CHECK-AFTER-ISEL-NEXT:   BL @bar_enabled, csr_aarch64_sve_aapcs, implicit-def dead $lr, implicit $sp, implicit $z0, implicit-def $sp
  ; CHECK-AFTER-ISEL-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
  ; CHECK-AFTER-ISEL-NEXT:   MSRpstatesvcrImm1 1, 0, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit $vg, implicit-def $vg, implicit-def $fpmr
  ; CHECK-AFTER-ISEL-NEXT:   RET_ReallyLR
entry:
  %v = load <vscale x 4 x i32>, ptr %arg, align 16
  tail call void @bar_enabled(<vscale x 4 x i32> %v) #0
  ret void
}

; Streaming -> calls non-streaming callee
define void @foo_streaming_pass_arg(ptr %arg) #0 {
  ; CHECK-BEFORE-ISEL-LABEL: name: foo_streaming_pass_arg
  ; CHECK-BEFORE-ISEL: bb.0.entry:
  ; CHECK-BEFORE-ISEL-NEXT:   liveins: $x0
  ; CHECK-BEFORE-ISEL-NEXT: {{  $}}
  ; CHECK-BEFORE-ISEL-NEXT:   [[COPY:%[0-9]+]]:gpr64common = COPY $x0
  ; CHECK-BEFORE-ISEL-NEXT:   [[LDR_ZXI:%[0-9]+]]:zpr = LDR_ZXI [[COPY]], 0 :: (load (<vscale x 1 x s128>) from %ir.arg)
  ; CHECK-BEFORE-ISEL-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
  ; CHECK-BEFORE-ISEL-NEXT:   MSRpstatesvcrImm1 1, 0, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit-def $sp, implicit-def $z0, implicit $vg, implicit-def $vg, implicit-def $fpmr
  ; CHECK-BEFORE-ISEL-NEXT:   CHECK_MATCHING_VL_PSEUDO
  ; CHECK-BEFORE-ISEL-NEXT:   $z0 = COPY [[LDR_ZXI]]
  ; CHECK-BEFORE-ISEL-NEXT:   BL @bar, csr_aarch64_sve_aapcs, implicit-def dead $lr, implicit $sp, implicit $z0, implicit-def $sp
  ; CHECK-BEFORE-ISEL-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
  ; CHECK-BEFORE-ISEL-NEXT:   MSRpstatesvcrImm1 1, 1, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit $vg, implicit-def $vg, implicit-def $fpmr
  ; CHECK-BEFORE-ISEL-NEXT:   RET_ReallyLR
  ;
  ; CHECK-AFTER-ISEL-LABEL: name: foo_streaming_pass_arg
  ; CHECK-AFTER-ISEL: bb.0.entry:
  ; CHECK-AFTER-ISEL-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
  ; CHECK-AFTER-ISEL-NEXT:   liveins: $x0
  ; CHECK-AFTER-ISEL-NEXT: {{  $}}
  ; CHECK-AFTER-ISEL-NEXT:   [[COPY:%[0-9]+]]:gpr64common = COPY $x0
  ; CHECK-AFTER-ISEL-NEXT:   [[LDR_ZXI:%[0-9]+]]:zpr = LDR_ZXI [[COPY]], 0 :: (load (<vscale x 1 x s128>) from %ir.arg)
  ; CHECK-AFTER-ISEL-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
  ; CHECK-AFTER-ISEL-NEXT:   MSRpstatesvcrImm1 1, 0, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit-def $sp, implicit-def $z0, implicit $vg, implicit-def $vg, implicit-def $fpmr
  ; CHECK-AFTER-ISEL-NEXT:   [[RDVLI_XI:%[0-9]+]]:gpr64 = RDVLI_XI 1, implicit $vg
  ; CHECK-AFTER-ISEL-NEXT:   [[COPY1:%[0-9]+]]:gpr64sp = COPY [[RDVLI_XI]]
  ; CHECK-AFTER-ISEL-NEXT:   [[ADDSVL_XXI:%[0-9]+]]:gpr64sp = ADDSVL_XXI [[COPY1]], -1, implicit $vg
  ; CHECK-AFTER-ISEL-NEXT:   [[COPY2:%[0-9]+]]:gpr64 = COPY [[ADDSVL_XXI]]
  ; CHECK-AFTER-ISEL-NEXT:   CBZX [[COPY2]], %bb.2
  ; CHECK-AFTER-ISEL-NEXT: {{  $}}
  ; CHECK-AFTER-ISEL-NEXT: bb.1.entry:
  ; CHECK-AFTER-ISEL-NEXT:   successors:
  ; CHECK-AFTER-ISEL-NEXT: {{  $}}
  ; CHECK-AFTER-ISEL-NEXT:   BRK 1
  ; CHECK-AFTER-ISEL-NEXT: {{  $}}
  ; CHECK-AFTER-ISEL-NEXT: bb.2.entry:
  ; CHECK-AFTER-ISEL-NEXT:   $z0 = COPY [[LDR_ZXI]]
  ; CHECK-AFTER-ISEL-NEXT:   BL @bar, csr_aarch64_sve_aapcs, implicit-def dead $lr, implicit $sp, implicit $z0, implicit-def $sp
  ; CHECK-AFTER-ISEL-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
  ; CHECK-AFTER-ISEL-NEXT:   MSRpstatesvcrImm1 1, 1, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit $vg, implicit-def $vg, implicit-def $fpmr
  ; CHECK-AFTER-ISEL-NEXT:   RET_ReallyLR
entry:
  %v = load <vscale x 4 x i32>, ptr %arg, align 16
  tail call void @bar(<vscale x 4 x i32> %v)
  ret void
}

; Non-streaming -> returns SVE value from streaming callee
define void @foo_non_streaming_retval(ptr %ptr) {
  ; CHECK-BEFORE-ISEL-LABEL: name: foo_non_streaming_retval
  ; CHECK-BEFORE-ISEL: bb.0.entry:
  ; CHECK-BEFORE-ISEL-NEXT:   liveins: $x0
  ; CHECK-BEFORE-ISEL-NEXT: {{  $}}
  ; CHECK-BEFORE-ISEL-NEXT:   [[COPY:%[0-9]+]]:gpr64common = COPY $x0
  ; CHECK-BEFORE-ISEL-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
  ; CHECK-BEFORE-ISEL-NEXT:   CHECK_MATCHING_VL_PSEUDO
  ; CHECK-BEFORE-ISEL-NEXT:   MSRpstatesvcrImm1 1, 1, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit-def $sp, implicit-def $z0, implicit $vg, implicit-def $vg, implicit-def $fpmr
  ; CHECK-BEFORE-ISEL-NEXT:   BL @bar_retv_enabled, csr_aarch64_sve_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def $z0
  ; CHECK-BEFORE-ISEL-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
  ; CHECK-BEFORE-ISEL-NEXT:   [[COPY1:%[0-9]+]]:zpr = COPY $z0
  ; CHECK-BEFORE-ISEL-NEXT:   MSRpstatesvcrImm1 1, 0, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit $vg, implicit-def $vg, implicit-def $fpmr
  ; CHECK-BEFORE-ISEL-NEXT:   [[COPY2:%[0-9]+]]:zpr = COPY [[COPY1]]
  ; CHECK-BEFORE-ISEL-NEXT:   STR_ZXI [[COPY2]], [[COPY]], 0 :: (store (<vscale x 1 x s128>) into %ir.ptr)
  ; CHECK-BEFORE-ISEL-NEXT:   RET_ReallyLR
  ;
  ; CHECK-AFTER-ISEL-LABEL: name: foo_non_streaming_retval
  ; CHECK-AFTER-ISEL: bb.0.entry:
  ; CHECK-AFTER-ISEL-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
  ; CHECK-AFTER-ISEL-NEXT:   liveins: $x0
  ; CHECK-AFTER-ISEL-NEXT: {{  $}}
  ; CHECK-AFTER-ISEL-NEXT:   [[COPY:%[0-9]+]]:gpr64common = COPY $x0
  ; CHECK-AFTER-ISEL-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
  ; CHECK-AFTER-ISEL-NEXT:   [[RDVLI_XI:%[0-9]+]]:gpr64 = RDVLI_XI 1, implicit $vg
  ; CHECK-AFTER-ISEL-NEXT:   [[COPY1:%[0-9]+]]:gpr64sp = COPY [[RDVLI_XI]]
  ; CHECK-AFTER-ISEL-NEXT:   [[ADDSVL_XXI:%[0-9]+]]:gpr64sp = ADDSVL_XXI [[COPY1]], -1, implicit $vg
  ; CHECK-AFTER-ISEL-NEXT:   [[COPY2:%[0-9]+]]:gpr64 = COPY [[ADDSVL_XXI]]
  ; CHECK-AFTER-ISEL-NEXT:   CBZX [[COPY2]], %bb.2
  ; CHECK-AFTER-ISEL-NEXT: {{  $}}
  ; CHECK-AFTER-ISEL-NEXT: bb.1.entry:
  ; CHECK-AFTER-ISEL-NEXT:   successors:
  ; CHECK-AFTER-ISEL-NEXT: {{  $}}
  ; CHECK-AFTER-ISEL-NEXT:   BRK 1
  ; CHECK-AFTER-ISEL-NEXT: {{  $}}
  ; CHECK-AFTER-ISEL-NEXT: bb.2.entry:
  ; CHECK-AFTER-ISEL-NEXT:   MSRpstatesvcrImm1 1, 1, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit-def $sp, implicit-def $z0, implicit $vg, implicit-def $vg, implicit-def $fpmr
  ; CHECK-AFTER-ISEL-NEXT:   BL @bar_retv_enabled, csr_aarch64_sve_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def $z0
  ; CHECK-AFTER-ISEL-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
  ; CHECK-AFTER-ISEL-NEXT:   [[COPY3:%[0-9]+]]:zpr = COPY $z0
  ; CHECK-AFTER-ISEL-NEXT:   MSRpstatesvcrImm1 1, 0, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit $vg, implicit-def $vg, implicit-def $fpmr
  ; CHECK-AFTER-ISEL-NEXT:   [[COPY4:%[0-9]+]]:zpr = COPY [[COPY3]]
  ; CHECK-AFTER-ISEL-NEXT:   STR_ZXI [[COPY4]], [[COPY]], 0 :: (store (<vscale x 1 x s128>) into %ir.ptr)
  ; CHECK-AFTER-ISEL-NEXT:   RET_ReallyLR
entry:
  %v = tail call <vscale x 4 x i32> @bar_retv_enabled() #0
  store <vscale x 4 x i32> %v, ptr %ptr, align 16
  ret void
}

; Streaming -> returns SVE value from non-streaming callee
define void @foo_streaming_retval(ptr %ptr) #0 {
  ; CHECK-BEFORE-ISEL-LABEL: name: foo_streaming_retval
  ; CHECK-BEFORE-ISEL: bb.0.entry:
  ; CHECK-BEFORE-ISEL-NEXT:   liveins: $x0
  ; CHECK-BEFORE-ISEL-NEXT: {{  $}}
  ; CHECK-BEFORE-ISEL-NEXT:   [[COPY:%[0-9]+]]:gpr64common = COPY $x0
  ; CHECK-BEFORE-ISEL-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
  ; CHECK-BEFORE-ISEL-NEXT:   MSRpstatesvcrImm1 1, 0, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit-def $sp, implicit-def $z0, implicit $vg, implicit-def $vg, implicit-def $fpmr
  ; CHECK-BEFORE-ISEL-NEXT:   CHECK_MATCHING_VL_PSEUDO
  ; CHECK-BEFORE-ISEL-NEXT:   BL @bar_retv, csr_aarch64_sve_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def $z0
  ; CHECK-BEFORE-ISEL-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
  ; CHECK-BEFORE-ISEL-NEXT:   [[COPY1:%[0-9]+]]:zpr = COPY $z0
  ; CHECK-BEFORE-ISEL-NEXT:   MSRpstatesvcrImm1 1, 1, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit $vg, implicit-def $vg, implicit-def $fpmr
  ; CHECK-BEFORE-ISEL-NEXT:   [[COPY2:%[0-9]+]]:zpr = COPY [[COPY1]]
  ; CHECK-BEFORE-ISEL-NEXT:   STR_ZXI [[COPY2]], [[COPY]], 0 :: (store (<vscale x 1 x s128>) into %ir.ptr)
  ; CHECK-BEFORE-ISEL-NEXT:   RET_ReallyLR
  ;
  ; CHECK-AFTER-ISEL-LABEL: name: foo_streaming_retval
  ; CHECK-AFTER-ISEL: bb.0.entry:
  ; CHECK-AFTER-ISEL-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
  ; CHECK-AFTER-ISEL-NEXT:   liveins: $x0
  ; CHECK-AFTER-ISEL-NEXT: {{  $}}
  ; CHECK-AFTER-ISEL-NEXT:   [[COPY:%[0-9]+]]:gpr64common = COPY $x0
  ; CHECK-AFTER-ISEL-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
  ; CHECK-AFTER-ISEL-NEXT:   MSRpstatesvcrImm1 1, 0, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit-def $sp, implicit-def $z0, implicit $vg, implicit-def $vg, implicit-def $fpmr
  ; CHECK-AFTER-ISEL-NEXT:   [[RDVLI_XI:%[0-9]+]]:gpr64 = RDVLI_XI 1, implicit $vg
  ; CHECK-AFTER-ISEL-NEXT:   [[COPY1:%[0-9]+]]:gpr64sp = COPY [[RDVLI_XI]]
  ; CHECK-AFTER-ISEL-NEXT:   [[ADDSVL_XXI:%[0-9]+]]:gpr64sp = ADDSVL_XXI [[COPY1]], -1, implicit $vg
  ; CHECK-AFTER-ISEL-NEXT:   [[COPY2:%[0-9]+]]:gpr64 = COPY [[ADDSVL_XXI]]
  ; CHECK-AFTER-ISEL-NEXT:   CBZX [[COPY2]], %bb.2
  ; CHECK-AFTER-ISEL-NEXT: {{  $}}
  ; CHECK-AFTER-ISEL-NEXT: bb.1.entry:
  ; CHECK-AFTER-ISEL-NEXT:   successors:
  ; CHECK-AFTER-ISEL-NEXT: {{  $}}
  ; CHECK-AFTER-ISEL-NEXT:   BRK 1
  ; CHECK-AFTER-ISEL-NEXT: {{  $}}
  ; CHECK-AFTER-ISEL-NEXT: bb.2.entry:
  ; CHECK-AFTER-ISEL-NEXT:   BL @bar_retv, csr_aarch64_sve_aapcs, implicit-def dead $lr, implicit $sp, implicit-def $sp, implicit-def $z0
  ; CHECK-AFTER-ISEL-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
  ; CHECK-AFTER-ISEL-NEXT:   [[COPY3:%[0-9]+]]:zpr = COPY $z0
  ; CHECK-AFTER-ISEL-NEXT:   MSRpstatesvcrImm1 1, 1, csr_aarch64_smstartstop, implicit-def dead $nzcv, implicit $vg, implicit-def $vg, implicit-def $fpmr
  ; CHECK-AFTER-ISEL-NEXT:   [[COPY4:%[0-9]+]]:zpr = COPY [[COPY3]]
  ; CHECK-AFTER-ISEL-NEXT:   STR_ZXI [[COPY4]], [[COPY]], 0 :: (store (<vscale x 1 x s128>) into %ir.ptr)
  ; CHECK-AFTER-ISEL-NEXT:   RET_ReallyLR
entry:
  %v = tail call <vscale x 4 x i32> @bar_retv()
  store <vscale x 4 x i32> %v, ptr %ptr, align 16
  ret void
}

attributes #0 = { "aarch64_pstate_sm_enabled" }
attributes #1 = { "aarch64_pstate_sm_compatible" }