; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5

; Test the -max-store-memset, -max-store-memcpy, and -max-store-memmove flags,
; which control the threshold for when memset/memcpy/memmove are inlined vs
; calling the library function.

; RUN: llc %s -o - -mtriple=aarch64 | FileCheck %s --check-prefix=DEFAULT
; RUN: llc %s -o - -mtriple=aarch64 -max-store-memset=7 | FileCheck %s --check-prefix=MEMSET-LIMIT-7
; RUN: llc %s -o - -mtriple=aarch64 -max-store-memset=8 | FileCheck %s --check-prefix=MEMSET-LIMIT-8
; RUN: llc %s -o - -mtriple=aarch64 -max-store-memcpy=7 | FileCheck %s --check-prefix=MEMCPY-LIMIT-7
; RUN: llc %s -o - -mtriple=aarch64 -max-store-memcpy=8 | FileCheck %s --check-prefix=MEMCPY-LIMIT-8
; RUN: llc %s -o - -mtriple=aarch64 -max-store-memmove=7 | FileCheck %s --check-prefix=MEMMOVE-LIMIT-7
; RUN: llc %s -o - -mtriple=aarch64 -max-store-memmove=8 | FileCheck %s --check-prefix=MEMMOVE-LIMIT-8

declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg)
declare void @llvm.memcpy.p0.p0.i64(ptr nocapture writeonly, ptr nocapture readonly, i64, i1)
declare void @llvm.memmove.p0.p0.i64(ptr nocapture writeonly, ptr nocapture readonly, i64, i1)

; memset tests

; Test memset with 128 bytes (the AArch64 backend counts stores in 16-bit)
; This should be inlined by default and with -max-store-memset=8
define void @memset_128(ptr %dst) {
; DEFAULT-LABEL: memset_128:
; DEFAULT:       // %bb.0:
; DEFAULT-NEXT:    movi v0.2d, #0000000000000000
; DEFAULT-NEXT:    stp q0, q0, [x0]
; DEFAULT-NEXT:    stp q0, q0, [x0, #32]
; DEFAULT-NEXT:    stp q0, q0, [x0, #64]
; DEFAULT-NEXT:    stp q0, q0, [x0, #96]
; DEFAULT-NEXT:    ret
;
; MEMSET-LIMIT-7-LABEL: memset_128:
; MEMSET-LIMIT-7:       // %bb.0:
; MEMSET-LIMIT-7-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; MEMSET-LIMIT-7-NEXT:    .cfi_def_cfa_offset 16
; MEMSET-LIMIT-7-NEXT:    .cfi_offset w30, -16
; MEMSET-LIMIT-7-NEXT:    mov w1, wzr
; MEMSET-LIMIT-7-NEXT:    mov w2, #128 // =0x80
; MEMSET-LIMIT-7-NEXT:    bl memset
; MEMSET-LIMIT-7-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; MEMSET-LIMIT-7-NEXT:    ret
;
; MEMSET-LIMIT-8-LABEL: memset_128:
; MEMSET-LIMIT-8:       // %bb.0:
; MEMSET-LIMIT-8-NEXT:    movi v0.2d, #0000000000000000
; MEMSET-LIMIT-8-NEXT:    stp q0, q0, [x0]
; MEMSET-LIMIT-8-NEXT:    stp q0, q0, [x0, #32]
; MEMSET-LIMIT-8-NEXT:    stp q0, q0, [x0, #64]
; MEMSET-LIMIT-8-NEXT:    stp q0, q0, [x0, #96]
; MEMSET-LIMIT-8-NEXT:    ret
;
; MEMCPY-LIMIT-7-LABEL: memset_128:
; MEMCPY-LIMIT-7:       // %bb.0:
; MEMCPY-LIMIT-7-NEXT:    movi v0.2d, #0000000000000000
; MEMCPY-LIMIT-7-NEXT:    stp q0, q0, [x0]
; MEMCPY-LIMIT-7-NEXT:    stp q0, q0, [x0, #32]
; MEMCPY-LIMIT-7-NEXT:    stp q0, q0, [x0, #64]
; MEMCPY-LIMIT-7-NEXT:    stp q0, q0, [x0, #96]
; MEMCPY-LIMIT-7-NEXT:    ret
;
; MEMCPY-LIMIT-8-LABEL: memset_128:
; MEMCPY-LIMIT-8:       // %bb.0:
; MEMCPY-LIMIT-8-NEXT:    movi v0.2d, #0000000000000000
; MEMCPY-LIMIT-8-NEXT:    stp q0, q0, [x0]
; MEMCPY-LIMIT-8-NEXT:    stp q0, q0, [x0, #32]
; MEMCPY-LIMIT-8-NEXT:    stp q0, q0, [x0, #64]
; MEMCPY-LIMIT-8-NEXT:    stp q0, q0, [x0, #96]
; MEMCPY-LIMIT-8-NEXT:    ret
;
; MEMMOVE-LIMIT-7-LABEL: memset_128:
; MEMMOVE-LIMIT-7:       // %bb.0:
; MEMMOVE-LIMIT-7-NEXT:    movi v0.2d, #0000000000000000
; MEMMOVE-LIMIT-7-NEXT:    stp q0, q0, [x0]
; MEMMOVE-LIMIT-7-NEXT:    stp q0, q0, [x0, #32]
; MEMMOVE-LIMIT-7-NEXT:    stp q0, q0, [x0, #64]
; MEMMOVE-LIMIT-7-NEXT:    stp q0, q0, [x0, #96]
; MEMMOVE-LIMIT-7-NEXT:    ret
;
; MEMMOVE-LIMIT-8-LABEL: memset_128:
; MEMMOVE-LIMIT-8:       // %bb.0:
; MEMMOVE-LIMIT-8-NEXT:    movi v0.2d, #0000000000000000
; MEMMOVE-LIMIT-8-NEXT:    stp q0, q0, [x0]
; MEMMOVE-LIMIT-8-NEXT:    stp q0, q0, [x0, #32]
; MEMMOVE-LIMIT-8-NEXT:    stp q0, q0, [x0, #64]
; MEMMOVE-LIMIT-8-NEXT:    stp q0, q0, [x0, #96]
; MEMMOVE-LIMIT-8-NEXT:    ret
  call void @llvm.memset.p0.i64(ptr align 16 %dst, i8 0, i64 128, i1 false)
  ret void
}

; Test memset in a function with optsize attribute
; With -max-store-memset=8, 128 bytes (8 stores) should still inline
define void @memset_128_optsize(ptr %dst) optsize {
; DEFAULT-LABEL: memset_128_optsize:
; DEFAULT:       // %bb.0:
; DEFAULT-NEXT:    movi v0.2d, #0000000000000000
; DEFAULT-NEXT:    stp q0, q0, [x0]
; DEFAULT-NEXT:    stp q0, q0, [x0, #32]
; DEFAULT-NEXT:    stp q0, q0, [x0, #64]
; DEFAULT-NEXT:    stp q0, q0, [x0, #96]
; DEFAULT-NEXT:    ret
;
; MEMSET-LIMIT-7-LABEL: memset_128_optsize:
; MEMSET-LIMIT-7:       // %bb.0:
; MEMSET-LIMIT-7-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; MEMSET-LIMIT-7-NEXT:    .cfi_def_cfa_offset 16
; MEMSET-LIMIT-7-NEXT:    .cfi_offset w30, -16
; MEMSET-LIMIT-7-NEXT:    mov w1, wzr
; MEMSET-LIMIT-7-NEXT:    mov w2, #128 // =0x80
; MEMSET-LIMIT-7-NEXT:    bl memset
; MEMSET-LIMIT-7-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; MEMSET-LIMIT-7-NEXT:    ret
;
; MEMSET-LIMIT-8-LABEL: memset_128_optsize:
; MEMSET-LIMIT-8:       // %bb.0:
; MEMSET-LIMIT-8-NEXT:    movi v0.2d, #0000000000000000
; MEMSET-LIMIT-8-NEXT:    stp q0, q0, [x0]
; MEMSET-LIMIT-8-NEXT:    stp q0, q0, [x0, #32]
; MEMSET-LIMIT-8-NEXT:    stp q0, q0, [x0, #64]
; MEMSET-LIMIT-8-NEXT:    stp q0, q0, [x0, #96]
; MEMSET-LIMIT-8-NEXT:    ret
;
; MEMCPY-LIMIT-7-LABEL: memset_128_optsize:
; MEMCPY-LIMIT-7:       // %bb.0:
; MEMCPY-LIMIT-7-NEXT:    movi v0.2d, #0000000000000000
; MEMCPY-LIMIT-7-NEXT:    stp q0, q0, [x0]
; MEMCPY-LIMIT-7-NEXT:    stp q0, q0, [x0, #32]
; MEMCPY-LIMIT-7-NEXT:    stp q0, q0, [x0, #64]
; MEMCPY-LIMIT-7-NEXT:    stp q0, q0, [x0, #96]
; MEMCPY-LIMIT-7-NEXT:    ret
;
; MEMCPY-LIMIT-8-LABEL: memset_128_optsize:
; MEMCPY-LIMIT-8:       // %bb.0:
; MEMCPY-LIMIT-8-NEXT:    movi v0.2d, #0000000000000000
; MEMCPY-LIMIT-8-NEXT:    stp q0, q0, [x0]
; MEMCPY-LIMIT-8-NEXT:    stp q0, q0, [x0, #32]
; MEMCPY-LIMIT-8-NEXT:    stp q0, q0, [x0, #64]
; MEMCPY-LIMIT-8-NEXT:    stp q0, q0, [x0, #96]
; MEMCPY-LIMIT-8-NEXT:    ret
;
; MEMMOVE-LIMIT-7-LABEL: memset_128_optsize:
; MEMMOVE-LIMIT-7:       // %bb.0:
; MEMMOVE-LIMIT-7-NEXT:    movi v0.2d, #0000000000000000
; MEMMOVE-LIMIT-7-NEXT:    stp q0, q0, [x0]
; MEMMOVE-LIMIT-7-NEXT:    stp q0, q0, [x0, #32]
; MEMMOVE-LIMIT-7-NEXT:    stp q0, q0, [x0, #64]
; MEMMOVE-LIMIT-7-NEXT:    stp q0, q0, [x0, #96]
; MEMMOVE-LIMIT-7-NEXT:    ret
;
; MEMMOVE-LIMIT-8-LABEL: memset_128_optsize:
; MEMMOVE-LIMIT-8:       // %bb.0:
; MEMMOVE-LIMIT-8-NEXT:    movi v0.2d, #0000000000000000
; MEMMOVE-LIMIT-8-NEXT:    stp q0, q0, [x0]
; MEMMOVE-LIMIT-8-NEXT:    stp q0, q0, [x0, #32]
; MEMMOVE-LIMIT-8-NEXT:    stp q0, q0, [x0, #64]
; MEMMOVE-LIMIT-8-NEXT:    stp q0, q0, [x0, #96]
; MEMMOVE-LIMIT-8-NEXT:    ret
  call void @llvm.memset.p0.i64(ptr align 16 %dst, i8 0, i64 128, i1 false)
  ret void
}

; memcpy tests

; Test memcpy with 128 bytes (8 stores of 16 bytes each)
; This should be inlined by default and with -max-store-memcpy=8
; but should call memcpy library with -max-store-memcpy=7
define void @memcpy_128(ptr %dst, ptr %src) {
; DEFAULT-LABEL: memcpy_128:
; DEFAULT:       // %bb.0:
; DEFAULT-NEXT:    ldp q1, q0, [x1, #32]
; DEFAULT-NEXT:    ldp q2, q3, [x1]
; DEFAULT-NEXT:    stp q1, q0, [x0, #32]
; DEFAULT-NEXT:    stp q2, q3, [x0]
; DEFAULT-NEXT:    ldp q1, q0, [x1, #96]
; DEFAULT-NEXT:    ldp q2, q3, [x1, #64]
; DEFAULT-NEXT:    stp q1, q0, [x0, #96]
; DEFAULT-NEXT:    stp q2, q3, [x0, #64]
; DEFAULT-NEXT:    ret
;
; MEMSET-LIMIT-7-LABEL: memcpy_128:
; MEMSET-LIMIT-7:       // %bb.0:
; MEMSET-LIMIT-7-NEXT:    ldp q1, q0, [x1, #32]
; MEMSET-LIMIT-7-NEXT:    ldp q2, q3, [x1]
; MEMSET-LIMIT-7-NEXT:    stp q1, q0, [x0, #32]
; MEMSET-LIMIT-7-NEXT:    stp q2, q3, [x0]
; MEMSET-LIMIT-7-NEXT:    ldp q1, q0, [x1, #96]
; MEMSET-LIMIT-7-NEXT:    ldp q2, q3, [x1, #64]
; MEMSET-LIMIT-7-NEXT:    stp q1, q0, [x0, #96]
; MEMSET-LIMIT-7-NEXT:    stp q2, q3, [x0, #64]
; MEMSET-LIMIT-7-NEXT:    ret
;
; MEMSET-LIMIT-8-LABEL: memcpy_128:
; MEMSET-LIMIT-8:       // %bb.0:
; MEMSET-LIMIT-8-NEXT:    ldp q1, q0, [x1, #32]
; MEMSET-LIMIT-8-NEXT:    ldp q2, q3, [x1]
; MEMSET-LIMIT-8-NEXT:    stp q1, q0, [x0, #32]
; MEMSET-LIMIT-8-NEXT:    stp q2, q3, [x0]
; MEMSET-LIMIT-8-NEXT:    ldp q1, q0, [x1, #96]
; MEMSET-LIMIT-8-NEXT:    ldp q2, q3, [x1, #64]
; MEMSET-LIMIT-8-NEXT:    stp q1, q0, [x0, #96]
; MEMSET-LIMIT-8-NEXT:    stp q2, q3, [x0, #64]
; MEMSET-LIMIT-8-NEXT:    ret
;
; MEMCPY-LIMIT-7-LABEL: memcpy_128:
; MEMCPY-LIMIT-7:       // %bb.0:
; MEMCPY-LIMIT-7-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; MEMCPY-LIMIT-7-NEXT:    .cfi_def_cfa_offset 16
; MEMCPY-LIMIT-7-NEXT:    .cfi_offset w30, -16
; MEMCPY-LIMIT-7-NEXT:    mov w2, #128 // =0x80
; MEMCPY-LIMIT-7-NEXT:    bl memcpy
; MEMCPY-LIMIT-7-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; MEMCPY-LIMIT-7-NEXT:    ret
;
; MEMCPY-LIMIT-8-LABEL: memcpy_128:
; MEMCPY-LIMIT-8:       // %bb.0:
; MEMCPY-LIMIT-8-NEXT:    ldp q1, q0, [x1, #32]
; MEMCPY-LIMIT-8-NEXT:    ldp q2, q3, [x1]
; MEMCPY-LIMIT-8-NEXT:    stp q1, q0, [x0, #32]
; MEMCPY-LIMIT-8-NEXT:    stp q2, q3, [x0]
; MEMCPY-LIMIT-8-NEXT:    ldp q1, q0, [x1, #96]
; MEMCPY-LIMIT-8-NEXT:    ldp q2, q3, [x1, #64]
; MEMCPY-LIMIT-8-NEXT:    stp q1, q0, [x0, #96]
; MEMCPY-LIMIT-8-NEXT:    stp q2, q3, [x0, #64]
; MEMCPY-LIMIT-8-NEXT:    ret
;
; MEMMOVE-LIMIT-7-LABEL: memcpy_128:
; MEMMOVE-LIMIT-7:       // %bb.0:
; MEMMOVE-LIMIT-7-NEXT:    ldp q1, q0, [x1, #32]
; MEMMOVE-LIMIT-7-NEXT:    ldp q2, q3, [x1]
; MEMMOVE-LIMIT-7-NEXT:    stp q1, q0, [x0, #32]
; MEMMOVE-LIMIT-7-NEXT:    stp q2, q3, [x0]
; MEMMOVE-LIMIT-7-NEXT:    ldp q1, q0, [x1, #96]
; MEMMOVE-LIMIT-7-NEXT:    ldp q2, q3, [x1, #64]
; MEMMOVE-LIMIT-7-NEXT:    stp q1, q0, [x0, #96]
; MEMMOVE-LIMIT-7-NEXT:    stp q2, q3, [x0, #64]
; MEMMOVE-LIMIT-7-NEXT:    ret
;
; MEMMOVE-LIMIT-8-LABEL: memcpy_128:
; MEMMOVE-LIMIT-8:       // %bb.0:
; MEMMOVE-LIMIT-8-NEXT:    ldp q1, q0, [x1, #32]
; MEMMOVE-LIMIT-8-NEXT:    ldp q2, q3, [x1]
; MEMMOVE-LIMIT-8-NEXT:    stp q1, q0, [x0, #32]
; MEMMOVE-LIMIT-8-NEXT:    stp q2, q3, [x0]
; MEMMOVE-LIMIT-8-NEXT:    ldp q1, q0, [x1, #96]
; MEMMOVE-LIMIT-8-NEXT:    ldp q2, q3, [x1, #64]
; MEMMOVE-LIMIT-8-NEXT:    stp q1, q0, [x0, #96]
; MEMMOVE-LIMIT-8-NEXT:    stp q2, q3, [x0, #64]
; MEMMOVE-LIMIT-8-NEXT:    ret
  call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 128, i1 false)
  ret void
}

; Test optsize memcpy with 128 bytes (8 stores of 16 bytes each)
; This should be inlined with -max-store-memcpy=8
; but should call memcpy library with -max-store-memcpy=7
define void @memcpy_128_optsize(ptr %dst, ptr %src) optsize {
; DEFAULT-LABEL: memcpy_128_optsize:
; DEFAULT:       // %bb.0:
; DEFAULT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; DEFAULT-NEXT:    .cfi_def_cfa_offset 16
; DEFAULT-NEXT:    .cfi_offset w30, -16
; DEFAULT-NEXT:    mov w2, #128 // =0x80
; DEFAULT-NEXT:    bl memcpy
; DEFAULT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; DEFAULT-NEXT:    ret
;
; MEMSET-LIMIT-7-LABEL: memcpy_128_optsize:
; MEMSET-LIMIT-7:       // %bb.0:
; MEMSET-LIMIT-7-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; MEMSET-LIMIT-7-NEXT:    .cfi_def_cfa_offset 16
; MEMSET-LIMIT-7-NEXT:    .cfi_offset w30, -16
; MEMSET-LIMIT-7-NEXT:    mov w2, #128 // =0x80
; MEMSET-LIMIT-7-NEXT:    bl memcpy
; MEMSET-LIMIT-7-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; MEMSET-LIMIT-7-NEXT:    ret
;
; MEMSET-LIMIT-8-LABEL: memcpy_128_optsize:
; MEMSET-LIMIT-8:       // %bb.0:
; MEMSET-LIMIT-8-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; MEMSET-LIMIT-8-NEXT:    .cfi_def_cfa_offset 16
; MEMSET-LIMIT-8-NEXT:    .cfi_offset w30, -16
; MEMSET-LIMIT-8-NEXT:    mov w2, #128 // =0x80
; MEMSET-LIMIT-8-NEXT:    bl memcpy
; MEMSET-LIMIT-8-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; MEMSET-LIMIT-8-NEXT:    ret
;
; MEMCPY-LIMIT-7-LABEL: memcpy_128_optsize:
; MEMCPY-LIMIT-7:       // %bb.0:
; MEMCPY-LIMIT-7-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; MEMCPY-LIMIT-7-NEXT:    .cfi_def_cfa_offset 16
; MEMCPY-LIMIT-7-NEXT:    .cfi_offset w30, -16
; MEMCPY-LIMIT-7-NEXT:    mov w2, #128 // =0x80
; MEMCPY-LIMIT-7-NEXT:    bl memcpy
; MEMCPY-LIMIT-7-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; MEMCPY-LIMIT-7-NEXT:    ret
;
; MEMCPY-LIMIT-8-LABEL: memcpy_128_optsize:
; MEMCPY-LIMIT-8:       // %bb.0:
; MEMCPY-LIMIT-8-NEXT:    ldp q1, q0, [x1, #32]
; MEMCPY-LIMIT-8-NEXT:    ldp q2, q3, [x1]
; MEMCPY-LIMIT-8-NEXT:    stp q1, q0, [x0, #32]
; MEMCPY-LIMIT-8-NEXT:    stp q2, q3, [x0]
; MEMCPY-LIMIT-8-NEXT:    ldp q1, q0, [x1, #96]
; MEMCPY-LIMIT-8-NEXT:    ldp q2, q3, [x1, #64]
; MEMCPY-LIMIT-8-NEXT:    stp q1, q0, [x0, #96]
; MEMCPY-LIMIT-8-NEXT:    stp q2, q3, [x0, #64]
; MEMCPY-LIMIT-8-NEXT:    ret
;
; MEMMOVE-LIMIT-7-LABEL: memcpy_128_optsize:
; MEMMOVE-LIMIT-7:       // %bb.0:
; MEMMOVE-LIMIT-7-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; MEMMOVE-LIMIT-7-NEXT:    .cfi_def_cfa_offset 16
; MEMMOVE-LIMIT-7-NEXT:    .cfi_offset w30, -16
; MEMMOVE-LIMIT-7-NEXT:    mov w2, #128 // =0x80
; MEMMOVE-LIMIT-7-NEXT:    bl memcpy
; MEMMOVE-LIMIT-7-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; MEMMOVE-LIMIT-7-NEXT:    ret
;
; MEMMOVE-LIMIT-8-LABEL: memcpy_128_optsize:
; MEMMOVE-LIMIT-8:       // %bb.0:
; MEMMOVE-LIMIT-8-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; MEMMOVE-LIMIT-8-NEXT:    .cfi_def_cfa_offset 16
; MEMMOVE-LIMIT-8-NEXT:    .cfi_offset w30, -16
; MEMMOVE-LIMIT-8-NEXT:    mov w2, #128 // =0x80
; MEMMOVE-LIMIT-8-NEXT:    bl memcpy
; MEMMOVE-LIMIT-8-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; MEMMOVE-LIMIT-8-NEXT:    ret
  call void @llvm.memcpy.p0.p0.i64(ptr %dst, ptr %src, i64 128, i1 false)
  ret void
}

; memmove tests

; Test memmove with 128 bytes (8 stores of 16 bytes each)
; This should be inlined by default and with -max-store-memmove=8
; but should call memmove library with -max-store-memmove=7
define void @memmove_128(ptr %dst, ptr %src) {
; DEFAULT-LABEL: memmove_128:
; DEFAULT:       // %bb.0:
; DEFAULT-NEXT:    ldp q0, q1, [x1, #96]
; DEFAULT-NEXT:    ldp q2, q3, [x1]
; DEFAULT-NEXT:    ldp q4, q5, [x1, #32]
; DEFAULT-NEXT:    ldp q6, q7, [x1, #64]
; DEFAULT-NEXT:    stp q0, q1, [x0, #96]
; DEFAULT-NEXT:    stp q2, q3, [x0]
; DEFAULT-NEXT:    stp q4, q5, [x0, #32]
; DEFAULT-NEXT:    stp q6, q7, [x0, #64]
; DEFAULT-NEXT:    ret
;
; MEMSET-LIMIT-7-LABEL: memmove_128:
; MEMSET-LIMIT-7:       // %bb.0:
; MEMSET-LIMIT-7-NEXT:    ldp q0, q1, [x1, #96]
; MEMSET-LIMIT-7-NEXT:    ldp q2, q3, [x1]
; MEMSET-LIMIT-7-NEXT:    ldp q4, q5, [x1, #32]
; MEMSET-LIMIT-7-NEXT:    ldp q6, q7, [x1, #64]
; MEMSET-LIMIT-7-NEXT:    stp q0, q1, [x0, #96]
; MEMSET-LIMIT-7-NEXT:    stp q2, q3, [x0]
; MEMSET-LIMIT-7-NEXT:    stp q4, q5, [x0, #32]
; MEMSET-LIMIT-7-NEXT:    stp q6, q7, [x0, #64]
; MEMSET-LIMIT-7-NEXT:    ret
;
; MEMSET-LIMIT-8-LABEL: memmove_128:
; MEMSET-LIMIT-8:       // %bb.0:
; MEMSET-LIMIT-8-NEXT:    ldp q0, q1, [x1, #96]
; MEMSET-LIMIT-8-NEXT:    ldp q2, q3, [x1]
; MEMSET-LIMIT-8-NEXT:    ldp q4, q5, [x1, #32]
; MEMSET-LIMIT-8-NEXT:    ldp q6, q7, [x1, #64]
; MEMSET-LIMIT-8-NEXT:    stp q0, q1, [x0, #96]
; MEMSET-LIMIT-8-NEXT:    stp q2, q3, [x0]
; MEMSET-LIMIT-8-NEXT:    stp q4, q5, [x0, #32]
; MEMSET-LIMIT-8-NEXT:    stp q6, q7, [x0, #64]
; MEMSET-LIMIT-8-NEXT:    ret
;
; MEMCPY-LIMIT-7-LABEL: memmove_128:
; MEMCPY-LIMIT-7:       // %bb.0:
; MEMCPY-LIMIT-7-NEXT:    ldp q0, q1, [x1, #96]
; MEMCPY-LIMIT-7-NEXT:    ldp q2, q3, [x1]
; MEMCPY-LIMIT-7-NEXT:    ldp q4, q5, [x1, #32]
; MEMCPY-LIMIT-7-NEXT:    ldp q6, q7, [x1, #64]
; MEMCPY-LIMIT-7-NEXT:    stp q0, q1, [x0, #96]
; MEMCPY-LIMIT-7-NEXT:    stp q2, q3, [x0]
; MEMCPY-LIMIT-7-NEXT:    stp q4, q5, [x0, #32]
; MEMCPY-LIMIT-7-NEXT:    stp q6, q7, [x0, #64]
; MEMCPY-LIMIT-7-NEXT:    ret
;
; MEMCPY-LIMIT-8-LABEL: memmove_128:
; MEMCPY-LIMIT-8:       // %bb.0:
; MEMCPY-LIMIT-8-NEXT:    ldp q0, q1, [x1, #96]
; MEMCPY-LIMIT-8-NEXT:    ldp q2, q3, [x1]
; MEMCPY-LIMIT-8-NEXT:    ldp q4, q5, [x1, #32]
; MEMCPY-LIMIT-8-NEXT:    ldp q6, q7, [x1, #64]
; MEMCPY-LIMIT-8-NEXT:    stp q0, q1, [x0, #96]
; MEMCPY-LIMIT-8-NEXT:    stp q2, q3, [x0]
; MEMCPY-LIMIT-8-NEXT:    stp q4, q5, [x0, #32]
; MEMCPY-LIMIT-8-NEXT:    stp q6, q7, [x0, #64]
; MEMCPY-LIMIT-8-NEXT:    ret
;
; MEMMOVE-LIMIT-7-LABEL: memmove_128:
; MEMMOVE-LIMIT-7:       // %bb.0:
; MEMMOVE-LIMIT-7-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; MEMMOVE-LIMIT-7-NEXT:    .cfi_def_cfa_offset 16
; MEMMOVE-LIMIT-7-NEXT:    .cfi_offset w30, -16
; MEMMOVE-LIMIT-7-NEXT:    mov w2, #128 // =0x80
; MEMMOVE-LIMIT-7-NEXT:    bl memmove
; MEMMOVE-LIMIT-7-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; MEMMOVE-LIMIT-7-NEXT:    ret
;
; MEMMOVE-LIMIT-8-LABEL: memmove_128:
; MEMMOVE-LIMIT-8:       // %bb.0:
; MEMMOVE-LIMIT-8-NEXT:    ldp q0, q1, [x1, #96]
; MEMMOVE-LIMIT-8-NEXT:    ldp q2, q3, [x1]
; MEMMOVE-LIMIT-8-NEXT:    ldp q4, q5, [x1, #32]
; MEMMOVE-LIMIT-8-NEXT:    ldp q6, q7, [x1, #64]
; MEMMOVE-LIMIT-8-NEXT:    stp q0, q1, [x0, #96]
; MEMMOVE-LIMIT-8-NEXT:    stp q2, q3, [x0]
; MEMMOVE-LIMIT-8-NEXT:    stp q4, q5, [x0, #32]
; MEMMOVE-LIMIT-8-NEXT:    stp q6, q7, [x0, #64]
; MEMMOVE-LIMIT-8-NEXT:    ret
  call void @llvm.memmove.p0.p0.i64(ptr %dst, ptr %src, i64 128, i1 false)
  ret void
}

; Test optsize memmove with 128 bytes (8 stores of 16 bytes each)
; This should be inlined with -max-store-memmove=8
; but should call memmove library with -max-store-memmove=7
define void @memmove_128_optsize(ptr %dst, ptr %src) optsize {
; DEFAULT-LABEL: memmove_128_optsize:
; DEFAULT:       // %bb.0:
; DEFAULT-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; DEFAULT-NEXT:    .cfi_def_cfa_offset 16
; DEFAULT-NEXT:    .cfi_offset w30, -16
; DEFAULT-NEXT:    mov w2, #128 // =0x80
; DEFAULT-NEXT:    bl memmove
; DEFAULT-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; DEFAULT-NEXT:    ret
;
; MEMSET-LIMIT-7-LABEL: memmove_128_optsize:
; MEMSET-LIMIT-7:       // %bb.0:
; MEMSET-LIMIT-7-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; MEMSET-LIMIT-7-NEXT:    .cfi_def_cfa_offset 16
; MEMSET-LIMIT-7-NEXT:    .cfi_offset w30, -16
; MEMSET-LIMIT-7-NEXT:    mov w2, #128 // =0x80
; MEMSET-LIMIT-7-NEXT:    bl memmove
; MEMSET-LIMIT-7-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; MEMSET-LIMIT-7-NEXT:    ret
;
; MEMSET-LIMIT-8-LABEL: memmove_128_optsize:
; MEMSET-LIMIT-8:       // %bb.0:
; MEMSET-LIMIT-8-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; MEMSET-LIMIT-8-NEXT:    .cfi_def_cfa_offset 16
; MEMSET-LIMIT-8-NEXT:    .cfi_offset w30, -16
; MEMSET-LIMIT-8-NEXT:    mov w2, #128 // =0x80
; MEMSET-LIMIT-8-NEXT:    bl memmove
; MEMSET-LIMIT-8-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; MEMSET-LIMIT-8-NEXT:    ret
;
; MEMCPY-LIMIT-7-LABEL: memmove_128_optsize:
; MEMCPY-LIMIT-7:       // %bb.0:
; MEMCPY-LIMIT-7-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; MEMCPY-LIMIT-7-NEXT:    .cfi_def_cfa_offset 16
; MEMCPY-LIMIT-7-NEXT:    .cfi_offset w30, -16
; MEMCPY-LIMIT-7-NEXT:    mov w2, #128 // =0x80
; MEMCPY-LIMIT-7-NEXT:    bl memmove
; MEMCPY-LIMIT-7-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; MEMCPY-LIMIT-7-NEXT:    ret
;
; MEMCPY-LIMIT-8-LABEL: memmove_128_optsize:
; MEMCPY-LIMIT-8:       // %bb.0:
; MEMCPY-LIMIT-8-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; MEMCPY-LIMIT-8-NEXT:    .cfi_def_cfa_offset 16
; MEMCPY-LIMIT-8-NEXT:    .cfi_offset w30, -16
; MEMCPY-LIMIT-8-NEXT:    mov w2, #128 // =0x80
; MEMCPY-LIMIT-8-NEXT:    bl memmove
; MEMCPY-LIMIT-8-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; MEMCPY-LIMIT-8-NEXT:    ret
;
; MEMMOVE-LIMIT-7-LABEL: memmove_128_optsize:
; MEMMOVE-LIMIT-7:       // %bb.0:
; MEMMOVE-LIMIT-7-NEXT:    str x30, [sp, #-16]! // 8-byte Folded Spill
; MEMMOVE-LIMIT-7-NEXT:    .cfi_def_cfa_offset 16
; MEMMOVE-LIMIT-7-NEXT:    .cfi_offset w30, -16
; MEMMOVE-LIMIT-7-NEXT:    mov w2, #128 // =0x80
; MEMMOVE-LIMIT-7-NEXT:    bl memmove
; MEMMOVE-LIMIT-7-NEXT:    ldr x30, [sp], #16 // 8-byte Folded Reload
; MEMMOVE-LIMIT-7-NEXT:    ret
;
; MEMMOVE-LIMIT-8-LABEL: memmove_128_optsize:
; MEMMOVE-LIMIT-8:       // %bb.0:
; MEMMOVE-LIMIT-8-NEXT:    ldp q0, q1, [x1, #96]
; MEMMOVE-LIMIT-8-NEXT:    ldp q2, q3, [x1]
; MEMMOVE-LIMIT-8-NEXT:    ldp q4, q5, [x1, #32]
; MEMMOVE-LIMIT-8-NEXT:    ldp q6, q7, [x1, #64]
; MEMMOVE-LIMIT-8-NEXT:    stp q0, q1, [x0, #96]
; MEMMOVE-LIMIT-8-NEXT:    stp q2, q3, [x0]
; MEMMOVE-LIMIT-8-NEXT:    stp q4, q5, [x0, #32]
; MEMMOVE-LIMIT-8-NEXT:    stp q6, q7, [x0, #64]
; MEMMOVE-LIMIT-8-NEXT:    ret
  call void @llvm.memmove.p0.p0.i64(ptr %dst, ptr %src, i64 128, i1 false)
  ret void
}