; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=aarch64-linux-gnu -aarch64-enable-atomic-cfg-tidy=0 -disable-lsr -verify-machineinstrs -o - %s | FileCheck --check-prefixes=CHECK,NOSTRICTALIGN %s
; RUN: llc -mtriple=aarch64-linux-gnu -mattr=+strict-align -aarch64-enable-atomic-cfg-tidy=0 -disable-lsr -verify-machineinstrs -o - %s | FileCheck --check-prefixes=CHECK,STRICTALIGN %s

; This file contains tests for the AArch64 load/store optimizer.

%padding = type { ptr, ptr, ptr, ptr }
%s.byte = type { i8, i8 }
%s.halfword = type { i16, i16 }
%s.word = type { i32, i32 }
%s.doubleword = type { i64, i32 }
%s.quadword = type { fp128, i32 }
%s.float = type { float, i32 }
%s.double = type { double, i32 }
%struct.byte = type { %padding, %s.byte }
%struct.halfword = type { %padding, %s.halfword }
%struct.word = type { %padding, %s.word }
%struct.doubleword = type { %padding, %s.doubleword }
%struct.quadword = type { %padding, %s.quadword }
%struct.float = type { %padding, %s.float }
%struct.double = type { %padding, %s.double }

; Check the following transform:
;
; (ldr|str) X, [x0, #32]
;  ...
; add x0, x0, #32
;  ->
; (ldr|str) X, [x0, #32]!
;
; with X being either w1, x1, s0, d0 or q0.

declare void @bar_byte(ptr, i8)

define void @load_pre_indexed_byte(ptr %ptr) nounwind {
; CHECK-LABEL: load_pre_indexed_byte:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    ldrb w1, [x0, #32]!
; CHECK-NEXT:    b bar_byte
entry:
  %a = getelementptr inbounds %struct.byte, ptr %ptr, i64 0, i32 1, i32 0
  %add = load i8, ptr %a, align 4
  br label %bar
bar:
  %c = getelementptr inbounds %struct.byte, ptr %ptr, i64 0, i32 1
  tail call void @bar_byte(ptr %c, i8 %add)
  ret void
}

define void @store_pre_indexed_byte(ptr %ptr, i8 %val) nounwind {
; CHECK-LABEL: store_pre_indexed_byte:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    strb w1, [x0, #32]!
; CHECK-NEXT:    b bar_byte
entry:
  %a = getelementptr inbounds %struct.byte, ptr %ptr, i64 0, i32 1, i32 0
  store i8 %val, ptr %a, align 4
  br label %bar
bar:
  %c = getelementptr inbounds %struct.byte, ptr %ptr, i64 0, i32 1
  tail call void @bar_byte(ptr %c, i8 %val)
  ret void
}

declare void @bar_halfword(ptr, i16)

define void @load_pre_indexed_halfword(ptr %ptr) nounwind {
; CHECK-LABEL: load_pre_indexed_halfword:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    ldrh w1, [x0, #32]!
; CHECK-NEXT:    b bar_halfword
entry:
  %a = getelementptr inbounds %struct.halfword, ptr %ptr, i64 0, i32 1, i32 0
  %add = load i16, ptr %a, align 4
  br label %bar
bar:
  %c = getelementptr inbounds %struct.halfword, ptr %ptr, i64 0, i32 1
  tail call void @bar_halfword(ptr %c, i16 %add)
  ret void
}

define void @store_pre_indexed_halfword(ptr %ptr, i16 %val) nounwind {
; CHECK-LABEL: store_pre_indexed_halfword:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    strh w1, [x0, #32]!
; CHECK-NEXT:    b bar_halfword
entry:
  %a = getelementptr inbounds %struct.halfword, ptr %ptr, i64 0, i32 1, i32 0
  store i16 %val, ptr %a, align 4
  br label %bar
bar:
  %c = getelementptr inbounds %struct.halfword, ptr %ptr, i64 0, i32 1
  tail call void @bar_halfword(ptr %c, i16 %val)
  ret void
}

declare void @bar_word(ptr, i32)

define void @load_pre_indexed_word(ptr %ptr) nounwind {
; CHECK-LABEL: load_pre_indexed_word:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    ldr w1, [x0, #32]!
; CHECK-NEXT:    b bar_word
entry:
  %a = getelementptr inbounds %struct.word, ptr %ptr, i64 0, i32 1, i32 0
  %add = load i32, ptr %a, align 4
  br label %bar
bar:
  %c = getelementptr inbounds %struct.word, ptr %ptr, i64 0, i32 1
  tail call void @bar_word(ptr %c, i32 %add)
  ret void
}

define void @store_pre_indexed_word(ptr %ptr, i32 %val) nounwind {
; CHECK-LABEL: store_pre_indexed_word:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    str w1, [x0, #32]!
; CHECK-NEXT:    b bar_word
entry:
  %a = getelementptr inbounds %struct.word, ptr %ptr, i64 0, i32 1, i32 0
  store i32 %val, ptr %a, align 4
  br label %bar
bar:
  %c = getelementptr inbounds %struct.word, ptr %ptr, i64 0, i32 1
  tail call void @bar_word(ptr %c, i32 %val)
  ret void
}

declare void @bar_doubleword(ptr, i64)

define void @load_pre_indexed_doubleword(ptr %ptr) nounwind {
; CHECK-LABEL: load_pre_indexed_doubleword:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    ldr x1, [x0, #32]!
; CHECK-NEXT:    b bar_doubleword
entry:
  %a = getelementptr inbounds %struct.doubleword, ptr %ptr, i64 0, i32 1, i32 0
  %add = load i64, ptr %a, align 8
  br label %bar
bar:
  %c = getelementptr inbounds %struct.doubleword, ptr %ptr, i64 0, i32 1
  tail call void @bar_doubleword(ptr %c, i64 %add)
  ret void
}

define void @store_pre_indexed_doubleword(ptr %ptr, i64 %val) nounwind {
; CHECK-LABEL: store_pre_indexed_doubleword:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    str x1, [x0, #32]!
; CHECK-NEXT:    b bar_doubleword
entry:
  %a = getelementptr inbounds %struct.doubleword, ptr %ptr, i64 0, i32 1, i32 0
  store i64 %val, ptr %a, align 8
  br label %bar
bar:
  %c = getelementptr inbounds %struct.doubleword, ptr %ptr, i64 0, i32 1
  tail call void @bar_doubleword(ptr %c, i64 %val)
  ret void
}

declare void @bar_quadword(ptr, fp128)

define void @load_pre_indexed_quadword(ptr %ptr) nounwind {
; CHECK-LABEL: load_pre_indexed_quadword:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    ldr q0, [x0, #32]!
; CHECK-NEXT:    b bar_quadword
entry:
  %a = getelementptr inbounds %struct.quadword, ptr %ptr, i64 0, i32 1, i32 0
  %add = load fp128, ptr %a, align 16
  br label %bar
bar:
  %c = getelementptr inbounds %struct.quadword, ptr %ptr, i64 0, i32 1
  tail call void @bar_quadword(ptr %c, fp128 %add)
  ret void
}

define void @store_pre_indexed_quadword(ptr %ptr, fp128 %val) nounwind {
; CHECK-LABEL: store_pre_indexed_quadword:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    str q0, [x0, #32]!
; CHECK-NEXT:    b bar_quadword
entry:
  %a = getelementptr inbounds %struct.quadword, ptr %ptr, i64 0, i32 1, i32 0
  store fp128 %val, ptr %a, align 16
  br label %bar
bar:
  %c = getelementptr inbounds %struct.quadword, ptr %ptr, i64 0, i32 1
  tail call void @bar_quadword(ptr %c, fp128 %val)
  ret void
}

declare void @bar_float(ptr, float)

define void @load_pre_indexed_float(ptr %ptr) nounwind {
; CHECK-LABEL: load_pre_indexed_float:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    ldr s0, [x0, #32]!
; CHECK-NEXT:    b bar_float
entry:
  %a = getelementptr inbounds %struct.float, ptr %ptr, i64 0, i32 1, i32 0
  %add = load float, ptr %a, align 4
  br label %bar
bar:
  %c = getelementptr inbounds %struct.float, ptr %ptr, i64 0, i32 1
  tail call void @bar_float(ptr %c, float %add)
  ret void
}

define void @store_pre_indexed_float(ptr %ptr, float %val) nounwind {
; CHECK-LABEL: store_pre_indexed_float:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    str s0, [x0, #32]!
; CHECK-NEXT:    b bar_float
entry:
  %a = getelementptr inbounds %struct.float, ptr %ptr, i64 0, i32 1, i32 0
  store float %val, ptr %a, align 4
  br label %bar
bar:
  %c = getelementptr inbounds %struct.float, ptr %ptr, i64 0, i32 1
  tail call void @bar_float(ptr %c, float %val)
  ret void
}

declare void @bar_double(ptr, double)

define void @load_pre_indexed_double(ptr %ptr) nounwind {
; CHECK-LABEL: load_pre_indexed_double:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    ldr d0, [x0, #32]!
; CHECK-NEXT:    b bar_double
entry:
  %a = getelementptr inbounds %struct.double, ptr %ptr, i64 0, i32 1, i32 0
  %add = load double, ptr %a, align 8
  br label %bar
bar:
  %c = getelementptr inbounds %struct.double, ptr %ptr, i64 0, i32 1
  tail call void @bar_double(ptr %c, double %add)
  ret void
}

define void @store_pre_indexed_double(ptr %ptr, double %val) nounwind {
; CHECK-LABEL: store_pre_indexed_double:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    str d0, [x0, #32]!
; CHECK-NEXT:    b bar_double
entry:
  %a = getelementptr inbounds %struct.double, ptr %ptr, i64 0, i32 1, i32 0
  store double %val, ptr %a, align 8
  br label %bar
bar:
  %c = getelementptr inbounds %struct.double, ptr %ptr, i64 0, i32 1
  tail call void @bar_double(ptr %c, double %val)
  ret void
}

; Check the following transform:
;
; (ldp|stp) w1, w2 [x0, #32]
;  ...
; add x0, x0, #32
;  ->
; (ldp|stp) w1, w2, [x0, #32]!
;

define void @load_pair_pre_indexed_word(ptr %ptr) nounwind {
; CHECK-LABEL: load_pair_pre_indexed_word:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    ldp w8, w9, [x0, #32]!
; CHECK-NEXT:    add w1, w8, w9
; CHECK-NEXT:    b bar_word
entry:
  %a = getelementptr inbounds %struct.word, ptr %ptr, i64 0, i32 1, i32 0
  %a1 = load i32, ptr %a, align 4
  %b = getelementptr inbounds %struct.word, ptr %ptr, i64 0, i32 1, i32 1
  %b1 = load i32, ptr %b, align 4
  %add = add i32 %a1, %b1
  br label %bar
bar:
  %c = getelementptr inbounds %struct.word, ptr %ptr, i64 0, i32 1
  tail call void @bar_word(ptr %c, i32 %add)
  ret void
}

define void @store_pair_pre_indexed_word(ptr %ptr, i32 %val) nounwind {
; CHECK-LABEL: store_pair_pre_indexed_word:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    stp w1, w1, [x0, #32]!
; CHECK-NEXT:    b bar_word
entry:
  %a = getelementptr inbounds %struct.word, ptr %ptr, i64 0, i32 1, i32 0
  store i32 %val, ptr %a, align 4
  %b = getelementptr inbounds %struct.word, ptr %ptr, i64 0, i32 1, i32 1
  store i32 %val, ptr %b, align 4
  br label %bar
bar:
  %c = getelementptr inbounds %struct.word, ptr %ptr, i64 0, i32 1
  tail call void @bar_word(ptr %c, i32 %val)
  ret void
}

; Check the following transform:
;
; add x8, x8, #16
;  ...
; ldr X, [x8]
;  ->
; ldr X, [x8, #16]!
;
; with X being either w0, x0, s0, d0 or q0.

%pre.struct.i32 = type { i32, i32, i32, i32, i32}
%pre.struct.i64 = type { i32, i64, i64, i64, i64}
%pre.struct.i128 = type { i32, <2 x i64>, <2 x i64>, <2 x i64>}
%pre.struct.float = type { i32, float, float, float}
%pre.struct.double = type { i32, double, double, double}

define i32 @load_pre_indexed_word2(ptr %this, i1 %cond, ptr %load2) nounwind {
; CHECK-LABEL: load_pre_indexed_word2:
; CHECK:       // %bb.0:
; CHECK-NEXT:    tbz w1, #0, .LBB16_2
; CHECK-NEXT:  // %bb.1: // %if.then
; CHECK-NEXT:    ldr x8, [x0]
; CHECK-NEXT:    ldr w0, [x8, #4]!
; CHECK-NEXT:    ret
; CHECK-NEXT:  .LBB16_2: // %if.end
; CHECK-NEXT:    add x8, x2, #8
; CHECK-NEXT:    ldr w0, [x8]
; CHECK-NEXT:    ret
  br i1 %cond, label %if.then, label %if.end
if.then:
  %load1 = load ptr, ptr %this
  %gep1 = getelementptr inbounds %pre.struct.i32, ptr %load1, i64 0, i32 1
  br label %return
if.end:
  %gep2 = getelementptr inbounds %pre.struct.i32, ptr %load2, i64 0, i32 2
  br label %return
return:
  %retptr = phi ptr [ %gep1, %if.then ], [ %gep2, %if.end ]
  %ret = load i32, ptr %retptr
  ret i32 %ret
}

define i64 @load_pre_indexed_doubleword2(ptr %this, i1 %cond, ptr %load2) nounwind {
; CHECK-LABEL: load_pre_indexed_doubleword2:
; CHECK:       // %bb.0:
; CHECK-NEXT:    tbz w1, #0, .LBB17_2
; CHECK-NEXT:  // %bb.1: // %if.then
; CHECK-NEXT:    ldr x8, [x0]
; CHECK-NEXT:    ldr x0, [x8, #8]!
; CHECK-NEXT:    ret
; CHECK-NEXT:  .LBB17_2: // %if.end
; CHECK-NEXT:    add x8, x2, #16
; CHECK-NEXT:    ldr x0, [x8]
; CHECK-NEXT:    ret
  br i1 %cond, label %if.then, label %if.end
if.then:
  %load1 = load ptr, ptr %this
  %gep1 = getelementptr inbounds %pre.struct.i64, ptr %load1, i64 0, i32 1
  br label %return
if.end:
  %gep2 = getelementptr inbounds %pre.struct.i64, ptr %load2, i64 0, i32 2
  br label %return
return:
  %retptr = phi ptr [ %gep1, %if.then ], [ %gep2, %if.end ]
  %ret = load i64, ptr %retptr
  ret i64 %ret
}

define <2 x i64> @load_pre_indexed_quadword2(ptr %this, i1 %cond, ptr %load2) nounwind {
; CHECK-LABEL: load_pre_indexed_quadword2:
; CHECK:       // %bb.0:
; CHECK-NEXT:    tbz w1, #0, .LBB18_2
; CHECK-NEXT:  // %bb.1: // %if.then
; CHECK-NEXT:    ldr x8, [x0]
; CHECK-NEXT:    ldr q0, [x8, #16]!
; CHECK-NEXT:    ret
; CHECK-NEXT:  .LBB18_2: // %if.end
; CHECK-NEXT:    add x8, x2, #32
; CHECK-NEXT:    ldr q0, [x8]
; CHECK-NEXT:    ret
  br i1 %cond, label %if.then, label %if.end
if.then:
  %load1 = load ptr, ptr %this
  %gep1 = getelementptr inbounds %pre.struct.i128, ptr %load1, i64 0, i32 1
  br label %return
if.end:
  %gep2 = getelementptr inbounds %pre.struct.i128, ptr %load2, i64 0, i32 2
  br label %return
return:
  %retptr = phi ptr [ %gep1, %if.then ], [ %gep2, %if.end ]
  %ret = load <2 x i64>, ptr %retptr
  ret <2 x i64> %ret
}

define float @load_pre_indexed_float2(ptr %this, i1 %cond, ptr %load2) nounwind {
; CHECK-LABEL: load_pre_indexed_float2:
; CHECK:       // %bb.0:
; CHECK-NEXT:    tbz w1, #0, .LBB19_2
; CHECK-NEXT:  // %bb.1: // %if.then
; CHECK-NEXT:    ldr x8, [x0]
; CHECK-NEXT:    ldr s0, [x8, #4]!
; CHECK-NEXT:    ret
; CHECK-NEXT:  .LBB19_2: // %if.end
; CHECK-NEXT:    add x8, x2, #8
; CHECK-NEXT:    ldr s0, [x8]
; CHECK-NEXT:    ret
  br i1 %cond, label %if.then, label %if.end
if.then:
  %load1 = load ptr, ptr %this
  %gep1 = getelementptr inbounds %pre.struct.float, ptr %load1, i64 0, i32 1
  br label %return
if.end:
  %gep2 = getelementptr inbounds %pre.struct.float, ptr %load2, i64 0, i32 2
  br label %return
return:
  %retptr = phi ptr [ %gep1, %if.then ], [ %gep2, %if.end ]
  %ret = load float, ptr %retptr
  ret float %ret
}

define double @load_pre_indexed_double2(ptr %this, i1 %cond, ptr %load2) nounwind {
; CHECK-LABEL: load_pre_indexed_double2:
; CHECK:       // %bb.0:
; CHECK-NEXT:    tbz w1, #0, .LBB20_2
; CHECK-NEXT:  // %bb.1: // %if.then
; CHECK-NEXT:    ldr x8, [x0]
; CHECK-NEXT:    ldr d0, [x8, #8]!
; CHECK-NEXT:    ret
; CHECK-NEXT:  .LBB20_2: // %if.end
; CHECK-NEXT:    add x8, x2, #16
; CHECK-NEXT:    ldr d0, [x8]
; CHECK-NEXT:    ret
  br i1 %cond, label %if.then, label %if.end
if.then:
  %load1 = load ptr, ptr %this
  %gep1 = getelementptr inbounds %pre.struct.double, ptr %load1, i64 0, i32 1
  br label %return
if.end:
  %gep2 = getelementptr inbounds %pre.struct.double, ptr %load2, i64 0, i32 2
  br label %return
return:
  %retptr = phi ptr [ %gep1, %if.then ], [ %gep2, %if.end ]
  %ret = load double, ptr %retptr
  ret double %ret
}

define i32 @load_pre_indexed_word3(ptr %this, i1 %cond, ptr %load2) nounwind {
; CHECK-LABEL: load_pre_indexed_word3:
; CHECK:       // %bb.0:
; CHECK-NEXT:    tbz w1, #0, .LBB21_2
; CHECK-NEXT:  // %bb.1: // %if.then
; CHECK-NEXT:    ldr x8, [x0]
; CHECK-NEXT:    ldr w0, [x8, #12]!
; CHECK-NEXT:    ret
; CHECK-NEXT:  .LBB21_2: // %if.end
; CHECK-NEXT:    add x8, x2, #16
; CHECK-NEXT:    ldr w0, [x8]
; CHECK-NEXT:    ret
  br i1 %cond, label %if.then, label %if.end
if.then:
  %load1 = load ptr, ptr %this
  %gep1 = getelementptr inbounds %pre.struct.i32, ptr %load1, i64 0, i32 3
  br label %return
if.end:
  %gep2 = getelementptr inbounds %pre.struct.i32, ptr %load2, i64 0, i32 4
  br label %return
return:
  %retptr = phi ptr [ %gep1, %if.then ], [ %gep2, %if.end ]
  %ret = load i32, ptr %retptr
  ret i32 %ret
}

define i64 @load_pre_indexed_doubleword3(ptr %this, i1 %cond, ptr %load2) nounwind {
; CHECK-LABEL: load_pre_indexed_doubleword3:
; CHECK:       // %bb.0:
; CHECK-NEXT:    tbz w1, #0, .LBB22_2
; CHECK-NEXT:  // %bb.1: // %if.then
; CHECK-NEXT:    ldr x8, [x0]
; CHECK-NEXT:    ldr x0, [x8, #16]!
; CHECK-NEXT:    ret
; CHECK-NEXT:  .LBB22_2: // %if.end
; CHECK-NEXT:    add x8, x2, #24
; CHECK-NEXT:    ldr x0, [x8]
; CHECK-NEXT:    ret
  br i1 %cond, label %if.then, label %if.end
if.then:
  %load1 = load ptr, ptr %this
  %gep1 = getelementptr inbounds %pre.struct.i64, ptr %load1, i64 0, i32 2
  br label %return
if.end:
  %gep2 = getelementptr inbounds %pre.struct.i64, ptr %load2, i64 0, i32 3
  br label %return
return:
  %retptr = phi ptr [ %gep1, %if.then ], [ %gep2, %if.end ]
  %ret = load i64, ptr %retptr
  ret i64 %ret
}

define <2 x i64> @load_pre_indexed_quadword3(ptr %this, i1 %cond, ptr %load2) nounwind {
; CHECK-LABEL: load_pre_indexed_quadword3:
; CHECK:       // %bb.0:
; CHECK-NEXT:    tbz w1, #0, .LBB23_2
; CHECK-NEXT:  // %bb.1: // %if.then
; CHECK-NEXT:    ldr x8, [x0]
; CHECK-NEXT:    ldr q0, [x8, #32]!
; CHECK-NEXT:    ret
; CHECK-NEXT:  .LBB23_2: // %if.end
; CHECK-NEXT:    add x8, x2, #48
; CHECK-NEXT:    ldr q0, [x8]
; CHECK-NEXT:    ret
  br i1 %cond, label %if.then, label %if.end
if.then:
  %load1 = load ptr, ptr %this
  %gep1 = getelementptr inbounds %pre.struct.i128, ptr %load1, i64 0, i32 2
  br label %return
if.end:
  %gep2 = getelementptr inbounds %pre.struct.i128, ptr %load2, i64 0, i32 3
  br label %return
return:
  %retptr = phi ptr [ %gep1, %if.then ], [ %gep2, %if.end ]
  %ret = load <2 x i64>, ptr %retptr
  ret <2 x i64> %ret
}

define float @load_pre_indexed_float3(ptr %this, i1 %cond, ptr %load2) nounwind {
; CHECK-LABEL: load_pre_indexed_float3:
; CHECK:       // %bb.0:
; CHECK-NEXT:    tbz w1, #0, .LBB24_2
; CHECK-NEXT:  // %bb.1: // %if.then
; CHECK-NEXT:    ldr x8, [x0]
; CHECK-NEXT:    ldr s0, [x8, #8]!
; CHECK-NEXT:    ret
; CHECK-NEXT:  .LBB24_2: // %if.end
; CHECK-NEXT:    add x8, x2, #12
; CHECK-NEXT:    ldr s0, [x8]
; CHECK-NEXT:    ret
  br i1 %cond, label %if.then, label %if.end
if.then:
  %load1 = load ptr, ptr %this
  %gep1 = getelementptr inbounds %pre.struct.float, ptr %load1, i64 0, i32 2
  br label %return
if.end:
  %gep2 = getelementptr inbounds %pre.struct.float, ptr %load2, i64 0, i32 3
  br label %return
return:
  %retptr = phi ptr [ %gep1, %if.then ], [ %gep2, %if.end ]
  %ret = load float, ptr %retptr
  ret float %ret
}

define double @load_pre_indexed_double3(ptr %this, i1 %cond, ptr %load2) nounwind {
; CHECK-LABEL: load_pre_indexed_double3:
; CHECK:       // %bb.0:
; CHECK-NEXT:    tbz w1, #0, .LBB25_2
; CHECK-NEXT:  // %bb.1: // %if.then
; CHECK-NEXT:    ldr x8, [x0]
; CHECK-NEXT:    ldr d0, [x8, #16]!
; CHECK-NEXT:    ret
; CHECK-NEXT:  .LBB25_2: // %if.end
; CHECK-NEXT:    add x8, x2, #24
; CHECK-NEXT:    ldr d0, [x8]
; CHECK-NEXT:    ret
  br i1 %cond, label %if.then, label %if.end
if.then:
  %load1 = load ptr, ptr %this
  %gep1 = getelementptr inbounds %pre.struct.double, ptr %load1, i64 0, i32 2
  br label %return
if.end:
  %gep2 = getelementptr inbounds %pre.struct.double, ptr %load2, i64 0, i32 3
  br label %return
return:
  %retptr = phi ptr [ %gep1, %if.then ], [ %gep2, %if.end ]
  %ret = load double, ptr %retptr
  ret double %ret
}

; Check the following transform:
;
; add x8, x8, #16
;  ...
; str X, [x8]
;  ->
; str X, [x8, #16]!
;
; with X being either w0, x0, s0, d0 or q0.

define void @store_pre_indexed_word2(ptr %this, i1 %cond, ptr %load2, i32 %val) nounwind {
; CHECK-LABEL: store_pre_indexed_word2:
; CHECK:       // %bb.0:
; CHECK-NEXT:    tbz w1, #0, .LBB26_2
; CHECK-NEXT:  // %bb.1: // %if.then
; CHECK-NEXT:    ldr x8, [x0]
; CHECK-NEXT:    str w3, [x8, #4]!
; CHECK-NEXT:    ret
; CHECK-NEXT:  .LBB26_2: // %if.end
; CHECK-NEXT:    add x8, x2, #8
; CHECK-NEXT:    str w3, [x8]
; CHECK-NEXT:    ret
  br i1 %cond, label %if.then, label %if.end
if.then:
  %load1 = load ptr, ptr %this
  %gep1 = getelementptr inbounds %pre.struct.i32, ptr %load1, i64 0, i32 1
  br label %return
if.end:
  %gep2 = getelementptr inbounds %pre.struct.i32, ptr %load2, i64 0, i32 2
  br label %return
return:
  %retptr = phi ptr [ %gep1, %if.then ], [ %gep2, %if.end ]
  store i32 %val, ptr %retptr
  ret void
}

define void @store_pre_indexed_doubleword2(ptr %this, i1 %cond, ptr %load2, i64 %val) nounwind {
; CHECK-LABEL: store_pre_indexed_doubleword2:
; CHECK:       // %bb.0:
; CHECK-NEXT:    tbz w1, #0, .LBB27_2
; CHECK-NEXT:  // %bb.1: // %if.then
; CHECK-NEXT:    ldr x8, [x0]
; CHECK-NEXT:    str x3, [x8, #8]!
; CHECK-NEXT:    ret
; CHECK-NEXT:  .LBB27_2: // %if.end
; CHECK-NEXT:    add x8, x2, #16
; CHECK-NEXT:    str x3, [x8]
; CHECK-NEXT:    ret
  br i1 %cond, label %if.then, label %if.end
if.then:
  %load1 = load ptr, ptr %this
  %gep1 = getelementptr inbounds %pre.struct.i64, ptr %load1, i64 0, i32 1
  br label %return
if.end:
  %gep2 = getelementptr inbounds %pre.struct.i64, ptr %load2, i64 0, i32 2
  br label %return
return:
  %retptr = phi ptr [ %gep1, %if.then ], [ %gep2, %if.end ]
  store i64 %val, ptr %retptr
  ret void
}

define void @store_pre_indexed_quadword2(ptr %this, i1 %cond, ptr %load2, <2 x i64> %val) nounwind {
; CHECK-LABEL: store_pre_indexed_quadword2:
; CHECK:       // %bb.0:
; CHECK-NEXT:    tbz w1, #0, .LBB28_2
; CHECK-NEXT:  // %bb.1: // %if.then
; CHECK-NEXT:    ldr x8, [x0]
; CHECK-NEXT:    str q0, [x8, #16]!
; CHECK-NEXT:    ret
; CHECK-NEXT:  .LBB28_2: // %if.end
; CHECK-NEXT:    add x8, x2, #32
; CHECK-NEXT:    str q0, [x8]
; CHECK-NEXT:    ret
  br i1 %cond, label %if.then, label %if.end
if.then:
  %load1 = load ptr, ptr %this
  %gep1 = getelementptr inbounds %pre.struct.i128, ptr %load1, i64 0, i32 1
  br label %return
if.end:
  %gep2 = getelementptr inbounds %pre.struct.i128, ptr %load2, i64 0, i32 2
  br label %return
return:
  %retptr = phi ptr [ %gep1, %if.then ], [ %gep2, %if.end ]
  store <2 x i64> %val, ptr %retptr
  ret void
}

define void @store_pre_indexed_float2(ptr %this, i1 %cond, ptr %load2, float %val) nounwind {
; CHECK-LABEL: store_pre_indexed_float2:
; CHECK:       // %bb.0:
; CHECK-NEXT:    tbz w1, #0, .LBB29_2
; CHECK-NEXT:  // %bb.1: // %if.then
; CHECK-NEXT:    ldr x8, [x0]
; CHECK-NEXT:    str s0, [x8, #4]!
; CHECK-NEXT:    ret
; CHECK-NEXT:  .LBB29_2: // %if.end
; CHECK-NEXT:    add x8, x2, #8
; CHECK-NEXT:    str s0, [x8]
; CHECK-NEXT:    ret
  br i1 %cond, label %if.then, label %if.end
if.then:
  %load1 = load ptr, ptr %this
  %gep1 = getelementptr inbounds %pre.struct.float, ptr %load1, i64 0, i32 1
  br label %return
if.end:
  %gep2 = getelementptr inbounds %pre.struct.float, ptr %load2, i64 0, i32 2
  br label %return
return:
  %retptr = phi ptr [ %gep1, %if.then ], [ %gep2, %if.end ]
  store float %val, ptr %retptr
  ret void
}

define void @store_pre_indexed_double2(ptr %this, i1 %cond, ptr %load2, double %val) nounwind {
; CHECK-LABEL: store_pre_indexed_double2:
; CHECK:       // %bb.0:
; CHECK-NEXT:    tbz w1, #0, .LBB30_2
; CHECK-NEXT:  // %bb.1: // %if.then
; CHECK-NEXT:    ldr x8, [x0]
; CHECK-NEXT:    str d0, [x8, #8]!
; CHECK-NEXT:    ret
; CHECK-NEXT:  .LBB30_2: // %if.end
; CHECK-NEXT:    add x8, x2, #16
; CHECK-NEXT:    str d0, [x8]
; CHECK-NEXT:    ret
  br i1 %cond, label %if.then, label %if.end
if.then:
  %load1 = load ptr, ptr %this
  %gep1 = getelementptr inbounds %pre.struct.double, ptr %load1, i64 0, i32 1
  br label %return
if.end:
  %gep2 = getelementptr inbounds %pre.struct.double, ptr %load2, i64 0, i32 2
  br label %return
return:
  %retptr = phi ptr [ %gep1, %if.then ], [ %gep2, %if.end ]
  store double %val, ptr %retptr
  ret void
}

define void @store_pre_indexed_word3(ptr %this, i1 %cond, ptr %load2, i32 %val) nounwind {
; CHECK-LABEL: store_pre_indexed_word3:
; CHECK:       // %bb.0:
; CHECK-NEXT:    tbz w1, #0, .LBB31_2
; CHECK-NEXT:  // %bb.1: // %if.then
; CHECK-NEXT:    ldr x8, [x0]
; CHECK-NEXT:    str w3, [x8, #12]!
; CHECK-NEXT:    ret
; CHECK-NEXT:  .LBB31_2: // %if.end
; CHECK-NEXT:    add x8, x2, #16
; CHECK-NEXT:    str w3, [x8]
; CHECK-NEXT:    ret
  br i1 %cond, label %if.then, label %if.end
if.then:
  %load1 = load ptr, ptr %this
  %gep1 = getelementptr inbounds %pre.struct.i32, ptr %load1, i64 0, i32 3
  br label %return
if.end:
  %gep2 = getelementptr inbounds %pre.struct.i32, ptr %load2, i64 0, i32 4
  br label %return
return:
  %retptr = phi ptr [ %gep1, %if.then ], [ %gep2, %if.end ]
  store i32 %val, ptr %retptr
  ret void
}

define void @store_pre_indexed_doubleword3(ptr %this, i1 %cond, ptr %load2, i64 %val) nounwind {
; CHECK-LABEL: store_pre_indexed_doubleword3:
; CHECK:       // %bb.0:
; CHECK-NEXT:    tbz w1, #0, .LBB32_2
; CHECK-NEXT:  // %bb.1: // %if.then
; CHECK-NEXT:    ldr x8, [x0]
; CHECK-NEXT:    str x3, [x8, #24]!
; CHECK-NEXT:    ret
; CHECK-NEXT:  .LBB32_2: // %if.end
; CHECK-NEXT:    add x8, x2, #32
; CHECK-NEXT:    str x3, [x8]
; CHECK-NEXT:    ret
  br i1 %cond, label %if.then, label %if.end
if.then:
  %load1 = load ptr, ptr %this
  %gep1 = getelementptr inbounds %pre.struct.i64, ptr %load1, i64 0, i32 3
  br label %return
if.end:
  %gep2 = getelementptr inbounds %pre.struct.i64, ptr %load2, i64 0, i32 4
  br label %return
return:
  %retptr = phi ptr [ %gep1, %if.then ], [ %gep2, %if.end ]
  store i64 %val, ptr %retptr
  ret void
}

define void @store_pre_indexed_quadword3(ptr %this, i1 %cond, ptr %load2, <2 x i64> %val) nounwind {
; CHECK-LABEL: store_pre_indexed_quadword3:
; CHECK:       // %bb.0:
; CHECK-NEXT:    tbz w1, #0, .LBB33_2
; CHECK-NEXT:  // %bb.1: // %if.then
; CHECK-NEXT:    ldr x8, [x0]
; CHECK-NEXT:    str q0, [x8, #32]!
; CHECK-NEXT:    ret
; CHECK-NEXT:  .LBB33_2: // %if.end
; CHECK-NEXT:    add x8, x2, #48
; CHECK-NEXT:    str q0, [x8]
; CHECK-NEXT:    ret
  br i1 %cond, label %if.then, label %if.end
if.then:
  %load1 = load ptr, ptr %this
  %gep1 = getelementptr inbounds %pre.struct.i128, ptr %load1, i64 0, i32 2
  br label %return
if.end:
  %gep2 = getelementptr inbounds %pre.struct.i128, ptr %load2, i64 0, i32 3
  br label %return
return:
  %retptr = phi ptr [ %gep1, %if.then ], [ %gep2, %if.end ]
  store <2 x i64> %val, ptr %retptr
  ret void
}

define void @store_pre_indexed_float3(ptr %this, i1 %cond, ptr %load2, float %val) nounwind {
; CHECK-LABEL: store_pre_indexed_float3:
; CHECK:       // %bb.0:
; CHECK-NEXT:    tbz w1, #0, .LBB34_2
; CHECK-NEXT:  // %bb.1: // %if.then
; CHECK-NEXT:    ldr x8, [x0]
; CHECK-NEXT:    str s0, [x8, #8]!
; CHECK-NEXT:    ret
; CHECK-NEXT:  .LBB34_2: // %if.end
; CHECK-NEXT:    add x8, x2, #12
; CHECK-NEXT:    str s0, [x8]
; CHECK-NEXT:    ret
  br i1 %cond, label %if.then, label %if.end
if.then:
  %load1 = load ptr, ptr %this
  %gep1 = getelementptr inbounds %pre.struct.float, ptr %load1, i64 0, i32 2
  br label %return
if.end:
  %gep2 = getelementptr inbounds %pre.struct.float, ptr %load2, i64 0, i32 3
  br label %return
return:
  %retptr = phi ptr [ %gep1, %if.then ], [ %gep2, %if.end ]
  store float %val, ptr %retptr
  ret void
}

define void @store_pre_indexed_double3(ptr %this, i1 %cond, ptr %load2, double %val) nounwind {
; CHECK-LABEL: store_pre_indexed_double3:
; CHECK:       // %bb.0:
; CHECK-NEXT:    tbz w1, #0, .LBB35_2
; CHECK-NEXT:  // %bb.1: // %if.then
; CHECK-NEXT:    ldr x8, [x0]
; CHECK-NEXT:    str d0, [x8, #16]!
; CHECK-NEXT:    ret
; CHECK-NEXT:  .LBB35_2: // %if.end
; CHECK-NEXT:    add x8, x2, #24
; CHECK-NEXT:    str d0, [x8]
; CHECK-NEXT:    ret
  br i1 %cond, label %if.then, label %if.end
if.then:
  %load1 = load ptr, ptr %this
  %gep1 = getelementptr inbounds %pre.struct.double, ptr %load1, i64 0, i32 2
  br label %return
if.end:
  %gep2 = getelementptr inbounds %pre.struct.double, ptr %load2, i64 0, i32 3
  br label %return
return:
  %retptr = phi ptr [ %gep1, %if.then ], [ %gep2, %if.end ]
  store double %val, ptr %retptr
  ret void
}

; Check the following transform:
;
; ldr X, [x20]
;  ...
; add x20, x20, #32
;  ->
; ldr X, [x20], #32
;
; with X being either w0, x0, s0, d0 or q0.

define void @load_post_indexed_byte(ptr %array, i64 %count) nounwind {
; CHECK-LABEL: load_post_indexed_byte:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    str x30, [sp, #-32]! // 8-byte Folded Spill
; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    mov x19, x1
; CHECK-NEXT:    add x20, x0, #2
; CHECK-NEXT:  .LBB36_1: // %body
; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldurb w0, [x20, #-1]
; CHECK-NEXT:    bl "use-byte"
; CHECK-NEXT:    ldrb w0, [x20], #4
; CHECK-NEXT:    bl "use-byte"
; CHECK-NEXT:    subs x19, x19, #4
; CHECK-NEXT:    b.ne .LBB36_1
; CHECK-NEXT:  // %bb.2: // %exit
; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
; CHECK-NEXT:    ret
entry:
  %gep1 = getelementptr i8, ptr %array, i64 2
  br label %body

body:
  %iv2 = phi ptr [ %gep3, %body ], [ %gep1, %entry ]
  %iv = phi i64 [ %iv.next, %body ], [ %count, %entry ]
  %gep2 = getelementptr i8, ptr %iv2, i64 -1
  %load = load i8, ptr %gep2
  call void @use-byte(i8 %load)
  %load2 = load i8, ptr %iv2
  call void @use-byte(i8 %load2)
  %iv.next = add i64 %iv, -4
  %gep3 = getelementptr i8, ptr %iv2, i64 4
  %cond = icmp eq i64 %iv.next, 0
  br i1 %cond, label %exit, label %body

exit:
  ret void
}

define void @load_post_indexed_halfword(ptr %array, i64 %count) nounwind {
; CHECK-LABEL: load_post_indexed_halfword:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    str x30, [sp, #-32]! // 8-byte Folded Spill
; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    mov x19, x1
; CHECK-NEXT:    add x20, x0, #4
; CHECK-NEXT:  .LBB37_1: // %body
; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldurh w0, [x20, #-2]
; CHECK-NEXT:    bl "use-halfword"
; CHECK-NEXT:    ldrh w0, [x20], #8
; CHECK-NEXT:    bl "use-halfword"
; CHECK-NEXT:    subs x19, x19, #4
; CHECK-NEXT:    b.ne .LBB37_1
; CHECK-NEXT:  // %bb.2: // %exit
; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
; CHECK-NEXT:    ret
entry:
  %gep1 = getelementptr i16, ptr %array, i64 2
  br label %body

body:
  %iv2 = phi ptr [ %gep3, %body ], [ %gep1, %entry ]
  %iv = phi i64 [ %iv.next, %body ], [ %count, %entry ]
  %gep2 = getelementptr i16, ptr %iv2, i64 -1
  %load = load i16, ptr %gep2
  call void @use-halfword(i16 %load)
  %load2 = load i16, ptr %iv2
  call void @use-halfword(i16 %load2)
  %iv.next = add i64 %iv, -4
  %gep3 = getelementptr i16, ptr %iv2, i64 4
  %cond = icmp eq i64 %iv.next, 0
  br i1 %cond, label %exit, label %body

exit:
  ret void
}

define void @load_post_indexed_word(ptr %array, i64 %count) nounwind {
; CHECK-LABEL: load_post_indexed_word:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    str x30, [sp, #-32]! // 8-byte Folded Spill
; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    mov x19, x1
; CHECK-NEXT:    add x20, x0, #8
; CHECK-NEXT:  .LBB38_1: // %body
; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldur w0, [x20, #-4]
; CHECK-NEXT:    bl "use-word"
; CHECK-NEXT:    ldr w0, [x20], #16
; CHECK-NEXT:    bl "use-word"
; CHECK-NEXT:    subs x19, x19, #4
; CHECK-NEXT:    b.ne .LBB38_1
; CHECK-NEXT:  // %bb.2: // %exit
; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
; CHECK-NEXT:    ret
entry:
  %gep1 = getelementptr i32, ptr %array, i64 2
  br label %body

body:
  %iv2 = phi ptr [ %gep3, %body ], [ %gep1, %entry ]
  %iv = phi i64 [ %iv.next, %body ], [ %count, %entry ]
  %gep2 = getelementptr i32, ptr %iv2, i64 -1
  %load = load i32, ptr %gep2
  call void @use-word(i32 %load)
  %load2 = load i32, ptr %iv2
  call void @use-word(i32 %load2)
  %iv.next = add i64 %iv, -4
  %gep3 = getelementptr i32, ptr %iv2, i64 4
  %cond = icmp eq i64 %iv.next, 0
  br i1 %cond, label %exit, label %body

exit:
  ret void
}

define void @load_post_indexed_doubleword(ptr %array, i64 %count) nounwind {
; CHECK-LABEL: load_post_indexed_doubleword:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    str x30, [sp, #-32]! // 8-byte Folded Spill
; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    mov x19, x1
; CHECK-NEXT:    add x20, x0, #16
; CHECK-NEXT:  .LBB39_1: // %body
; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldur x0, [x20, #-8]
; CHECK-NEXT:    bl "use-doubleword"
; CHECK-NEXT:    ldr x0, [x20], #32
; CHECK-NEXT:    bl "use-doubleword"
; CHECK-NEXT:    subs x19, x19, #4
; CHECK-NEXT:    b.ne .LBB39_1
; CHECK-NEXT:  // %bb.2: // %exit
; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
; CHECK-NEXT:    ret
entry:
  %gep1 = getelementptr i64, ptr %array, i64 2
  br label %body

body:
  %iv2 = phi ptr [ %gep3, %body ], [ %gep1, %entry ]
  %iv = phi i64 [ %iv.next, %body ], [ %count, %entry ]
  %gep2 = getelementptr i64, ptr %iv2, i64 -1
  %load = load i64, ptr %gep2
  call void @use-doubleword(i64 %load)
  %load2 = load i64, ptr %iv2
  call void @use-doubleword(i64 %load2)
  %iv.next = add i64 %iv, -4
  %gep3 = getelementptr i64, ptr %iv2, i64 4
  %cond = icmp eq i64 %iv.next, 0
  br i1 %cond, label %exit, label %body

exit:
  ret void
}

define void @load_post_indexed_quadword(ptr %array, i64 %count) nounwind {
; CHECK-LABEL: load_post_indexed_quadword:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    str x30, [sp, #-32]! // 8-byte Folded Spill
; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    mov x19, x1
; CHECK-NEXT:    add x20, x0, #32
; CHECK-NEXT:  .LBB40_1: // %body
; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldur q0, [x20, #-16]
; CHECK-NEXT:    bl "use-quadword"
; CHECK-NEXT:    ldr q0, [x20], #64
; CHECK-NEXT:    bl "use-quadword"
; CHECK-NEXT:    subs x19, x19, #4
; CHECK-NEXT:    b.ne .LBB40_1
; CHECK-NEXT:  // %bb.2: // %exit
; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
; CHECK-NEXT:    ret
entry:
  %gep1 = getelementptr <2 x i64>, ptr %array, i64 2
  br label %body

body:
  %iv2 = phi ptr [ %gep3, %body ], [ %gep1, %entry ]
  %iv = phi i64 [ %iv.next, %body ], [ %count, %entry ]
  %gep2 = getelementptr <2 x i64>, ptr %iv2, i64 -1
  %load = load <2 x i64>, ptr %gep2
  call void @use-quadword(<2 x i64> %load)
  %load2 = load <2 x i64>, ptr %iv2
  call void @use-quadword(<2 x i64> %load2)
  %iv.next = add i64 %iv, -4
  %gep3 = getelementptr <2 x i64>, ptr %iv2, i64 4
  %cond = icmp eq i64 %iv.next, 0
  br i1 %cond, label %exit, label %body

exit:
  ret void
}

define void @load_post_indexed_float(ptr %array, i64 %count) nounwind {
; CHECK-LABEL: load_post_indexed_float:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    str x30, [sp, #-32]! // 8-byte Folded Spill
; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    mov x19, x1
; CHECK-NEXT:    add x20, x0, #8
; CHECK-NEXT:  .LBB41_1: // %body
; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldur s0, [x20, #-4]
; CHECK-NEXT:    bl "use-float"
; CHECK-NEXT:    ldr s0, [x20], #16
; CHECK-NEXT:    bl "use-float"
; CHECK-NEXT:    subs x19, x19, #4
; CHECK-NEXT:    b.ne .LBB41_1
; CHECK-NEXT:  // %bb.2: // %exit
; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
; CHECK-NEXT:    ret
entry:
  %gep1 = getelementptr float, ptr %array, i64 2
  br label %body

body:
  %iv2 = phi ptr [ %gep3, %body ], [ %gep1, %entry ]
  %iv = phi i64 [ %iv.next, %body ], [ %count, %entry ]
  %gep2 = getelementptr float, ptr %iv2, i64 -1
  %load = load float, ptr %gep2
  call void @use-float(float %load)
  %load2 = load float, ptr %iv2
  call void @use-float(float %load2)
  %iv.next = add i64 %iv, -4
  %gep3 = getelementptr float, ptr %iv2, i64 4
  %cond = icmp eq i64 %iv.next, 0
  br i1 %cond, label %exit, label %body

exit:
  ret void
}

define void @load_post_indexed_double(ptr %array, i64 %count) nounwind {
; CHECK-LABEL: load_post_indexed_double:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    str x30, [sp, #-32]! // 8-byte Folded Spill
; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    mov x19, x1
; CHECK-NEXT:    add x20, x0, #16
; CHECK-NEXT:  .LBB42_1: // %body
; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldur d0, [x20, #-8]
; CHECK-NEXT:    bl "use-double"
; CHECK-NEXT:    ldr d0, [x20], #32
; CHECK-NEXT:    bl "use-double"
; CHECK-NEXT:    subs x19, x19, #4
; CHECK-NEXT:    b.ne .LBB42_1
; CHECK-NEXT:  // %bb.2: // %exit
; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldr x30, [sp], #32 // 8-byte Folded Reload
; CHECK-NEXT:    ret
entry:
  %gep1 = getelementptr double, ptr %array, i64 2
  br label %body

body:
  %iv2 = phi ptr [ %gep3, %body ], [ %gep1, %entry ]
  %iv = phi i64 [ %iv.next, %body ], [ %count, %entry ]
  %gep2 = getelementptr double, ptr %iv2, i64 -1
  %load = load double, ptr %gep2
  call void @use-double(double %load)
  %load2 = load double, ptr %iv2
  call void @use-double(double %load2)
  %iv.next = add i64 %iv, -4
  %gep3 = getelementptr double, ptr %iv2, i64 4
  %cond = icmp eq i64 %iv.next, 0
  br i1 %cond, label %exit, label %body

exit:
  ret void
}

; Check the following transform:
;
; str X, [x20]
;  ...
; add x20, x20, #32
;  ->
; str X, [x20], #32
;
; with X being either w0, x0, s0, d0 or q0.

define void @store_post_indexed_byte(ptr %array, i64 %count, i8 %val) nounwind {
; CHECK-LABEL: store_post_indexed_byte:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    stp x30, x21, [sp, #-32]! // 16-byte Folded Spill
; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    mov w19, w2
; CHECK-NEXT:    mov x20, x1
; CHECK-NEXT:    add x21, x0, #2
; CHECK-NEXT:  .LBB43_1: // %body
; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldurb w0, [x21, #-1]
; CHECK-NEXT:    bl "use-byte"
; CHECK-NEXT:    subs x20, x20, #4
; CHECK-NEXT:    strb w19, [x21], #4
; CHECK-NEXT:    b.ne .LBB43_1
; CHECK-NEXT:  // %bb.2: // %exit
; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldp x30, x21, [sp], #32 // 16-byte Folded Reload
; CHECK-NEXT:    ret
entry:
  %gep1 = getelementptr i8, ptr %array, i64 2
  br label %body

body:
  %iv2 = phi ptr [ %gep3, %body ], [ %gep1, %entry ]
  %iv = phi i64 [ %iv.next, %body ], [ %count, %entry ]
  %gep2 = getelementptr i8, ptr %iv2, i64 -1
  %load = load i8, ptr %gep2
  call void @use-byte(i8 %load)
  store i8 %val, ptr %iv2
  %iv.next = add i64 %iv, -4
  %gep3 = getelementptr i8, ptr %iv2, i64 4
  %cond = icmp eq i64 %iv.next, 0
  br i1 %cond, label %exit, label %body

exit:
  ret void
}

define void @store_post_indexed_halfword(ptr %array, i64 %count, i16 %val) nounwind {
; CHECK-LABEL: store_post_indexed_halfword:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    stp x30, x21, [sp, #-32]! // 16-byte Folded Spill
; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    mov w19, w2
; CHECK-NEXT:    mov x20, x1
; CHECK-NEXT:    add x21, x0, #4
; CHECK-NEXT:  .LBB44_1: // %body
; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldurh w0, [x21, #-2]
; CHECK-NEXT:    bl "use-halfword"
; CHECK-NEXT:    subs x20, x20, #4
; CHECK-NEXT:    strh w19, [x21], #8
; CHECK-NEXT:    b.ne .LBB44_1
; CHECK-NEXT:  // %bb.2: // %exit
; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldp x30, x21, [sp], #32 // 16-byte Folded Reload
; CHECK-NEXT:    ret
entry:
  %gep1 = getelementptr i16, ptr %array, i64 2
  br label %body

body:
  %iv2 = phi ptr [ %gep3, %body ], [ %gep1, %entry ]
  %iv = phi i64 [ %iv.next, %body ], [ %count, %entry ]
  %gep2 = getelementptr i16, ptr %iv2, i64 -1
  %load = load i16, ptr %gep2
  call void @use-halfword(i16 %load)
  store i16 %val, ptr %iv2
  %iv.next = add i64 %iv, -4
  %gep3 = getelementptr i16, ptr %iv2, i64 4
  %cond = icmp eq i64 %iv.next, 0
  br i1 %cond, label %exit, label %body

exit:
  ret void
}

define void @store_post_indexed_word(ptr %array, i64 %count, i32 %val) nounwind {
; CHECK-LABEL: store_post_indexed_word:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    stp x30, x21, [sp, #-32]! // 16-byte Folded Spill
; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    mov w19, w2
; CHECK-NEXT:    mov x20, x1
; CHECK-NEXT:    add x21, x0, #8
; CHECK-NEXT:  .LBB45_1: // %body
; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldur w0, [x21, #-4]
; CHECK-NEXT:    bl "use-word"
; CHECK-NEXT:    subs x20, x20, #4
; CHECK-NEXT:    str w19, [x21], #16
; CHECK-NEXT:    b.ne .LBB45_1
; CHECK-NEXT:  // %bb.2: // %exit
; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldp x30, x21, [sp], #32 // 16-byte Folded Reload
; CHECK-NEXT:    ret
entry:
  %gep1 = getelementptr i32, ptr %array, i64 2
  br label %body

body:
  %iv2 = phi ptr [ %gep3, %body ], [ %gep1, %entry ]
  %iv = phi i64 [ %iv.next, %body ], [ %count, %entry ]
  %gep2 = getelementptr i32, ptr %iv2, i64 -1
  %load = load i32, ptr %gep2
  call void @use-word(i32 %load)
  store i32 %val, ptr %iv2
  %iv.next = add i64 %iv, -4
  %gep3 = getelementptr i32, ptr %iv2, i64 4
  %cond = icmp eq i64 %iv.next, 0
  br i1 %cond, label %exit, label %body

exit:
  ret void
}

define void @store_post_indexed_doubleword(ptr %array, i64 %count, i64 %val) nounwind {
; CHECK-LABEL: store_post_indexed_doubleword:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    stp x30, x21, [sp, #-32]! // 16-byte Folded Spill
; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    mov x19, x2
; CHECK-NEXT:    mov x20, x1
; CHECK-NEXT:    add x21, x0, #16
; CHECK-NEXT:  .LBB46_1: // %body
; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldur x0, [x21, #-8]
; CHECK-NEXT:    bl "use-doubleword"
; CHECK-NEXT:    subs x20, x20, #4
; CHECK-NEXT:    str x19, [x21], #32
; CHECK-NEXT:    b.ne .LBB46_1
; CHECK-NEXT:  // %bb.2: // %exit
; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldp x30, x21, [sp], #32 // 16-byte Folded Reload
; CHECK-NEXT:    ret
entry:
  %gep1 = getelementptr i64, ptr %array, i64 2
  br label %body

body:
  %iv2 = phi ptr [ %gep3, %body ], [ %gep1, %entry ]
  %iv = phi i64 [ %iv.next, %body ], [ %count, %entry ]
  %gep2 = getelementptr i64, ptr %iv2, i64 -1
  %load = load i64, ptr %gep2
  call void @use-doubleword(i64 %load)
  store i64 %val, ptr %iv2
  %iv.next = add i64 %iv, -4
  %gep3 = getelementptr i64, ptr %iv2, i64 4
  %cond = icmp eq i64 %iv.next, 0
  br i1 %cond, label %exit, label %body

exit:
  ret void
}

define void @store_post_indexed_quadword(ptr %array, i64 %count, <2 x i64> %val) nounwind {
; CHECK-LABEL: store_post_indexed_quadword:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    sub sp, sp, #48
; CHECK-NEXT:    stp x20, x19, [sp, #32] // 16-byte Folded Spill
; CHECK-NEXT:    mov x19, x1
; CHECK-NEXT:    add x20, x0, #32
; CHECK-NEXT:    str x30, [sp, #16] // 8-byte Spill
; CHECK-NEXT:    str q0, [sp] // 16-byte Spill
; CHECK-NEXT:  .LBB47_1: // %body
; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldur q0, [x20, #-16]
; CHECK-NEXT:    bl "use-quadword"
; CHECK-NEXT:    ldr q0, [sp] // 16-byte Reload
; CHECK-NEXT:    subs x19, x19, #4
; CHECK-NEXT:    str q0, [x20], #64
; CHECK-NEXT:    b.ne .LBB47_1
; CHECK-NEXT:  // %bb.2: // %exit
; CHECK-NEXT:    ldp x20, x19, [sp, #32] // 16-byte Folded Reload
; CHECK-NEXT:    ldr x30, [sp, #16] // 8-byte Reload
; CHECK-NEXT:    add sp, sp, #48
; CHECK-NEXT:    ret
entry:
  %gep1 = getelementptr <2 x i64>, ptr %array, i64 2
  br label %body

body:
  %iv2 = phi ptr [ %gep3, %body ], [ %gep1, %entry ]
  %iv = phi i64 [ %iv.next, %body ], [ %count, %entry ]
  %gep2 = getelementptr <2 x i64>, ptr %iv2, i64 -1
  %load = load <2 x i64>, ptr %gep2
  call void @use-quadword(<2 x i64> %load)
  store <2 x i64> %val, ptr %iv2
  %iv.next = add i64 %iv, -4
  %gep3 = getelementptr <2 x i64>, ptr %iv2, i64 4
  %cond = icmp eq i64 %iv.next, 0
  br i1 %cond, label %exit, label %body

exit:
  ret void
}

define void @store_post_indexed_float(ptr %array, i64 %count, float %val) nounwind {
; CHECK-LABEL: store_post_indexed_float:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
; CHECK-NEXT:    fmov s8, s0
; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    mov x19, x1
; CHECK-NEXT:    add x20, x0, #8
; CHECK-NEXT:    str x30, [sp, #8] // 8-byte Spill
; CHECK-NEXT:  .LBB48_1: // %body
; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldur s0, [x20, #-4]
; CHECK-NEXT:    bl "use-float"
; CHECK-NEXT:    subs x19, x19, #4
; CHECK-NEXT:    str s8, [x20], #16
; CHECK-NEXT:    b.ne .LBB48_1
; CHECK-NEXT:  // %bb.2: // %exit
; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldr x30, [sp, #8] // 8-byte Reload
; CHECK-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
; CHECK-NEXT:    ret
entry:
  %gep1 = getelementptr float, ptr %array, i64 2
  br label %body

body:
  %iv2 = phi ptr [ %gep3, %body ], [ %gep1, %entry ]
  %iv = phi i64 [ %iv.next, %body ], [ %count, %entry ]
  %gep2 = getelementptr float, ptr %iv2, i64 -1
  %load = load float, ptr %gep2
  call void @use-float(float %load)
  store float %val, ptr %iv2
  %iv.next = add i64 %iv, -4
  %gep3 = getelementptr float, ptr %iv2, i64 4
  %cond = icmp eq i64 %iv.next, 0
  br i1 %cond, label %exit, label %body

exit:
  ret void
}

define void @store_post_indexed_double(ptr %array, i64 %count, double %val) nounwind {
; CHECK-LABEL: store_post_indexed_double:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    str d8, [sp, #-32]! // 8-byte Folded Spill
; CHECK-NEXT:    fmov d8, d0
; CHECK-NEXT:    stp x20, x19, [sp, #16] // 16-byte Folded Spill
; CHECK-NEXT:    mov x19, x1
; CHECK-NEXT:    add x20, x0, #16
; CHECK-NEXT:    str x30, [sp, #8] // 8-byte Spill
; CHECK-NEXT:  .LBB49_1: // %body
; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldur d0, [x20, #-8]
; CHECK-NEXT:    bl "use-double"
; CHECK-NEXT:    subs x19, x19, #4
; CHECK-NEXT:    str d8, [x20], #32
; CHECK-NEXT:    b.ne .LBB49_1
; CHECK-NEXT:  // %bb.2: // %exit
; CHECK-NEXT:    ldp x20, x19, [sp, #16] // 16-byte Folded Reload
; CHECK-NEXT:    ldr x30, [sp, #8] // 8-byte Reload
; CHECK-NEXT:    ldr d8, [sp], #32 // 8-byte Folded Reload
; CHECK-NEXT:    ret
entry:
  %gep1 = getelementptr double, ptr %array, i64 2
  br label %body

body:
  %iv2 = phi ptr [ %gep3, %body ], [ %gep1, %entry ]
  %iv = phi i64 [ %iv.next, %body ], [ %count, %entry ]
  %gep2 = getelementptr double, ptr %iv2, i64 -1
  %load = load double, ptr %gep2
  call void @use-double(double %load)
  store double %val, ptr %iv2
  %iv.next = add i64 %iv, -4
  %gep3 = getelementptr double, ptr %iv2, i64 4
  %cond = icmp eq i64 %iv.next, 0
  br i1 %cond, label %exit, label %body

exit:
  ret void
}

declare void @use-byte(i8)
declare void @use-halfword(i16)
declare void @use-word(i32)
declare void @use-doubleword(i64)
declare void @use-quadword(<2 x i64>)
declare void @use-float(float)
declare void @use-double(double)

; Check the following transform:
;
; stp w0, [x20]
;  ...
; add x20, x20, #32
;  ->
; stp w0, [x20], #32

define void @store_pair_post_indexed_word() nounwind {
; CHECK-LABEL: store_pair_post_indexed_word:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub sp, sp, #16
; CHECK-NEXT:    ldp w8, w9, [sp, #8]
; CHECK-NEXT:    stp w8, w9, [sp], #16
; CHECK-NEXT:    ret
  %src = alloca { i32, i32 }, align 8
  %dst = alloca { i32, i32 }, align 8

  %src.realp = getelementptr inbounds { i32, i32 }, ptr %src, i32 0, i32 0
  %src.real = load i32, ptr %src.realp
  %src.imagp = getelementptr inbounds { i32, i32 }, ptr %src, i32 0, i32 1
  %src.imag = load i32, ptr %src.imagp

  %dst.realp = getelementptr inbounds { i32, i32 }, ptr %dst, i32 0, i32 0
  %dst.imagp = getelementptr inbounds { i32, i32 }, ptr %dst, i32 0, i32 1
  store i32 %src.real, ptr %dst.realp
  store i32 %src.imag, ptr %dst.imagp
  ret void
}

define void @store_pair_post_indexed_doubleword() nounwind {
; CHECK-LABEL: store_pair_post_indexed_doubleword:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub sp, sp, #32
; CHECK-NEXT:    ldp x8, x9, [sp, #16]
; CHECK-NEXT:    stp x8, x9, [sp], #32
; CHECK-NEXT:    ret
  %src = alloca { i64, i64 }, align 8
  %dst = alloca { i64, i64 }, align 8

  %src.realp = getelementptr inbounds { i64, i64 }, ptr %src, i32 0, i32 0
  %src.real = load i64, ptr %src.realp
  %src.imagp = getelementptr inbounds { i64, i64 }, ptr %src, i32 0, i32 1
  %src.imag = load i64, ptr %src.imagp

  %dst.realp = getelementptr inbounds { i64, i64 }, ptr %dst, i32 0, i32 0
  %dst.imagp = getelementptr inbounds { i64, i64 }, ptr %dst, i32 0, i32 1
  store i64 %src.real, ptr %dst.realp
  store i64 %src.imag, ptr %dst.imagp
  ret void
}

define void @store_pair_post_indexed_float() nounwind {
; CHECK-LABEL: store_pair_post_indexed_float:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub sp, sp, #16
; CHECK-NEXT:    ldp s0, s1, [sp, #8]
; CHECK-NEXT:    stp s0, s1, [sp], #16
; CHECK-NEXT:    ret
  %src = alloca { float, float }, align 8
  %dst = alloca { float, float }, align 8

  %src.realp = getelementptr inbounds { float, float }, ptr %src, i32 0, i32 0
  %src.real = load float, ptr %src.realp
  %src.imagp = getelementptr inbounds { float, float }, ptr %src, i32 0, i32 1
  %src.imag = load float, ptr %src.imagp

  %dst.realp = getelementptr inbounds { float, float }, ptr %dst, i32 0, i32 0
  %dst.imagp = getelementptr inbounds { float, float }, ptr %dst, i32 0, i32 1
  store float %src.real, ptr %dst.realp
  store float %src.imag, ptr %dst.imagp
  ret void
}

define void @store_pair_post_indexed_double() nounwind {
; CHECK-LABEL: store_pair_post_indexed_double:
; CHECK:       // %bb.0:
; CHECK-NEXT:    sub sp, sp, #32
; CHECK-NEXT:    ldp d0, d1, [sp, #16]
; CHECK-NEXT:    stp d0, d1, [sp], #32
; CHECK-NEXT:    ret
  %src = alloca { double, double }, align 8
  %dst = alloca { double, double }, align 8

  %src.realp = getelementptr inbounds { double, double }, ptr %src, i32 0, i32 0
  %src.real = load double, ptr %src.realp
  %src.imagp = getelementptr inbounds { double, double }, ptr %src, i32 0, i32 1
  %src.imag = load double, ptr %src.imagp

  %dst.realp = getelementptr inbounds { double, double }, ptr %dst, i32 0, i32 0
  %dst.imagp = getelementptr inbounds { double, double }, ptr %dst, i32 0, i32 1
  store double %src.real, ptr %dst.realp
  store double %src.imag, ptr %dst.imagp
  ret void
}

; Check the following transform:
;
; (ldr|str) X, [x20]
;  ...
; sub x20, x20, #16
;  ->
; (ldr|str) X, [x20], #-16
;
; with X being either w0, x0, s0, d0 or q0.

define void @post_indexed_sub_word(ptr %a, ptr %b, i64 %count) nounwind {
; CHECK-LABEL: post_indexed_sub_word:
; CHECK:       // %bb.0:
; CHECK-NEXT:  .LBB54_1: // %for.body
; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldur w8, [x1, #-4]
; CHECK-NEXT:    subs x2, x2, #1
; CHECK-NEXT:    stur w8, [x0, #-4]
; CHECK-NEXT:    ldr w8, [x1], #-8
; CHECK-NEXT:    str w8, [x0], #-8
; CHECK-NEXT:    b.gt .LBB54_1
; CHECK-NEXT:  // %bb.2: // %end
; CHECK-NEXT:    ret
  br label %for.body
for.body:
  %phi1 = phi ptr [ %gep4, %for.body ], [ %b, %0 ]
  %phi2 = phi ptr [ %gep3, %for.body ], [ %a, %0 ]
  %i = phi i64 [ %dec.i, %for.body], [ %count, %0 ]
  %gep1 = getelementptr i32, ptr %phi1, i64 -1
  %load1 = load i32, ptr %gep1
  %gep2 = getelementptr i32, ptr %phi2, i64 -1
  store i32 %load1, ptr %gep2
  %load2 = load i32, ptr %phi1
  store i32 %load2, ptr %phi2
  %dec.i = add nsw i64 %i, -1
  %gep3 = getelementptr i32, ptr %phi2, i64 -2
  %gep4 = getelementptr i32, ptr %phi1, i64 -2
  %cond = icmp sgt i64 %dec.i, 0
  br i1 %cond, label %for.body, label %end
end:
  ret void
}

define void @post_indexed_sub_doubleword(ptr %a, ptr %b, i64 %count) nounwind {
; CHECK-LABEL: post_indexed_sub_doubleword:
; CHECK:       // %bb.0:
; CHECK-NEXT:  .LBB55_1: // %for.body
; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldur x8, [x1, #-8]
; CHECK-NEXT:    subs x2, x2, #1
; CHECK-NEXT:    stur x8, [x0, #-8]
; CHECK-NEXT:    ldr x8, [x1], #-16
; CHECK-NEXT:    str x8, [x0], #-16
; CHECK-NEXT:    b.gt .LBB55_1
; CHECK-NEXT:  // %bb.2: // %end
; CHECK-NEXT:    ret
  br label %for.body
for.body:
  %phi1 = phi ptr [ %gep4, %for.body ], [ %b, %0 ]
  %phi2 = phi ptr [ %gep3, %for.body ], [ %a, %0 ]
  %i = phi i64 [ %dec.i, %for.body], [ %count, %0 ]
  %gep1 = getelementptr i64, ptr %phi1, i64 -1
  %load1 = load i64, ptr %gep1
  %gep2 = getelementptr i64, ptr %phi2, i64 -1
  store i64 %load1, ptr %gep2
  %load2 = load i64, ptr %phi1
  store i64 %load2, ptr %phi2
  %dec.i = add nsw i64 %i, -1
  %gep3 = getelementptr i64, ptr %phi2, i64 -2
  %gep4 = getelementptr i64, ptr %phi1, i64 -2
  %cond = icmp sgt i64 %dec.i, 0
  br i1 %cond, label %for.body, label %end
end:
  ret void
}

define void @post_indexed_sub_quadword(ptr %a, ptr %b, i64 %count) nounwind {
; CHECK-LABEL: post_indexed_sub_quadword:
; CHECK:       // %bb.0:
; CHECK-NEXT:  .LBB56_1: // %for.body
; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldur q0, [x1, #-16]
; CHECK-NEXT:    subs x2, x2, #1
; CHECK-NEXT:    stur q0, [x0, #-16]
; CHECK-NEXT:    ldr q0, [x1], #-32
; CHECK-NEXT:    str q0, [x0], #-32
; CHECK-NEXT:    b.gt .LBB56_1
; CHECK-NEXT:  // %bb.2: // %end
; CHECK-NEXT:    ret
  br label %for.body
for.body:
  %phi1 = phi ptr [ %gep4, %for.body ], [ %b, %0 ]
  %phi2 = phi ptr [ %gep3, %for.body ], [ %a, %0 ]
  %i = phi i64 [ %dec.i, %for.body], [ %count, %0 ]
  %gep1 = getelementptr <2 x i64>, ptr %phi1, i64 -1
  %load1 = load <2 x i64>, ptr %gep1
  %gep2 = getelementptr <2 x i64>, ptr %phi2, i64 -1
  store <2 x i64> %load1, ptr %gep2
  %load2 = load <2 x i64>, ptr %phi1
  store <2 x i64> %load2, ptr %phi2
  %dec.i = add nsw i64 %i, -1
  %gep3 = getelementptr <2 x i64>, ptr %phi2, i64 -2
  %gep4 = getelementptr <2 x i64>, ptr %phi1, i64 -2
  %cond = icmp sgt i64 %dec.i, 0
  br i1 %cond, label %for.body, label %end
end:
  ret void
}

define void @post_indexed_sub_float(ptr %a, ptr %b, i64 %count) nounwind {
; CHECK-LABEL: post_indexed_sub_float:
; CHECK:       // %bb.0:
; CHECK-NEXT:  .LBB57_1: // %for.body
; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldur s0, [x1, #-4]
; CHECK-NEXT:    subs x2, x2, #1
; CHECK-NEXT:    stur s0, [x0, #-4]
; CHECK-NEXT:    ldr s0, [x1], #-8
; CHECK-NEXT:    str s0, [x0], #-8
; CHECK-NEXT:    b.gt .LBB57_1
; CHECK-NEXT:  // %bb.2: // %end
; CHECK-NEXT:    ret
  br label %for.body
for.body:
  %phi1 = phi ptr [ %gep4, %for.body ], [ %b, %0 ]
  %phi2 = phi ptr [ %gep3, %for.body ], [ %a, %0 ]
  %i = phi i64 [ %dec.i, %for.body], [ %count, %0 ]
  %gep1 = getelementptr float, ptr %phi1, i64 -1
  %load1 = load float, ptr %gep1
  %gep2 = getelementptr float, ptr %phi2, i64 -1
  store float %load1, ptr %gep2
  %load2 = load float, ptr %phi1
  store float %load2, ptr %phi2
  %dec.i = add nsw i64 %i, -1
  %gep3 = getelementptr float, ptr %phi2, i64 -2
  %gep4 = getelementptr float, ptr %phi1, i64 -2
  %cond = icmp sgt i64 %dec.i, 0
  br i1 %cond, label %for.body, label %end
end:
  ret void
}

define void @post_indexed_sub_double(ptr %a, ptr %b, i64 %count) nounwind {
; CHECK-LABEL: post_indexed_sub_double:
; CHECK:       // %bb.0:
; CHECK-NEXT:  .LBB58_1: // %for.body
; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldur d0, [x1, #-8]
; CHECK-NEXT:    subs x2, x2, #1
; CHECK-NEXT:    stur d0, [x0, #-8]
; CHECK-NEXT:    ldr d0, [x1], #-16
; CHECK-NEXT:    str d0, [x0], #-16
; CHECK-NEXT:    b.gt .LBB58_1
; CHECK-NEXT:  // %bb.2: // %end
; CHECK-NEXT:    ret
  br label %for.body
for.body:
  %phi1 = phi ptr [ %gep4, %for.body ], [ %b, %0 ]
  %phi2 = phi ptr [ %gep3, %for.body ], [ %a, %0 ]
  %i = phi i64 [ %dec.i, %for.body], [ %count, %0 ]
  %gep1 = getelementptr double, ptr %phi1, i64 -1
  %load1 = load double, ptr %gep1
  %gep2 = getelementptr double, ptr %phi2, i64 -1
  store double %load1, ptr %gep2
  %load2 = load double, ptr %phi1
  store double %load2, ptr %phi2
  %dec.i = add nsw i64 %i, -1
  %gep3 = getelementptr double, ptr %phi2, i64 -2
  %gep4 = getelementptr double, ptr %phi1, i64 -2
  %cond = icmp sgt i64 %dec.i, 0
  br i1 %cond, label %for.body, label %end
end:
  ret void
}

define void @post_indexed_sub_doubleword_offset_min(ptr %a, ptr %b, i64 %count) nounwind {
; CHECK-LABEL: post_indexed_sub_doubleword_offset_min:
; CHECK:       // %bb.0:
; CHECK-NEXT:  .LBB59_1: // %for.body
; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldr x8, [x1, #8]
; CHECK-NEXT:    subs x2, x2, #1
; CHECK-NEXT:    str x8, [x0, #8]
; CHECK-NEXT:    ldr x8, [x1], #-256
; CHECK-NEXT:    str x8, [x0], #-256
; CHECK-NEXT:    b.gt .LBB59_1
; CHECK-NEXT:  // %bb.2: // %end
; CHECK-NEXT:    ret
  br label %for.body
for.body:
  %phi1 = phi ptr [ %gep4, %for.body ], [ %b, %0 ]
  %phi2 = phi ptr [ %gep3, %for.body ], [ %a, %0 ]
  %i = phi i64 [ %dec.i, %for.body], [ %count, %0 ]
  %gep1 = getelementptr i64, ptr %phi1, i64 1
  %load1 = load i64, ptr %gep1
  %gep2 = getelementptr i64, ptr %phi2, i64 1
  store i64 %load1, ptr %gep2
  %load2 = load i64, ptr %phi1
  store i64 %load2, ptr %phi2
  %dec.i = add nsw i64 %i, -1
  %gep3 = getelementptr i64, ptr %phi2, i64 -32
  %gep4 = getelementptr i64, ptr %phi1, i64 -32
  %cond = icmp sgt i64 %dec.i, 0
  br i1 %cond, label %for.body, label %end
end:
  ret void
}

define void @post_indexed_doubleword_offset_out_of_range(ptr %a, ptr %b, i64 %count) nounwind {
; CHECK-LABEL: post_indexed_doubleword_offset_out_of_range:
; CHECK:       // %bb.0:
; CHECK-NEXT:  .LBB60_1: // %for.body
; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldr x8, [x1, #8]
; CHECK-NEXT:    subs x2, x2, #1
; CHECK-NEXT:    str x8, [x0, #8]
; CHECK-NEXT:    ldr x8, [x1]
; CHECK-NEXT:    add x1, x1, #256
; CHECK-NEXT:    str x8, [x0]
; CHECK-NEXT:    add x0, x0, #256
; CHECK-NEXT:    b.gt .LBB60_1
; CHECK-NEXT:  // %bb.2: // %end
; CHECK-NEXT:    ret
  br label %for.body
for.body:
  %phi1 = phi ptr [ %gep4, %for.body ], [ %b, %0 ]
  %phi2 = phi ptr [ %gep3, %for.body ], [ %a, %0 ]
  %i = phi i64 [ %dec.i, %for.body], [ %count, %0 ]
  %gep1 = getelementptr i64, ptr %phi1, i64 1
  %load1 = load i64, ptr %gep1
  %gep2 = getelementptr i64, ptr %phi2, i64 1
  store i64 %load1, ptr %gep2
  %load2 = load i64, ptr %phi1
  store i64 %load2, ptr %phi2
  %dec.i = add nsw i64 %i, -1
  %gep3 = getelementptr i64, ptr %phi2, i64 32
  %gep4 = getelementptr i64, ptr %phi1, i64 32
  %cond = icmp sgt i64 %dec.i, 0
  br i1 %cond, label %for.body, label %end
end:
  ret void
}

define void @post_indexed_paired_min_offset(ptr %a, ptr %b, i64 %count) nounwind {
; CHECK-LABEL: post_indexed_paired_min_offset:
; CHECK:       // %bb.0:
; CHECK-NEXT:  .LBB61_1: // %for.body
; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldp x9, x8, [x1], #-512
; CHECK-NEXT:    subs x2, x2, #1
; CHECK-NEXT:    stp x9, x8, [x0], #-512
; CHECK-NEXT:    b.gt .LBB61_1
; CHECK-NEXT:  // %bb.2: // %end
; CHECK-NEXT:    ret
  br label %for.body
for.body:
  %phi1 = phi ptr [ %gep4, %for.body ], [ %b, %0 ]
  %phi2 = phi ptr [ %gep3, %for.body ], [ %a, %0 ]
  %i = phi i64 [ %dec.i, %for.body], [ %count, %0 ]
  %gep1 = getelementptr i64, ptr %phi1, i64 1
  %load1 = load i64, ptr %gep1
  %gep2 = getelementptr i64, ptr %phi2, i64 1
  %load2 = load i64, ptr %phi1
  store i64 %load1, ptr %gep2
  store i64 %load2, ptr %phi2
  %dec.i = add nsw i64 %i, -1
  %gep3 = getelementptr i64, ptr %phi2, i64 -64
  %gep4 = getelementptr i64, ptr %phi1, i64 -64
  %cond = icmp sgt i64 %dec.i, 0
  br i1 %cond, label %for.body, label %end
end:
  ret void
}

define void @post_indexed_paired_offset_out_of_range(ptr %a, ptr %b, i64 %count) nounwind {
; CHECK-LABEL: post_indexed_paired_offset_out_of_range:
; CHECK:       // %bb.0:
; CHECK-NEXT:  .LBB62_1: // %for.body
; CHECK-NEXT:    // =>This Inner Loop Header: Depth=1
; CHECK-NEXT:    ldp x8, x9, [x1]
; CHECK-NEXT:    subs x2, x2, #1
; CHECK-NEXT:    add x1, x1, #512
; CHECK-NEXT:    stp x9, x8, [x0]
; CHECK-NEXT:    add x0, x0, #512
; CHECK-NEXT:    b.gt .LBB62_1
; CHECK-NEXT:  // %bb.2: // %end
; CHECK-NEXT:    ret
  br label %for.body
for.body:
  %phi1 = phi ptr [ %gep4, %for.body ], [ %b, %0 ]
  %phi2 = phi ptr [ %gep3, %for.body ], [ %a, %0 ]
  %i = phi i64 [ %dec.i, %for.body], [ %count, %0 ]
  %gep1 = getelementptr i64, ptr %phi1, i64 1
  %load1 = load i64, ptr %phi1
  %gep2 = getelementptr i64, ptr %phi2, i64 1
  %load2 = load i64, ptr %gep1
  store i64 %load1, ptr %gep2
  store i64 %load2, ptr %phi2
  %dec.i = add nsw i64 %i, -1
  %gep3 = getelementptr i64, ptr %phi2, i64 64
  %gep4 = getelementptr i64, ptr %phi1, i64 64
  %cond = icmp sgt i64 %dec.i, 0
  br i1 %cond, label %for.body, label %end
end:
  ret void
}

; DAGCombiner::MergeConsecutiveStores merges this into a vector store,
; replaceZeroVectorStore should split the vector store back into
; scalar stores which should get merged by AArch64LoadStoreOptimizer.
define void @merge_zr32(ptr %p) {
; NOSTRICTALIGN-LABEL: merge_zr32:
; NOSTRICTALIGN:       // %bb.0: // %entry
; NOSTRICTALIGN-NEXT:    str xzr, [x0]
; NOSTRICTALIGN-NEXT:    ret
;
; STRICTALIGN-LABEL: merge_zr32:
; STRICTALIGN:       // %bb.0: // %entry
; STRICTALIGN-NEXT:    stp wzr, wzr, [x0]
; STRICTALIGN-NEXT:    ret
entry:
  store i32 0, ptr %p
  %p1 = getelementptr i32, ptr %p, i32 1
  store i32 0, ptr %p1
  ret void
}

; Same as merge_zr32 but the merged stores should also get paried.
define void @merge_zr32_2(ptr %p) {
; NOSTRICTALIGN-LABEL: merge_zr32_2:
; NOSTRICTALIGN:       // %bb.0: // %entry
; NOSTRICTALIGN-NEXT:    stp xzr, xzr, [x0]
; NOSTRICTALIGN-NEXT:    ret
;
; STRICTALIGN-LABEL: merge_zr32_2:
; STRICTALIGN:       // %bb.0: // %entry
; STRICTALIGN-NEXT:    stp wzr, wzr, [x0]
; STRICTALIGN-NEXT:    stp wzr, wzr, [x0, #8]
; STRICTALIGN-NEXT:    ret
entry:
  store i32 0, ptr %p
  %p1 = getelementptr i32, ptr %p, i32 1
  store i32 0, ptr %p1
  %p2 = getelementptr i32, ptr %p, i64 2
  store i32 0, ptr %p2
  %p3 = getelementptr i32, ptr %p, i64 3
  store i32 0, ptr %p3
  ret void
}

; Like merge_zr32_2, but checking the largest allowed stp immediate offset.
define void @merge_zr32_2_offset(ptr %p) {
; NOSTRICTALIGN-LABEL: merge_zr32_2_offset:
; NOSTRICTALIGN:       // %bb.0: // %entry
; NOSTRICTALIGN-NEXT:    stp xzr, xzr, [x0, #504]
; NOSTRICTALIGN-NEXT:    ret
;
; STRICTALIGN-LABEL: merge_zr32_2_offset:
; STRICTALIGN:       // %bb.0: // %entry
; STRICTALIGN-NEXT:    str wzr, [x0, #504]
; STRICTALIGN-NEXT:    str wzr, [x0, #508]
; STRICTALIGN-NEXT:    str wzr, [x0, #512]
; STRICTALIGN-NEXT:    str wzr, [x0, #516]
; STRICTALIGN-NEXT:    ret
entry:
  %p0 = getelementptr i32, ptr %p, i32 126
  store i32 0, ptr %p0
  %p1 = getelementptr i32, ptr %p, i32 127
  store i32 0, ptr %p1
  %p2 = getelementptr i32, ptr %p, i64 128
  store i32 0, ptr %p2
  %p3 = getelementptr i32, ptr %p, i64 129
  store i32 0, ptr %p3
  ret void
}

; Like merge_zr32, but replaceZeroVectorStore should not split this
; vector store since the address offset is too large for the stp
; instruction.
define void @no_merge_zr32_2_offset(ptr %p) {
; NOSTRICTALIGN-LABEL: no_merge_zr32_2_offset:
; NOSTRICTALIGN:       // %bb.0: // %entry
; NOSTRICTALIGN-NEXT:    movi v0.2d, #0000000000000000
; NOSTRICTALIGN-NEXT:    str q0, [x0, #4096]
; NOSTRICTALIGN-NEXT:    ret
;
; STRICTALIGN-LABEL: no_merge_zr32_2_offset:
; STRICTALIGN:       // %bb.0: // %entry
; STRICTALIGN-NEXT:    str wzr, [x0, #4096]
; STRICTALIGN-NEXT:    str wzr, [x0, #4100]
; STRICTALIGN-NEXT:    str wzr, [x0, #4104]
; STRICTALIGN-NEXT:    str wzr, [x0, #4108]
; STRICTALIGN-NEXT:    ret
entry:
  %p0 = getelementptr i32, ptr %p, i32 1024
  store i32 0, ptr %p0
  %p1 = getelementptr i32, ptr %p, i32 1025
  store i32 0, ptr %p1
  %p2 = getelementptr i32, ptr %p, i64 1026
  store i32 0, ptr %p2
  %p3 = getelementptr i32, ptr %p, i64 1027
  store i32 0, ptr %p3
  ret void
}

; Like merge_zr32, but replaceZeroVectorStore should not split the
; vector store since the zero constant vector has multiple uses, so we
; err on the side that allows for stp q instruction generation.
define void @merge_zr32_3(ptr %p) {
; NOSTRICTALIGN-LABEL: merge_zr32_3:
; NOSTRICTALIGN:       // %bb.0: // %entry
; NOSTRICTALIGN-NEXT:    movi v0.2d, #0000000000000000
; NOSTRICTALIGN-NEXT:    stp q0, q0, [x0]
; NOSTRICTALIGN-NEXT:    ret
;
; STRICTALIGN-LABEL: merge_zr32_3:
; STRICTALIGN:       // %bb.0: // %entry
; STRICTALIGN-NEXT:    stp wzr, wzr, [x0]
; STRICTALIGN-NEXT:    stp wzr, wzr, [x0, #8]
; STRICTALIGN-NEXT:    stp wzr, wzr, [x0, #16]
; STRICTALIGN-NEXT:    stp wzr, wzr, [x0, #24]
; STRICTALIGN-NEXT:    ret
entry:
  store i32 0, ptr %p
  %p1 = getelementptr i32, ptr %p, i32 1
  store i32 0, ptr %p1
  %p2 = getelementptr i32, ptr %p, i64 2
  store i32 0, ptr %p2
  %p3 = getelementptr i32, ptr %p, i64 3
  store i32 0, ptr %p3
  %p4 = getelementptr i32, ptr %p, i64 4
  store i32 0, ptr %p4
  %p5 = getelementptr i32, ptr %p, i64 5
  store i32 0, ptr %p5
  %p6 = getelementptr i32, ptr %p, i64 6
  store i32 0, ptr %p6
  %p7 = getelementptr i32, ptr %p, i64 7
  store i32 0, ptr %p7
  ret void
}

; Like merge_zr32, but with 2-vector type.
define void @merge_zr32_2vec(ptr %p) {
; NOSTRICTALIGN-LABEL: merge_zr32_2vec:
; NOSTRICTALIGN:       // %bb.0: // %entry
; NOSTRICTALIGN-NEXT:    str xzr, [x0]
; NOSTRICTALIGN-NEXT:    ret
;
; STRICTALIGN-LABEL: merge_zr32_2vec:
; STRICTALIGN:       // %bb.0: // %entry
; STRICTALIGN-NEXT:    stp wzr, wzr, [x0]
; STRICTALIGN-NEXT:    ret
entry:
  store <2 x i32> zeroinitializer, ptr %p
  ret void
}

; Like merge_zr32, but with 3-vector type.
define void @merge_zr32_3vec(ptr %p) {
; NOSTRICTALIGN-LABEL: merge_zr32_3vec:
; NOSTRICTALIGN:       // %bb.0: // %entry
; NOSTRICTALIGN-NEXT:    str wzr, [x0, #8]
; NOSTRICTALIGN-NEXT:    str xzr, [x0]
; NOSTRICTALIGN-NEXT:    ret
;
; STRICTALIGN-LABEL: merge_zr32_3vec:
; STRICTALIGN:       // %bb.0: // %entry
; STRICTALIGN-NEXT:    stp wzr, wzr, [x0, #4]
; STRICTALIGN-NEXT:    str wzr, [x0]
; STRICTALIGN-NEXT:    ret
entry:
  store <3 x i32> zeroinitializer, ptr %p
  ret void
}

; Like merge_zr32, but with 4-vector type.
define void @merge_zr32_4vec(ptr %p) {
; NOSTRICTALIGN-LABEL: merge_zr32_4vec:
; NOSTRICTALIGN:       // %bb.0: // %entry
; NOSTRICTALIGN-NEXT:    stp xzr, xzr, [x0]
; NOSTRICTALIGN-NEXT:    ret
;
; STRICTALIGN-LABEL: merge_zr32_4vec:
; STRICTALIGN:       // %bb.0: // %entry
; STRICTALIGN-NEXT:    stp wzr, wzr, [x0, #8]
; STRICTALIGN-NEXT:    stp wzr, wzr, [x0]
; STRICTALIGN-NEXT:    ret
entry:
  store <4 x i32> zeroinitializer, ptr %p
  ret void
}

; Like merge_zr32, but with 2-vector float type.
define void @merge_zr32_2vecf(ptr %p) {
; NOSTRICTALIGN-LABEL: merge_zr32_2vecf:
; NOSTRICTALIGN:       // %bb.0: // %entry
; NOSTRICTALIGN-NEXT:    str xzr, [x0]
; NOSTRICTALIGN-NEXT:    ret
;
; STRICTALIGN-LABEL: merge_zr32_2vecf:
; STRICTALIGN:       // %bb.0: // %entry
; STRICTALIGN-NEXT:    stp wzr, wzr, [x0]
; STRICTALIGN-NEXT:    ret
entry:
  store <2 x float> zeroinitializer, ptr %p
  ret void
}

; Like merge_zr32, but with 4-vector float type.
define void @merge_zr32_4vecf(ptr %p) {
; NOSTRICTALIGN-LABEL: merge_zr32_4vecf:
; NOSTRICTALIGN:       // %bb.0: // %entry
; NOSTRICTALIGN-NEXT:    stp xzr, xzr, [x0]
; NOSTRICTALIGN-NEXT:    ret
;
; STRICTALIGN-LABEL: merge_zr32_4vecf:
; STRICTALIGN:       // %bb.0: // %entry
; STRICTALIGN-NEXT:    stp wzr, wzr, [x0, #8]
; STRICTALIGN-NEXT:    stp wzr, wzr, [x0]
; STRICTALIGN-NEXT:    ret
entry:
  store <4 x float> zeroinitializer, ptr %p
  ret void
}

; Similar to merge_zr32, but for 64-bit values.
define void @merge_zr64(ptr %p) {
; CHECK-LABEL: merge_zr64:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    stp xzr, xzr, [x0]
; CHECK-NEXT:    ret
entry:
  store i64 0, ptr %p
  %p1 = getelementptr i64, ptr %p, i64 1
  store i64 0, ptr %p1
  ret void
}

; Similar to merge_zr32, but for 64-bit values and with unaligned stores.
define void @merge_zr64_unalign(ptr %p) {
; NOSTRICTALIGN-LABEL: merge_zr64_unalign:
; NOSTRICTALIGN:       // %bb.0: // %entry
; NOSTRICTALIGN-NEXT:    stp xzr, xzr, [x0]
; NOSTRICTALIGN-NEXT:    ret
;
; STRICTALIGN-LABEL: merge_zr64_unalign:
; STRICTALIGN:       // %bb.0: // %entry
; STRICTALIGN-NEXT:    lsr x8, xzr, #40
; STRICTALIGN-NEXT:    lsr x9, xzr, #24
; STRICTALIGN-NEXT:    lsr x11, xzr, #16
; STRICTALIGN-NEXT:    mov x10, x0
; STRICTALIGN-NEXT:    lsr x12, xzr, #8
; STRICTALIGN-NEXT:    strb wzr, [x0]
; STRICTALIGN-NEXT:    strb w8, [x0, #13]
; STRICTALIGN-NEXT:    strb wzr, [x10, #8]!
; STRICTALIGN-NEXT:    strb w9, [x10, #3]
; STRICTALIGN-NEXT:    strb w11, [x10, #2]
; STRICTALIGN-NEXT:    strb w8, [x0, #5]
; STRICTALIGN-NEXT:    lsr x8, xzr, #32
; STRICTALIGN-NEXT:    strb w9, [x0, #3]
; STRICTALIGN-NEXT:    lsr x9, xzr, #56
; STRICTALIGN-NEXT:    strb w11, [x0, #2]
; STRICTALIGN-NEXT:    lsr x11, xzr, #48
; STRICTALIGN-NEXT:    strb w12, [x0, #9]
; STRICTALIGN-NEXT:    strb w12, [x0, #1]
; STRICTALIGN-NEXT:    strb w8, [x10, #4]!
; STRICTALIGN-NEXT:    strb w9, [x10, #3]
; STRICTALIGN-NEXT:    strb w11, [x10, #2]
; STRICTALIGN-NEXT:    strb w8, [x0, #4]!
; STRICTALIGN-NEXT:    strb w9, [x0, #3]
; STRICTALIGN-NEXT:    strb w11, [x0, #2]
; STRICTALIGN-NEXT:    ret
entry:
  store <2 x i64> zeroinitializer, ptr %p, align 1
  ret void
}

; Similar to merge_zr32_3, replaceZeroVectorStore should not split the
; vector store since the zero constant vector has multiple uses.
define void @merge_zr64_2(ptr %p) {
; NOSTRICTALIGN-LABEL: merge_zr64_2:
; NOSTRICTALIGN:       // %bb.0: // %entry
; NOSTRICTALIGN-NEXT:    movi v0.2d, #0000000000000000
; NOSTRICTALIGN-NEXT:    stp q0, q0, [x0]
; NOSTRICTALIGN-NEXT:    ret
;
; STRICTALIGN-LABEL: merge_zr64_2:
; STRICTALIGN:       // %bb.0: // %entry
; STRICTALIGN-NEXT:    stp xzr, xzr, [x0]
; STRICTALIGN-NEXT:    stp xzr, xzr, [x0, #16]
; STRICTALIGN-NEXT:    ret
entry:
  store i64 0, ptr %p
  %p1 = getelementptr i64, ptr %p, i64 1
  store i64 0, ptr %p1
  %p2 = getelementptr i64, ptr %p, i64 2
  store i64 0, ptr %p2
  %p3 = getelementptr i64, ptr %p, i64 3
  store i64 0, ptr %p3
  ret void
}

; Like merge_zr64, but with 2-vector double type.
define void @merge_zr64_2vecd(ptr %p) {
; CHECK-LABEL: merge_zr64_2vecd:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    stp xzr, xzr, [x0]
; CHECK-NEXT:    ret
entry:
  store <2 x double> zeroinitializer, ptr %p
  ret void
}

; Like merge_zr64, but with 3-vector i64 type.
define void @merge_zr64_3vec(ptr %p) {
; CHECK-LABEL: merge_zr64_3vec:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    stp xzr, xzr, [x0, #8]
; CHECK-NEXT:    str xzr, [x0]
; CHECK-NEXT:    ret
entry:
  store <3 x i64> zeroinitializer, ptr %p
  ret void
}

; Like merge_zr64_2, but with 4-vector double type.
define void @merge_zr64_4vecd(ptr %p) {
; CHECK-LABEL: merge_zr64_4vecd:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    movi v0.2d, #0000000000000000
; CHECK-NEXT:    stp q0, q0, [x0]
; CHECK-NEXT:    ret
entry:
  store <4 x double> zeroinitializer, ptr %p
  ret void
}

; Verify that non-consecutive merges do not generate q0
define void @merge_multiple_128bit_stores(ptr %p) {
; NOSTRICTALIGN-LABEL: merge_multiple_128bit_stores:
; NOSTRICTALIGN:       // %bb.0: // %entry
; NOSTRICTALIGN-NEXT:    movi v0.2d, #0000000000000000
; NOSTRICTALIGN-NEXT:    str q0, [x0]
; NOSTRICTALIGN-NEXT:    stur q0, [x0, #24]
; NOSTRICTALIGN-NEXT:    str q0, [x0, #48]
; NOSTRICTALIGN-NEXT:    ret
;
; STRICTALIGN-LABEL: merge_multiple_128bit_stores:
; STRICTALIGN:       // %bb.0: // %entry
; STRICTALIGN-NEXT:    stp xzr, xzr, [x0]
; STRICTALIGN-NEXT:    stp xzr, xzr, [x0, #24]
; STRICTALIGN-NEXT:    stp xzr, xzr, [x0, #48]
; STRICTALIGN-NEXT:    ret
entry:
  store i64 0, ptr %p
  %p1 = getelementptr i64, ptr %p, i64 1
  store i64 0, ptr %p1
  %p3 = getelementptr i64, ptr %p, i64 3
  store i64 0, ptr %p3
  %p4 = getelementptr i64, ptr %p, i64 4
  store i64 0, ptr %p4
  %p6 = getelementptr i64, ptr %p, i64 6
  store i64 0, ptr %p6
  %p7 = getelementptr i64, ptr %p, i64 7
  store i64 0, ptr %p7
  ret void
}

; Verify that large stores generate stp q
define void @merge_multiple_128bit_stores_consec(ptr %p) {
; NOSTRICTALIGN-LABEL: merge_multiple_128bit_stores_consec:
; NOSTRICTALIGN:       // %bb.0: // %entry
; NOSTRICTALIGN-NEXT:    movi v0.2d, #0000000000000000
; NOSTRICTALIGN-NEXT:    stp q0, q0, [x0]
; NOSTRICTALIGN-NEXT:    stp q0, q0, [x0, #32]
; NOSTRICTALIGN-NEXT:    ret
;
; STRICTALIGN-LABEL: merge_multiple_128bit_stores_consec:
; STRICTALIGN:       // %bb.0: // %entry
; STRICTALIGN-NEXT:    stp xzr, xzr, [x0]
; STRICTALIGN-NEXT:    stp xzr, xzr, [x0, #16]
; STRICTALIGN-NEXT:    stp xzr, xzr, [x0, #32]
; STRICTALIGN-NEXT:    stp xzr, xzr, [x0, #48]
; STRICTALIGN-NEXT:    ret
entry:
  store i64 0, ptr %p
  %p1 = getelementptr i64, ptr %p, i64 1
  store i64 0, ptr %p1
  %p2 = getelementptr i64, ptr %p, i64 2
  store i64 0, ptr %p2
  %p3 = getelementptr i64, ptr %p, i64 3
  store i64 0, ptr %p3
  %p4 = getelementptr i64, ptr %p, i64 4
  store i64 0, ptr %p4
  %p5 = getelementptr i64, ptr %p, i64 5
  store i64 0, ptr %p5
  %p6 = getelementptr i64, ptr %p, i64 6
  store i64 0, ptr %p6
  %p7 = getelementptr i64, ptr %p, i64 7
  store i64 0, ptr %p7
  ret void
}

; Check for bug 34674 where invalid add of xzr was being generated.
define i64 @bug34674(ptr %p) {
; CHECK-LABEL: bug34674:
; CHECK:       // %bb.0: // %entry
; CHECK-NEXT:    mov x9, xzr
; CHECK-NEXT:    mov x8, x0
; CHECK-NEXT:    add x0, x9, #1
; CHECK-NEXT:    stp xzr, xzr, [x8]
; CHECK-NEXT:    ret
entry:
  store <2 x i64> zeroinitializer, ptr %p
  %ld = load i64, ptr %p
  %add = add i64 %ld, 1
  ret i64 %add
}

define void @trunc_splat_zero(ptr %ptr) {
; CHECK-LABEL: trunc_splat_zero:
; CHECK:       // %bb.0:
; CHECK-NEXT:    strh wzr, [x0]
; CHECK-NEXT:    ret
  store <2 x i8> zeroinitializer, ptr %ptr, align 2
  ret void
}

define void @trunc_splat(ptr %ptr) {
; CHECK-LABEL: trunc_splat:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w8, #42 // =0x2a
; CHECK-NEXT:    movk w8, #42, lsl #16
; CHECK-NEXT:    str w8, [x0]
; CHECK-NEXT:    ret
  store <2 x i16> <i16 42, i16 42>, ptr %ptr, align 4
  ret void
}