; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
; RUN: llc < %s -mtriple=aarch64 -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI

define i32 @haddu_base(i32 %src1, i32 %src2) {
; CHECK-SD-LABEL: haddu_base:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    mov w8, w0
; CHECK-SD-NEXT:    add x8, x8, w1, uxtw
; CHECK-SD-NEXT:    lsr x0, x8, #1
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: haddu_base:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    mov w8, w1
; CHECK-GI-NEXT:    add x8, x8, w0, uxtw
; CHECK-GI-NEXT:    lsr x0, x8, #1
; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-GI-NEXT:    ret
  %zextsrc1 = zext i32 %src1 to i64
  %zextsrc2 = zext i32 %src2 to i64
  %add = add i64 %zextsrc1, %zextsrc2
  %resulti32 = lshr i64 %add, 1
  %result = trunc i64 %resulti32 to i32
  ret i32 %result
}

define i32 @haddu_const(i32 %src1) {
; CHECK-LABEL: haddu_const:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w8, w0
; CHECK-NEXT:    add x8, x8, #1
; CHECK-NEXT:    lsr x0, x8, #1
; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-NEXT:    ret
  %zextsrc1 = zext i32 %src1 to i64
  %add = add i64 %zextsrc1, 1
  %resulti32 = lshr i64 %add, 1
  %result = trunc i64 %resulti32 to i32
  ret i32 %result
}

define i32 @haddu_const_lhs(i32 %src1) {
; CHECK-LABEL: haddu_const_lhs:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w8, w0
; CHECK-NEXT:    add x8, x8, #1
; CHECK-NEXT:    lsr x0, x8, #1
; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-NEXT:    ret
  %zextsrc1 = zext i32 %src1 to i64
  %add = add i64 1, %zextsrc1
  %resulti32 = lshr i64 %add, 1
  %result = trunc i64 %resulti32 to i32
  ret i32 %result
}

define i32 @haddu_const_zero(i32 %src1) {
; CHECK-LABEL: haddu_const_zero:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w8, w0
; CHECK-NEXT:    lsr x0, x8, #1
; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-NEXT:    ret
  %zextsrc1 = zext i32 %src1 to i64
  %add = add i64 0, %zextsrc1
  %resulti32 = lshr i64 %add, 1
  %result = trunc i64 %resulti32 to i32
  ret i32 %result
}

define i32 @haddu_poison(i32 %src1) {
; CHECK-SD-LABEL: haddu_poison:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    mov w0, wzr
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: haddu_poison:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    mov w8, w0
; CHECK-GI-NEXT:    lsr x0, x8, #1
; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-GI-NEXT:    ret
  %zextsrc1 = zext i32 %src1 to i64
  %zextsrc2 = zext i32 poison to i64
  %add = add i64 %zextsrc2, %zextsrc1
  %resulti32 = lshr i64 %add, 1
  %result = trunc i64 %resulti32 to i32
  ret i32 %result
}



define i32 @hadds_base(i32 %src1, i32 %src2) {
; CHECK-SD-LABEL: hadds_base:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
; CHECK-SD-NEXT:    sxtw x8, w0
; CHECK-SD-NEXT:    add x8, x8, w1, sxtw
; CHECK-SD-NEXT:    lsr x0, x8, #1
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: hadds_base:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
; CHECK-GI-NEXT:    sxtw x8, w1
; CHECK-GI-NEXT:    add x8, x8, w0, sxtw
; CHECK-GI-NEXT:    asr x0, x8, #1
; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-GI-NEXT:    ret
  %zextsrc1 = sext i32 %src1 to i64
  %zextsrc2 = sext i32 %src2 to i64
  %add = add i64 %zextsrc1, %zextsrc2
  %resulti32 = ashr i64 %add, 1
  %result = trunc i64 %resulti32 to i32
  ret i32 %result
}

define i32 @hadds_const(i32 %src1) {
; CHECK-SD-LABEL: hadds_const:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
; CHECK-SD-NEXT:    sxtw x8, w0
; CHECK-SD-NEXT:    add x8, x8, #1
; CHECK-SD-NEXT:    lsr x0, x8, #1
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: hadds_const:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
; CHECK-GI-NEXT:    sxtw x8, w0
; CHECK-GI-NEXT:    add x8, x8, #1
; CHECK-GI-NEXT:    asr x0, x8, #1
; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-GI-NEXT:    ret
  %zextsrc1 = sext i32 %src1 to i64
  %add = add i64 %zextsrc1, 1
  %resulti32 = ashr i64 %add, 1
  %result = trunc i64 %resulti32 to i32
  ret i32 %result
}

define i32 @hadds_const_lhs(i32 %src1) {
; CHECK-SD-LABEL: hadds_const_lhs:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
; CHECK-SD-NEXT:    sxtw x8, w0
; CHECK-SD-NEXT:    add x8, x8, #1
; CHECK-SD-NEXT:    lsr x0, x8, #1
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: hadds_const_lhs:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
; CHECK-GI-NEXT:    sxtw x8, w0
; CHECK-GI-NEXT:    add x8, x8, #1
; CHECK-GI-NEXT:    asr x0, x8, #1
; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-GI-NEXT:    ret
  %zextsrc1 = sext i32 %src1 to i64
  %add = add i64 1, %zextsrc1
  %resulti32 = ashr i64 %add, 1
  %result = trunc i64 %resulti32 to i32
  ret i32 %result
}

define i32 @hadds_const_zero(i32 %src1) {
; CHECK-SD-LABEL: hadds_const_zero:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
; CHECK-SD-NEXT:    sxtw x8, w0
; CHECK-SD-NEXT:    lsr x0, x8, #1
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: hadds_const_zero:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
; CHECK-GI-NEXT:    sbfx x0, x0, #1, #31
; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-GI-NEXT:    ret
  %zextsrc1 = sext i32 %src1 to i64
  %add = add i64 0, %zextsrc1
  %resulti32 = ashr i64 %add, 1
  %result = trunc i64 %resulti32 to i32
  ret i32 %result
}

define i32 @hadds_poison(i32 %src1) {
; CHECK-SD-LABEL: hadds_poison:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    mov w0, wzr
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: hadds_poison:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
; CHECK-GI-NEXT:    sbfx x0, x0, #1, #31
; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-GI-NEXT:    ret
  %zextsrc1 = sext i32 %src1 to i64
  %zextsrc2 = sext i32 poison to i64
  %add = add i64 %zextsrc2, %zextsrc1
  %resulti32 = ashr i64 %add, 1
  %result = trunc i64 %resulti32 to i32
  ret i32 %result
}

define i32 @sub_i64(i32 %a0, i32 %a1)  {
; CHECK-SD-LABEL: sub_i64:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    mov w8, w0
; CHECK-SD-NEXT:    add x8, x8, w1, uxtw
; CHECK-SD-NEXT:    add x8, x8, #1
; CHECK-SD-NEXT:    lsr x0, x8, #1
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: sub_i64:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    orr w8, w0, w1
; CHECK-GI-NEXT:    eor w9, w0, w1
; CHECK-GI-NEXT:    sub w0, w8, w9, lsr #1
; CHECK-GI-NEXT:    ret
  %or = or i32 %a0, %a1
  %xor = xor i32 %a0, %a1
  %srl = lshr i32 %xor, 1
  %res = sub i32 %or, %srl
  ret i32 %res
}

define i32 @srhadd_i32(i32 %a0, i32 %a1)  {
; CHECK-SD-LABEL: srhadd_i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
; CHECK-SD-NEXT:    sxtw x8, w0
; CHECK-SD-NEXT:    add x8, x8, w1, sxtw
; CHECK-SD-NEXT:    add x8, x8, #1
; CHECK-SD-NEXT:    lsr x0, x8, #1
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: srhadd_i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    orr w8, w0, w1
; CHECK-GI-NEXT:    eor w9, w0, w1
; CHECK-GI-NEXT:    sub w0, w8, w9, asr #1
; CHECK-GI-NEXT:    ret
  %or = or i32 %a0, %a1
  %xor = xor i32 %a0, %a1
  %srl = ashr i32 %xor, 1
  %res = sub i32 %or, %srl
  ret i32 %res
}

define i32 @rhaddu_base(i32 %src1, i32 %src2) {
; CHECK-SD-LABEL: rhaddu_base:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    mov w8, w0
; CHECK-SD-NEXT:    add x8, x8, w1, uxtw
; CHECK-SD-NEXT:    add x8, x8, #1
; CHECK-SD-NEXT:    lsr x0, x8, #1
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rhaddu_base:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    mov w8, w1
; CHECK-GI-NEXT:    add x8, x8, w0, uxtw
; CHECK-GI-NEXT:    add x8, x8, #1
; CHECK-GI-NEXT:    lsr x0, x8, #1
; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-GI-NEXT:    ret
  %zextsrc1 = zext i32 %src1 to i64
  %zextsrc2 = zext i32 %src2 to i64
  %add1 = add i64 %zextsrc1, %zextsrc2
  %add = add i64 %add1, 1
  %resulti32 = lshr i64 %add, 1
  %result = trunc i64 %resulti32 to i32
  ret i32 %result
}

define i32 @rhaddu_const(i32 %src1) {
; CHECK-LABEL: rhaddu_const:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w8, w0
; CHECK-NEXT:    add x8, x8, #2
; CHECK-NEXT:    lsr x0, x8, #1
; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-NEXT:    ret
  %zextsrc1 = zext i32 %src1 to i64
  %add1 = add i64 %zextsrc1, 1
  %add = add i64 %add1, 1
  %resulti32 = lshr i64 %add, 1
  %result = trunc i64 %resulti32 to i32
  ret i32 %result
}

define i32 @rhaddu_const_lhs(i32 %src1) {
; CHECK-LABEL: rhaddu_const_lhs:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w8, w0
; CHECK-NEXT:    add x8, x8, #2
; CHECK-NEXT:    lsr x0, x8, #1
; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-NEXT:    ret
  %zextsrc1 = zext i32 %src1 to i64
  %add1 = add i64 1, %zextsrc1
  %add = add i64 %add1, 1
  %resulti32 = lshr i64 %add, 1
  %result = trunc i64 %resulti32 to i32
  ret i32 %result
}

define i32 @rhaddu_const_zero(i32 %src1) {
; CHECK-LABEL: rhaddu_const_zero:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w8, w0
; CHECK-NEXT:    add x8, x8, #1
; CHECK-NEXT:    lsr x0, x8, #1
; CHECK-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-NEXT:    ret
  %zextsrc1 = zext i32 %src1 to i64
  %add1 = add i64 0, %zextsrc1
  %add = add i64 %add1, 1
  %resulti32 = lshr i64 %add, 1
  %result = trunc i64 %resulti32 to i32
  ret i32 %result
}

define i32 @rhaddu_poison(i32 %src1) {
; CHECK-SD-LABEL: rhaddu_poison:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    mov w0, wzr
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rhaddu_poison:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    mov w8, w0
; CHECK-GI-NEXT:    add x8, x8, #1
; CHECK-GI-NEXT:    lsr x0, x8, #1
; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-GI-NEXT:    ret
  %zextsrc1 = zext i32 %src1 to i64
  %zextsrc2 = zext i32 poison to i64
  %add1 = add i64 %zextsrc2, %zextsrc1
  %add = add i64 %add1, 1
  %resulti32 = lshr i64 %add, 1
  %result = trunc i64 %resulti32 to i32
  ret i32 %result
}





define i32 @rhadds_base(i32 %src1, i32 %src2) {
; CHECK-SD-LABEL: rhadds_base:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
; CHECK-SD-NEXT:    sxtw x8, w0
; CHECK-SD-NEXT:    add x8, x8, w1, sxtw
; CHECK-SD-NEXT:    add x8, x8, #1
; CHECK-SD-NEXT:    lsr x0, x8, #1
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rhadds_base:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $w1 killed $w1 def $x1
; CHECK-GI-NEXT:    sxtw x8, w1
; CHECK-GI-NEXT:    add x8, x8, w0, sxtw
; CHECK-GI-NEXT:    add x8, x8, #1
; CHECK-GI-NEXT:    asr x0, x8, #1
; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-GI-NEXT:    ret
  %zextsrc1 = sext i32 %src1 to i64
  %zextsrc2 = sext i32 %src2 to i64
  %add1 = add i64 %zextsrc1, %zextsrc2
  %add = add i64 %add1, 1
  %resulti32 = ashr i64 %add, 1
  %result = trunc i64 %resulti32 to i32
  ret i32 %result
}

define i32 @rhadds_const(i32 %src1) {
; CHECK-SD-LABEL: rhadds_const:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
; CHECK-SD-NEXT:    sxtw x8, w0
; CHECK-SD-NEXT:    add x8, x8, #2
; CHECK-SD-NEXT:    lsr x0, x8, #1
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rhadds_const:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
; CHECK-GI-NEXT:    sxtw x8, w0
; CHECK-GI-NEXT:    add x8, x8, #2
; CHECK-GI-NEXT:    asr x0, x8, #1
; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-GI-NEXT:    ret
  %zextsrc1 = sext i32 %src1 to i64
  %add1 = add i64 %zextsrc1, 1
  %add = add i64 %add1, 1
  %resulti32 = ashr i64 %add, 1
  %result = trunc i64 %resulti32 to i32
  ret i32 %result
}

define i32 @rhadds_const_lhs(i32 %src1) {
; CHECK-SD-LABEL: rhadds_const_lhs:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
; CHECK-SD-NEXT:    sxtw x8, w0
; CHECK-SD-NEXT:    add x8, x8, #2
; CHECK-SD-NEXT:    lsr x0, x8, #1
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rhadds_const_lhs:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
; CHECK-GI-NEXT:    sxtw x8, w0
; CHECK-GI-NEXT:    add x8, x8, #2
; CHECK-GI-NEXT:    asr x0, x8, #1
; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-GI-NEXT:    ret
  %zextsrc1 = sext i32 %src1 to i64
  %add1 = add i64 1, %zextsrc1
  %add = add i64 %add1, 1
  %resulti32 = ashr i64 %add, 1
  %result = trunc i64 %resulti32 to i32
  ret i32 %result
}

define i32 @rhadds_const_zero(i32 %src1) {
; CHECK-SD-LABEL: rhadds_const_zero:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
; CHECK-SD-NEXT:    sxtw x8, w0
; CHECK-SD-NEXT:    add x8, x8, #1
; CHECK-SD-NEXT:    lsr x0, x8, #1
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rhadds_const_zero:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
; CHECK-GI-NEXT:    sxtw x8, w0
; CHECK-GI-NEXT:    add x8, x8, #1
; CHECK-GI-NEXT:    asr x0, x8, #1
; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-GI-NEXT:    ret
  %zextsrc1 = sext i32 %src1 to i64
  %add1 = add i64 0, %zextsrc1
  %add = add i64 %add1, 1
  %resulti32 = ashr i64 %add, 1
  %result = trunc i64 %resulti32 to i32
  ret i32 %result
}

define i32 @rhadds_const_both() {
; CHECK-LABEL: rhadds_const_both:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w0, #2 // =0x2
; CHECK-NEXT:    ret
  %add1 = add i64 3, 1
  %add = add i64 %add1, 1
  %resulti32 = ashr i64 %add, 1
  %result = trunc i64 %resulti32 to i32
  ret i32 %result
}

define i32 @rhadds_const_bothhigh() {
; CHECK-LABEL: rhadds_const_bothhigh:
; CHECK:       // %bb.0:
; CHECK-NEXT:    mov w0, #32767 // =0x7fff
; CHECK-NEXT:    ret
  %ext1 = sext i32 32766 to i64
  %ext2 = sext i32 32767 to i64
  %add1 = add i64 %ext1, %ext2
  %add = add i64 %add1, 1
  %resulti32 = ashr i64 %add, 1
  %result = trunc i64 %resulti32 to i32
  ret i32 %result
}

define i32 @rhadds_poison(i32 %src1) {
; CHECK-SD-LABEL: rhadds_poison:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    mov w0, wzr
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: rhadds_poison:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 def $x0
; CHECK-GI-NEXT:    sxtw x8, w0
; CHECK-GI-NEXT:    add x8, x8, #1
; CHECK-GI-NEXT:    asr x0, x8, #1
; CHECK-GI-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-GI-NEXT:    ret
  %zextsrc1 = sext i32 %src1 to i64
  %zextsrc2 = sext i32 poison to i64
  %add1 = add i64 %zextsrc2, %zextsrc1
  %add = add i64 %add1, 1
  %resulti32 = ashr i64 %add, 1
  %result = trunc i64 %resulti32 to i32
  ret i32 %result
}


define i32 @uhadd_i64(i32 %a0, i32 %a1)  {
; CHECK-SD-LABEL: uhadd_i64:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    mov w8, w0
; CHECK-SD-NEXT:    add x8, x8, w1, uxtw
; CHECK-SD-NEXT:    lsr x0, x8, #1
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: uhadd_i64:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    and w8, w0, w1
; CHECK-GI-NEXT:    eor w9, w0, w1
; CHECK-GI-NEXT:    add w0, w8, w9, lsr #1
; CHECK-GI-NEXT:    ret
  %and = and i32 %a0, %a1
  %xor = xor i32 %a0, %a1
  %srl = lshr i32 %xor, 1
  %res = add i32 %and, %srl
  ret i32 %res
}

define i32 @shadd_i32(i32 %a0, i32 %a1)  {
; CHECK-SD-LABEL: shadd_i32:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
; CHECK-SD-NEXT:    sxtw x8, w0
; CHECK-SD-NEXT:    add x8, x8, w1, sxtw
; CHECK-SD-NEXT:    lsr x0, x8, #1
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: shadd_i32:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    and w8, w0, w1
; CHECK-GI-NEXT:    eor w9, w0, w1
; CHECK-GI-NEXT:    add w0, w8, w9, asr #1
; CHECK-GI-NEXT:    ret
  %and = and i32 %a0, %a1
  %xor = xor i32 %a0, %a1
  %srl = ashr i32 %xor, 1
  %res = add i32 %and, %srl
  ret i32 %res
}

define i64 @shadd_signbits_i64(i64 %a0, i64 %a1, ptr %p2) {
; CHECK-SD-LABEL: shadd_signbits_i64:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    asr x8, x0, #17
; CHECK-SD-NEXT:    add x8, x8, x1, asr #17
; CHECK-SD-NEXT:    asr x0, x8, #1
; CHECK-SD-NEXT:    str x0, [x2]
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: shadd_signbits_i64:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    asr x8, x0, #17
; CHECK-GI-NEXT:    asr x9, x1, #17
; CHECK-GI-NEXT:    and x10, x8, x9
; CHECK-GI-NEXT:    eor x8, x8, x9
; CHECK-GI-NEXT:    add x8, x10, x8, asr #1
; CHECK-GI-NEXT:    sbfx x0, x8, #0, #47
; CHECK-GI-NEXT:    str x8, [x2]
; CHECK-GI-NEXT:    ret
  %x0 = ashr i64 %a0, 17
  %x1 = ashr i64 %a1, 17
  %m = and i64 %x0, %x1
  %s = xor i64 %x0, %x1
  %x = ashr i64 %s, 1
  %avg = add i64 %m, %x
  %avg1 = shl i64 %avg, 17
  %avg2 = ashr i64 %avg1, 17
  store i64 %avg, ptr %p2 ; extra use
  ret i64 %avg2
}

define i64 @srhadd_signbits_i64(i64 %a0, i64 %a1, ptr %p2) {
; CHECK-SD-LABEL: srhadd_signbits_i64:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    asr x8, x0, #17
; CHECK-SD-NEXT:    add x8, x8, x1, asr #17
; CHECK-SD-NEXT:    add x8, x8, #1
; CHECK-SD-NEXT:    asr x0, x8, #1
; CHECK-SD-NEXT:    str x0, [x2]
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: srhadd_signbits_i64:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    asr x8, x0, #17
; CHECK-GI-NEXT:    asr x9, x1, #17
; CHECK-GI-NEXT:    orr x10, x8, x9
; CHECK-GI-NEXT:    eor x8, x8, x9
; CHECK-GI-NEXT:    sub x8, x10, x8, asr #1
; CHECK-GI-NEXT:    sbfx x0, x8, #0, #47
; CHECK-GI-NEXT:    str x8, [x2]
; CHECK-GI-NEXT:    ret
  %x0 = ashr i64 %a0, 17
  %x1 = ashr i64 %a1, 17
  %m = or i64 %x0, %x1
  %s = xor i64 %x0, %x1
  %x = ashr i64 %s, 1
  %avg = sub i64 %m, %x
  %avg1 = shl i64 %avg, 17
  %avg2 = ashr i64 %avg1, 17
  store i64 %avg, ptr %p2 ; extra use
  ret i64 %avg2
}

; negative test - not enough signbits to remove sign_extend_inreg after srhadd
define i64 @srhadd_signbits_i64_negative(i64 %a0, i64 %a1, ptr %p2) {
; CHECK-SD-LABEL: srhadd_signbits_i64_negative:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    asr x8, x0, #17
; CHECK-SD-NEXT:    add x8, x8, x1, asr #17
; CHECK-SD-NEXT:    add x8, x8, #1
; CHECK-SD-NEXT:    asr x8, x8, #1
; CHECK-SD-NEXT:    sbfx x0, x8, #0, #42
; CHECK-SD-NEXT:    str x8, [x2]
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: srhadd_signbits_i64_negative:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    asr x8, x0, #17
; CHECK-GI-NEXT:    asr x9, x1, #17
; CHECK-GI-NEXT:    orr x10, x8, x9
; CHECK-GI-NEXT:    eor x8, x8, x9
; CHECK-GI-NEXT:    sub x8, x10, x8, asr #1
; CHECK-GI-NEXT:    sbfx x0, x8, #0, #42
; CHECK-GI-NEXT:    str x8, [x2]
; CHECK-GI-NEXT:    ret
  %x0 = ashr i64 %a0, 17
  %x1 = ashr i64 %a1, 17
  %m = or i64 %x0, %x1
  %s = xor i64 %x0, %x1
  %x = ashr i64 %s, 1
  %avg = sub i64 %m, %x
  %avg1 = shl i64 %avg, 22
  %avg2 = ashr i64 %avg1, 22
  store i64 %avg, ptr %p2 ; extra use
  ret i64 %avg2
}


define i32 @haddu_nuw(i32 %src1, i32 %src2) {
; CHECK-LABEL: haddu_nuw:
; CHECK:       // %bb.0:
; CHECK-NEXT:    add w8, w0, w1
; CHECK-NEXT:    lsr w0, w8, #1
; CHECK-NEXT:    ret
  %add = add nuw i32 %src1, %src2
  %result = lshr i32 %add, 1
  ret i32 %result
}

define i32 @hadds_nsw(i32 %src1, i32 %src2) {
; CHECK-LABEL: hadds_nsw:
; CHECK:       // %bb.0:
; CHECK-NEXT:    add w8, w0, w1
; CHECK-NEXT:    asr w0, w8, #1
; CHECK-NEXT:    ret
  %add = add nsw i32 %src1, %src2
  %result = ashr i32 %add, 1
  ret i32 %result
}

define i32 @hraddu_nuw(i32 %src1, i32 %src2) {
; CHECK-LABEL: hraddu_nuw:
; CHECK:       // %bb.0:
; CHECK-NEXT:    add w8, w0, w1
; CHECK-NEXT:    add w8, w8, #1
; CHECK-NEXT:    lsr w0, w8, #1
; CHECK-NEXT:    ret
  %add = add nuw i32 %src1, %src2
  %add2 = add nsw i32 %add, 1
  %result = lshr i32 %add2, 1
  ret i32 %result
}

define i32 @hradds_nsw(i32 %src1, i32 %src2) {
; CHECK-LABEL: hradds_nsw:
; CHECK:       // %bb.0:
; CHECK-NEXT:    add w8, w0, w1
; CHECK-NEXT:    add w8, w8, #1
; CHECK-NEXT:    asr w0, w8, #1
; CHECK-NEXT:    ret
  %add = add nsw i32 %src1, %src2
  %add2 = add nsw i32 %add, 1
  %result = ashr i32 %add2, 1
  ret i32 %result
}


; avgfloor(A, B) from (A & B) + ((A ^ B) >> 1)
define i32 @avgflooru_from_andaddshift(i32 %a, i32 %b)  {
; CHECK-SD-LABEL: avgflooru_from_andaddshift:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    mov w8, w0
; CHECK-SD-NEXT:    add x8, x8, w1, uxtw
; CHECK-SD-NEXT:    lsr x0, x8, #1
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: avgflooru_from_andaddshift:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    eor w8, w0, w1
; CHECK-GI-NEXT:    and w9, w0, w1
; CHECK-GI-NEXT:    add w0, w9, w8, lsr #1
; CHECK-GI-NEXT:    ret
  %c = xor i32 %a, %b
  %d = lshr i32 %c, 1
  %e = and i32 %a, %b
  %f = add i32 %d, %e
  ret i32 %f
}

; avgfloor(A, B) from (A & B) + ((A ^ B) >> 1)
define i32 @avgfloors_from_andaddshift(i32 %a, i32 %b)  {
; CHECK-SD-LABEL: avgfloors_from_andaddshift:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
; CHECK-SD-NEXT:    sxtw x8, w0
; CHECK-SD-NEXT:    add x8, x8, w1, sxtw
; CHECK-SD-NEXT:    lsr x0, x8, #1
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: avgfloors_from_andaddshift:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    eor w8, w0, w1
; CHECK-GI-NEXT:    and w9, w0, w1
; CHECK-GI-NEXT:    add w0, w9, w8, asr #1
; CHECK-GI-NEXT:    ret
  %c = xor i32 %a, %b
  %d = ashr i32 %c, 1
  %e = and i32 %a, %b
  %f = add i32 %d, %e
  ret i32 %f
}

; avgfloor(A, B) from ((A >> 1) + (B >> 1)) + (A & B & 1)
define i32 @avgflooru_from_shiftaddand(i32 %a, i32 %b)  {
; CHECK-SD-LABEL: avgflooru_from_shiftaddand:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    mov w8, w0
; CHECK-SD-NEXT:    add x8, x8, w1, uxtw
; CHECK-SD-NEXT:    lsr x0, x8, #1
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: avgflooru_from_shiftaddand:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    lsr w8, w1, #1
; CHECK-GI-NEXT:    and w9, w0, w1
; CHECK-GI-NEXT:    and w9, w9, #0x1
; CHECK-GI-NEXT:    add w8, w8, w0, lsr #1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
  %c = lshr i32 %a, 1
  %d = lshr i32 %b, 1
  %e = add i32 %c, %d
  %f = and i32 %a, %b
  %g = and i32 %f, 1
  %h = add i32 %e, %g
  ret i32 %h
}

; avgfloor(A, B) from ((A >> 1) + (B >> 1)) + (A & B & 1)
define i32 @avgfloors_from_shiftaddand(i32 %a, i32 %b)  {
; CHECK-SD-LABEL: avgfloors_from_shiftaddand:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
; CHECK-SD-NEXT:    sxtw x8, w0
; CHECK-SD-NEXT:    add x8, x8, w1, sxtw
; CHECK-SD-NEXT:    lsr x0, x8, #1
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: avgfloors_from_shiftaddand:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    asr w8, w1, #1
; CHECK-GI-NEXT:    and w9, w0, w1
; CHECK-GI-NEXT:    and w9, w9, #0x1
; CHECK-GI-NEXT:    add w8, w8, w0, asr #1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
  %c = ashr i32 %a, 1
  %d = ashr i32 %b, 1
  %e = add i32 %c, %d
  %f = and i32 %a, %b
  %g = and i32 %f, 1
  %h = add i32 %e, %g
  ret i32 %h
}

; avgceil(A, B) from ((A >> 1) + (B >> 1)) + ((A | B) & 1)
define i32 @avgceilu_from_shiftaddand(i32 %a, i32 %b)  {
; CHECK-SD-LABEL: avgceilu_from_shiftaddand:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    mov w8, w0
; CHECK-SD-NEXT:    add x8, x8, w1, uxtw
; CHECK-SD-NEXT:    add x8, x8, #1
; CHECK-SD-NEXT:    lsr x0, x8, #1
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: avgceilu_from_shiftaddand:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    lsr w8, w1, #1
; CHECK-GI-NEXT:    orr w9, w0, w1
; CHECK-GI-NEXT:    and w9, w9, #0x1
; CHECK-GI-NEXT:    add w8, w8, w0, lsr #1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
  %c = lshr i32 %a, 1
  %d = lshr i32 %b, 1
  %e = add i32 %c, %d
  %f = or i32 %a, %b
  %g = and i32 %f, 1
  %h = add i32 %e, %g
  ret i32 %h
}


; avgceil(A, B) from ((A >> 1) + (B >> 1)) + ((A | B) & 1)
define i32 @avgceils_from_shiftaddand(i32 %a, i32 %b)  {
; CHECK-SD-LABEL: avgceils_from_shiftaddand:
; CHECK-SD:       // %bb.0:
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 def $x0
; CHECK-SD-NEXT:    sxtw x8, w0
; CHECK-SD-NEXT:    add x8, x8, w1, sxtw
; CHECK-SD-NEXT:    add x8, x8, #1
; CHECK-SD-NEXT:    lsr x0, x8, #1
; CHECK-SD-NEXT:    // kill: def $w0 killed $w0 killed $x0
; CHECK-SD-NEXT:    ret
;
; CHECK-GI-LABEL: avgceils_from_shiftaddand:
; CHECK-GI:       // %bb.0:
; CHECK-GI-NEXT:    asr w8, w1, #1
; CHECK-GI-NEXT:    orr w9, w0, w1
; CHECK-GI-NEXT:    and w9, w9, #0x1
; CHECK-GI-NEXT:    add w8, w8, w0, asr #1
; CHECK-GI-NEXT:    add w0, w8, w9
; CHECK-GI-NEXT:    ret
  %c = ashr i32 %a, 1
  %d = ashr i32 %b, 1
  %e = add i32 %c, %d
  %f = or i32 %a, %b
  %g = and i32 %f, 1
  %h = add i32 %e, %g
  ret i32 %h
}