; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2

declare <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16>, <8 x i16>) nounwind readnone
declare <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8>, <16 x i8>) nounwind readnone

define <4 x i32> @combine_pmaddwd_zero(<8 x i16> %a0, <8 x i16> %a1) {
; SSE-LABEL: combine_pmaddwd_zero:
; SSE:       # %bb.0:
; SSE-NEXT:    xorps %xmm0, %xmm0
; SSE-NEXT:    retq
;
; AVX-LABEL: combine_pmaddwd_zero:
; AVX:       # %bb.0:
; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
; AVX-NEXT:    retq
  %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a0, <8 x i16> zeroinitializer)
  ret <4 x i32> %1
}

define <4 x i32> @combine_pmaddwd_zero_commute(<8 x i16> %a0, <8 x i16> %a1) {
; SSE-LABEL: combine_pmaddwd_zero_commute:
; SSE:       # %bb.0:
; SSE-NEXT:    xorps %xmm0, %xmm0
; SSE-NEXT:    retq
;
; AVX-LABEL: combine_pmaddwd_zero_commute:
; AVX:       # %bb.0:
; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
; AVX-NEXT:    retq
  %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> zeroinitializer, <8 x i16> %a0)
  ret <4 x i32> %1
}

define <8 x i32> @combine_pmaddwd_concat(<8 x i16> %a0, <8 x i16> %a1, <8 x i16> %a2, <8 x i16> %a3) {
; SSE-LABEL: combine_pmaddwd_concat:
; SSE:       # %bb.0:
; SSE-NEXT:    pmaddwd %xmm1, %xmm0
; SSE-NEXT:    pmaddwd %xmm3, %xmm2
; SSE-NEXT:    movdqa %xmm2, %xmm1
; SSE-NEXT:    retq
;
; AVX1-LABEL: combine_pmaddwd_concat:
; AVX1:       # %bb.0:
; AVX1-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm0
; AVX1-NEXT:    vpmaddwd %xmm3, %xmm2, %xmm1
; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX1-NEXT:    retq
;
; AVX2-LABEL: combine_pmaddwd_concat:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm0
; AVX2-NEXT:    vpmaddwd %xmm3, %xmm2, %xmm1
; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
; AVX2-NEXT:    retq
  %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a0, <8 x i16> %a1)
  %2 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a2, <8 x i16> %a3)
  %3 = shufflevector <4 x i32> %1, <4 x i32> %2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
  ret <8 x i32> %3
}

define <8 x i32> @combine_pmaddwd_concat_freeze(<8 x i16> %a0, <8 x i16> %a1) {
; SSE-LABEL: combine_pmaddwd_concat_freeze:
; SSE:       # %bb.0:
; SSE-NEXT:    pmovsxbw {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1]
; SSE-NEXT:    pmaddwd %xmm2, %xmm0
; SSE-NEXT:    pmaddwd %xmm2, %xmm1
; SSE-NEXT:    retq
;
; AVX1-LABEL: combine_pmaddwd_concat_freeze:
; AVX1:       # %bb.0:
; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1]
; AVX1-NEXT:    vpmaddwd %xmm2, %xmm0, %xmm0
; AVX1-NEXT:    vpmaddwd %xmm2, %xmm1, %xmm1
; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX1-NEXT:    retq
;
; AVX2-LABEL: combine_pmaddwd_concat_freeze:
; AVX2:       # %bb.0:
; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
; AVX2-NEXT:    vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; AVX2-NEXT:    retq
  %lo = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a0, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)
  %hi = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %a1, <8 x i16> <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>)
  %flo = freeze <4 x i32> %lo
  %fhi = freeze <4 x i32> %hi
  %res = shufflevector <4 x i32> %flo, <4 x i32> %fhi, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
  ret <8 x i32> %res
}

define <4 x i32> @combine_pmaddwd_demandedelts(<8 x i16> %a0, <8 x i16> %a1) {
; SSE-LABEL: combine_pmaddwd_demandedelts:
; SSE:       # %bb.0:
; SSE-NEXT:    pmaddwd %xmm1, %xmm0
; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
; SSE-NEXT:    retq
;
; AVX1-LABEL: combine_pmaddwd_demandedelts:
; AVX1:       # %bb.0:
; AVX1-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm0
; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
; AVX1-NEXT:    retq
;
; AVX2-LABEL: combine_pmaddwd_demandedelts:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpmaddwd %xmm1, %xmm0, %xmm0
; AVX2-NEXT:    vpbroadcastd %xmm0, %xmm0
; AVX2-NEXT:    retq
  %1 = shufflevector <8 x i16> %a0, <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 4, i32 4, i32 4>
  %2 = shufflevector <8 x i16> %a1, <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 7, i32 7, i32 7, i32 7>
  %3 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %1, <8 x i16> %2)
  %4 = shufflevector <4 x i32> %3, <4 x i32> poison, <4 x i32> zeroinitializer
  ret <4 x i32> %4
}

; [2]: (-5*13)+(6*-15) = -155 = 4294967141
define <4 x i32> @combine_pmaddwd_constant() {
; SSE-LABEL: combine_pmaddwd_constant:
; SSE:       # %bb.0:
; SSE-NEXT:    movaps {{.*#+}} xmm0 = [19,17,4294967141,271]
; SSE-NEXT:    retq
;
; AVX-LABEL: combine_pmaddwd_constant:
; AVX:       # %bb.0:
; AVX-NEXT:    vmovaps {{.*#+}} xmm0 = [19,17,4294967141,271]
; AVX-NEXT:    retq
  %1 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> <i16 -1, i16 2, i16 3, i16 -4, i16 -5, i16 6, i16 7, i16 -8>, <8 x i16> <i16 -5, i16 7, i16 -9, i16 -11, i16 13, i16 -15, i16 17, i16 -19>)
  ret <4 x i32> %1
}

; ensure we don't assume pmaddwd performs add nsw
; [0]: (-32768*-32768)+(-32768*-32768) = 0x80000000 = 2147483648
define <4 x i32> @combine_pmaddwd_constant_nsw() {
; SSE-LABEL: combine_pmaddwd_constant_nsw:
; SSE:       # %bb.0:
; SSE-NEXT:    movaps {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648]
; SSE-NEXT:    retq
;
; AVX-LABEL: combine_pmaddwd_constant_nsw:
; AVX:       # %bb.0:
; AVX-NEXT:    vbroadcastss {{.*#+}} xmm0 = [2147483648,2147483648,2147483648,2147483648]
; AVX-NEXT:    retq
  %1 = insertelement <8 x i16> undef, i16 32768, i32 0
  %2 = shufflevector <8 x i16> %1, <8 x i16> undef, <8 x i32> zeroinitializer
  %3 = call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> %2, <8 x i16> %2)
  ret <4 x i32> %3
}

define <8 x i16> @combine_pmaddubsw_zero(<16 x i8> %a0, <16 x i8> %a1) {
; SSE-LABEL: combine_pmaddubsw_zero:
; SSE:       # %bb.0:
; SSE-NEXT:    xorps %xmm0, %xmm0
; SSE-NEXT:    retq
;
; AVX-LABEL: combine_pmaddubsw_zero:
; AVX:       # %bb.0:
; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
; AVX-NEXT:    retq
  %1 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %a0, <16 x i8> zeroinitializer)
  ret <8 x i16> %1
}

define <8 x i16> @combine_pmaddubsw_zero_commute(<16 x i8> %a0, <16 x i8> %a1) {
; SSE-LABEL: combine_pmaddubsw_zero_commute:
; SSE:       # %bb.0:
; SSE-NEXT:    xorps %xmm0, %xmm0
; SSE-NEXT:    retq
;
; AVX-LABEL: combine_pmaddubsw_zero_commute:
; AVX:       # %bb.0:
; AVX-NEXT:    vxorps %xmm0, %xmm0, %xmm0
; AVX-NEXT:    retq
  %1 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> zeroinitializer, <16 x i8> %a0)
  ret <8 x i16> %1
}

define <16 x i16> @combine_pmaddubsw_concat(<32 x i8> %a0, <32 x i8> %a1) {
; SSE-LABEL: combine_pmaddubsw_concat:
; SSE:       # %bb.0:
; SSE-NEXT:    pmaddubsw %xmm2, %xmm0
; SSE-NEXT:    pmaddubsw %xmm3, %xmm1
; SSE-NEXT:    retq
;
; AVX1-LABEL: combine_pmaddubsw_concat:
; AVX1:       # %bb.0:
; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm2
; AVX1-NEXT:    vextractf128 $1, %ymm1, %xmm3
; AVX1-NEXT:    vpmaddubsw %xmm3, %xmm2, %xmm2
; AVX1-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm0
; AVX1-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
; AVX1-NEXT:    retq
;
; AVX2-LABEL: combine_pmaddubsw_concat:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpmaddubsw %ymm1, %ymm0, %ymm0
; AVX2-NEXT:    retq
  %lo0 = shufflevector <32 x i8> %a0, <32 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
  %lo1 = shufflevector <32 x i8> %a1, <32 x i8> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
  %hi0 = shufflevector <32 x i8> %a0, <32 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
  %hi1 = shufflevector <32 x i8> %a1, <32 x i8> undef, <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
  %lo = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %lo0, <16 x i8> %lo1)
  %hi = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %hi0, <16 x i8> %hi1)
  %res = shufflevector <8 x i16> %lo, <8 x i16> %hi, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
  ret <16 x i16> %res
}

; Not beneficial to concatenate both inputs just to create a 256-bit pmaddubsw
define <16 x i16> @combine_pmaddubsw_concat_unecessary(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %a2, <16 x i8> %a3) {
; SSE-LABEL: combine_pmaddubsw_concat_unecessary:
; SSE:       # %bb.0:
; SSE-NEXT:    pmaddubsw %xmm1, %xmm0
; SSE-NEXT:    pmaddubsw %xmm3, %xmm2
; SSE-NEXT:    movdqa %xmm2, %xmm1
; SSE-NEXT:    retq
;
; AVX1-LABEL: combine_pmaddubsw_concat_unecessary:
; AVX1:       # %bb.0:
; AVX1-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm0
; AVX1-NEXT:    vpmaddubsw %xmm3, %xmm2, %xmm1
; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX1-NEXT:    retq
;
; AVX2-LABEL: combine_pmaddubsw_concat_unecessary:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm0
; AVX2-NEXT:    vpmaddubsw %xmm3, %xmm2, %xmm1
; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
; AVX2-NEXT:    retq
  %1 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %a0, <16 x i8> %a1)
  %2 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %a2, <16 x i8> %a3)
  %3 = shufflevector <8 x i16> %1, <8 x i16> %2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
  ret <16 x i16> %3
}

define <16 x i16> @combine_pmaddubsw_concat_freeze(<16 x i8> %a0, <16 x i8> %a1) {
; SSE-LABEL: combine_pmaddubsw_concat_freeze:
; SSE:       # %bb.0:
; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; SSE-NEXT:    pmaddubsw %xmm2, %xmm0
; SSE-NEXT:    pmaddubsw %xmm2, %xmm1
; SSE-NEXT:    retq
;
; AVX1-LABEL: combine_pmaddubsw_concat_freeze:
; AVX1:       # %bb.0:
; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; AVX1-NEXT:    vpmaddubsw %xmm2, %xmm0, %xmm0
; AVX1-NEXT:    vpmaddubsw %xmm2, %xmm1, %xmm1
; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX1-NEXT:    retq
;
; AVX2-LABEL: combine_pmaddubsw_concat_freeze:
; AVX2:       # %bb.0:
; AVX2-NEXT:    # kill: def $xmm0 killed $xmm0 def $ymm0
; AVX2-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm0
; AVX2-NEXT:    vpmaddubsw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; AVX2-NEXT:    retq
  %lo = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %a0, <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>)
  %hi = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %a1, <16 x i8> <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>)
  %flo = freeze <8 x i16> %lo
  %fhi = freeze <8 x i16> %hi
  %res = shufflevector <8 x i16> %flo, <8 x i16> %fhi, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
  ret <16 x i16> %res
}

define <8 x i16> @combine_pmaddubsw_demandedelts(<16 x i8> %a0, <16 x i8> %a1) {
; SSE-LABEL: combine_pmaddubsw_demandedelts:
; SSE:       # %bb.0:
; SSE-NEXT:    pmaddubsw %xmm1, %xmm0
; SSE-NEXT:    pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
; SSE-NEXT:    retq
;
; AVX1-LABEL: combine_pmaddubsw_demandedelts:
; AVX1:       # %bb.0:
; AVX1-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm0
; AVX1-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
; AVX1-NEXT:    retq
;
; AVX2-LABEL: combine_pmaddubsw_demandedelts:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpmaddubsw %xmm1, %xmm0, %xmm0
; AVX2-NEXT:    vpbroadcastd %xmm0, %xmm0
; AVX2-NEXT:    retq
  %1 = shufflevector <16 x i8> %a0, <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8, i32 8>
  %2 = shufflevector <16 x i8> %a1, <16 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
  %3 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %1, <16 x i8> %2)
  %4 = shufflevector <8 x i16> %3, <8 x i16> poison, <8 x i32> <i32 0, i32 1, i32 0, i32 1, i32 0, i32 1, i32 0, i32 1>
  ret <8 x i16> %4
}

; [3]: ((uint16_t)-6*7)+(7*-8) = (250*7)+(7*-8) = 1694
define i32 @combine_pmaddubsw_constant() {
; CHECK-LABEL: combine_pmaddubsw_constant:
; CHECK:       # %bb.0:
; CHECK-NEXT:    movl $1694, %eax # imm = 0x69E
; CHECK-NEXT:    retq
  %1 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> <i8 0, i8 1, i8 2, i8 3, i8 4, i8 5, i8 -6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, <16 x i8> <i8 1, i8 2, i8 3, i8 4, i8 5, i8 6, i8 7, i8 -8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16>)
  %2 = extractelement <8 x i16> %1, i32 3
  %3 = sext i16 %2 to i32
  ret i32 %3
}

; [0]: add_sat_i16(((uint16_t)-1*-128),((uint16_t)-1*-128)_ = add_sat_i16(255*-128),(255*-128)) = sat_i16(-65280) = -32768
define i32 @combine_pmaddubsw_constant_sat() {
; CHECK-LABEL: combine_pmaddubsw_constant_sat:
; CHECK:       # %bb.0:
; CHECK-NEXT:    movl $-32768, %eax # imm = 0x8000
; CHECK-NEXT:    retq
  %1 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> <i8 -1, i8 -1, i8 2, i8 3, i8 4, i8 5, i8 -6, i8 7, i8 8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15>, <16 x i8> <i8 -128, i8 -128, i8 3, i8 4, i8 5, i8 6, i8 7, i8 -8, i8 9, i8 10, i8 11, i8 12, i8 13, i8 14, i8 15, i8 16>)
  %2 = extractelement <8 x i16> %1, i32 0
  %3 = sext i16 %2 to i32
  ret i32 %3
}

; Constant folding PMADDWD was causing an infinite loop in the PCMPGT commuting between 2 constant values.
define i1 @pmaddwd_pcmpgt_infinite_loop() {
; CHECK-LABEL: pmaddwd_pcmpgt_infinite_loop:
; CHECK:       # %bb.0:
; CHECK-NEXT:    movb $1, %al
; CHECK-NEXT:    retq
  %1 = tail call <4 x i32> @llvm.x86.sse2.pmadd.wd(<8 x i16> <i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768>, <8 x i16> <i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768, i16 -32768>)
  %2 = icmp eq <4 x i32> %1, <i32 -2147483648, i32 -2147483648, i32 -2147483648, i32 -2147483648>
  %3 = select <4 x i1> %2, <4 x i32> <i32 2147483647, i32 2147483647, i32 2147483647, i32 2147483647>, <4 x i32> zeroinitializer
  %4 = add <4 x i32> %3, <i32 -8, i32 -9, i32 -10, i32 -11>
  %.not = trunc <4 x i32> %3 to <4 x i1>
  %5 = icmp sgt <4 x i32> %4, <i32 2147483640, i32 2147483639, i32 2147483638, i32 2147483637>
  %6 = select <4 x i1> %.not, <4 x i1> %5, <4 x i1> zeroinitializer
  %7 = bitcast <4 x i1> %6 to i4
  %8 = icmp eq i4 %7, 0
  ret i1 %8
}

; If the shuffle matches, but there is no multiply, introduce a trivial multiply by 1.
define <8 x i32> @sext_pairwise_add(<16 x i16> %x) {
; SSE-LABEL: sext_pairwise_add:
; SSE:       # %bb.0:
; SSE-NEXT:    pmovsxbw {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1]
; SSE-NEXT:    pmaddwd %xmm2, %xmm0
; SSE-NEXT:    pmaddwd %xmm2, %xmm1
; SSE-NEXT:    retq
;
; AVX1-LABEL: sext_pairwise_add:
; AVX1:       # %bb.0:
; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [1,1,1,1,1,1,1,1]
; AVX1-NEXT:    vpmaddwd %xmm2, %xmm1, %xmm1
; AVX1-NEXT:    vpmaddwd %xmm2, %xmm0, %xmm0
; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX1-NEXT:    retq
;
; AVX2-LABEL: sext_pairwise_add:
; AVX2:       # %bb.0:
; AVX2-NEXT:    vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; AVX2-NEXT:    retq
  %1 = sext <16 x i16> %x to <16 x i32>
  %2 = shufflevector <16 x i32> %1, <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
  %3 = shufflevector <16 x i32> %1, <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
  %4 = add nsw <8 x i32> %2, %3
  ret <8 x i32> %4
}

define <8 x i32> @combine_with_mul(<16 x i16> %v) {
; SSE-LABEL: combine_with_mul:
; SSE:       # %bb.0: # %bb1
; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [4096,1,4096,1,4096,1,4096,1]
; SSE-NEXT:    pmaddwd %xmm2, %xmm0
; SSE-NEXT:    pmaddwd %xmm2, %xmm1
; SSE-NEXT:    retq
;
; AVX1-LABEL: combine_with_mul:
; AVX1:       # %bb.0: # %bb1
; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [4096,1,4096,1,4096,1,4096,1]
; AVX1-NEXT:    vpmaddwd %xmm2, %xmm1, %xmm1
; AVX1-NEXT:    vpmaddwd %xmm2, %xmm0, %xmm0
; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX1-NEXT:    retq
;
; AVX2-LABEL: combine_with_mul:
; AVX2:       # %bb.0: # %bb1
; AVX2-NEXT:    vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [4096,1,4096,1,4096,1,4096,1,4096,1,4096,1,4096,1,4096,1]
; AVX2-NEXT:    retq
bb1:
  %0 = sext <16 x i16> %v to <16 x i32>
  %1 = mul nsw <16 x i32> %0, <i32 4096, i32 1, i32 4096, i32 1, i32 4096, i32 1, i32 4096, i32 1, i32 4096, i32 1, i32 4096, i32 1, i32 4096, i32 1, i32 4096, i32 1>
  %2 = shufflevector <16 x i32> %1, <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
  %3 = shufflevector <16 x i32> %1, <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
  %4 = add <8 x i32> %2, %3
  ret <8 x i32> %4
}

define <8 x i32> @combine_with_shl(<16 x i16> %v) {
; SSE-LABEL: combine_with_shl:
; SSE:       # %bb.0: # %bb1
; SSE-NEXT:    movdqa {{.*#+}} xmm2 = [4096,1,4096,1,4096,1,4096,1]
; SSE-NEXT:    pmaddwd %xmm2, %xmm0
; SSE-NEXT:    pmaddwd %xmm2, %xmm1
; SSE-NEXT:    retq
;
; AVX1-LABEL: combine_with_shl:
; AVX1:       # %bb.0: # %bb1
; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
; AVX1-NEXT:    vbroadcastss {{.*#+}} xmm2 = [4096,1,4096,1,4096,1,4096,1]
; AVX1-NEXT:    vpmaddwd %xmm2, %xmm1, %xmm1
; AVX1-NEXT:    vpmaddwd %xmm2, %xmm0, %xmm0
; AVX1-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX1-NEXT:    retq
;
; AVX2-LABEL: combine_with_shl:
; AVX2:       # %bb.0: # %bb1
; AVX2-NEXT:    vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [4096,1,4096,1,4096,1,4096,1,4096,1,4096,1,4096,1,4096,1]
; AVX2-NEXT:    retq
bb1:
  %0 = sext <16 x i16> %v to <16 x i32>
  %1 = shl nsw <16 x i32> %0, <i32 12, i32 0, i32 12, i32 0, i32 12, i32 0, i32 12, i32 0, i32 12, i32 0, i32 12, i32 0, i32 12, i32 0, i32 12, i32 0>
  %2 = shufflevector <16 x i32> %1, <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
  %3 = shufflevector <16 x i32> %1, <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
  %4 = add <8 x i32> %2, %3
  ret <8 x i32> %4
}

; This version cannot use `vpmaddwd` because the multiply would overflow an i16.
define <8 x i32> @combine_with_shl_overflow(<16 x i16> %v) {
; SSE-LABEL: combine_with_shl_overflow:
; SSE:       # %bb.0: # %bb1
; SSE-NEXT:    pxor %xmm2, %xmm2
; SSE-NEXT:    pxor %xmm4, %xmm4
; SSE-NEXT:    punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
; SSE-NEXT:    pxor %xmm3, %xmm3
; SSE-NEXT:    punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
; SSE-NEXT:    phaddd %xmm4, %xmm3
; SSE-NEXT:    pxor %xmm1, %xmm1
; SSE-NEXT:    punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
; SSE-NEXT:    punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
; SSE-NEXT:    phaddd %xmm1, %xmm2
; SSE-NEXT:    movdqa %xmm2, %xmm0
; SSE-NEXT:    movdqa %xmm3, %xmm1
; SSE-NEXT:    retq
;
; AVX1-LABEL: combine_with_shl_overflow:
; AVX1:       # %bb.0: # %bb1
; AVX1-NEXT:    vextractf128 $1, %ymm0, %xmm1
; AVX1-NEXT:    vpxor %xmm2, %xmm2, %xmm2
; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; AVX1-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
; AVX1-NEXT:    vphaddd %xmm3, %xmm4, %xmm3
; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
; AVX1-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
; AVX1-NEXT:    vphaddd %xmm1, %xmm0, %xmm0
; AVX1-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
; AVX1-NEXT:    vinsertf128 $1, %xmm3, %ymm3, %ymm1
; AVX1-NEXT:    vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[3],ymm0[3]
; AVX1-NEXT:    retq
;
; AVX2-LABEL: combine_with_shl_overflow:
; AVX2:       # %bb.0: # %bb1
; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
; AVX2-NEXT:    vextracti128 $1, %ymm0, %xmm0
; AVX2-NEXT:    vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
; AVX2-NEXT:    vpslld $16, %ymm0, %ymm0
; AVX2-NEXT:    vpslld $16, %ymm1, %ymm1
; AVX2-NEXT:    vphaddd %ymm0, %ymm1, %ymm0
; AVX2-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX2-NEXT:    retq
bb1:
  %0 = sext <16 x i16> %v to <16 x i32>
  %1 = shl nsw <16 x i32> %0, splat (i32 16)
  %2 = shufflevector <16 x i32> %1, <16 x i32> poison, <8 x i32> <i32 0, i32 2, i32 4, i32 6, i32 8, i32 10, i32 12, i32 14>
  %3 = shufflevector <16 x i32> %1, <16 x i32> poison, <8 x i32> <i32 1, i32 3, i32 5, i32 7, i32 9, i32 11, i32 13, i32 15>
  %4 = add <8 x i32> %2, %3
  ret <8 x i32> %4
}