; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 ; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD ; RUN: llc -mtriple=aarch64-none-eabi -global-isel -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-GI define i8 @rotl_i8(i8 %a, i8 %c) { ; CHECK-SD-LABEL: rotl_i8: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: neg w8, w1 ; CHECK-SD-NEXT: and w9, w0, #0xff ; CHECK-SD-NEXT: and w10, w1, #0x7 ; CHECK-SD-NEXT: and w8, w8, #0x7 ; CHECK-SD-NEXT: lsl w10, w0, w10 ; CHECK-SD-NEXT: lsr w8, w9, w8 ; CHECK-SD-NEXT: orr w0, w10, w8 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotl_i8: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: mov x8, xzr ; CHECK-GI-NEXT: neg w9, w1 ; CHECK-GI-NEXT: and w10, w0, #0xff ; CHECK-GI-NEXT: sub x8, x8, w9, uxtb ; CHECK-GI-NEXT: and x9, x9, #0x7 ; CHECK-GI-NEXT: lsr w9, w10, w9 ; CHECK-GI-NEXT: and x8, x8, #0x7 ; CHECK-GI-NEXT: lsl w8, w0, w8 ; CHECK-GI-NEXT: orr w0, w9, w8 ; CHECK-GI-NEXT: ret entry: %d = call i8 @llvm.fshl(i8 %a, i8 %a, i8 %c) ret i8 %d } define i8 @rotr_i8(i8 %a, i8 %c) { ; CHECK-SD-LABEL: rotr_i8: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: neg w8, w1 ; CHECK-SD-NEXT: and w9, w0, #0xff ; CHECK-SD-NEXT: and w10, w1, #0x7 ; CHECK-SD-NEXT: and w8, w8, #0x7 ; CHECK-SD-NEXT: lsr w9, w9, w10 ; CHECK-SD-NEXT: lsl w8, w0, w8 ; CHECK-SD-NEXT: orr w0, w9, w8 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotr_i8: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: mov x8, xzr ; CHECK-GI-NEXT: // kill: def $w1 killed $w1 def $x1 ; CHECK-GI-NEXT: and x9, x1, #0x7 ; CHECK-GI-NEXT: and w10, w0, #0xff ; CHECK-GI-NEXT: sub x8, x8, w1, uxtb ; CHECK-GI-NEXT: lsr w9, w10, w9 ; CHECK-GI-NEXT: and x8, x8, #0x7 ; CHECK-GI-NEXT: lsl w8, w0, w8 ; CHECK-GI-NEXT: orr w0, w9, w8 ; CHECK-GI-NEXT: ret entry: %d = call i8 @llvm.fshr(i8 %a, i8 %a, i8 %c) ret i8 %d } define i16 @rotl_i16(i16 %a, i16 %c) { ; CHECK-SD-LABEL: rotl_i16: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: neg w8, w1 ; CHECK-SD-NEXT: and w9, w0, #0xffff ; CHECK-SD-NEXT: and w10, w1, #0xf ; CHECK-SD-NEXT: and w8, w8, #0xf ; CHECK-SD-NEXT: lsl w10, w0, w10 ; CHECK-SD-NEXT: lsr w8, w9, w8 ; CHECK-SD-NEXT: orr w0, w10, w8 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotl_i16: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: mov x8, xzr ; CHECK-GI-NEXT: neg w9, w1 ; CHECK-GI-NEXT: and w10, w0, #0xffff ; CHECK-GI-NEXT: sub x8, x8, w9, uxth ; CHECK-GI-NEXT: and x9, x9, #0xf ; CHECK-GI-NEXT: lsr w9, w10, w9 ; CHECK-GI-NEXT: and x8, x8, #0xf ; CHECK-GI-NEXT: lsl w8, w0, w8 ; CHECK-GI-NEXT: orr w0, w9, w8 ; CHECK-GI-NEXT: ret entry: %d = call i16 @llvm.fshl(i16 %a, i16 %a, i16 %c) ret i16 %d } define i16 @rotr_i16(i16 %a, i16 %c) { ; CHECK-SD-LABEL: rotr_i16: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: neg w8, w1 ; CHECK-SD-NEXT: and w9, w0, #0xffff ; CHECK-SD-NEXT: and w10, w1, #0xf ; CHECK-SD-NEXT: and w8, w8, #0xf ; CHECK-SD-NEXT: lsr w9, w9, w10 ; CHECK-SD-NEXT: lsl w8, w0, w8 ; CHECK-SD-NEXT: orr w0, w9, w8 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotr_i16: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: mov x8, xzr ; CHECK-GI-NEXT: // kill: def $w1 killed $w1 def $x1 ; CHECK-GI-NEXT: and x9, x1, #0xf ; CHECK-GI-NEXT: and w10, w0, #0xffff ; CHECK-GI-NEXT: sub x8, x8, w1, uxth ; CHECK-GI-NEXT: lsr w9, w10, w9 ; CHECK-GI-NEXT: and x8, x8, #0xf ; CHECK-GI-NEXT: lsl w8, w0, w8 ; CHECK-GI-NEXT: orr w0, w9, w8 ; CHECK-GI-NEXT: ret entry: %d = call i16 @llvm.fshr(i16 %a, i16 %a, i16 %c) ret i16 %d } define i32 @rotl_i32(i32 %a, i32 %c) { ; CHECK-LABEL: rotl_i32: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: neg w8, w1 ; CHECK-NEXT: ror w0, w0, w8 ; CHECK-NEXT: ret entry: %d = call i32 @llvm.fshl(i32 %a, i32 %a, i32 %c) ret i32 %d } define i32 @rotr_i32(i32 %a, i32 %c) { ; CHECK-LABEL: rotr_i32: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ror w0, w0, w1 ; CHECK-NEXT: ret entry: %d = call i32 @llvm.fshr(i32 %a, i32 %a, i32 %c) ret i32 %d } define i64 @rotl_i64(i64 %a, i64 %c) { ; CHECK-SD-LABEL: rotl_i64: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: neg w8, w1 ; CHECK-SD-NEXT: ror x0, x0, x8 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotl_i64: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: neg x8, x1 ; CHECK-GI-NEXT: ror x0, x0, x8 ; CHECK-GI-NEXT: ret entry: %d = call i64 @llvm.fshl(i64 %a, i64 %a, i64 %c) ret i64 %d } define i64 @rotr_i64(i64 %a, i64 %c) { ; CHECK-LABEL: rotr_i64: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ror x0, x0, x1 ; CHECK-NEXT: ret entry: %d = call i64 @llvm.fshr(i64 %a, i64 %a, i64 %c) ret i64 %d } define i128 @rotl_i128(i128 %a, i128 %c) { ; CHECK-SD-LABEL: rotl_i128: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: tst x2, #0x40 ; CHECK-SD-NEXT: mvn w12, w2 ; CHECK-SD-NEXT: csel x8, x1, x0, ne ; CHECK-SD-NEXT: csel x9, x0, x1, ne ; CHECK-SD-NEXT: lsr x10, x9, #1 ; CHECK-SD-NEXT: lsr x11, x8, #1 ; CHECK-SD-NEXT: lsl x8, x8, x2 ; CHECK-SD-NEXT: lsl x9, x9, x2 ; CHECK-SD-NEXT: lsr x10, x10, x12 ; CHECK-SD-NEXT: lsr x11, x11, x12 ; CHECK-SD-NEXT: orr x0, x8, x10 ; CHECK-SD-NEXT: orr x1, x9, x11 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotl_i128: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: mov w8, #64 // =0x40 ; CHECK-GI-NEXT: and x9, x2, #0x7f ; CHECK-GI-NEXT: neg x10, x2 ; CHECK-GI-NEXT: sub x12, x8, x9 ; CHECK-GI-NEXT: sub x11, x9, #64 ; CHECK-GI-NEXT: lsl x14, x1, x9 ; CHECK-GI-NEXT: lsr x12, x0, x12 ; CHECK-GI-NEXT: lsl x13, x0, x9 ; CHECK-GI-NEXT: cmp x9, #64 ; CHECK-GI-NEXT: and x9, x10, #0x7f ; CHECK-GI-NEXT: lsl x11, x0, x11 ; CHECK-GI-NEXT: orr x12, x12, x14 ; CHECK-GI-NEXT: sub x8, x8, x9 ; CHECK-GI-NEXT: sub x14, x9, #64 ; CHECK-GI-NEXT: csel x11, x12, x11, lo ; CHECK-GI-NEXT: lsr x12, x0, x9 ; CHECK-GI-NEXT: lsl x8, x1, x8 ; CHECK-GI-NEXT: csel x13, x13, xzr, lo ; CHECK-GI-NEXT: tst x2, #0x7f ; CHECK-GI-NEXT: lsr x14, x1, x14 ; CHECK-GI-NEXT: csel x11, x1, x11, eq ; CHECK-GI-NEXT: orr x8, x12, x8 ; CHECK-GI-NEXT: cmp x9, #64 ; CHECK-GI-NEXT: lsr x12, x1, x9 ; CHECK-GI-NEXT: csel x8, x8, x14, lo ; CHECK-GI-NEXT: tst x10, #0x7f ; CHECK-GI-NEXT: csel x8, x0, x8, eq ; CHECK-GI-NEXT: cmp x9, #64 ; CHECK-GI-NEXT: csel x9, x12, xzr, lo ; CHECK-GI-NEXT: orr x0, x13, x8 ; CHECK-GI-NEXT: orr x1, x11, x9 ; CHECK-GI-NEXT: ret entry: %d = call i128 @llvm.fshl(i128 %a, i128 %a, i128 %c) ret i128 %d } define i128 @rotr_i128(i128 %a, i128 %c) { ; CHECK-SD-LABEL: rotr_i128: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: tst x2, #0x40 ; CHECK-SD-NEXT: mvn w12, w2 ; CHECK-SD-NEXT: csel x8, x0, x1, eq ; CHECK-SD-NEXT: csel x9, x1, x0, eq ; CHECK-SD-NEXT: lsl x10, x9, #1 ; CHECK-SD-NEXT: lsl x11, x8, #1 ; CHECK-SD-NEXT: lsr x8, x8, x2 ; CHECK-SD-NEXT: lsr x9, x9, x2 ; CHECK-SD-NEXT: lsl x10, x10, x12 ; CHECK-SD-NEXT: lsl x11, x11, x12 ; CHECK-SD-NEXT: orr x0, x10, x8 ; CHECK-SD-NEXT: orr x1, x11, x9 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotr_i128: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: mov w8, #64 // =0x40 ; CHECK-GI-NEXT: and x9, x2, #0x7f ; CHECK-GI-NEXT: neg x13, x2 ; CHECK-GI-NEXT: sub x10, x8, x9 ; CHECK-GI-NEXT: sub x11, x9, #64 ; CHECK-GI-NEXT: lsr x12, x0, x9 ; CHECK-GI-NEXT: lsl x10, x1, x10 ; CHECK-GI-NEXT: lsr x11, x1, x11 ; CHECK-GI-NEXT: and x14, x13, #0x7f ; CHECK-GI-NEXT: cmp x9, #64 ; CHECK-GI-NEXT: sub x8, x8, x14 ; CHECK-GI-NEXT: lsl x15, x1, x14 ; CHECK-GI-NEXT: orr x10, x12, x10 ; CHECK-GI-NEXT: lsr x12, x1, x9 ; CHECK-GI-NEXT: lsr x8, x0, x8 ; CHECK-GI-NEXT: csel x10, x10, x11, lo ; CHECK-GI-NEXT: sub x11, x14, #64 ; CHECK-GI-NEXT: tst x2, #0x7f ; CHECK-GI-NEXT: csel x10, x0, x10, eq ; CHECK-GI-NEXT: cmp x9, #64 ; CHECK-GI-NEXT: lsl x9, x0, x14 ; CHECK-GI-NEXT: lsl x11, x0, x11 ; CHECK-GI-NEXT: csel x12, x12, xzr, lo ; CHECK-GI-NEXT: orr x8, x8, x15 ; CHECK-GI-NEXT: cmp x14, #64 ; CHECK-GI-NEXT: csel x9, x9, xzr, lo ; CHECK-GI-NEXT: csel x8, x8, x11, lo ; CHECK-GI-NEXT: tst x13, #0x7f ; CHECK-GI-NEXT: csel x8, x1, x8, eq ; CHECK-GI-NEXT: orr x0, x10, x9 ; CHECK-GI-NEXT: orr x1, x12, x8 ; CHECK-GI-NEXT: ret entry: %d = call i128 @llvm.fshr(i128 %a, i128 %a, i128 %c) ret i128 %d } define i8 @fshl_i8(i8 %a, i8 %b, i8 %c) { ; CHECK-SD-LABEL: fshl_i8: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: and w8, w2, #0x7 ; CHECK-SD-NEXT: ubfiz w9, w1, #23, #8 ; CHECK-SD-NEXT: mvn w10, w8 ; CHECK-SD-NEXT: lsl w8, w0, w8 ; CHECK-SD-NEXT: lsr w9, w9, w10 ; CHECK-SD-NEXT: orr w0, w8, w9 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshl_i8: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: mov w8, #7 // =0x7 ; CHECK-GI-NEXT: ubfx w9, w1, #1, #7 ; CHECK-GI-NEXT: and w10, w2, #0x7 ; CHECK-GI-NEXT: bic w8, w8, w2 ; CHECK-GI-NEXT: lsl w10, w0, w10 ; CHECK-GI-NEXT: lsr w8, w9, w8 ; CHECK-GI-NEXT: orr w0, w10, w8 ; CHECK-GI-NEXT: ret entry: %d = call i8 @llvm.fshl(i8 %a, i8 %b, i8 %c) ret i8 %d } define i8 @fshr_i8(i8 %a, i8 %b, i8 %c) { ; CHECK-SD-LABEL: fshr_i8: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: mov w8, #24 // =0x18 ; CHECK-SD-NEXT: lsl w9, w1, #24 ; CHECK-SD-NEXT: mvn w10, w2 ; CHECK-SD-NEXT: bfxil w8, w2, #0, #3 ; CHECK-SD-NEXT: lsl w11, w0, #1 ; CHECK-SD-NEXT: and x10, x10, #0x7 ; CHECK-SD-NEXT: lsr w8, w9, w8 ; CHECK-SD-NEXT: lsl w9, w11, w10 ; CHECK-SD-NEXT: orr w0, w9, w8 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_i8: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: mov w8, #7 // =0x7 ; CHECK-GI-NEXT: lsl w9, w0, #1 ; CHECK-GI-NEXT: and w10, w2, #0x7 ; CHECK-GI-NEXT: bic w8, w8, w2 ; CHECK-GI-NEXT: and w11, w1, #0xff ; CHECK-GI-NEXT: lsl w8, w9, w8 ; CHECK-GI-NEXT: lsr w9, w11, w10 ; CHECK-GI-NEXT: orr w0, w8, w9 ; CHECK-GI-NEXT: ret entry: %d = call i8 @llvm.fshr(i8 %a, i8 %b, i8 %c) ret i8 %d } define i16 @fshl_i16(i16 %a, i16 %b, i16 %c) { ; CHECK-SD-LABEL: fshl_i16: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: and w8, w2, #0xf ; CHECK-SD-NEXT: ubfiz w9, w1, #15, #16 ; CHECK-SD-NEXT: mvn w10, w8 ; CHECK-SD-NEXT: lsl w8, w0, w8 ; CHECK-SD-NEXT: lsr w9, w9, w10 ; CHECK-SD-NEXT: orr w0, w8, w9 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshl_i16: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: mov w8, #15 // =0xf ; CHECK-GI-NEXT: ubfx w9, w1, #1, #15 ; CHECK-GI-NEXT: and w10, w2, #0xf ; CHECK-GI-NEXT: bic w8, w8, w2 ; CHECK-GI-NEXT: lsl w10, w0, w10 ; CHECK-GI-NEXT: lsr w8, w9, w8 ; CHECK-GI-NEXT: orr w0, w10, w8 ; CHECK-GI-NEXT: ret entry: %d = call i16 @llvm.fshl(i16 %a, i16 %b, i16 %c) ret i16 %d } define i16 @fshr_i16(i16 %a, i16 %b, i16 %c) { ; CHECK-SD-LABEL: fshr_i16: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: mov w8, #16 // =0x10 ; CHECK-SD-NEXT: lsl w9, w1, #16 ; CHECK-SD-NEXT: mvn w10, w2 ; CHECK-SD-NEXT: bfxil w8, w2, #0, #4 ; CHECK-SD-NEXT: lsl w11, w0, #1 ; CHECK-SD-NEXT: and x10, x10, #0xf ; CHECK-SD-NEXT: lsr w8, w9, w8 ; CHECK-SD-NEXT: lsl w9, w11, w10 ; CHECK-SD-NEXT: orr w0, w9, w8 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_i16: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: mov w8, #15 // =0xf ; CHECK-GI-NEXT: lsl w9, w0, #1 ; CHECK-GI-NEXT: and w10, w2, #0xf ; CHECK-GI-NEXT: bic w8, w8, w2 ; CHECK-GI-NEXT: and w11, w1, #0xffff ; CHECK-GI-NEXT: lsl w8, w9, w8 ; CHECK-GI-NEXT: lsr w9, w11, w10 ; CHECK-GI-NEXT: orr w0, w8, w9 ; CHECK-GI-NEXT: ret entry: %d = call i16 @llvm.fshr(i16 %a, i16 %b, i16 %c) ret i16 %d } define i32 @fshl_i32(i32 %a, i32 %b, i32 %c) { ; CHECK-SD-LABEL: fshl_i32: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: lsr w8, w1, #1 ; CHECK-SD-NEXT: mvn w9, w2 ; CHECK-SD-NEXT: lsl w10, w0, w2 ; CHECK-SD-NEXT: lsr w8, w8, w9 ; CHECK-SD-NEXT: orr w0, w10, w8 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshl_i32: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: mov w8, #31 // =0x1f ; CHECK-GI-NEXT: lsr w9, w1, #1 ; CHECK-GI-NEXT: and w10, w2, #0x1f ; CHECK-GI-NEXT: bic w8, w8, w2 ; CHECK-GI-NEXT: lsl w10, w0, w10 ; CHECK-GI-NEXT: lsr w8, w9, w8 ; CHECK-GI-NEXT: orr w0, w10, w8 ; CHECK-GI-NEXT: ret entry: %d = call i32 @llvm.fshl(i32 %a, i32 %b, i32 %c) ret i32 %d } define i32 @fshr_i32(i32 %a, i32 %b, i32 %c) { ; CHECK-SD-LABEL: fshr_i32: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: lsl w8, w0, #1 ; CHECK-SD-NEXT: mvn w9, w2 ; CHECK-SD-NEXT: lsr w10, w1, w2 ; CHECK-SD-NEXT: lsl w8, w8, w9 ; CHECK-SD-NEXT: orr w0, w8, w10 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_i32: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: mov w8, #31 // =0x1f ; CHECK-GI-NEXT: lsl w9, w0, #1 ; CHECK-GI-NEXT: and w10, w2, #0x1f ; CHECK-GI-NEXT: bic w8, w8, w2 ; CHECK-GI-NEXT: lsl w8, w9, w8 ; CHECK-GI-NEXT: lsr w9, w1, w10 ; CHECK-GI-NEXT: orr w0, w8, w9 ; CHECK-GI-NEXT: ret entry: %d = call i32 @llvm.fshr(i32 %a, i32 %b, i32 %c) ret i32 %d } define i64 @fshl_i64(i64 %a, i64 %b, i64 %c) { ; CHECK-SD-LABEL: fshl_i64: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: lsr x8, x1, #1 ; CHECK-SD-NEXT: mvn w9, w2 ; CHECK-SD-NEXT: lsl x10, x0, x2 ; CHECK-SD-NEXT: lsr x8, x8, x9 ; CHECK-SD-NEXT: orr x0, x10, x8 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshl_i64: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: mov w8, #63 // =0x3f ; CHECK-GI-NEXT: lsr x9, x1, #1 ; CHECK-GI-NEXT: and x10, x2, #0x3f ; CHECK-GI-NEXT: bic x8, x8, x2 ; CHECK-GI-NEXT: lsl x10, x0, x10 ; CHECK-GI-NEXT: lsr x8, x9, x8 ; CHECK-GI-NEXT: orr x0, x10, x8 ; CHECK-GI-NEXT: ret entry: %d = call i64 @llvm.fshl(i64 %a, i64 %b, i64 %c) ret i64 %d } define i64 @fshr_i64(i64 %a, i64 %b, i64 %c) { ; CHECK-SD-LABEL: fshr_i64: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: lsl x8, x0, #1 ; CHECK-SD-NEXT: mvn w9, w2 ; CHECK-SD-NEXT: lsr x10, x1, x2 ; CHECK-SD-NEXT: lsl x8, x8, x9 ; CHECK-SD-NEXT: orr x0, x8, x10 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_i64: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: mov w8, #63 // =0x3f ; CHECK-GI-NEXT: lsl x9, x0, #1 ; CHECK-GI-NEXT: and x10, x2, #0x3f ; CHECK-GI-NEXT: bic x8, x8, x2 ; CHECK-GI-NEXT: lsl x8, x9, x8 ; CHECK-GI-NEXT: lsr x9, x1, x10 ; CHECK-GI-NEXT: orr x0, x8, x9 ; CHECK-GI-NEXT: ret entry: %d = call i64 @llvm.fshr(i64 %a, i64 %b, i64 %c) ret i64 %d } define i128 @fshl_i128(i128 %a, i128 %b, i128 %c) { ; CHECK-SD-LABEL: fshl_i128: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: tst x4, #0x40 ; CHECK-SD-NEXT: mvn w11, w4 ; CHECK-SD-NEXT: csel x8, x3, x0, ne ; CHECK-SD-NEXT: csel x9, x2, x3, ne ; CHECK-SD-NEXT: csel x12, x0, x1, ne ; CHECK-SD-NEXT: lsr x9, x9, #1 ; CHECK-SD-NEXT: lsr x10, x8, #1 ; CHECK-SD-NEXT: lsl x8, x8, x4 ; CHECK-SD-NEXT: lsl x12, x12, x4 ; CHECK-SD-NEXT: lsr x9, x9, x11 ; CHECK-SD-NEXT: lsr x10, x10, x11 ; CHECK-SD-NEXT: orr x0, x8, x9 ; CHECK-SD-NEXT: orr x1, x12, x10 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshl_i128: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: mov w8, #64 // =0x40 ; CHECK-GI-NEXT: and x9, x4, #0x7f ; CHECK-GI-NEXT: mov w10, #127 // =0x7f ; CHECK-GI-NEXT: sub x12, x8, x9 ; CHECK-GI-NEXT: lsl x13, x1, x9 ; CHECK-GI-NEXT: bic x10, x10, x4 ; CHECK-GI-NEXT: lsr x12, x0, x12 ; CHECK-GI-NEXT: sub x14, x9, #64 ; CHECK-GI-NEXT: lsl x15, x0, x9 ; CHECK-GI-NEXT: extr x16, x3, x2, #1 ; CHECK-GI-NEXT: cmp x9, #64 ; CHECK-GI-NEXT: sub x8, x8, x10 ; CHECK-GI-NEXT: orr x9, x12, x13 ; CHECK-GI-NEXT: lsr x12, x3, #1 ; CHECK-GI-NEXT: lsl x13, x0, x14 ; CHECK-GI-NEXT: csel x14, x15, xzr, lo ; CHECK-GI-NEXT: sub x15, x10, #64 ; CHECK-GI-NEXT: lsr x17, x16, x10 ; CHECK-GI-NEXT: lsl x8, x12, x8 ; CHECK-GI-NEXT: csel x9, x9, x13, lo ; CHECK-GI-NEXT: tst x4, #0x7f ; CHECK-GI-NEXT: lsr x13, x12, x15 ; CHECK-GI-NEXT: mvn x11, x4 ; CHECK-GI-NEXT: csel x9, x1, x9, eq ; CHECK-GI-NEXT: orr x8, x17, x8 ; CHECK-GI-NEXT: cmp x10, #64 ; CHECK-GI-NEXT: lsr x12, x12, x10 ; CHECK-GI-NEXT: csel x8, x8, x13, lo ; CHECK-GI-NEXT: tst x11, #0x7f ; CHECK-GI-NEXT: csel x8, x16, x8, eq ; CHECK-GI-NEXT: cmp x10, #64 ; CHECK-GI-NEXT: csel x10, x12, xzr, lo ; CHECK-GI-NEXT: orr x0, x14, x8 ; CHECK-GI-NEXT: orr x1, x9, x10 ; CHECK-GI-NEXT: ret entry: %d = call i128 @llvm.fshl(i128 %a, i128 %b, i128 %c) ret i128 %d } define i128 @fshr_i128(i128 %a, i128 %b, i128 %c) { ; CHECK-SD-LABEL: fshr_i128: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: tst x4, #0x40 ; CHECK-SD-NEXT: mvn w12, w4 ; CHECK-SD-NEXT: csel x8, x3, x0, eq ; CHECK-SD-NEXT: csel x9, x0, x1, eq ; CHECK-SD-NEXT: csel x10, x2, x3, eq ; CHECK-SD-NEXT: lsl x11, x8, #1 ; CHECK-SD-NEXT: lsl x9, x9, #1 ; CHECK-SD-NEXT: lsr x10, x10, x4 ; CHECK-SD-NEXT: lsr x8, x8, x4 ; CHECK-SD-NEXT: lsl x11, x11, x12 ; CHECK-SD-NEXT: lsl x9, x9, x12 ; CHECK-SD-NEXT: orr x0, x11, x10 ; CHECK-SD-NEXT: orr x1, x9, x8 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_i128: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: mov w8, #127 // =0x7f ; CHECK-GI-NEXT: lsl x9, x0, #1 ; CHECK-GI-NEXT: extr x10, x1, x0, #63 ; CHECK-GI-NEXT: bic x8, x8, x4 ; CHECK-GI-NEXT: mov w11, #64 // =0x40 ; CHECK-GI-NEXT: and x14, x4, #0x7f ; CHECK-GI-NEXT: sub x12, x11, x8 ; CHECK-GI-NEXT: lsl x13, x10, x8 ; CHECK-GI-NEXT: lsl x16, x9, x8 ; CHECK-GI-NEXT: lsr x12, x9, x12 ; CHECK-GI-NEXT: sub x17, x8, #64 ; CHECK-GI-NEXT: cmp x8, #64 ; CHECK-GI-NEXT: lsl x8, x9, x17 ; CHECK-GI-NEXT: sub x11, x11, x14 ; CHECK-GI-NEXT: mvn x15, x4 ; CHECK-GI-NEXT: orr x12, x12, x13 ; CHECK-GI-NEXT: csel x9, x16, xzr, lo ; CHECK-GI-NEXT: sub x13, x14, #64 ; CHECK-GI-NEXT: lsr x16, x2, x14 ; CHECK-GI-NEXT: lsl x11, x3, x11 ; CHECK-GI-NEXT: csel x8, x12, x8, lo ; CHECK-GI-NEXT: tst x15, #0x7f ; CHECK-GI-NEXT: lsr x12, x3, x13 ; CHECK-GI-NEXT: csel x8, x10, x8, eq ; CHECK-GI-NEXT: orr x10, x16, x11 ; CHECK-GI-NEXT: cmp x14, #64 ; CHECK-GI-NEXT: lsr x11, x3, x14 ; CHECK-GI-NEXT: csel x10, x10, x12, lo ; CHECK-GI-NEXT: tst x4, #0x7f ; CHECK-GI-NEXT: csel x10, x2, x10, eq ; CHECK-GI-NEXT: cmp x14, #64 ; CHECK-GI-NEXT: csel x11, x11, xzr, lo ; CHECK-GI-NEXT: orr x0, x9, x10 ; CHECK-GI-NEXT: orr x1, x8, x11 ; CHECK-GI-NEXT: ret entry: %d = call i128 @llvm.fshr(i128 %a, i128 %b, i128 %c) ret i128 %d } define i8 @rotl_i8_c(i8 %a) { ; CHECK-LABEL: rotl_i8_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ubfx w8, w0, #5, #3 ; CHECK-NEXT: orr w0, w8, w0, lsl #3 ; CHECK-NEXT: ret entry: %d = call i8 @llvm.fshl(i8 %a, i8 %a, i8 3) ret i8 %d } define i8 @rotr_i8_c(i8 %a) { ; CHECK-SD-LABEL: rotr_i8_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: lsl w8, w0, #5 ; CHECK-SD-NEXT: bfxil w8, w0, #3, #5 ; CHECK-SD-NEXT: mov w0, w8 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotr_i8_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: ubfx w8, w0, #3, #5 ; CHECK-GI-NEXT: orr w0, w8, w0, lsl #5 ; CHECK-GI-NEXT: ret entry: %d = call i8 @llvm.fshr(i8 %a, i8 %a, i8 3) ret i8 %d } define i16 @rotl_i16_c(i16 %a) { ; CHECK-LABEL: rotl_i16_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ubfx w8, w0, #13, #3 ; CHECK-NEXT: orr w0, w8, w0, lsl #3 ; CHECK-NEXT: ret entry: %d = call i16 @llvm.fshl(i16 %a, i16 %a, i16 3) ret i16 %d } define i16 @rotr_i16_c(i16 %a) { ; CHECK-SD-LABEL: rotr_i16_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: lsl w8, w0, #13 ; CHECK-SD-NEXT: bfxil w8, w0, #3, #13 ; CHECK-SD-NEXT: mov w0, w8 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotr_i16_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: ubfx w8, w0, #3, #13 ; CHECK-GI-NEXT: orr w0, w8, w0, lsl #13 ; CHECK-GI-NEXT: ret entry: %d = call i16 @llvm.fshr(i16 %a, i16 %a, i16 3) ret i16 %d } define i32 @rotl_i32_c(i32 %a) { ; CHECK-LABEL: rotl_i32_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ror w0, w0, #29 ; CHECK-NEXT: ret entry: %d = call i32 @llvm.fshl(i32 %a, i32 %a, i32 3) ret i32 %d } define i32 @rotr_i32_c(i32 %a) { ; CHECK-LABEL: rotr_i32_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ror w0, w0, #3 ; CHECK-NEXT: ret entry: %d = call i32 @llvm.fshr(i32 %a, i32 %a, i32 3) ret i32 %d } define i64 @rotl_i64_c(i64 %a) { ; CHECK-LABEL: rotl_i64_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ror x0, x0, #61 ; CHECK-NEXT: ret entry: %d = call i64 @llvm.fshl(i64 %a, i64 %a, i64 3) ret i64 %d } define i64 @rotr_i64_c(i64 %a) { ; CHECK-LABEL: rotr_i64_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: ror x0, x0, #3 ; CHECK-NEXT: ret entry: %d = call i64 @llvm.fshr(i64 %a, i64 %a, i64 3) ret i64 %d } define i128 @rotl_i128_c(i128 %a) { ; CHECK-SD-LABEL: rotl_i128_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: extr x8, x0, x1, #61 ; CHECK-SD-NEXT: extr x1, x1, x0, #61 ; CHECK-SD-NEXT: mov x0, x8 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotl_i128_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: extr x8, x1, x0, #61 ; CHECK-GI-NEXT: extr x0, x0, x1, #61 ; CHECK-GI-NEXT: mov x1, x8 ; CHECK-GI-NEXT: ret entry: %d = call i128 @llvm.fshl(i128 %a, i128 %a, i128 3) ret i128 %d } define i128 @rotr_i128_c(i128 %a) { ; CHECK-LABEL: rotr_i128_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: extr x8, x1, x0, #3 ; CHECK-NEXT: extr x1, x0, x1, #3 ; CHECK-NEXT: mov x0, x8 ; CHECK-NEXT: ret entry: %d = call i128 @llvm.fshr(i128 %a, i128 %a, i128 3) ret i128 %d } define i8 @fshl_i8_c(i8 %a, i8 %b) { ; CHECK-SD-LABEL: fshl_i8_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: lsl w8, w1, #24 ; CHECK-SD-NEXT: extr w0, w0, w8, #29 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshl_i8_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: ubfx w8, w1, #5, #3 ; CHECK-GI-NEXT: orr w0, w8, w0, lsl #3 ; CHECK-GI-NEXT: ret entry: %d = call i8 @llvm.fshl(i8 %a, i8 %b, i8 3) ret i8 %d } define i8 @fshr_i8_c(i8 %a, i8 %b) { ; CHECK-SD-LABEL: fshr_i8_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: lsl w8, w1, #24 ; CHECK-SD-NEXT: extr w0, w0, w8, #27 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_i8_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: ubfx w8, w1, #3, #5 ; CHECK-GI-NEXT: orr w0, w8, w0, lsl #5 ; CHECK-GI-NEXT: ret entry: %d = call i8 @llvm.fshr(i8 %a, i8 %b, i8 3) ret i8 %d } define i16 @fshl_i16_c(i16 %a, i16 %b) { ; CHECK-SD-LABEL: fshl_i16_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: lsl w8, w1, #16 ; CHECK-SD-NEXT: extr w0, w0, w8, #29 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshl_i16_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: ubfx w8, w1, #13, #3 ; CHECK-GI-NEXT: orr w0, w8, w0, lsl #3 ; CHECK-GI-NEXT: ret entry: %d = call i16 @llvm.fshl(i16 %a, i16 %b, i16 3) ret i16 %d } define i16 @fshr_i16_c(i16 %a, i16 %b) { ; CHECK-SD-LABEL: fshr_i16_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: lsl w8, w1, #16 ; CHECK-SD-NEXT: extr w0, w0, w8, #19 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_i16_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: ubfx w8, w1, #3, #13 ; CHECK-GI-NEXT: orr w0, w8, w0, lsl #13 ; CHECK-GI-NEXT: ret entry: %d = call i16 @llvm.fshr(i16 %a, i16 %b, i16 3) ret i16 %d } define i32 @fshl_i32_c(i32 %a, i32 %b) { ; CHECK-LABEL: fshl_i32_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: extr w0, w0, w1, #29 ; CHECK-NEXT: ret entry: %d = call i32 @llvm.fshl(i32 %a, i32 %b, i32 3) ret i32 %d } define i32 @fshr_i32_c(i32 %a, i32 %b) { ; CHECK-LABEL: fshr_i32_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: extr w0, w0, w1, #3 ; CHECK-NEXT: ret entry: %d = call i32 @llvm.fshr(i32 %a, i32 %b, i32 3) ret i32 %d } define i64 @fshl_i64_c(i64 %a, i64 %b) { ; CHECK-LABEL: fshl_i64_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: extr x0, x0, x1, #61 ; CHECK-NEXT: ret entry: %d = call i64 @llvm.fshl(i64 %a, i64 %b, i64 3) ret i64 %d } define i64 @fshr_i64_c(i64 %a, i64 %b) { ; CHECK-LABEL: fshr_i64_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: extr x0, x0, x1, #3 ; CHECK-NEXT: ret entry: %d = call i64 @llvm.fshr(i64 %a, i64 %b, i64 3) ret i64 %d } define i128 @fshl_i128_c(i128 %a, i128 %b) { ; CHECK-SD-LABEL: fshl_i128_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: extr x8, x0, x3, #61 ; CHECK-SD-NEXT: extr x1, x1, x0, #61 ; CHECK-SD-NEXT: mov x0, x8 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshl_i128_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: extr x1, x1, x0, #61 ; CHECK-GI-NEXT: extr x0, x0, x3, #61 ; CHECK-GI-NEXT: ret entry: %d = call i128 @llvm.fshl(i128 %a, i128 %b, i128 3) ret i128 %d } define i128 @fshr_i128_c(i128 %a, i128 %b) { ; CHECK-LABEL: fshr_i128_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: extr x8, x3, x2, #3 ; CHECK-NEXT: extr x1, x0, x3, #3 ; CHECK-NEXT: mov x0, x8 ; CHECK-NEXT: ret entry: %d = call i128 @llvm.fshr(i128 %a, i128 %b, i128 3) ret i128 %d } define <8 x i8> @rotl_v8i8(<8 x i8> %a, <8 x i8> %c) { ; CHECK-LABEL: rotl_v8i8: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: movi v2.8b, #7 ; CHECK-NEXT: neg v3.8b, v1.8b ; CHECK-NEXT: and v3.8b, v3.8b, v2.8b ; CHECK-NEXT: and v1.8b, v1.8b, v2.8b ; CHECK-NEXT: neg v2.8b, v3.8b ; CHECK-NEXT: ushl v1.8b, v0.8b, v1.8b ; CHECK-NEXT: ushl v0.8b, v0.8b, v2.8b ; CHECK-NEXT: orr v0.8b, v1.8b, v0.8b ; CHECK-NEXT: ret entry: %d = call <8 x i8> @llvm.fshl(<8 x i8> %a, <8 x i8> %a, <8 x i8> %c) ret <8 x i8> %d } define <8 x i8> @rotr_v8i8(<8 x i8> %a, <8 x i8> %c) { ; CHECK-SD-LABEL: rotr_v8i8: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: movi v2.8b, #7 ; CHECK-SD-NEXT: neg v3.8b, v1.8b ; CHECK-SD-NEXT: and v1.8b, v1.8b, v2.8b ; CHECK-SD-NEXT: and v2.8b, v3.8b, v2.8b ; CHECK-SD-NEXT: neg v1.8b, v1.8b ; CHECK-SD-NEXT: ushl v2.8b, v0.8b, v2.8b ; CHECK-SD-NEXT: ushl v0.8b, v0.8b, v1.8b ; CHECK-SD-NEXT: orr v0.8b, v0.8b, v2.8b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotr_v8i8: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: movi v2.8b, #7 ; CHECK-GI-NEXT: neg v3.8b, v1.8b ; CHECK-GI-NEXT: and v1.8b, v1.8b, v2.8b ; CHECK-GI-NEXT: and v2.8b, v3.8b, v2.8b ; CHECK-GI-NEXT: neg v1.8b, v1.8b ; CHECK-GI-NEXT: ushl v1.8b, v0.8b, v1.8b ; CHECK-GI-NEXT: ushl v0.8b, v0.8b, v2.8b ; CHECK-GI-NEXT: orr v0.8b, v1.8b, v0.8b ; CHECK-GI-NEXT: ret entry: %d = call <8 x i8> @llvm.fshr(<8 x i8> %a, <8 x i8> %a, <8 x i8> %c) ret <8 x i8> %d } define <16 x i8> @rotl_v16i8(<16 x i8> %a, <16 x i8> %c) { ; CHECK-LABEL: rotl_v16i8: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: movi v2.16b, #7 ; CHECK-NEXT: neg v3.16b, v1.16b ; CHECK-NEXT: and v3.16b, v3.16b, v2.16b ; CHECK-NEXT: and v1.16b, v1.16b, v2.16b ; CHECK-NEXT: neg v2.16b, v3.16b ; CHECK-NEXT: ushl v1.16b, v0.16b, v1.16b ; CHECK-NEXT: ushl v0.16b, v0.16b, v2.16b ; CHECK-NEXT: orr v0.16b, v1.16b, v0.16b ; CHECK-NEXT: ret entry: %d = call <16 x i8> @llvm.fshl(<16 x i8> %a, <16 x i8> %a, <16 x i8> %c) ret <16 x i8> %d } define <16 x i8> @rotr_v16i8(<16 x i8> %a, <16 x i8> %c) { ; CHECK-SD-LABEL: rotr_v16i8: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: movi v2.16b, #7 ; CHECK-SD-NEXT: neg v3.16b, v1.16b ; CHECK-SD-NEXT: and v1.16b, v1.16b, v2.16b ; CHECK-SD-NEXT: and v2.16b, v3.16b, v2.16b ; CHECK-SD-NEXT: neg v1.16b, v1.16b ; CHECK-SD-NEXT: ushl v2.16b, v0.16b, v2.16b ; CHECK-SD-NEXT: ushl v0.16b, v0.16b, v1.16b ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v2.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotr_v16i8: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: movi v2.16b, #7 ; CHECK-GI-NEXT: neg v3.16b, v1.16b ; CHECK-GI-NEXT: and v1.16b, v1.16b, v2.16b ; CHECK-GI-NEXT: and v2.16b, v3.16b, v2.16b ; CHECK-GI-NEXT: neg v1.16b, v1.16b ; CHECK-GI-NEXT: ushl v1.16b, v0.16b, v1.16b ; CHECK-GI-NEXT: ushl v0.16b, v0.16b, v2.16b ; CHECK-GI-NEXT: orr v0.16b, v1.16b, v0.16b ; CHECK-GI-NEXT: ret entry: %d = call <16 x i8> @llvm.fshr(<16 x i8> %a, <16 x i8> %a, <16 x i8> %c) ret <16 x i8> %d } define <4 x i16> @rotl_v4i16(<4 x i16> %a, <4 x i16> %c) { ; CHECK-LABEL: rotl_v4i16: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: movi v2.4h, #15 ; CHECK-NEXT: neg v3.4h, v1.4h ; CHECK-NEXT: and v3.8b, v3.8b, v2.8b ; CHECK-NEXT: and v1.8b, v1.8b, v2.8b ; CHECK-NEXT: neg v2.4h, v3.4h ; CHECK-NEXT: ushl v1.4h, v0.4h, v1.4h ; CHECK-NEXT: ushl v0.4h, v0.4h, v2.4h ; CHECK-NEXT: orr v0.8b, v1.8b, v0.8b ; CHECK-NEXT: ret entry: %d = call <4 x i16> @llvm.fshl(<4 x i16> %a, <4 x i16> %a, <4 x i16> %c) ret <4 x i16> %d } define <4 x i16> @rotr_v4i16(<4 x i16> %a, <4 x i16> %c) { ; CHECK-SD-LABEL: rotr_v4i16: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: movi v2.4h, #15 ; CHECK-SD-NEXT: neg v3.4h, v1.4h ; CHECK-SD-NEXT: and v1.8b, v1.8b, v2.8b ; CHECK-SD-NEXT: and v2.8b, v3.8b, v2.8b ; CHECK-SD-NEXT: neg v1.4h, v1.4h ; CHECK-SD-NEXT: ushl v2.4h, v0.4h, v2.4h ; CHECK-SD-NEXT: ushl v0.4h, v0.4h, v1.4h ; CHECK-SD-NEXT: orr v0.8b, v0.8b, v2.8b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotr_v4i16: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: movi v2.4h, #15 ; CHECK-GI-NEXT: neg v3.4h, v1.4h ; CHECK-GI-NEXT: and v1.8b, v1.8b, v2.8b ; CHECK-GI-NEXT: and v2.8b, v3.8b, v2.8b ; CHECK-GI-NEXT: neg v1.4h, v1.4h ; CHECK-GI-NEXT: ushl v1.4h, v0.4h, v1.4h ; CHECK-GI-NEXT: ushl v0.4h, v0.4h, v2.4h ; CHECK-GI-NEXT: orr v0.8b, v1.8b, v0.8b ; CHECK-GI-NEXT: ret entry: %d = call <4 x i16> @llvm.fshr(<4 x i16> %a, <4 x i16> %a, <4 x i16> %c) ret <4 x i16> %d } define <7 x i16> @rotl_v7i16(<7 x i16> %a, <7 x i16> %c) { ; CHECK-SD-LABEL: rotl_v7i16: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: movi v2.8h, #15 ; CHECK-SD-NEXT: neg v3.8h, v1.8h ; CHECK-SD-NEXT: and v3.16b, v3.16b, v2.16b ; CHECK-SD-NEXT: and v1.16b, v1.16b, v2.16b ; CHECK-SD-NEXT: neg v2.8h, v3.8h ; CHECK-SD-NEXT: ushl v1.8h, v0.8h, v1.8h ; CHECK-SD-NEXT: ushl v0.8h, v0.8h, v2.8h ; CHECK-SD-NEXT: orr v0.16b, v1.16b, v0.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotl_v7i16: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: movi v2.2d, #0000000000000000 ; CHECK-GI-NEXT: movi v3.8h, #15 ; CHECK-GI-NEXT: sub v2.8h, v2.8h, v1.8h ; CHECK-GI-NEXT: and v1.16b, v1.16b, v3.16b ; CHECK-GI-NEXT: and v2.16b, v2.16b, v3.16b ; CHECK-GI-NEXT: ushl v1.8h, v0.8h, v1.8h ; CHECK-GI-NEXT: neg v2.8h, v2.8h ; CHECK-GI-NEXT: ushl v0.8h, v0.8h, v2.8h ; CHECK-GI-NEXT: orr v0.16b, v1.16b, v0.16b ; CHECK-GI-NEXT: ret entry: %d = call <7 x i16> @llvm.fshl(<7 x i16> %a, <7 x i16> %a, <7 x i16> %c) ret <7 x i16> %d } define <7 x i16> @rotr_v7i16(<7 x i16> %a, <7 x i16> %c) { ; CHECK-SD-LABEL: rotr_v7i16: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: movi v2.8h, #15 ; CHECK-SD-NEXT: neg v3.8h, v1.8h ; CHECK-SD-NEXT: and v1.16b, v1.16b, v2.16b ; CHECK-SD-NEXT: and v2.16b, v3.16b, v2.16b ; CHECK-SD-NEXT: neg v1.8h, v1.8h ; CHECK-SD-NEXT: ushl v2.8h, v0.8h, v2.8h ; CHECK-SD-NEXT: ushl v0.8h, v0.8h, v1.8h ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v2.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotr_v7i16: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: movi v2.2d, #0000000000000000 ; CHECK-GI-NEXT: movi v3.8h, #15 ; CHECK-GI-NEXT: sub v2.8h, v2.8h, v1.8h ; CHECK-GI-NEXT: and v1.16b, v1.16b, v3.16b ; CHECK-GI-NEXT: neg v1.8h, v1.8h ; CHECK-GI-NEXT: and v2.16b, v2.16b, v3.16b ; CHECK-GI-NEXT: ushl v1.8h, v0.8h, v1.8h ; CHECK-GI-NEXT: ushl v0.8h, v0.8h, v2.8h ; CHECK-GI-NEXT: orr v0.16b, v1.16b, v0.16b ; CHECK-GI-NEXT: ret entry: %d = call <7 x i16> @llvm.fshr(<7 x i16> %a, <7 x i16> %a, <7 x i16> %c) ret <7 x i16> %d } define <8 x i16> @rotl_v8i16(<8 x i16> %a, <8 x i16> %c) { ; CHECK-LABEL: rotl_v8i16: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: movi v2.8h, #15 ; CHECK-NEXT: neg v3.8h, v1.8h ; CHECK-NEXT: and v3.16b, v3.16b, v2.16b ; CHECK-NEXT: and v1.16b, v1.16b, v2.16b ; CHECK-NEXT: neg v2.8h, v3.8h ; CHECK-NEXT: ushl v1.8h, v0.8h, v1.8h ; CHECK-NEXT: ushl v0.8h, v0.8h, v2.8h ; CHECK-NEXT: orr v0.16b, v1.16b, v0.16b ; CHECK-NEXT: ret entry: %d = call <8 x i16> @llvm.fshl(<8 x i16> %a, <8 x i16> %a, <8 x i16> %c) ret <8 x i16> %d } define <8 x i16> @rotr_v8i16(<8 x i16> %a, <8 x i16> %c) { ; CHECK-SD-LABEL: rotr_v8i16: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: movi v2.8h, #15 ; CHECK-SD-NEXT: neg v3.8h, v1.8h ; CHECK-SD-NEXT: and v1.16b, v1.16b, v2.16b ; CHECK-SD-NEXT: and v2.16b, v3.16b, v2.16b ; CHECK-SD-NEXT: neg v1.8h, v1.8h ; CHECK-SD-NEXT: ushl v2.8h, v0.8h, v2.8h ; CHECK-SD-NEXT: ushl v0.8h, v0.8h, v1.8h ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v2.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotr_v8i16: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: movi v2.8h, #15 ; CHECK-GI-NEXT: neg v3.8h, v1.8h ; CHECK-GI-NEXT: and v1.16b, v1.16b, v2.16b ; CHECK-GI-NEXT: and v2.16b, v3.16b, v2.16b ; CHECK-GI-NEXT: neg v1.8h, v1.8h ; CHECK-GI-NEXT: ushl v1.8h, v0.8h, v1.8h ; CHECK-GI-NEXT: ushl v0.8h, v0.8h, v2.8h ; CHECK-GI-NEXT: orr v0.16b, v1.16b, v0.16b ; CHECK-GI-NEXT: ret entry: %d = call <8 x i16> @llvm.fshr(<8 x i16> %a, <8 x i16> %a, <8 x i16> %c) ret <8 x i16> %d } define <16 x i16> @rotl_v16i16(<16 x i16> %a, <16 x i16> %c) { ; CHECK-LABEL: rotl_v16i16: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: movi v4.8h, #15 ; CHECK-NEXT: neg v5.8h, v2.8h ; CHECK-NEXT: neg v6.8h, v3.8h ; CHECK-NEXT: and v5.16b, v5.16b, v4.16b ; CHECK-NEXT: and v6.16b, v6.16b, v4.16b ; CHECK-NEXT: and v2.16b, v2.16b, v4.16b ; CHECK-NEXT: and v3.16b, v3.16b, v4.16b ; CHECK-NEXT: neg v4.8h, v5.8h ; CHECK-NEXT: neg v5.8h, v6.8h ; CHECK-NEXT: ushl v2.8h, v0.8h, v2.8h ; CHECK-NEXT: ushl v3.8h, v1.8h, v3.8h ; CHECK-NEXT: ushl v0.8h, v0.8h, v4.8h ; CHECK-NEXT: ushl v1.8h, v1.8h, v5.8h ; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b ; CHECK-NEXT: orr v1.16b, v3.16b, v1.16b ; CHECK-NEXT: ret entry: %d = call <16 x i16> @llvm.fshl(<16 x i16> %a, <16 x i16> %a, <16 x i16> %c) ret <16 x i16> %d } define <16 x i16> @rotr_v16i16(<16 x i16> %a, <16 x i16> %c) { ; CHECK-SD-LABEL: rotr_v16i16: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: movi v4.8h, #15 ; CHECK-SD-NEXT: neg v5.8h, v2.8h ; CHECK-SD-NEXT: neg v6.8h, v3.8h ; CHECK-SD-NEXT: and v2.16b, v2.16b, v4.16b ; CHECK-SD-NEXT: and v3.16b, v3.16b, v4.16b ; CHECK-SD-NEXT: and v5.16b, v5.16b, v4.16b ; CHECK-SD-NEXT: and v4.16b, v6.16b, v4.16b ; CHECK-SD-NEXT: neg v2.8h, v2.8h ; CHECK-SD-NEXT: neg v3.8h, v3.8h ; CHECK-SD-NEXT: ushl v5.8h, v0.8h, v5.8h ; CHECK-SD-NEXT: ushl v0.8h, v0.8h, v2.8h ; CHECK-SD-NEXT: ushl v2.8h, v1.8h, v4.8h ; CHECK-SD-NEXT: ushl v1.8h, v1.8h, v3.8h ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v5.16b ; CHECK-SD-NEXT: orr v1.16b, v1.16b, v2.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotr_v16i16: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: movi v4.8h, #15 ; CHECK-GI-NEXT: neg v6.8h, v3.8h ; CHECK-GI-NEXT: and v5.16b, v2.16b, v4.16b ; CHECK-GI-NEXT: neg v2.8h, v2.8h ; CHECK-GI-NEXT: and v3.16b, v3.16b, v4.16b ; CHECK-GI-NEXT: neg v5.8h, v5.8h ; CHECK-GI-NEXT: and v2.16b, v2.16b, v4.16b ; CHECK-GI-NEXT: and v4.16b, v6.16b, v4.16b ; CHECK-GI-NEXT: neg v3.8h, v3.8h ; CHECK-GI-NEXT: ushl v5.8h, v0.8h, v5.8h ; CHECK-GI-NEXT: ushl v0.8h, v0.8h, v2.8h ; CHECK-GI-NEXT: ushl v3.8h, v1.8h, v3.8h ; CHECK-GI-NEXT: ushl v1.8h, v1.8h, v4.8h ; CHECK-GI-NEXT: orr v0.16b, v5.16b, v0.16b ; CHECK-GI-NEXT: orr v1.16b, v3.16b, v1.16b ; CHECK-GI-NEXT: ret entry: %d = call <16 x i16> @llvm.fshr(<16 x i16> %a, <16 x i16> %a, <16 x i16> %c) ret <16 x i16> %d } define <2 x i32> @rotl_v2i32(<2 x i32> %a, <2 x i32> %c) { ; CHECK-LABEL: rotl_v2i32: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: movi v2.2s, #31 ; CHECK-NEXT: neg v3.2s, v1.2s ; CHECK-NEXT: and v3.8b, v3.8b, v2.8b ; CHECK-NEXT: and v1.8b, v1.8b, v2.8b ; CHECK-NEXT: neg v2.2s, v3.2s ; CHECK-NEXT: ushl v1.2s, v0.2s, v1.2s ; CHECK-NEXT: ushl v0.2s, v0.2s, v2.2s ; CHECK-NEXT: orr v0.8b, v1.8b, v0.8b ; CHECK-NEXT: ret entry: %d = call <2 x i32> @llvm.fshl(<2 x i32> %a, <2 x i32> %a, <2 x i32> %c) ret <2 x i32> %d } define <2 x i32> @rotr_v2i32(<2 x i32> %a, <2 x i32> %c) { ; CHECK-SD-LABEL: rotr_v2i32: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: movi v2.2s, #31 ; CHECK-SD-NEXT: neg v3.2s, v1.2s ; CHECK-SD-NEXT: and v1.8b, v1.8b, v2.8b ; CHECK-SD-NEXT: and v2.8b, v3.8b, v2.8b ; CHECK-SD-NEXT: neg v1.2s, v1.2s ; CHECK-SD-NEXT: ushl v2.2s, v0.2s, v2.2s ; CHECK-SD-NEXT: ushl v0.2s, v0.2s, v1.2s ; CHECK-SD-NEXT: orr v0.8b, v0.8b, v2.8b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotr_v2i32: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: movi v2.2s, #31 ; CHECK-GI-NEXT: neg v3.2s, v1.2s ; CHECK-GI-NEXT: and v1.8b, v1.8b, v2.8b ; CHECK-GI-NEXT: and v2.8b, v3.8b, v2.8b ; CHECK-GI-NEXT: neg v1.2s, v1.2s ; CHECK-GI-NEXT: ushl v1.2s, v0.2s, v1.2s ; CHECK-GI-NEXT: ushl v0.2s, v0.2s, v2.2s ; CHECK-GI-NEXT: orr v0.8b, v1.8b, v0.8b ; CHECK-GI-NEXT: ret entry: %d = call <2 x i32> @llvm.fshr(<2 x i32> %a, <2 x i32> %a, <2 x i32> %c) ret <2 x i32> %d } define <4 x i32> @rotl_v4i32(<4 x i32> %a, <4 x i32> %c) { ; CHECK-LABEL: rotl_v4i32: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: movi v2.4s, #31 ; CHECK-NEXT: neg v3.4s, v1.4s ; CHECK-NEXT: and v3.16b, v3.16b, v2.16b ; CHECK-NEXT: and v1.16b, v1.16b, v2.16b ; CHECK-NEXT: neg v2.4s, v3.4s ; CHECK-NEXT: ushl v1.4s, v0.4s, v1.4s ; CHECK-NEXT: ushl v0.4s, v0.4s, v2.4s ; CHECK-NEXT: orr v0.16b, v1.16b, v0.16b ; CHECK-NEXT: ret entry: %d = call <4 x i32> @llvm.fshl(<4 x i32> %a, <4 x i32> %a, <4 x i32> %c) ret <4 x i32> %d } define <4 x i32> @rotr_v4i32(<4 x i32> %a, <4 x i32> %c) { ; CHECK-SD-LABEL: rotr_v4i32: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: movi v2.4s, #31 ; CHECK-SD-NEXT: neg v3.4s, v1.4s ; CHECK-SD-NEXT: and v1.16b, v1.16b, v2.16b ; CHECK-SD-NEXT: and v2.16b, v3.16b, v2.16b ; CHECK-SD-NEXT: neg v1.4s, v1.4s ; CHECK-SD-NEXT: ushl v2.4s, v0.4s, v2.4s ; CHECK-SD-NEXT: ushl v0.4s, v0.4s, v1.4s ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v2.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotr_v4i32: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: movi v2.4s, #31 ; CHECK-GI-NEXT: neg v3.4s, v1.4s ; CHECK-GI-NEXT: and v1.16b, v1.16b, v2.16b ; CHECK-GI-NEXT: and v2.16b, v3.16b, v2.16b ; CHECK-GI-NEXT: neg v1.4s, v1.4s ; CHECK-GI-NEXT: ushl v1.4s, v0.4s, v1.4s ; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v2.4s ; CHECK-GI-NEXT: orr v0.16b, v1.16b, v0.16b ; CHECK-GI-NEXT: ret entry: %d = call <4 x i32> @llvm.fshr(<4 x i32> %a, <4 x i32> %a, <4 x i32> %c) ret <4 x i32> %d } define <7 x i32> @rotl_v7i32(<7 x i32> %a, <7 x i32> %c) { ; CHECK-SD-LABEL: rotl_v7i32: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fmov s0, w7 ; CHECK-SD-NEXT: mov x8, sp ; CHECK-SD-NEXT: ldr s1, [sp, #24] ; CHECK-SD-NEXT: fmov s2, w0 ; CHECK-SD-NEXT: add x9, sp, #40 ; CHECK-SD-NEXT: fmov s5, w4 ; CHECK-SD-NEXT: movi v3.4s, #31 ; CHECK-SD-NEXT: ld1 { v0.s }[1], [x8] ; CHECK-SD-NEXT: add x8, sp, #8 ; CHECK-SD-NEXT: mov v2.s[1], w1 ; CHECK-SD-NEXT: mov v5.s[1], w5 ; CHECK-SD-NEXT: ld1 { v0.s }[2], [x8] ; CHECK-SD-NEXT: add x8, sp, #32 ; CHECK-SD-NEXT: ld1 { v1.s }[1], [x8] ; CHECK-SD-NEXT: add x8, sp, #16 ; CHECK-SD-NEXT: mov v2.s[2], w2 ; CHECK-SD-NEXT: mov v5.s[2], w6 ; CHECK-SD-NEXT: ld1 { v0.s }[3], [x8] ; CHECK-SD-NEXT: ld1 { v1.s }[2], [x9] ; CHECK-SD-NEXT: neg v4.4s, v0.4s ; CHECK-SD-NEXT: mov v2.s[3], w3 ; CHECK-SD-NEXT: and v0.16b, v0.16b, v3.16b ; CHECK-SD-NEXT: neg v6.4s, v1.4s ; CHECK-SD-NEXT: and v1.16b, v1.16b, v3.16b ; CHECK-SD-NEXT: and v4.16b, v4.16b, v3.16b ; CHECK-SD-NEXT: and v6.16b, v6.16b, v3.16b ; CHECK-SD-NEXT: ushl v0.4s, v2.4s, v0.4s ; CHECK-SD-NEXT: ushl v1.4s, v5.4s, v1.4s ; CHECK-SD-NEXT: neg v4.4s, v4.4s ; CHECK-SD-NEXT: neg v3.4s, v6.4s ; CHECK-SD-NEXT: ushl v2.4s, v2.4s, v4.4s ; CHECK-SD-NEXT: ushl v3.4s, v5.4s, v3.4s ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v2.16b ; CHECK-SD-NEXT: orr v1.16b, v1.16b, v3.16b ; CHECK-SD-NEXT: mov w1, v0.s[1] ; CHECK-SD-NEXT: mov w2, v0.s[2] ; CHECK-SD-NEXT: mov w3, v0.s[3] ; CHECK-SD-NEXT: mov w5, v1.s[1] ; CHECK-SD-NEXT: mov w6, v1.s[2] ; CHECK-SD-NEXT: fmov w0, s0 ; CHECK-SD-NEXT: fmov w4, s1 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotl_v7i32: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: ldr s0, [sp, #24] ; CHECK-GI-NEXT: fmov s2, w7 ; CHECK-GI-NEXT: ldr s3, [sp, #32] ; CHECK-GI-NEXT: mov x8, sp ; CHECK-GI-NEXT: fmov s1, w7 ; CHECK-GI-NEXT: ldr s6, [sp] ; CHECK-GI-NEXT: mov v5.16b, v0.16b ; CHECK-GI-NEXT: ldr s4, [sp, #40] ; CHECK-GI-NEXT: fmov s16, w0 ; CHECK-GI-NEXT: ld1 { v2.s }[1], [x8] ; CHECK-GI-NEXT: add x8, sp, #8 ; CHECK-GI-NEXT: fmov s17, w0 ; CHECK-GI-NEXT: mov v1.s[1], v6.s[0] ; CHECK-GI-NEXT: movi v6.2d, #0000000000000000 ; CHECK-GI-NEXT: ldr s7, [sp, #8] ; CHECK-GI-NEXT: mov v5.s[1], v3.s[0] ; CHECK-GI-NEXT: mov v16.s[1], w1 ; CHECK-GI-NEXT: mov v0.s[1], v3.s[0] ; CHECK-GI-NEXT: ld1 { v2.s }[2], [x8] ; CHECK-GI-NEXT: add x8, sp, #16 ; CHECK-GI-NEXT: mov v17.s[1], w1 ; CHECK-GI-NEXT: ldr s18, [sp, #16] ; CHECK-GI-NEXT: mov v1.s[2], v7.s[0] ; CHECK-GI-NEXT: movi v7.4s, #31 ; CHECK-GI-NEXT: mov v5.s[2], v4.s[0] ; CHECK-GI-NEXT: ld1 { v2.s }[3], [x8] ; CHECK-GI-NEXT: mov v16.s[2], w2 ; CHECK-GI-NEXT: mov v17.s[2], w2 ; CHECK-GI-NEXT: mov v0.s[2], v4.s[0] ; CHECK-GI-NEXT: neg v2.4s, v2.4s ; CHECK-GI-NEXT: mov v1.s[3], v18.s[0] ; CHECK-GI-NEXT: sub v3.4s, v6.4s, v5.4s ; CHECK-GI-NEXT: fmov s5, w4 ; CHECK-GI-NEXT: fmov s6, w4 ; CHECK-GI-NEXT: mov v17.s[3], w3 ; CHECK-GI-NEXT: mov v16.s[3], w3 ; CHECK-GI-NEXT: and v0.16b, v0.16b, v7.16b ; CHECK-GI-NEXT: and v2.16b, v2.16b, v7.16b ; CHECK-GI-NEXT: mov v6.s[1], w5 ; CHECK-GI-NEXT: mov v5.s[1], w5 ; CHECK-GI-NEXT: and v3.16b, v3.16b, v7.16b ; CHECK-GI-NEXT: and v1.16b, v1.16b, v7.16b ; CHECK-GI-NEXT: neg v2.4s, v2.4s ; CHECK-GI-NEXT: neg v3.4s, v3.4s ; CHECK-GI-NEXT: mov v6.s[2], w6 ; CHECK-GI-NEXT: mov v5.s[2], w6 ; CHECK-GI-NEXT: ushl v1.4s, v17.4s, v1.4s ; CHECK-GI-NEXT: ushl v2.4s, v16.4s, v2.4s ; CHECK-GI-NEXT: ushl v0.4s, v6.4s, v0.4s ; CHECK-GI-NEXT: ushl v3.4s, v5.4s, v3.4s ; CHECK-GI-NEXT: orr v1.16b, v1.16b, v2.16b ; CHECK-GI-NEXT: orr v0.16b, v0.16b, v3.16b ; CHECK-GI-NEXT: mov s2, v1.s[1] ; CHECK-GI-NEXT: mov s3, v1.s[2] ; CHECK-GI-NEXT: mov s4, v1.s[3] ; CHECK-GI-NEXT: fmov w0, s1 ; CHECK-GI-NEXT: mov s5, v0.s[1] ; CHECK-GI-NEXT: mov s6, v0.s[2] ; CHECK-GI-NEXT: fmov w4, s0 ; CHECK-GI-NEXT: fmov w1, s2 ; CHECK-GI-NEXT: fmov w2, s3 ; CHECK-GI-NEXT: fmov w3, s4 ; CHECK-GI-NEXT: fmov w5, s5 ; CHECK-GI-NEXT: fmov w6, s6 ; CHECK-GI-NEXT: ret entry: %d = call <7 x i32> @llvm.fshl(<7 x i32> %a, <7 x i32> %a, <7 x i32> %c) ret <7 x i32> %d } define <7 x i32> @rotr_v7i32(<7 x i32> %a, <7 x i32> %c) { ; CHECK-SD-LABEL: rotr_v7i32: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fmov s0, w7 ; CHECK-SD-NEXT: mov x8, sp ; CHECK-SD-NEXT: fmov s2, w0 ; CHECK-SD-NEXT: ldr s1, [sp, #24] ; CHECK-SD-NEXT: add x9, sp, #32 ; CHECK-SD-NEXT: fmov s4, w4 ; CHECK-SD-NEXT: movi v3.4s, #31 ; CHECK-SD-NEXT: ld1 { v0.s }[1], [x8] ; CHECK-SD-NEXT: add x8, sp, #8 ; CHECK-SD-NEXT: mov v2.s[1], w1 ; CHECK-SD-NEXT: ld1 { v1.s }[1], [x9] ; CHECK-SD-NEXT: add x9, sp, #40 ; CHECK-SD-NEXT: mov v4.s[1], w5 ; CHECK-SD-NEXT: ld1 { v0.s }[2], [x8] ; CHECK-SD-NEXT: add x8, sp, #16 ; CHECK-SD-NEXT: ld1 { v1.s }[2], [x9] ; CHECK-SD-NEXT: mov v2.s[2], w2 ; CHECK-SD-NEXT: mov v4.s[2], w6 ; CHECK-SD-NEXT: ld1 { v0.s }[3], [x8] ; CHECK-SD-NEXT: neg v6.4s, v1.4s ; CHECK-SD-NEXT: and v1.16b, v1.16b, v3.16b ; CHECK-SD-NEXT: mov v2.s[3], w3 ; CHECK-SD-NEXT: neg v5.4s, v0.4s ; CHECK-SD-NEXT: and v0.16b, v0.16b, v3.16b ; CHECK-SD-NEXT: neg v1.4s, v1.4s ; CHECK-SD-NEXT: and v5.16b, v5.16b, v3.16b ; CHECK-SD-NEXT: neg v0.4s, v0.4s ; CHECK-SD-NEXT: and v3.16b, v6.16b, v3.16b ; CHECK-SD-NEXT: ushl v1.4s, v4.4s, v1.4s ; CHECK-SD-NEXT: ushl v5.4s, v2.4s, v5.4s ; CHECK-SD-NEXT: ushl v0.4s, v2.4s, v0.4s ; CHECK-SD-NEXT: ushl v2.4s, v4.4s, v3.4s ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v5.16b ; CHECK-SD-NEXT: orr v1.16b, v1.16b, v2.16b ; CHECK-SD-NEXT: mov w1, v0.s[1] ; CHECK-SD-NEXT: mov w2, v0.s[2] ; CHECK-SD-NEXT: mov w3, v0.s[3] ; CHECK-SD-NEXT: mov w5, v1.s[1] ; CHECK-SD-NEXT: mov w6, v1.s[2] ; CHECK-SD-NEXT: fmov w0, s0 ; CHECK-SD-NEXT: fmov w4, s1 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotr_v7i32: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: fmov s1, w7 ; CHECK-GI-NEXT: ldr s2, [sp] ; CHECK-GI-NEXT: ldr s0, [sp, #24] ; CHECK-GI-NEXT: fmov s3, w7 ; CHECK-GI-NEXT: ldr s4, [sp, #32] ; CHECK-GI-NEXT: ldr s5, [sp, #8] ; CHECK-GI-NEXT: mov x8, sp ; CHECK-GI-NEXT: fmov s7, w0 ; CHECK-GI-NEXT: add x9, sp, #8 ; CHECK-GI-NEXT: mov v1.s[1], v2.s[0] ; CHECK-GI-NEXT: mov v2.16b, v0.16b ; CHECK-GI-NEXT: mov v0.s[1], v4.s[0] ; CHECK-GI-NEXT: ld1 { v3.s }[1], [x8] ; CHECK-GI-NEXT: ldr s6, [sp, #16] ; CHECK-GI-NEXT: fmov s17, w4 ; CHECK-GI-NEXT: mov v7.s[1], w1 ; CHECK-GI-NEXT: add x8, sp, #16 ; CHECK-GI-NEXT: movi v16.2d, #0000000000000000 ; CHECK-GI-NEXT: mov v2.s[1], v4.s[0] ; CHECK-GI-NEXT: ldr s4, [sp, #40] ; CHECK-GI-NEXT: mov v1.s[2], v5.s[0] ; CHECK-GI-NEXT: fmov s5, w0 ; CHECK-GI-NEXT: ld1 { v3.s }[2], [x9] ; CHECK-GI-NEXT: mov v0.s[2], v4.s[0] ; CHECK-GI-NEXT: mov v17.s[1], w5 ; CHECK-GI-NEXT: mov v7.s[2], w2 ; CHECK-GI-NEXT: mov v5.s[1], w1 ; CHECK-GI-NEXT: mov v2.s[2], v4.s[0] ; CHECK-GI-NEXT: fmov s4, w4 ; CHECK-GI-NEXT: mov v1.s[3], v6.s[0] ; CHECK-GI-NEXT: movi v6.4s, #31 ; CHECK-GI-NEXT: ld1 { v3.s }[3], [x8] ; CHECK-GI-NEXT: mov v17.s[2], w6 ; CHECK-GI-NEXT: mov v4.s[1], w5 ; CHECK-GI-NEXT: mov v7.s[3], w3 ; CHECK-GI-NEXT: mov v5.s[2], w2 ; CHECK-GI-NEXT: neg v3.4s, v3.4s ; CHECK-GI-NEXT: sub v2.4s, v16.4s, v2.4s ; CHECK-GI-NEXT: and v1.16b, v1.16b, v6.16b ; CHECK-GI-NEXT: and v0.16b, v0.16b, v6.16b ; CHECK-GI-NEXT: mov v4.s[2], w6 ; CHECK-GI-NEXT: and v3.16b, v3.16b, v6.16b ; CHECK-GI-NEXT: and v2.16b, v2.16b, v6.16b ; CHECK-GI-NEXT: mov v5.s[3], w3 ; CHECK-GI-NEXT: neg v1.4s, v1.4s ; CHECK-GI-NEXT: neg v0.4s, v0.4s ; CHECK-GI-NEXT: ushl v1.4s, v7.4s, v1.4s ; CHECK-GI-NEXT: ushl v0.4s, v17.4s, v0.4s ; CHECK-GI-NEXT: ushl v2.4s, v4.4s, v2.4s ; CHECK-GI-NEXT: ushl v3.4s, v5.4s, v3.4s ; CHECK-GI-NEXT: orr v0.16b, v0.16b, v2.16b ; CHECK-GI-NEXT: orr v1.16b, v1.16b, v3.16b ; CHECK-GI-NEXT: mov s5, v0.s[1] ; CHECK-GI-NEXT: mov s6, v0.s[2] ; CHECK-GI-NEXT: fmov w4, s0 ; CHECK-GI-NEXT: mov s2, v1.s[1] ; CHECK-GI-NEXT: mov s3, v1.s[2] ; CHECK-GI-NEXT: mov s4, v1.s[3] ; CHECK-GI-NEXT: fmov w0, s1 ; CHECK-GI-NEXT: fmov w5, s5 ; CHECK-GI-NEXT: fmov w6, s6 ; CHECK-GI-NEXT: fmov w1, s2 ; CHECK-GI-NEXT: fmov w2, s3 ; CHECK-GI-NEXT: fmov w3, s4 ; CHECK-GI-NEXT: ret entry: %d = call <7 x i32> @llvm.fshr(<7 x i32> %a, <7 x i32> %a, <7 x i32> %c) ret <7 x i32> %d } define <8 x i32> @rotl_v8i32(<8 x i32> %a, <8 x i32> %c) { ; CHECK-LABEL: rotl_v8i32: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: movi v4.4s, #31 ; CHECK-NEXT: neg v5.4s, v2.4s ; CHECK-NEXT: neg v6.4s, v3.4s ; CHECK-NEXT: and v5.16b, v5.16b, v4.16b ; CHECK-NEXT: and v6.16b, v6.16b, v4.16b ; CHECK-NEXT: and v2.16b, v2.16b, v4.16b ; CHECK-NEXT: and v3.16b, v3.16b, v4.16b ; CHECK-NEXT: neg v4.4s, v5.4s ; CHECK-NEXT: neg v5.4s, v6.4s ; CHECK-NEXT: ushl v2.4s, v0.4s, v2.4s ; CHECK-NEXT: ushl v3.4s, v1.4s, v3.4s ; CHECK-NEXT: ushl v0.4s, v0.4s, v4.4s ; CHECK-NEXT: ushl v1.4s, v1.4s, v5.4s ; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b ; CHECK-NEXT: orr v1.16b, v3.16b, v1.16b ; CHECK-NEXT: ret entry: %d = call <8 x i32> @llvm.fshl(<8 x i32> %a, <8 x i32> %a, <8 x i32> %c) ret <8 x i32> %d } define <8 x i32> @rotr_v8i32(<8 x i32> %a, <8 x i32> %c) { ; CHECK-SD-LABEL: rotr_v8i32: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: movi v4.4s, #31 ; CHECK-SD-NEXT: neg v5.4s, v2.4s ; CHECK-SD-NEXT: neg v6.4s, v3.4s ; CHECK-SD-NEXT: and v2.16b, v2.16b, v4.16b ; CHECK-SD-NEXT: and v3.16b, v3.16b, v4.16b ; CHECK-SD-NEXT: and v5.16b, v5.16b, v4.16b ; CHECK-SD-NEXT: and v4.16b, v6.16b, v4.16b ; CHECK-SD-NEXT: neg v2.4s, v2.4s ; CHECK-SD-NEXT: neg v3.4s, v3.4s ; CHECK-SD-NEXT: ushl v5.4s, v0.4s, v5.4s ; CHECK-SD-NEXT: ushl v0.4s, v0.4s, v2.4s ; CHECK-SD-NEXT: ushl v2.4s, v1.4s, v4.4s ; CHECK-SD-NEXT: ushl v1.4s, v1.4s, v3.4s ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v5.16b ; CHECK-SD-NEXT: orr v1.16b, v1.16b, v2.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotr_v8i32: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: movi v4.4s, #31 ; CHECK-GI-NEXT: neg v6.4s, v3.4s ; CHECK-GI-NEXT: and v5.16b, v2.16b, v4.16b ; CHECK-GI-NEXT: neg v2.4s, v2.4s ; CHECK-GI-NEXT: and v3.16b, v3.16b, v4.16b ; CHECK-GI-NEXT: neg v5.4s, v5.4s ; CHECK-GI-NEXT: and v2.16b, v2.16b, v4.16b ; CHECK-GI-NEXT: and v4.16b, v6.16b, v4.16b ; CHECK-GI-NEXT: neg v3.4s, v3.4s ; CHECK-GI-NEXT: ushl v5.4s, v0.4s, v5.4s ; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v2.4s ; CHECK-GI-NEXT: ushl v3.4s, v1.4s, v3.4s ; CHECK-GI-NEXT: ushl v1.4s, v1.4s, v4.4s ; CHECK-GI-NEXT: orr v0.16b, v5.16b, v0.16b ; CHECK-GI-NEXT: orr v1.16b, v3.16b, v1.16b ; CHECK-GI-NEXT: ret entry: %d = call <8 x i32> @llvm.fshr(<8 x i32> %a, <8 x i32> %a, <8 x i32> %c) ret <8 x i32> %d } define <2 x i64> @rotl_v2i64(<2 x i64> %a, <2 x i64> %c) { ; CHECK-SD-LABEL: rotl_v2i64: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: mov w8, #63 // =0x3f ; CHECK-SD-NEXT: neg v3.2d, v1.2d ; CHECK-SD-NEXT: dup v2.2d, x8 ; CHECK-SD-NEXT: and v3.16b, v3.16b, v2.16b ; CHECK-SD-NEXT: and v1.16b, v1.16b, v2.16b ; CHECK-SD-NEXT: neg v2.2d, v3.2d ; CHECK-SD-NEXT: ushl v1.2d, v0.2d, v1.2d ; CHECK-SD-NEXT: ushl v0.2d, v0.2d, v2.2d ; CHECK-SD-NEXT: orr v0.16b, v1.16b, v0.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotl_v2i64: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: adrp x8, .LCPI60_0 ; CHECK-GI-NEXT: neg v2.2d, v1.2d ; CHECK-GI-NEXT: ldr q3, [x8, :lo12:.LCPI60_0] ; CHECK-GI-NEXT: and v2.16b, v2.16b, v3.16b ; CHECK-GI-NEXT: and v1.16b, v1.16b, v3.16b ; CHECK-GI-NEXT: neg v2.2d, v2.2d ; CHECK-GI-NEXT: ushl v1.2d, v0.2d, v1.2d ; CHECK-GI-NEXT: ushl v0.2d, v0.2d, v2.2d ; CHECK-GI-NEXT: orr v0.16b, v1.16b, v0.16b ; CHECK-GI-NEXT: ret entry: %d = call <2 x i64> @llvm.fshl(<2 x i64> %a, <2 x i64> %a, <2 x i64> %c) ret <2 x i64> %d } define <2 x i64> @rotr_v2i64(<2 x i64> %a, <2 x i64> %c) { ; CHECK-SD-LABEL: rotr_v2i64: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: mov w8, #63 // =0x3f ; CHECK-SD-NEXT: neg v3.2d, v1.2d ; CHECK-SD-NEXT: dup v2.2d, x8 ; CHECK-SD-NEXT: and v1.16b, v1.16b, v2.16b ; CHECK-SD-NEXT: and v2.16b, v3.16b, v2.16b ; CHECK-SD-NEXT: neg v1.2d, v1.2d ; CHECK-SD-NEXT: ushl v2.2d, v0.2d, v2.2d ; CHECK-SD-NEXT: ushl v0.2d, v0.2d, v1.2d ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v2.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotr_v2i64: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: adrp x8, .LCPI61_0 ; CHECK-GI-NEXT: neg v3.2d, v1.2d ; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI61_0] ; CHECK-GI-NEXT: and v1.16b, v1.16b, v2.16b ; CHECK-GI-NEXT: and v2.16b, v3.16b, v2.16b ; CHECK-GI-NEXT: neg v1.2d, v1.2d ; CHECK-GI-NEXT: ushl v1.2d, v0.2d, v1.2d ; CHECK-GI-NEXT: ushl v0.2d, v0.2d, v2.2d ; CHECK-GI-NEXT: orr v0.16b, v1.16b, v0.16b ; CHECK-GI-NEXT: ret entry: %d = call <2 x i64> @llvm.fshr(<2 x i64> %a, <2 x i64> %a, <2 x i64> %c) ret <2 x i64> %d } define <4 x i64> @rotl_v4i64(<4 x i64> %a, <4 x i64> %c) { ; CHECK-SD-LABEL: rotl_v4i64: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: mov w8, #63 // =0x3f ; CHECK-SD-NEXT: neg v5.2d, v2.2d ; CHECK-SD-NEXT: neg v6.2d, v3.2d ; CHECK-SD-NEXT: dup v4.2d, x8 ; CHECK-SD-NEXT: and v5.16b, v5.16b, v4.16b ; CHECK-SD-NEXT: and v6.16b, v6.16b, v4.16b ; CHECK-SD-NEXT: and v2.16b, v2.16b, v4.16b ; CHECK-SD-NEXT: and v3.16b, v3.16b, v4.16b ; CHECK-SD-NEXT: neg v4.2d, v5.2d ; CHECK-SD-NEXT: neg v5.2d, v6.2d ; CHECK-SD-NEXT: ushl v2.2d, v0.2d, v2.2d ; CHECK-SD-NEXT: ushl v3.2d, v1.2d, v3.2d ; CHECK-SD-NEXT: ushl v0.2d, v0.2d, v4.2d ; CHECK-SD-NEXT: ushl v1.2d, v1.2d, v5.2d ; CHECK-SD-NEXT: orr v0.16b, v2.16b, v0.16b ; CHECK-SD-NEXT: orr v1.16b, v3.16b, v1.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotl_v4i64: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: adrp x8, .LCPI62_0 ; CHECK-GI-NEXT: neg v4.2d, v2.2d ; CHECK-GI-NEXT: neg v5.2d, v3.2d ; CHECK-GI-NEXT: ldr q6, [x8, :lo12:.LCPI62_0] ; CHECK-GI-NEXT: and v4.16b, v4.16b, v6.16b ; CHECK-GI-NEXT: and v5.16b, v5.16b, v6.16b ; CHECK-GI-NEXT: and v2.16b, v2.16b, v6.16b ; CHECK-GI-NEXT: and v3.16b, v3.16b, v6.16b ; CHECK-GI-NEXT: neg v4.2d, v4.2d ; CHECK-GI-NEXT: neg v5.2d, v5.2d ; CHECK-GI-NEXT: ushl v2.2d, v0.2d, v2.2d ; CHECK-GI-NEXT: ushl v3.2d, v1.2d, v3.2d ; CHECK-GI-NEXT: ushl v0.2d, v0.2d, v4.2d ; CHECK-GI-NEXT: ushl v1.2d, v1.2d, v5.2d ; CHECK-GI-NEXT: orr v0.16b, v2.16b, v0.16b ; CHECK-GI-NEXT: orr v1.16b, v3.16b, v1.16b ; CHECK-GI-NEXT: ret entry: %d = call <4 x i64> @llvm.fshl(<4 x i64> %a, <4 x i64> %a, <4 x i64> %c) ret <4 x i64> %d } define <4 x i64> @rotr_v4i64(<4 x i64> %a, <4 x i64> %c) { ; CHECK-SD-LABEL: rotr_v4i64: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: mov w8, #63 // =0x3f ; CHECK-SD-NEXT: neg v5.2d, v2.2d ; CHECK-SD-NEXT: neg v6.2d, v3.2d ; CHECK-SD-NEXT: dup v4.2d, x8 ; CHECK-SD-NEXT: and v2.16b, v2.16b, v4.16b ; CHECK-SD-NEXT: and v3.16b, v3.16b, v4.16b ; CHECK-SD-NEXT: and v5.16b, v5.16b, v4.16b ; CHECK-SD-NEXT: and v4.16b, v6.16b, v4.16b ; CHECK-SD-NEXT: neg v2.2d, v2.2d ; CHECK-SD-NEXT: neg v3.2d, v3.2d ; CHECK-SD-NEXT: ushl v5.2d, v0.2d, v5.2d ; CHECK-SD-NEXT: ushl v0.2d, v0.2d, v2.2d ; CHECK-SD-NEXT: ushl v2.2d, v1.2d, v4.2d ; CHECK-SD-NEXT: ushl v1.2d, v1.2d, v3.2d ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v5.16b ; CHECK-SD-NEXT: orr v1.16b, v1.16b, v2.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotr_v4i64: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: adrp x8, .LCPI63_0 ; CHECK-GI-NEXT: neg v6.2d, v3.2d ; CHECK-GI-NEXT: ldr q4, [x8, :lo12:.LCPI63_0] ; CHECK-GI-NEXT: and v5.16b, v2.16b, v4.16b ; CHECK-GI-NEXT: neg v2.2d, v2.2d ; CHECK-GI-NEXT: and v3.16b, v3.16b, v4.16b ; CHECK-GI-NEXT: neg v5.2d, v5.2d ; CHECK-GI-NEXT: and v2.16b, v2.16b, v4.16b ; CHECK-GI-NEXT: and v4.16b, v6.16b, v4.16b ; CHECK-GI-NEXT: neg v3.2d, v3.2d ; CHECK-GI-NEXT: ushl v5.2d, v0.2d, v5.2d ; CHECK-GI-NEXT: ushl v0.2d, v0.2d, v2.2d ; CHECK-GI-NEXT: ushl v3.2d, v1.2d, v3.2d ; CHECK-GI-NEXT: ushl v1.2d, v1.2d, v4.2d ; CHECK-GI-NEXT: orr v0.16b, v5.16b, v0.16b ; CHECK-GI-NEXT: orr v1.16b, v3.16b, v1.16b ; CHECK-GI-NEXT: ret entry: %d = call <4 x i64> @llvm.fshr(<4 x i64> %a, <4 x i64> %a, <4 x i64> %c) ret <4 x i64> %d } define <2 x i128> @rotl_v2i128(<2 x i128> %a, <2 x i128> %c) { ; CHECK-SD-LABEL: rotl_v2i128: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: tst x4, #0x40 ; CHECK-SD-NEXT: mvn w9, w4 ; CHECK-SD-NEXT: csel x8, x1, x0, ne ; CHECK-SD-NEXT: csel x10, x0, x1, ne ; CHECK-SD-NEXT: tst x6, #0x40 ; CHECK-SD-NEXT: lsl x11, x8, x4 ; CHECK-SD-NEXT: lsr x12, x10, #1 ; CHECK-SD-NEXT: lsr x8, x8, #1 ; CHECK-SD-NEXT: csel x13, x3, x2, ne ; CHECK-SD-NEXT: csel x14, x2, x3, ne ; CHECK-SD-NEXT: lsl x10, x10, x4 ; CHECK-SD-NEXT: lsr x15, x14, #1 ; CHECK-SD-NEXT: lsr x16, x13, #1 ; CHECK-SD-NEXT: lsr x12, x12, x9 ; CHECK-SD-NEXT: lsr x8, x8, x9 ; CHECK-SD-NEXT: lsl x9, x13, x6 ; CHECK-SD-NEXT: mvn w13, w6 ; CHECK-SD-NEXT: lsr x15, x15, x13 ; CHECK-SD-NEXT: lsl x14, x14, x6 ; CHECK-SD-NEXT: lsr x13, x16, x13 ; CHECK-SD-NEXT: orr x0, x11, x12 ; CHECK-SD-NEXT: orr x1, x10, x8 ; CHECK-SD-NEXT: orr x2, x9, x15 ; CHECK-SD-NEXT: orr x3, x14, x13 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotl_v2i128: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: and x8, x4, #0x7f ; CHECK-GI-NEXT: mov w9, #64 // =0x40 ; CHECK-GI-NEXT: neg x13, x4 ; CHECK-GI-NEXT: sub x10, x9, x8 ; CHECK-GI-NEXT: lsl x12, x1, x8 ; CHECK-GI-NEXT: sub x11, x8, #64 ; CHECK-GI-NEXT: lsr x10, x0, x10 ; CHECK-GI-NEXT: lsl x14, x0, x8 ; CHECK-GI-NEXT: lsl x11, x0, x11 ; CHECK-GI-NEXT: cmp x8, #64 ; CHECK-GI-NEXT: neg x15, x6 ; CHECK-GI-NEXT: orr x8, x10, x12 ; CHECK-GI-NEXT: and x10, x6, #0x7f ; CHECK-GI-NEXT: csel x12, x14, xzr, lo ; CHECK-GI-NEXT: sub x14, x9, x10 ; CHECK-GI-NEXT: csel x8, x8, x11, lo ; CHECK-GI-NEXT: tst x4, #0x7f ; CHECK-GI-NEXT: sub x11, x10, #64 ; CHECK-GI-NEXT: lsl x16, x2, x10 ; CHECK-GI-NEXT: lsr x14, x2, x14 ; CHECK-GI-NEXT: lsl x17, x3, x10 ; CHECK-GI-NEXT: csel x8, x1, x8, eq ; CHECK-GI-NEXT: lsl x11, x2, x11 ; CHECK-GI-NEXT: cmp x10, #64 ; CHECK-GI-NEXT: and x4, x15, #0x7f ; CHECK-GI-NEXT: orr x10, x14, x17 ; CHECK-GI-NEXT: csel x14, x16, xzr, lo ; CHECK-GI-NEXT: and x16, x13, #0x7f ; CHECK-GI-NEXT: csel x10, x10, x11, lo ; CHECK-GI-NEXT: sub x11, x9, x16 ; CHECK-GI-NEXT: sub x17, x16, #64 ; CHECK-GI-NEXT: lsr x18, x0, x16 ; CHECK-GI-NEXT: lsl x11, x1, x11 ; CHECK-GI-NEXT: tst x6, #0x7f ; CHECK-GI-NEXT: lsr x17, x1, x17 ; CHECK-GI-NEXT: csel x10, x3, x10, eq ; CHECK-GI-NEXT: cmp x16, #64 ; CHECK-GI-NEXT: orr x11, x18, x11 ; CHECK-GI-NEXT: sub x9, x9, x4 ; CHECK-GI-NEXT: lsr x1, x1, x16 ; CHECK-GI-NEXT: csel x11, x11, x17, lo ; CHECK-GI-NEXT: tst x13, #0x7f ; CHECK-GI-NEXT: sub x13, x4, #64 ; CHECK-GI-NEXT: lsr x17, x2, x4 ; CHECK-GI-NEXT: lsl x9, x3, x9 ; CHECK-GI-NEXT: csel x11, x0, x11, eq ; CHECK-GI-NEXT: cmp x16, #64 ; CHECK-GI-NEXT: lsr x13, x3, x13 ; CHECK-GI-NEXT: orr x0, x12, x11 ; CHECK-GI-NEXT: csel x16, x1, xzr, lo ; CHECK-GI-NEXT: orr x9, x17, x9 ; CHECK-GI-NEXT: cmp x4, #64 ; CHECK-GI-NEXT: lsr x17, x3, x4 ; CHECK-GI-NEXT: csel x9, x9, x13, lo ; CHECK-GI-NEXT: tst x15, #0x7f ; CHECK-GI-NEXT: csel x9, x2, x9, eq ; CHECK-GI-NEXT: cmp x4, #64 ; CHECK-GI-NEXT: orr x1, x8, x16 ; CHECK-GI-NEXT: csel x13, x17, xzr, lo ; CHECK-GI-NEXT: orr x2, x14, x9 ; CHECK-GI-NEXT: orr x3, x10, x13 ; CHECK-GI-NEXT: ret entry: %d = call <2 x i128> @llvm.fshl(<2 x i128> %a, <2 x i128> %a, <2 x i128> %c) ret <2 x i128> %d } define <2 x i128> @rotr_v2i128(<2 x i128> %a, <2 x i128> %c) { ; CHECK-SD-LABEL: rotr_v2i128: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: tst x4, #0x40 ; CHECK-SD-NEXT: mvn w9, w4 ; CHECK-SD-NEXT: csel x8, x0, x1, eq ; CHECK-SD-NEXT: csel x10, x1, x0, eq ; CHECK-SD-NEXT: tst x6, #0x40 ; CHECK-SD-NEXT: lsr x11, x8, x4 ; CHECK-SD-NEXT: lsl x12, x10, #1 ; CHECK-SD-NEXT: lsl x8, x8, #1 ; CHECK-SD-NEXT: csel x13, x2, x3, eq ; CHECK-SD-NEXT: csel x14, x3, x2, eq ; CHECK-SD-NEXT: lsr x10, x10, x4 ; CHECK-SD-NEXT: lsl x15, x14, #1 ; CHECK-SD-NEXT: lsl x16, x13, #1 ; CHECK-SD-NEXT: lsl x12, x12, x9 ; CHECK-SD-NEXT: lsl x8, x8, x9 ; CHECK-SD-NEXT: lsr x9, x13, x6 ; CHECK-SD-NEXT: mvn w13, w6 ; CHECK-SD-NEXT: lsl x15, x15, x13 ; CHECK-SD-NEXT: lsr x14, x14, x6 ; CHECK-SD-NEXT: lsl x13, x16, x13 ; CHECK-SD-NEXT: orr x0, x12, x11 ; CHECK-SD-NEXT: orr x1, x8, x10 ; CHECK-SD-NEXT: orr x2, x15, x9 ; CHECK-SD-NEXT: orr x3, x13, x14 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotr_v2i128: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: and x8, x4, #0x7f ; CHECK-GI-NEXT: mov w9, #64 // =0x40 ; CHECK-GI-NEXT: and x14, x6, #0x7f ; CHECK-GI-NEXT: sub x10, x9, x8 ; CHECK-GI-NEXT: sub x11, x8, #64 ; CHECK-GI-NEXT: lsr x12, x0, x8 ; CHECK-GI-NEXT: lsl x10, x1, x10 ; CHECK-GI-NEXT: lsr x11, x1, x11 ; CHECK-GI-NEXT: cmp x8, #64 ; CHECK-GI-NEXT: sub x15, x9, x14 ; CHECK-GI-NEXT: neg x13, x4 ; CHECK-GI-NEXT: lsr x17, x3, x14 ; CHECK-GI-NEXT: orr x10, x12, x10 ; CHECK-GI-NEXT: lsr x12, x1, x8 ; CHECK-GI-NEXT: lsl x15, x3, x15 ; CHECK-GI-NEXT: csel x10, x10, x11, lo ; CHECK-GI-NEXT: tst x4, #0x7f ; CHECK-GI-NEXT: sub x11, x14, #64 ; CHECK-GI-NEXT: csel x10, x0, x10, eq ; CHECK-GI-NEXT: cmp x8, #64 ; CHECK-GI-NEXT: lsr x8, x2, x14 ; CHECK-GI-NEXT: lsr x11, x3, x11 ; CHECK-GI-NEXT: csel x12, x12, xzr, lo ; CHECK-GI-NEXT: cmp x14, #64 ; CHECK-GI-NEXT: orr x8, x8, x15 ; CHECK-GI-NEXT: neg x16, x6 ; CHECK-GI-NEXT: csel x8, x8, x11, lo ; CHECK-GI-NEXT: tst x6, #0x7f ; CHECK-GI-NEXT: and x11, x13, #0x7f ; CHECK-GI-NEXT: csel x8, x2, x8, eq ; CHECK-GI-NEXT: cmp x14, #64 ; CHECK-GI-NEXT: sub x14, x9, x11 ; CHECK-GI-NEXT: sub x15, x11, #64 ; CHECK-GI-NEXT: lsr x14, x0, x14 ; CHECK-GI-NEXT: lsl x18, x1, x11 ; CHECK-GI-NEXT: lsl x4, x0, x11 ; CHECK-GI-NEXT: lsl x15, x0, x15 ; CHECK-GI-NEXT: and x0, x16, #0x7f ; CHECK-GI-NEXT: csel x17, x17, xzr, lo ; CHECK-GI-NEXT: orr x14, x14, x18 ; CHECK-GI-NEXT: cmp x11, #64 ; CHECK-GI-NEXT: sub x9, x9, x0 ; CHECK-GI-NEXT: csel x14, x14, x15, lo ; CHECK-GI-NEXT: sub x15, x0, #64 ; CHECK-GI-NEXT: lsr x9, x2, x9 ; CHECK-GI-NEXT: lsl x18, x3, x0 ; CHECK-GI-NEXT: csel x11, x4, xzr, lo ; CHECK-GI-NEXT: tst x13, #0x7f ; CHECK-GI-NEXT: lsl x13, x2, x0 ; CHECK-GI-NEXT: lsl x15, x2, x15 ; CHECK-GI-NEXT: csel x14, x1, x14, eq ; CHECK-GI-NEXT: orr x9, x9, x18 ; CHECK-GI-NEXT: cmp x0, #64 ; CHECK-GI-NEXT: csel x13, x13, xzr, lo ; CHECK-GI-NEXT: csel x9, x9, x15, lo ; CHECK-GI-NEXT: tst x16, #0x7f ; CHECK-GI-NEXT: csel x9, x3, x9, eq ; CHECK-GI-NEXT: orr x0, x10, x11 ; CHECK-GI-NEXT: orr x1, x12, x14 ; CHECK-GI-NEXT: orr x2, x8, x13 ; CHECK-GI-NEXT: orr x3, x17, x9 ; CHECK-GI-NEXT: ret entry: %d = call <2 x i128> @llvm.fshr(<2 x i128> %a, <2 x i128> %a, <2 x i128> %c) ret <2 x i128> %d } define <8 x i8> @fshl_v8i8(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c) { ; CHECK-LABEL: fshl_v8i8: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: movi v3.8b, #7 ; CHECK-NEXT: ushr v1.8b, v1.8b, #1 ; CHECK-NEXT: bic v4.8b, v3.8b, v2.8b ; CHECK-NEXT: and v2.8b, v2.8b, v3.8b ; CHECK-NEXT: neg v3.8b, v4.8b ; CHECK-NEXT: ushl v0.8b, v0.8b, v2.8b ; CHECK-NEXT: ushl v1.8b, v1.8b, v3.8b ; CHECK-NEXT: orr v0.8b, v0.8b, v1.8b ; CHECK-NEXT: ret entry: %d = call <8 x i8> @llvm.fshl(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c) ret <8 x i8> %d } define <8 x i8> @fshr_v8i8(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c) { ; CHECK-SD-LABEL: fshr_v8i8: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: movi v3.8b, #7 ; CHECK-SD-NEXT: add v0.8b, v0.8b, v0.8b ; CHECK-SD-NEXT: and v4.8b, v2.8b, v3.8b ; CHECK-SD-NEXT: bic v2.8b, v3.8b, v2.8b ; CHECK-SD-NEXT: neg v3.8b, v4.8b ; CHECK-SD-NEXT: ushl v0.8b, v0.8b, v2.8b ; CHECK-SD-NEXT: ushl v1.8b, v1.8b, v3.8b ; CHECK-SD-NEXT: orr v0.8b, v0.8b, v1.8b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_v8i8: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: movi v3.8b, #7 ; CHECK-GI-NEXT: shl v0.8b, v0.8b, #1 ; CHECK-GI-NEXT: and v4.8b, v2.8b, v3.8b ; CHECK-GI-NEXT: bic v2.8b, v3.8b, v2.8b ; CHECK-GI-NEXT: neg v3.8b, v4.8b ; CHECK-GI-NEXT: ushl v0.8b, v0.8b, v2.8b ; CHECK-GI-NEXT: ushl v1.8b, v1.8b, v3.8b ; CHECK-GI-NEXT: orr v0.8b, v0.8b, v1.8b ; CHECK-GI-NEXT: ret entry: %d = call <8 x i8> @llvm.fshr(<8 x i8> %a, <8 x i8> %b, <8 x i8> %c) ret <8 x i8> %d } define <16 x i8> @fshl_v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) { ; CHECK-LABEL: fshl_v16i8: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: movi v3.16b, #7 ; CHECK-NEXT: ushr v1.16b, v1.16b, #1 ; CHECK-NEXT: bic v4.16b, v3.16b, v2.16b ; CHECK-NEXT: and v2.16b, v2.16b, v3.16b ; CHECK-NEXT: neg v3.16b, v4.16b ; CHECK-NEXT: ushl v0.16b, v0.16b, v2.16b ; CHECK-NEXT: ushl v1.16b, v1.16b, v3.16b ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret entry: %d = call <16 x i8> @llvm.fshl(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) ret <16 x i8> %d } define <16 x i8> @fshr_v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) { ; CHECK-SD-LABEL: fshr_v16i8: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: movi v3.16b, #7 ; CHECK-SD-NEXT: add v0.16b, v0.16b, v0.16b ; CHECK-SD-NEXT: and v4.16b, v2.16b, v3.16b ; CHECK-SD-NEXT: bic v2.16b, v3.16b, v2.16b ; CHECK-SD-NEXT: neg v3.16b, v4.16b ; CHECK-SD-NEXT: ushl v0.16b, v0.16b, v2.16b ; CHECK-SD-NEXT: ushl v1.16b, v1.16b, v3.16b ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_v16i8: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: movi v3.16b, #7 ; CHECK-GI-NEXT: shl v0.16b, v0.16b, #1 ; CHECK-GI-NEXT: and v4.16b, v2.16b, v3.16b ; CHECK-GI-NEXT: bic v2.16b, v3.16b, v2.16b ; CHECK-GI-NEXT: neg v3.16b, v4.16b ; CHECK-GI-NEXT: ushl v0.16b, v0.16b, v2.16b ; CHECK-GI-NEXT: ushl v1.16b, v1.16b, v3.16b ; CHECK-GI-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-GI-NEXT: ret entry: %d = call <16 x i8> @llvm.fshr(<16 x i8> %a, <16 x i8> %b, <16 x i8> %c) ret <16 x i8> %d } define <4 x i16> @fshl_v4i16(<4 x i16> %a, <4 x i16> %b, <4 x i16> %c) { ; CHECK-LABEL: fshl_v4i16: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: movi v3.4h, #15 ; CHECK-NEXT: ushr v1.4h, v1.4h, #1 ; CHECK-NEXT: bic v4.8b, v3.8b, v2.8b ; CHECK-NEXT: and v2.8b, v2.8b, v3.8b ; CHECK-NEXT: neg v3.4h, v4.4h ; CHECK-NEXT: ushl v0.4h, v0.4h, v2.4h ; CHECK-NEXT: ushl v1.4h, v1.4h, v3.4h ; CHECK-NEXT: orr v0.8b, v0.8b, v1.8b ; CHECK-NEXT: ret entry: %d = call <4 x i16> @llvm.fshl(<4 x i16> %a, <4 x i16> %b, <4 x i16> %c) ret <4 x i16> %d } define <4 x i16> @fshr_v4i16(<4 x i16> %a, <4 x i16> %b, <4 x i16> %c) { ; CHECK-SD-LABEL: fshr_v4i16: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: movi v3.4h, #15 ; CHECK-SD-NEXT: add v0.4h, v0.4h, v0.4h ; CHECK-SD-NEXT: and v4.8b, v2.8b, v3.8b ; CHECK-SD-NEXT: bic v2.8b, v3.8b, v2.8b ; CHECK-SD-NEXT: neg v3.4h, v4.4h ; CHECK-SD-NEXT: ushl v0.4h, v0.4h, v2.4h ; CHECK-SD-NEXT: ushl v1.4h, v1.4h, v3.4h ; CHECK-SD-NEXT: orr v0.8b, v0.8b, v1.8b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_v4i16: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: movi v3.4h, #15 ; CHECK-GI-NEXT: shl v0.4h, v0.4h, #1 ; CHECK-GI-NEXT: and v4.8b, v2.8b, v3.8b ; CHECK-GI-NEXT: bic v2.8b, v3.8b, v2.8b ; CHECK-GI-NEXT: neg v3.4h, v4.4h ; CHECK-GI-NEXT: ushl v0.4h, v0.4h, v2.4h ; CHECK-GI-NEXT: ushl v1.4h, v1.4h, v3.4h ; CHECK-GI-NEXT: orr v0.8b, v0.8b, v1.8b ; CHECK-GI-NEXT: ret entry: %d = call <4 x i16> @llvm.fshr(<4 x i16> %a, <4 x i16> %b, <4 x i16> %c) ret <4 x i16> %d } define <7 x i16> @fshl_v7i16(<7 x i16> %a, <7 x i16> %b, <7 x i16> %c) { ; CHECK-SD-LABEL: fshl_v7i16: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: movi v3.8h, #15 ; CHECK-SD-NEXT: ushr v1.8h, v1.8h, #1 ; CHECK-SD-NEXT: bic v4.16b, v3.16b, v2.16b ; CHECK-SD-NEXT: and v2.16b, v2.16b, v3.16b ; CHECK-SD-NEXT: neg v3.8h, v4.8h ; CHECK-SD-NEXT: ushl v0.8h, v0.8h, v2.8h ; CHECK-SD-NEXT: ushl v1.8h, v1.8h, v3.8h ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshl_v7i16: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: movi v3.2d, #0xffffffffffffffff ; CHECK-GI-NEXT: movi v4.8h, #15 ; CHECK-GI-NEXT: ushr v1.8h, v1.8h, #1 ; CHECK-GI-NEXT: eor v3.16b, v2.16b, v3.16b ; CHECK-GI-NEXT: and v2.16b, v2.16b, v4.16b ; CHECK-GI-NEXT: and v3.16b, v3.16b, v4.16b ; CHECK-GI-NEXT: ushl v0.8h, v0.8h, v2.8h ; CHECK-GI-NEXT: neg v3.8h, v3.8h ; CHECK-GI-NEXT: ushl v1.8h, v1.8h, v3.8h ; CHECK-GI-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-GI-NEXT: ret entry: %d = call <7 x i16> @llvm.fshl(<7 x i16> %a, <7 x i16> %b, <7 x i16> %c) ret <7 x i16> %d } define <7 x i16> @fshr_v7i16(<7 x i16> %a, <7 x i16> %b, <7 x i16> %c) { ; CHECK-SD-LABEL: fshr_v7i16: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: movi v3.8h, #15 ; CHECK-SD-NEXT: add v0.8h, v0.8h, v0.8h ; CHECK-SD-NEXT: and v4.16b, v2.16b, v3.16b ; CHECK-SD-NEXT: bic v2.16b, v3.16b, v2.16b ; CHECK-SD-NEXT: neg v3.8h, v4.8h ; CHECK-SD-NEXT: ushl v0.8h, v0.8h, v2.8h ; CHECK-SD-NEXT: ushl v1.8h, v1.8h, v3.8h ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_v7i16: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: movi v3.8h, #15 ; CHECK-GI-NEXT: movi v4.2d, #0xffffffffffffffff ; CHECK-GI-NEXT: shl v0.8h, v0.8h, #1 ; CHECK-GI-NEXT: and v5.16b, v2.16b, v3.16b ; CHECK-GI-NEXT: eor v2.16b, v2.16b, v4.16b ; CHECK-GI-NEXT: and v2.16b, v2.16b, v3.16b ; CHECK-GI-NEXT: neg v3.8h, v5.8h ; CHECK-GI-NEXT: ushl v0.8h, v0.8h, v2.8h ; CHECK-GI-NEXT: ushl v1.8h, v1.8h, v3.8h ; CHECK-GI-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-GI-NEXT: ret entry: %d = call <7 x i16> @llvm.fshr(<7 x i16> %a, <7 x i16> %b, <7 x i16> %c) ret <7 x i16> %d } define <8 x i16> @fshl_v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) { ; CHECK-LABEL: fshl_v8i16: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: movi v3.8h, #15 ; CHECK-NEXT: ushr v1.8h, v1.8h, #1 ; CHECK-NEXT: bic v4.16b, v3.16b, v2.16b ; CHECK-NEXT: and v2.16b, v2.16b, v3.16b ; CHECK-NEXT: neg v3.8h, v4.8h ; CHECK-NEXT: ushl v0.8h, v0.8h, v2.8h ; CHECK-NEXT: ushl v1.8h, v1.8h, v3.8h ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret entry: %d = call <8 x i16> @llvm.fshl(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) ret <8 x i16> %d } define <8 x i16> @fshr_v8i16(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) { ; CHECK-SD-LABEL: fshr_v8i16: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: movi v3.8h, #15 ; CHECK-SD-NEXT: add v0.8h, v0.8h, v0.8h ; CHECK-SD-NEXT: and v4.16b, v2.16b, v3.16b ; CHECK-SD-NEXT: bic v2.16b, v3.16b, v2.16b ; CHECK-SD-NEXT: neg v3.8h, v4.8h ; CHECK-SD-NEXT: ushl v0.8h, v0.8h, v2.8h ; CHECK-SD-NEXT: ushl v1.8h, v1.8h, v3.8h ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_v8i16: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: movi v3.8h, #15 ; CHECK-GI-NEXT: shl v0.8h, v0.8h, #1 ; CHECK-GI-NEXT: and v4.16b, v2.16b, v3.16b ; CHECK-GI-NEXT: bic v2.16b, v3.16b, v2.16b ; CHECK-GI-NEXT: neg v3.8h, v4.8h ; CHECK-GI-NEXT: ushl v0.8h, v0.8h, v2.8h ; CHECK-GI-NEXT: ushl v1.8h, v1.8h, v3.8h ; CHECK-GI-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-GI-NEXT: ret entry: %d = call <8 x i16> @llvm.fshr(<8 x i16> %a, <8 x i16> %b, <8 x i16> %c) ret <8 x i16> %d } define <16 x i16> @fshl_v16i16(<16 x i16> %a, <16 x i16> %b, <16 x i16> %c) { ; CHECK-LABEL: fshl_v16i16: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: movi v6.8h, #15 ; CHECK-NEXT: ushr v2.8h, v2.8h, #1 ; CHECK-NEXT: ushr v3.8h, v3.8h, #1 ; CHECK-NEXT: bic v7.16b, v6.16b, v4.16b ; CHECK-NEXT: bic v16.16b, v6.16b, v5.16b ; CHECK-NEXT: and v4.16b, v4.16b, v6.16b ; CHECK-NEXT: and v5.16b, v5.16b, v6.16b ; CHECK-NEXT: neg v6.8h, v7.8h ; CHECK-NEXT: neg v7.8h, v16.8h ; CHECK-NEXT: ushl v0.8h, v0.8h, v4.8h ; CHECK-NEXT: ushl v1.8h, v1.8h, v5.8h ; CHECK-NEXT: ushl v2.8h, v2.8h, v6.8h ; CHECK-NEXT: ushl v3.8h, v3.8h, v7.8h ; CHECK-NEXT: orr v0.16b, v0.16b, v2.16b ; CHECK-NEXT: orr v1.16b, v1.16b, v3.16b ; CHECK-NEXT: ret entry: %d = call <16 x i16> @llvm.fshl(<16 x i16> %a, <16 x i16> %b, <16 x i16> %c) ret <16 x i16> %d } define <16 x i16> @fshr_v16i16(<16 x i16> %a, <16 x i16> %b, <16 x i16> %c) { ; CHECK-SD-LABEL: fshr_v16i16: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: movi v6.8h, #15 ; CHECK-SD-NEXT: add v0.8h, v0.8h, v0.8h ; CHECK-SD-NEXT: add v1.8h, v1.8h, v1.8h ; CHECK-SD-NEXT: and v7.16b, v4.16b, v6.16b ; CHECK-SD-NEXT: and v16.16b, v5.16b, v6.16b ; CHECK-SD-NEXT: bic v4.16b, v6.16b, v4.16b ; CHECK-SD-NEXT: bic v5.16b, v6.16b, v5.16b ; CHECK-SD-NEXT: neg v6.8h, v7.8h ; CHECK-SD-NEXT: neg v7.8h, v16.8h ; CHECK-SD-NEXT: ushl v0.8h, v0.8h, v4.8h ; CHECK-SD-NEXT: ushl v1.8h, v1.8h, v5.8h ; CHECK-SD-NEXT: ushl v2.8h, v2.8h, v6.8h ; CHECK-SD-NEXT: ushl v3.8h, v3.8h, v7.8h ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v2.16b ; CHECK-SD-NEXT: orr v1.16b, v1.16b, v3.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_v16i16: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: movi v6.8h, #15 ; CHECK-GI-NEXT: shl v0.8h, v0.8h, #1 ; CHECK-GI-NEXT: shl v1.8h, v1.8h, #1 ; CHECK-GI-NEXT: and v7.16b, v4.16b, v6.16b ; CHECK-GI-NEXT: and v16.16b, v5.16b, v6.16b ; CHECK-GI-NEXT: bic v4.16b, v6.16b, v4.16b ; CHECK-GI-NEXT: bic v5.16b, v6.16b, v5.16b ; CHECK-GI-NEXT: neg v6.8h, v7.8h ; CHECK-GI-NEXT: neg v7.8h, v16.8h ; CHECK-GI-NEXT: ushl v0.8h, v0.8h, v4.8h ; CHECK-GI-NEXT: ushl v1.8h, v1.8h, v5.8h ; CHECK-GI-NEXT: ushl v2.8h, v2.8h, v6.8h ; CHECK-GI-NEXT: ushl v3.8h, v3.8h, v7.8h ; CHECK-GI-NEXT: orr v0.16b, v0.16b, v2.16b ; CHECK-GI-NEXT: orr v1.16b, v1.16b, v3.16b ; CHECK-GI-NEXT: ret entry: %d = call <16 x i16> @llvm.fshr(<16 x i16> %a, <16 x i16> %b, <16 x i16> %c) ret <16 x i16> %d } define <2 x i32> @fshl_v2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c) { ; CHECK-LABEL: fshl_v2i32: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: movi v3.2s, #31 ; CHECK-NEXT: ushr v1.2s, v1.2s, #1 ; CHECK-NEXT: bic v4.8b, v3.8b, v2.8b ; CHECK-NEXT: and v2.8b, v2.8b, v3.8b ; CHECK-NEXT: neg v3.2s, v4.2s ; CHECK-NEXT: ushl v0.2s, v0.2s, v2.2s ; CHECK-NEXT: ushl v1.2s, v1.2s, v3.2s ; CHECK-NEXT: orr v0.8b, v0.8b, v1.8b ; CHECK-NEXT: ret entry: %d = call <2 x i32> @llvm.fshl(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c) ret <2 x i32> %d } define <2 x i32> @fshr_v2i32(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c) { ; CHECK-SD-LABEL: fshr_v2i32: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: movi v3.2s, #31 ; CHECK-SD-NEXT: add v0.2s, v0.2s, v0.2s ; CHECK-SD-NEXT: and v4.8b, v2.8b, v3.8b ; CHECK-SD-NEXT: bic v2.8b, v3.8b, v2.8b ; CHECK-SD-NEXT: neg v3.2s, v4.2s ; CHECK-SD-NEXT: ushl v0.2s, v0.2s, v2.2s ; CHECK-SD-NEXT: ushl v1.2s, v1.2s, v3.2s ; CHECK-SD-NEXT: orr v0.8b, v0.8b, v1.8b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_v2i32: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: movi v3.2s, #31 ; CHECK-GI-NEXT: shl v0.2s, v0.2s, #1 ; CHECK-GI-NEXT: and v4.8b, v2.8b, v3.8b ; CHECK-GI-NEXT: bic v2.8b, v3.8b, v2.8b ; CHECK-GI-NEXT: neg v3.2s, v4.2s ; CHECK-GI-NEXT: ushl v0.2s, v0.2s, v2.2s ; CHECK-GI-NEXT: ushl v1.2s, v1.2s, v3.2s ; CHECK-GI-NEXT: orr v0.8b, v0.8b, v1.8b ; CHECK-GI-NEXT: ret entry: %d = call <2 x i32> @llvm.fshr(<2 x i32> %a, <2 x i32> %b, <2 x i32> %c) ret <2 x i32> %d } define <4 x i32> @fshl_v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) { ; CHECK-LABEL: fshl_v4i32: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: movi v3.4s, #31 ; CHECK-NEXT: ushr v1.4s, v1.4s, #1 ; CHECK-NEXT: bic v4.16b, v3.16b, v2.16b ; CHECK-NEXT: and v2.16b, v2.16b, v3.16b ; CHECK-NEXT: neg v3.4s, v4.4s ; CHECK-NEXT: ushl v0.4s, v0.4s, v2.4s ; CHECK-NEXT: ushl v1.4s, v1.4s, v3.4s ; CHECK-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret entry: %d = call <4 x i32> @llvm.fshl(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) ret <4 x i32> %d } define <4 x i32> @fshr_v4i32(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) { ; CHECK-SD-LABEL: fshr_v4i32: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: movi v3.4s, #31 ; CHECK-SD-NEXT: add v0.4s, v0.4s, v0.4s ; CHECK-SD-NEXT: and v4.16b, v2.16b, v3.16b ; CHECK-SD-NEXT: bic v2.16b, v3.16b, v2.16b ; CHECK-SD-NEXT: neg v3.4s, v4.4s ; CHECK-SD-NEXT: ushl v0.4s, v0.4s, v2.4s ; CHECK-SD-NEXT: ushl v1.4s, v1.4s, v3.4s ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_v4i32: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: movi v3.4s, #31 ; CHECK-GI-NEXT: shl v0.4s, v0.4s, #1 ; CHECK-GI-NEXT: and v4.16b, v2.16b, v3.16b ; CHECK-GI-NEXT: bic v2.16b, v3.16b, v2.16b ; CHECK-GI-NEXT: neg v3.4s, v4.4s ; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v2.4s ; CHECK-GI-NEXT: ushl v1.4s, v1.4s, v3.4s ; CHECK-GI-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-GI-NEXT: ret entry: %d = call <4 x i32> @llvm.fshr(<4 x i32> %a, <4 x i32> %b, <4 x i32> %c) ret <4 x i32> %d } define <7 x i32> @fshl_v7i32(<7 x i32> %a, <7 x i32> %b, <7 x i32> %c) { ; CHECK-SD-LABEL: fshl_v7i32: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: ldr s0, [sp, #48] ; CHECK-SD-NEXT: add x8, sp, #56 ; CHECK-SD-NEXT: fmov s3, w7 ; CHECK-SD-NEXT: fmov s4, w0 ; CHECK-SD-NEXT: ldr s2, [sp, #80] ; CHECK-SD-NEXT: mov x9, sp ; CHECK-SD-NEXT: ld1 { v0.s }[1], [x8] ; CHECK-SD-NEXT: add x10, sp, #88 ; CHECK-SD-NEXT: ldr s1, [sp, #24] ; CHECK-SD-NEXT: ld1 { v3.s }[1], [x9] ; CHECK-SD-NEXT: add x9, sp, #64 ; CHECK-SD-NEXT: ld1 { v2.s }[1], [x10] ; CHECK-SD-NEXT: mov v4.s[1], w1 ; CHECK-SD-NEXT: add x8, sp, #32 ; CHECK-SD-NEXT: fmov s5, w4 ; CHECK-SD-NEXT: ld1 { v0.s }[2], [x9] ; CHECK-SD-NEXT: ld1 { v1.s }[1], [x8] ; CHECK-SD-NEXT: add x8, sp, #8 ; CHECK-SD-NEXT: add x9, sp, #72 ; CHECK-SD-NEXT: movi v6.4s, #31 ; CHECK-SD-NEXT: add x10, sp, #96 ; CHECK-SD-NEXT: ld1 { v3.s }[2], [x8] ; CHECK-SD-NEXT: ld1 { v2.s }[2], [x10] ; CHECK-SD-NEXT: mov v5.s[1], w5 ; CHECK-SD-NEXT: ld1 { v0.s }[3], [x9] ; CHECK-SD-NEXT: mov v4.s[2], w2 ; CHECK-SD-NEXT: add x8, sp, #16 ; CHECK-SD-NEXT: add x9, sp, #40 ; CHECK-SD-NEXT: ld1 { v3.s }[3], [x8] ; CHECK-SD-NEXT: ld1 { v1.s }[2], [x9] ; CHECK-SD-NEXT: bic v16.16b, v6.16b, v2.16b ; CHECK-SD-NEXT: bic v7.16b, v6.16b, v0.16b ; CHECK-SD-NEXT: mov v5.s[2], w6 ; CHECK-SD-NEXT: and v0.16b, v0.16b, v6.16b ; CHECK-SD-NEXT: mov v4.s[3], w3 ; CHECK-SD-NEXT: and v2.16b, v2.16b, v6.16b ; CHECK-SD-NEXT: ushr v3.4s, v3.4s, #1 ; CHECK-SD-NEXT: ushr v1.4s, v1.4s, #1 ; CHECK-SD-NEXT: neg v6.4s, v16.4s ; CHECK-SD-NEXT: neg v7.4s, v7.4s ; CHECK-SD-NEXT: ushl v2.4s, v5.4s, v2.4s ; CHECK-SD-NEXT: ushl v0.4s, v4.4s, v0.4s ; CHECK-SD-NEXT: ushl v1.4s, v1.4s, v6.4s ; CHECK-SD-NEXT: ushl v3.4s, v3.4s, v7.4s ; CHECK-SD-NEXT: orr v1.16b, v2.16b, v1.16b ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v3.16b ; CHECK-SD-NEXT: mov w5, v1.s[1] ; CHECK-SD-NEXT: mov w6, v1.s[2] ; CHECK-SD-NEXT: fmov w4, s1 ; CHECK-SD-NEXT: mov w1, v0.s[1] ; CHECK-SD-NEXT: mov w2, v0.s[2] ; CHECK-SD-NEXT: mov w3, v0.s[3] ; CHECK-SD-NEXT: fmov w0, s0 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshl_v7i32: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: ldr s1, [sp, #48] ; CHECK-GI-NEXT: ldr s7, [sp, #56] ; CHECK-GI-NEXT: add x9, sp, #64 ; CHECK-GI-NEXT: ldr s5, [sp, #80] ; CHECK-GI-NEXT: ldr s16, [sp, #88] ; CHECK-GI-NEXT: add x8, sp, #56 ; CHECK-GI-NEXT: mov v1.s[1], v7.s[0] ; CHECK-GI-NEXT: ldr s6, [sp] ; CHECK-GI-NEXT: ldr s7, [sp, #64] ; CHECK-GI-NEXT: mov v5.s[1], v16.s[0] ; CHECK-GI-NEXT: fmov s16, w7 ; CHECK-GI-NEXT: ldr s17, [sp, #48] ; CHECK-GI-NEXT: ldr s18, [x9] ; CHECK-GI-NEXT: add x10, sp, #72 ; CHECK-GI-NEXT: ldr s19, [sp, #96] ; CHECK-GI-NEXT: ld1 { v17.s }[1], [x8] ; CHECK-GI-NEXT: ldr s4, [sp, #8] ; CHECK-GI-NEXT: ldr s0, [sp, #24] ; CHECK-GI-NEXT: mov v16.s[1], v6.s[0] ; CHECK-GI-NEXT: mov v1.s[2], v7.s[0] ; CHECK-GI-NEXT: fmov s7, w0 ; CHECK-GI-NEXT: ld1 { v18.s }[1], [x10] ; CHECK-GI-NEXT: ldr s3, [sp, #32] ; CHECK-GI-NEXT: ldr s6, [sp, #72] ; CHECK-GI-NEXT: mov v5.s[2], v19.s[0] ; CHECK-GI-NEXT: movi v19.2d, #0xffffffffffffffff ; CHECK-GI-NEXT: ldr s20, [sp, #80] ; CHECK-GI-NEXT: mov v7.s[1], w1 ; CHECK-GI-NEXT: mov v0.s[1], v3.s[0] ; CHECK-GI-NEXT: add x8, sp, #88 ; CHECK-GI-NEXT: mov v16.s[2], v4.s[0] ; CHECK-GI-NEXT: mov v1.s[3], v6.s[0] ; CHECK-GI-NEXT: zip1 v6.2d, v17.2d, v18.2d ; CHECK-GI-NEXT: fmov s17, w4 ; CHECK-GI-NEXT: movi v4.4s, #31 ; CHECK-GI-NEXT: ldr s2, [sp, #16] ; CHECK-GI-NEXT: ldr s3, [sp, #40] ; CHECK-GI-NEXT: ld1 { v20.s }[1], [x8] ; CHECK-GI-NEXT: eor v5.16b, v5.16b, v19.16b ; CHECK-GI-NEXT: mov v7.s[2], w2 ; CHECK-GI-NEXT: add x8, sp, #96 ; CHECK-GI-NEXT: mov v17.s[1], w5 ; CHECK-GI-NEXT: mov v16.s[3], v2.s[0] ; CHECK-GI-NEXT: mov v0.s[2], v3.s[0] ; CHECK-GI-NEXT: bic v1.16b, v4.16b, v1.16b ; CHECK-GI-NEXT: ld1 { v20.s }[2], [x8] ; CHECK-GI-NEXT: and v2.16b, v5.16b, v4.16b ; CHECK-GI-NEXT: and v3.16b, v6.16b, v4.16b ; CHECK-GI-NEXT: mov v7.s[3], w3 ; CHECK-GI-NEXT: mov v17.s[2], w6 ; CHECK-GI-NEXT: ushr v5.4s, v16.4s, #1 ; CHECK-GI-NEXT: neg v1.4s, v1.4s ; CHECK-GI-NEXT: and v4.16b, v20.16b, v4.16b ; CHECK-GI-NEXT: ushr v0.4s, v0.4s, #1 ; CHECK-GI-NEXT: neg v2.4s, v2.4s ; CHECK-GI-NEXT: ushl v3.4s, v7.4s, v3.4s ; CHECK-GI-NEXT: ushl v1.4s, v5.4s, v1.4s ; CHECK-GI-NEXT: ushl v4.4s, v17.4s, v4.4s ; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v2.4s ; CHECK-GI-NEXT: orr v1.16b, v3.16b, v1.16b ; CHECK-GI-NEXT: orr v0.16b, v4.16b, v0.16b ; CHECK-GI-NEXT: mov s2, v1.s[1] ; CHECK-GI-NEXT: mov s3, v1.s[2] ; CHECK-GI-NEXT: mov s4, v1.s[3] ; CHECK-GI-NEXT: mov s5, v0.s[1] ; CHECK-GI-NEXT: mov s6, v0.s[2] ; CHECK-GI-NEXT: fmov w0, s1 ; CHECK-GI-NEXT: fmov w4, s0 ; CHECK-GI-NEXT: fmov w1, s2 ; CHECK-GI-NEXT: fmov w2, s3 ; CHECK-GI-NEXT: fmov w3, s4 ; CHECK-GI-NEXT: fmov w5, s5 ; CHECK-GI-NEXT: fmov w6, s6 ; CHECK-GI-NEXT: ret entry: %d = call <7 x i32> @llvm.fshl(<7 x i32> %a, <7 x i32> %b, <7 x i32> %c) ret <7 x i32> %d } define <7 x i32> @fshr_v7i32(<7 x i32> %a, <7 x i32> %b, <7 x i32> %c) { ; CHECK-SD-LABEL: fshr_v7i32: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fmov s1, w0 ; CHECK-SD-NEXT: ldr s0, [sp, #48] ; CHECK-SD-NEXT: add x8, sp, #56 ; CHECK-SD-NEXT: ldr s2, [sp, #80] ; CHECK-SD-NEXT: fmov s3, w4 ; CHECK-SD-NEXT: add x9, sp, #64 ; CHECK-SD-NEXT: ld1 { v0.s }[1], [x8] ; CHECK-SD-NEXT: add x8, sp, #88 ; CHECK-SD-NEXT: fmov s4, w7 ; CHECK-SD-NEXT: mov v1.s[1], w1 ; CHECK-SD-NEXT: ld1 { v2.s }[1], [x8] ; CHECK-SD-NEXT: mov x10, sp ; CHECK-SD-NEXT: mov v3.s[1], w5 ; CHECK-SD-NEXT: add x8, sp, #72 ; CHECK-SD-NEXT: movi v5.4s, #31 ; CHECK-SD-NEXT: ld1 { v0.s }[2], [x9] ; CHECK-SD-NEXT: add x9, sp, #96 ; CHECK-SD-NEXT: ld1 { v4.s }[1], [x10] ; CHECK-SD-NEXT: ld1 { v2.s }[2], [x9] ; CHECK-SD-NEXT: ldr s6, [sp, #24] ; CHECK-SD-NEXT: add x9, sp, #8 ; CHECK-SD-NEXT: mov v1.s[2], w2 ; CHECK-SD-NEXT: ld1 { v0.s }[3], [x8] ; CHECK-SD-NEXT: add x8, sp, #32 ; CHECK-SD-NEXT: mov v3.s[2], w6 ; CHECK-SD-NEXT: ld1 { v4.s }[2], [x9] ; CHECK-SD-NEXT: ld1 { v6.s }[1], [x8] ; CHECK-SD-NEXT: and v16.16b, v2.16b, v5.16b ; CHECK-SD-NEXT: add x8, sp, #40 ; CHECK-SD-NEXT: add x9, sp, #16 ; CHECK-SD-NEXT: bic v2.16b, v5.16b, v2.16b ; CHECK-SD-NEXT: mov v1.s[3], w3 ; CHECK-SD-NEXT: and v7.16b, v0.16b, v5.16b ; CHECK-SD-NEXT: bic v0.16b, v5.16b, v0.16b ; CHECK-SD-NEXT: ld1 { v4.s }[3], [x9] ; CHECK-SD-NEXT: ld1 { v6.s }[2], [x8] ; CHECK-SD-NEXT: add v3.4s, v3.4s, v3.4s ; CHECK-SD-NEXT: neg v5.4s, v7.4s ; CHECK-SD-NEXT: neg v7.4s, v16.4s ; CHECK-SD-NEXT: add v1.4s, v1.4s, v1.4s ; CHECK-SD-NEXT: ushl v4.4s, v4.4s, v5.4s ; CHECK-SD-NEXT: ushl v0.4s, v1.4s, v0.4s ; CHECK-SD-NEXT: ushl v1.4s, v3.4s, v2.4s ; CHECK-SD-NEXT: ushl v2.4s, v6.4s, v7.4s ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v4.16b ; CHECK-SD-NEXT: orr v1.16b, v1.16b, v2.16b ; CHECK-SD-NEXT: mov w1, v0.s[1] ; CHECK-SD-NEXT: mov w2, v0.s[2] ; CHECK-SD-NEXT: mov w3, v0.s[3] ; CHECK-SD-NEXT: mov w5, v1.s[1] ; CHECK-SD-NEXT: mov w6, v1.s[2] ; CHECK-SD-NEXT: fmov w0, s0 ; CHECK-SD-NEXT: fmov w4, s1 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_v7i32: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: ldr s1, [sp, #48] ; CHECK-GI-NEXT: ldr s17, [sp, #56] ; CHECK-GI-NEXT: add x9, sp, #64 ; CHECK-GI-NEXT: ldr s18, [sp, #80] ; CHECK-GI-NEXT: ldr s19, [sp, #88] ; CHECK-GI-NEXT: add x8, sp, #56 ; CHECK-GI-NEXT: mov v1.s[1], v17.s[0] ; CHECK-GI-NEXT: fmov s17, w0 ; CHECK-GI-NEXT: ldr s4, [sp] ; CHECK-GI-NEXT: mov v18.s[1], v19.s[0] ; CHECK-GI-NEXT: fmov s19, w7 ; CHECK-GI-NEXT: ldr s7, [sp, #48] ; CHECK-GI-NEXT: ldr s16, [x9] ; CHECK-GI-NEXT: add x10, sp, #72 ; CHECK-GI-NEXT: ldr s20, [sp, #80] ; CHECK-GI-NEXT: mov v17.s[1], w1 ; CHECK-GI-NEXT: ldr s6, [sp, #64] ; CHECK-GI-NEXT: add x9, sp, #88 ; CHECK-GI-NEXT: mov v19.s[1], v4.s[0] ; CHECK-GI-NEXT: fmov s4, w4 ; CHECK-GI-NEXT: ld1 { v7.s }[1], [x8] ; CHECK-GI-NEXT: ld1 { v16.s }[1], [x10] ; CHECK-GI-NEXT: ldr s21, [sp, #96] ; CHECK-GI-NEXT: ld1 { v20.s }[1], [x9] ; CHECK-GI-NEXT: mov v1.s[2], v6.s[0] ; CHECK-GI-NEXT: ldr s3, [sp, #8] ; CHECK-GI-NEXT: ldr s0, [sp, #24] ; CHECK-GI-NEXT: mov v17.s[2], w2 ; CHECK-GI-NEXT: mov v4.s[1], w5 ; CHECK-GI-NEXT: ldr s2, [sp, #32] ; CHECK-GI-NEXT: zip1 v6.2d, v7.2d, v16.2d ; CHECK-GI-NEXT: movi v7.4s, #31 ; CHECK-GI-NEXT: add x8, sp, #96 ; CHECK-GI-NEXT: mov v18.s[2], v21.s[0] ; CHECK-GI-NEXT: movi v16.2d, #0xffffffffffffffff ; CHECK-GI-NEXT: ldr s5, [sp, #72] ; CHECK-GI-NEXT: ld1 { v20.s }[2], [x8] ; CHECK-GI-NEXT: mov v19.s[2], v3.s[0] ; CHECK-GI-NEXT: mov v0.s[1], v2.s[0] ; CHECK-GI-NEXT: mov v1.s[3], v5.s[0] ; CHECK-GI-NEXT: mov v17.s[3], w3 ; CHECK-GI-NEXT: mov v4.s[2], w6 ; CHECK-GI-NEXT: ldr s2, [sp, #16] ; CHECK-GI-NEXT: and v3.16b, v6.16b, v7.16b ; CHECK-GI-NEXT: ldr s5, [sp, #40] ; CHECK-GI-NEXT: and v6.16b, v20.16b, v7.16b ; CHECK-GI-NEXT: eor v16.16b, v18.16b, v16.16b ; CHECK-GI-NEXT: mov v19.s[3], v2.s[0] ; CHECK-GI-NEXT: mov v0.s[2], v5.s[0] ; CHECK-GI-NEXT: bic v1.16b, v7.16b, v1.16b ; CHECK-GI-NEXT: shl v2.4s, v17.4s, #1 ; CHECK-GI-NEXT: neg v3.4s, v3.4s ; CHECK-GI-NEXT: and v5.16b, v16.16b, v7.16b ; CHECK-GI-NEXT: shl v4.4s, v4.4s, #1 ; CHECK-GI-NEXT: neg v6.4s, v6.4s ; CHECK-GI-NEXT: ushl v1.4s, v2.4s, v1.4s ; CHECK-GI-NEXT: ushl v2.4s, v19.4s, v3.4s ; CHECK-GI-NEXT: ushl v3.4s, v4.4s, v5.4s ; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v6.4s ; CHECK-GI-NEXT: orr v1.16b, v1.16b, v2.16b ; CHECK-GI-NEXT: orr v0.16b, v3.16b, v0.16b ; CHECK-GI-NEXT: mov s2, v1.s[1] ; CHECK-GI-NEXT: mov s3, v1.s[2] ; CHECK-GI-NEXT: mov s4, v1.s[3] ; CHECK-GI-NEXT: mov s5, v0.s[1] ; CHECK-GI-NEXT: mov s6, v0.s[2] ; CHECK-GI-NEXT: fmov w0, s1 ; CHECK-GI-NEXT: fmov w4, s0 ; CHECK-GI-NEXT: fmov w1, s2 ; CHECK-GI-NEXT: fmov w2, s3 ; CHECK-GI-NEXT: fmov w3, s4 ; CHECK-GI-NEXT: fmov w5, s5 ; CHECK-GI-NEXT: fmov w6, s6 ; CHECK-GI-NEXT: ret entry: %d = call <7 x i32> @llvm.fshr(<7 x i32> %a, <7 x i32> %b, <7 x i32> %c) ret <7 x i32> %d } define <8 x i32> @fshl_v8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c) { ; CHECK-LABEL: fshl_v8i32: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: movi v6.4s, #31 ; CHECK-NEXT: ushr v2.4s, v2.4s, #1 ; CHECK-NEXT: ushr v3.4s, v3.4s, #1 ; CHECK-NEXT: bic v7.16b, v6.16b, v4.16b ; CHECK-NEXT: bic v16.16b, v6.16b, v5.16b ; CHECK-NEXT: and v4.16b, v4.16b, v6.16b ; CHECK-NEXT: and v5.16b, v5.16b, v6.16b ; CHECK-NEXT: neg v6.4s, v7.4s ; CHECK-NEXT: neg v7.4s, v16.4s ; CHECK-NEXT: ushl v0.4s, v0.4s, v4.4s ; CHECK-NEXT: ushl v1.4s, v1.4s, v5.4s ; CHECK-NEXT: ushl v2.4s, v2.4s, v6.4s ; CHECK-NEXT: ushl v3.4s, v3.4s, v7.4s ; CHECK-NEXT: orr v0.16b, v0.16b, v2.16b ; CHECK-NEXT: orr v1.16b, v1.16b, v3.16b ; CHECK-NEXT: ret entry: %d = call <8 x i32> @llvm.fshl(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c) ret <8 x i32> %d } define <8 x i32> @fshr_v8i32(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c) { ; CHECK-SD-LABEL: fshr_v8i32: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: movi v6.4s, #31 ; CHECK-SD-NEXT: add v0.4s, v0.4s, v0.4s ; CHECK-SD-NEXT: add v1.4s, v1.4s, v1.4s ; CHECK-SD-NEXT: and v7.16b, v4.16b, v6.16b ; CHECK-SD-NEXT: and v16.16b, v5.16b, v6.16b ; CHECK-SD-NEXT: bic v4.16b, v6.16b, v4.16b ; CHECK-SD-NEXT: bic v5.16b, v6.16b, v5.16b ; CHECK-SD-NEXT: neg v6.4s, v7.4s ; CHECK-SD-NEXT: neg v7.4s, v16.4s ; CHECK-SD-NEXT: ushl v0.4s, v0.4s, v4.4s ; CHECK-SD-NEXT: ushl v1.4s, v1.4s, v5.4s ; CHECK-SD-NEXT: ushl v2.4s, v2.4s, v6.4s ; CHECK-SD-NEXT: ushl v3.4s, v3.4s, v7.4s ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v2.16b ; CHECK-SD-NEXT: orr v1.16b, v1.16b, v3.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_v8i32: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: movi v6.4s, #31 ; CHECK-GI-NEXT: shl v0.4s, v0.4s, #1 ; CHECK-GI-NEXT: shl v1.4s, v1.4s, #1 ; CHECK-GI-NEXT: and v7.16b, v4.16b, v6.16b ; CHECK-GI-NEXT: and v16.16b, v5.16b, v6.16b ; CHECK-GI-NEXT: bic v4.16b, v6.16b, v4.16b ; CHECK-GI-NEXT: bic v5.16b, v6.16b, v5.16b ; CHECK-GI-NEXT: neg v6.4s, v7.4s ; CHECK-GI-NEXT: neg v7.4s, v16.4s ; CHECK-GI-NEXT: ushl v0.4s, v0.4s, v4.4s ; CHECK-GI-NEXT: ushl v1.4s, v1.4s, v5.4s ; CHECK-GI-NEXT: ushl v2.4s, v2.4s, v6.4s ; CHECK-GI-NEXT: ushl v3.4s, v3.4s, v7.4s ; CHECK-GI-NEXT: orr v0.16b, v0.16b, v2.16b ; CHECK-GI-NEXT: orr v1.16b, v1.16b, v3.16b ; CHECK-GI-NEXT: ret entry: %d = call <8 x i32> @llvm.fshr(<8 x i32> %a, <8 x i32> %b, <8 x i32> %c) ret <8 x i32> %d } define <2 x i64> @fshl_v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) { ; CHECK-SD-LABEL: fshl_v2i64: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: mov w8, #63 // =0x3f ; CHECK-SD-NEXT: ushr v1.2d, v1.2d, #1 ; CHECK-SD-NEXT: dup v3.2d, x8 ; CHECK-SD-NEXT: bic v4.16b, v3.16b, v2.16b ; CHECK-SD-NEXT: and v2.16b, v2.16b, v3.16b ; CHECK-SD-NEXT: neg v3.2d, v4.2d ; CHECK-SD-NEXT: ushl v0.2d, v0.2d, v2.2d ; CHECK-SD-NEXT: ushl v1.2d, v1.2d, v3.2d ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshl_v2i64: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: adrp x8, .LCPI86_0 ; CHECK-GI-NEXT: ushr v1.2d, v1.2d, #1 ; CHECK-GI-NEXT: ldr q3, [x8, :lo12:.LCPI86_0] ; CHECK-GI-NEXT: bic v4.16b, v3.16b, v2.16b ; CHECK-GI-NEXT: and v2.16b, v2.16b, v3.16b ; CHECK-GI-NEXT: neg v3.2d, v4.2d ; CHECK-GI-NEXT: ushl v0.2d, v0.2d, v2.2d ; CHECK-GI-NEXT: ushl v1.2d, v1.2d, v3.2d ; CHECK-GI-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-GI-NEXT: ret entry: %d = call <2 x i64> @llvm.fshl(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) ret <2 x i64> %d } define <2 x i64> @fshr_v2i64(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) { ; CHECK-SD-LABEL: fshr_v2i64: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: mov w8, #63 // =0x3f ; CHECK-SD-NEXT: add v0.2d, v0.2d, v0.2d ; CHECK-SD-NEXT: dup v3.2d, x8 ; CHECK-SD-NEXT: and v4.16b, v2.16b, v3.16b ; CHECK-SD-NEXT: bic v2.16b, v3.16b, v2.16b ; CHECK-SD-NEXT: neg v3.2d, v4.2d ; CHECK-SD-NEXT: ushl v0.2d, v0.2d, v2.2d ; CHECK-SD-NEXT: ushl v1.2d, v1.2d, v3.2d ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_v2i64: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: adrp x8, .LCPI87_0 ; CHECK-GI-NEXT: shl v0.2d, v0.2d, #1 ; CHECK-GI-NEXT: ldr q3, [x8, :lo12:.LCPI87_0] ; CHECK-GI-NEXT: and v4.16b, v2.16b, v3.16b ; CHECK-GI-NEXT: bic v2.16b, v3.16b, v2.16b ; CHECK-GI-NEXT: neg v3.2d, v4.2d ; CHECK-GI-NEXT: ushl v0.2d, v0.2d, v2.2d ; CHECK-GI-NEXT: ushl v1.2d, v1.2d, v3.2d ; CHECK-GI-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-GI-NEXT: ret entry: %d = call <2 x i64> @llvm.fshr(<2 x i64> %a, <2 x i64> %b, <2 x i64> %c) ret <2 x i64> %d } define <4 x i64> @fshl_v4i64(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c) { ; CHECK-SD-LABEL: fshl_v4i64: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: mov w8, #63 // =0x3f ; CHECK-SD-NEXT: ushr v2.2d, v2.2d, #1 ; CHECK-SD-NEXT: ushr v3.2d, v3.2d, #1 ; CHECK-SD-NEXT: dup v6.2d, x8 ; CHECK-SD-NEXT: bic v7.16b, v6.16b, v4.16b ; CHECK-SD-NEXT: bic v16.16b, v6.16b, v5.16b ; CHECK-SD-NEXT: and v4.16b, v4.16b, v6.16b ; CHECK-SD-NEXT: and v5.16b, v5.16b, v6.16b ; CHECK-SD-NEXT: neg v6.2d, v7.2d ; CHECK-SD-NEXT: neg v7.2d, v16.2d ; CHECK-SD-NEXT: ushl v0.2d, v0.2d, v4.2d ; CHECK-SD-NEXT: ushl v1.2d, v1.2d, v5.2d ; CHECK-SD-NEXT: ushl v2.2d, v2.2d, v6.2d ; CHECK-SD-NEXT: ushl v3.2d, v3.2d, v7.2d ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v2.16b ; CHECK-SD-NEXT: orr v1.16b, v1.16b, v3.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshl_v4i64: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: adrp x8, .LCPI88_0 ; CHECK-GI-NEXT: ushr v2.2d, v2.2d, #1 ; CHECK-GI-NEXT: ushr v3.2d, v3.2d, #1 ; CHECK-GI-NEXT: ldr q6, [x8, :lo12:.LCPI88_0] ; CHECK-GI-NEXT: bic v7.16b, v6.16b, v4.16b ; CHECK-GI-NEXT: bic v16.16b, v6.16b, v5.16b ; CHECK-GI-NEXT: and v4.16b, v4.16b, v6.16b ; CHECK-GI-NEXT: and v5.16b, v5.16b, v6.16b ; CHECK-GI-NEXT: neg v6.2d, v7.2d ; CHECK-GI-NEXT: neg v7.2d, v16.2d ; CHECK-GI-NEXT: ushl v0.2d, v0.2d, v4.2d ; CHECK-GI-NEXT: ushl v1.2d, v1.2d, v5.2d ; CHECK-GI-NEXT: ushl v2.2d, v2.2d, v6.2d ; CHECK-GI-NEXT: ushl v3.2d, v3.2d, v7.2d ; CHECK-GI-NEXT: orr v0.16b, v0.16b, v2.16b ; CHECK-GI-NEXT: orr v1.16b, v1.16b, v3.16b ; CHECK-GI-NEXT: ret entry: %d = call <4 x i64> @llvm.fshl(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c) ret <4 x i64> %d } define <4 x i64> @fshr_v4i64(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c) { ; CHECK-SD-LABEL: fshr_v4i64: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: mov w8, #63 // =0x3f ; CHECK-SD-NEXT: add v0.2d, v0.2d, v0.2d ; CHECK-SD-NEXT: add v1.2d, v1.2d, v1.2d ; CHECK-SD-NEXT: dup v6.2d, x8 ; CHECK-SD-NEXT: and v7.16b, v4.16b, v6.16b ; CHECK-SD-NEXT: and v16.16b, v5.16b, v6.16b ; CHECK-SD-NEXT: bic v4.16b, v6.16b, v4.16b ; CHECK-SD-NEXT: bic v5.16b, v6.16b, v5.16b ; CHECK-SD-NEXT: neg v6.2d, v7.2d ; CHECK-SD-NEXT: neg v7.2d, v16.2d ; CHECK-SD-NEXT: ushl v0.2d, v0.2d, v4.2d ; CHECK-SD-NEXT: ushl v1.2d, v1.2d, v5.2d ; CHECK-SD-NEXT: ushl v2.2d, v2.2d, v6.2d ; CHECK-SD-NEXT: ushl v3.2d, v3.2d, v7.2d ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v2.16b ; CHECK-SD-NEXT: orr v1.16b, v1.16b, v3.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_v4i64: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: adrp x8, .LCPI89_0 ; CHECK-GI-NEXT: shl v0.2d, v0.2d, #1 ; CHECK-GI-NEXT: shl v1.2d, v1.2d, #1 ; CHECK-GI-NEXT: ldr q6, [x8, :lo12:.LCPI89_0] ; CHECK-GI-NEXT: and v7.16b, v4.16b, v6.16b ; CHECK-GI-NEXT: and v16.16b, v5.16b, v6.16b ; CHECK-GI-NEXT: bic v4.16b, v6.16b, v4.16b ; CHECK-GI-NEXT: bic v5.16b, v6.16b, v5.16b ; CHECK-GI-NEXT: neg v6.2d, v7.2d ; CHECK-GI-NEXT: neg v7.2d, v16.2d ; CHECK-GI-NEXT: ushl v0.2d, v0.2d, v4.2d ; CHECK-GI-NEXT: ushl v1.2d, v1.2d, v5.2d ; CHECK-GI-NEXT: ushl v2.2d, v2.2d, v6.2d ; CHECK-GI-NEXT: ushl v3.2d, v3.2d, v7.2d ; CHECK-GI-NEXT: orr v0.16b, v0.16b, v2.16b ; CHECK-GI-NEXT: orr v1.16b, v1.16b, v3.16b ; CHECK-GI-NEXT: ret entry: %d = call <4 x i64> @llvm.fshr(<4 x i64> %a, <4 x i64> %b, <4 x i64> %c) ret <4 x i64> %d } define <2 x i128> @fshl_v2i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c) { ; CHECK-SD-LABEL: fshl_v2i128: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: ldr x8, [sp] ; CHECK-SD-NEXT: ldr x10, [sp, #16] ; CHECK-SD-NEXT: tst x8, #0x40 ; CHECK-SD-NEXT: mvn w13, w8 ; CHECK-SD-NEXT: mvn w16, w10 ; CHECK-SD-NEXT: csel x9, x5, x0, ne ; CHECK-SD-NEXT: csel x11, x4, x5, ne ; CHECK-SD-NEXT: csel x14, x0, x1, ne ; CHECK-SD-NEXT: lsl x12, x9, x8 ; CHECK-SD-NEXT: lsr x11, x11, #1 ; CHECK-SD-NEXT: lsr x9, x9, #1 ; CHECK-SD-NEXT: tst x10, #0x40 ; CHECK-SD-NEXT: lsl x8, x14, x8 ; CHECK-SD-NEXT: csel x14, x7, x2, ne ; CHECK-SD-NEXT: csel x15, x6, x7, ne ; CHECK-SD-NEXT: lsr x11, x11, x13 ; CHECK-SD-NEXT: lsr x9, x9, x13 ; CHECK-SD-NEXT: lsr x13, x15, #1 ; CHECK-SD-NEXT: lsr x15, x14, #1 ; CHECK-SD-NEXT: csel x17, x2, x3, ne ; CHECK-SD-NEXT: lsl x14, x14, x10 ; CHECK-SD-NEXT: orr x0, x12, x11 ; CHECK-SD-NEXT: lsr x13, x13, x16 ; CHECK-SD-NEXT: lsl x10, x17, x10 ; CHECK-SD-NEXT: lsr x15, x15, x16 ; CHECK-SD-NEXT: orr x1, x8, x9 ; CHECK-SD-NEXT: orr x2, x14, x13 ; CHECK-SD-NEXT: orr x3, x10, x15 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshl_v2i128: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: str x19, [sp, #-16]! // 8-byte Folded Spill ; CHECK-GI-NEXT: .cfi_def_cfa_offset 16 ; CHECK-GI-NEXT: .cfi_offset w19, -16 ; CHECK-GI-NEXT: ldr x11, [sp, #16] ; CHECK-GI-NEXT: mov w9, #64 // =0x40 ; CHECK-GI-NEXT: ldr x12, [sp, #32] ; CHECK-GI-NEXT: mov w13, #127 // =0x7f ; CHECK-GI-NEXT: and x8, x11, #0x7f ; CHECK-GI-NEXT: and x14, x12, #0x7f ; CHECK-GI-NEXT: mvn x18, x11 ; CHECK-GI-NEXT: sub x10, x9, x8 ; CHECK-GI-NEXT: sub x15, x8, #64 ; CHECK-GI-NEXT: lsl x17, x1, x8 ; CHECK-GI-NEXT: lsr x16, x0, x10 ; CHECK-GI-NEXT: lsl x15, x0, x15 ; CHECK-GI-NEXT: cmp x8, #64 ; CHECK-GI-NEXT: lsl x19, x0, x8 ; CHECK-GI-NEXT: lsl x0, x3, x14 ; CHECK-GI-NEXT: mvn x10, x12 ; CHECK-GI-NEXT: orr x16, x16, x17 ; CHECK-GI-NEXT: sub x17, x14, #64 ; CHECK-GI-NEXT: csel x15, x16, x15, lo ; CHECK-GI-NEXT: sub x16, x9, x14 ; CHECK-GI-NEXT: csel x8, x19, xzr, lo ; CHECK-GI-NEXT: lsr x16, x2, x16 ; CHECK-GI-NEXT: tst x11, #0x7f ; CHECK-GI-NEXT: lsl x19, x2, x14 ; CHECK-GI-NEXT: lsl x17, x2, x17 ; CHECK-GI-NEXT: csel x15, x1, x15, eq ; CHECK-GI-NEXT: cmp x14, #64 ; CHECK-GI-NEXT: orr x16, x16, x0 ; CHECK-GI-NEXT: bic x11, x13, x11 ; CHECK-GI-NEXT: csel x14, x19, xzr, lo ; CHECK-GI-NEXT: csel x16, x16, x17, lo ; CHECK-GI-NEXT: tst x12, #0x7f ; CHECK-GI-NEXT: lsr x17, x5, #1 ; CHECK-GI-NEXT: extr x0, x5, x4, #1 ; CHECK-GI-NEXT: bic x12, x13, x12 ; CHECK-GI-NEXT: csel x13, x3, x16, eq ; CHECK-GI-NEXT: sub x16, x9, x11 ; CHECK-GI-NEXT: sub x1, x11, #64 ; CHECK-GI-NEXT: lsr x3, x7, #1 ; CHECK-GI-NEXT: lsr x2, x0, x11 ; CHECK-GI-NEXT: lsl x16, x17, x16 ; CHECK-GI-NEXT: extr x4, x7, x6, #1 ; CHECK-GI-NEXT: lsr x1, x17, x1 ; CHECK-GI-NEXT: cmp x11, #64 ; CHECK-GI-NEXT: sub x9, x9, x12 ; CHECK-GI-NEXT: orr x16, x2, x16 ; CHECK-GI-NEXT: lsr x17, x17, x11 ; CHECK-GI-NEXT: lsl x9, x3, x9 ; CHECK-GI-NEXT: csel x16, x16, x1, lo ; CHECK-GI-NEXT: tst x18, #0x7f ; CHECK-GI-NEXT: sub x18, x12, #64 ; CHECK-GI-NEXT: lsr x1, x4, x12 ; CHECK-GI-NEXT: csel x16, x0, x16, eq ; CHECK-GI-NEXT: cmp x11, #64 ; CHECK-GI-NEXT: lsr x11, x3, x18 ; CHECK-GI-NEXT: csel x17, x17, xzr, lo ; CHECK-GI-NEXT: cmp x12, #64 ; CHECK-GI-NEXT: orr x9, x1, x9 ; CHECK-GI-NEXT: lsr x18, x3, x12 ; CHECK-GI-NEXT: orr x0, x8, x16 ; CHECK-GI-NEXT: csel x9, x9, x11, lo ; CHECK-GI-NEXT: tst x10, #0x7f ; CHECK-GI-NEXT: orr x1, x15, x17 ; CHECK-GI-NEXT: csel x9, x4, x9, eq ; CHECK-GI-NEXT: cmp x12, #64 ; CHECK-GI-NEXT: csel x10, x18, xzr, lo ; CHECK-GI-NEXT: orr x2, x14, x9 ; CHECK-GI-NEXT: orr x3, x13, x10 ; CHECK-GI-NEXT: ldr x19, [sp], #16 // 8-byte Folded Reload ; CHECK-GI-NEXT: ret entry: %d = call <2 x i128> @llvm.fshl(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c) ret <2 x i128> %d } define <2 x i128> @fshr_v2i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c) { ; CHECK-SD-LABEL: fshr_v2i128: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: ldr x8, [sp] ; CHECK-SD-NEXT: ldr x10, [sp, #16] ; CHECK-SD-NEXT: tst x8, #0x40 ; CHECK-SD-NEXT: mvn w13, w8 ; CHECK-SD-NEXT: csel x9, x4, x5, eq ; CHECK-SD-NEXT: csel x11, x5, x0, eq ; CHECK-SD-NEXT: csel x14, x0, x1, eq ; CHECK-SD-NEXT: tst x10, #0x40 ; CHECK-SD-NEXT: lsr x9, x9, x8 ; CHECK-SD-NEXT: lsl x12, x11, #1 ; CHECK-SD-NEXT: lsr x8, x11, x8 ; CHECK-SD-NEXT: lsl x11, x14, #1 ; CHECK-SD-NEXT: csel x14, x7, x2, eq ; CHECK-SD-NEXT: csel x15, x2, x3, eq ; CHECK-SD-NEXT: csel x16, x6, x7, eq ; CHECK-SD-NEXT: lsl x17, x14, #1 ; CHECK-SD-NEXT: lsl x15, x15, #1 ; CHECK-SD-NEXT: lsl x12, x12, x13 ; CHECK-SD-NEXT: lsl x11, x11, x13 ; CHECK-SD-NEXT: lsr x13, x16, x10 ; CHECK-SD-NEXT: mvn w16, w10 ; CHECK-SD-NEXT: lsr x10, x14, x10 ; CHECK-SD-NEXT: lsl x17, x17, x16 ; CHECK-SD-NEXT: lsl x14, x15, x16 ; CHECK-SD-NEXT: orr x0, x12, x9 ; CHECK-SD-NEXT: orr x1, x11, x8 ; CHECK-SD-NEXT: orr x2, x17, x13 ; CHECK-SD-NEXT: orr x3, x14, x10 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_v2i128: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: ldr x9, [sp] ; CHECK-GI-NEXT: mov w10, #127 // =0x7f ; CHECK-GI-NEXT: mov w12, #64 // =0x40 ; CHECK-GI-NEXT: lsl x13, x0, #1 ; CHECK-GI-NEXT: extr x14, x1, x0, #63 ; CHECK-GI-NEXT: ldr x8, [sp, #16] ; CHECK-GI-NEXT: bic x11, x10, x9 ; CHECK-GI-NEXT: mvn x16, x9 ; CHECK-GI-NEXT: and x15, x9, #0x7f ; CHECK-GI-NEXT: sub x17, x12, x11 ; CHECK-GI-NEXT: sub x18, x11, #64 ; CHECK-GI-NEXT: lsl x0, x14, x11 ; CHECK-GI-NEXT: lsr x17, x13, x17 ; CHECK-GI-NEXT: lsl x1, x13, x11 ; CHECK-GI-NEXT: lsl x13, x13, x18 ; CHECK-GI-NEXT: bic x10, x10, x8 ; CHECK-GI-NEXT: lsl x18, x2, #1 ; CHECK-GI-NEXT: cmp x11, #64 ; CHECK-GI-NEXT: orr x17, x17, x0 ; CHECK-GI-NEXT: extr x11, x3, x2, #63 ; CHECK-GI-NEXT: csel x0, x1, xzr, lo ; CHECK-GI-NEXT: csel x13, x17, x13, lo ; CHECK-GI-NEXT: sub x17, x12, x10 ; CHECK-GI-NEXT: tst x16, #0x7f ; CHECK-GI-NEXT: sub x16, x10, #64 ; CHECK-GI-NEXT: lsr x17, x18, x17 ; CHECK-GI-NEXT: lsl x2, x11, x10 ; CHECK-GI-NEXT: lsl x1, x18, x10 ; CHECK-GI-NEXT: csel x13, x14, x13, eq ; CHECK-GI-NEXT: lsl x14, x18, x16 ; CHECK-GI-NEXT: cmp x10, #64 ; CHECK-GI-NEXT: mvn x10, x8 ; CHECK-GI-NEXT: orr x16, x17, x2 ; CHECK-GI-NEXT: csel x17, x1, xzr, lo ; CHECK-GI-NEXT: csel x14, x16, x14, lo ; CHECK-GI-NEXT: tst x10, #0x7f ; CHECK-GI-NEXT: sub x10, x12, x15 ; CHECK-GI-NEXT: sub x16, x15, #64 ; CHECK-GI-NEXT: lsr x18, x4, x15 ; CHECK-GI-NEXT: lsl x10, x5, x10 ; CHECK-GI-NEXT: csel x11, x11, x14, eq ; CHECK-GI-NEXT: lsr x14, x5, x16 ; CHECK-GI-NEXT: and x1, x8, #0x7f ; CHECK-GI-NEXT: cmp x15, #64 ; CHECK-GI-NEXT: lsr x16, x5, x15 ; CHECK-GI-NEXT: orr x10, x18, x10 ; CHECK-GI-NEXT: csel x10, x10, x14, lo ; CHECK-GI-NEXT: tst x9, #0x7f ; CHECK-GI-NEXT: sub x9, x12, x1 ; CHECK-GI-NEXT: sub x12, x1, #64 ; CHECK-GI-NEXT: lsr x14, x6, x1 ; CHECK-GI-NEXT: lsl x9, x7, x9 ; CHECK-GI-NEXT: csel x10, x4, x10, eq ; CHECK-GI-NEXT: cmp x15, #64 ; CHECK-GI-NEXT: lsr x12, x7, x12 ; CHECK-GI-NEXT: csel x15, x16, xzr, lo ; CHECK-GI-NEXT: orr x9, x14, x9 ; CHECK-GI-NEXT: cmp x1, #64 ; CHECK-GI-NEXT: lsr x14, x7, x1 ; CHECK-GI-NEXT: csel x9, x9, x12, lo ; CHECK-GI-NEXT: tst x8, #0x7f ; CHECK-GI-NEXT: csel x8, x6, x9, eq ; CHECK-GI-NEXT: cmp x1, #64 ; CHECK-GI-NEXT: orr x0, x0, x10 ; CHECK-GI-NEXT: csel x9, x14, xzr, lo ; CHECK-GI-NEXT: orr x1, x13, x15 ; CHECK-GI-NEXT: orr x2, x17, x8 ; CHECK-GI-NEXT: orr x3, x11, x9 ; CHECK-GI-NEXT: ret entry: %d = call <2 x i128> @llvm.fshr(<2 x i128> %a, <2 x i128> %b, <2 x i128> %c) ret <2 x i128> %d } define <8 x i8> @rotl_v8i8_c(<8 x i8> %a) { ; CHECK-LABEL: rotl_v8i8_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v1.8b, v0.8b, #3 ; CHECK-NEXT: usra v1.8b, v0.8b, #5 ; CHECK-NEXT: fmov d0, d1 ; CHECK-NEXT: ret entry: %d = call <8 x i8> @llvm.fshl(<8 x i8> %a, <8 x i8> %a, <8 x i8> ) ret <8 x i8> %d } define <8 x i8> @rotr_v8i8_c(<8 x i8> %a) { ; CHECK-LABEL: rotr_v8i8_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v1.8b, v0.8b, #5 ; CHECK-NEXT: usra v1.8b, v0.8b, #3 ; CHECK-NEXT: fmov d0, d1 ; CHECK-NEXT: ret entry: %d = call <8 x i8> @llvm.fshr(<8 x i8> %a, <8 x i8> %a, <8 x i8> ) ret <8 x i8> %d } define <16 x i8> @rotl_v16i8_c(<16 x i8> %a) { ; CHECK-LABEL: rotl_v16i8_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v1.16b, v0.16b, #3 ; CHECK-NEXT: usra v1.16b, v0.16b, #5 ; CHECK-NEXT: mov v0.16b, v1.16b ; CHECK-NEXT: ret entry: %d = call <16 x i8> @llvm.fshl(<16 x i8> %a, <16 x i8> %a, <16 x i8> ) ret <16 x i8> %d } define <16 x i8> @rotr_v16i8_c(<16 x i8> %a) { ; CHECK-LABEL: rotr_v16i8_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v1.16b, v0.16b, #5 ; CHECK-NEXT: usra v1.16b, v0.16b, #3 ; CHECK-NEXT: mov v0.16b, v1.16b ; CHECK-NEXT: ret entry: %d = call <16 x i8> @llvm.fshr(<16 x i8> %a, <16 x i8> %a, <16 x i8> ) ret <16 x i8> %d } define <4 x i16> @rotl_v4i16_c(<4 x i16> %a) { ; CHECK-LABEL: rotl_v4i16_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v1.4h, v0.4h, #3 ; CHECK-NEXT: usra v1.4h, v0.4h, #13 ; CHECK-NEXT: fmov d0, d1 ; CHECK-NEXT: ret entry: %d = call <4 x i16> @llvm.fshl(<4 x i16> %a, <4 x i16> %a, <4 x i16> ) ret <4 x i16> %d } define <4 x i16> @rotr_v4i16_c(<4 x i16> %a) { ; CHECK-LABEL: rotr_v4i16_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v1.4h, v0.4h, #13 ; CHECK-NEXT: usra v1.4h, v0.4h, #3 ; CHECK-NEXT: fmov d0, d1 ; CHECK-NEXT: ret entry: %d = call <4 x i16> @llvm.fshr(<4 x i16> %a, <4 x i16> %a, <4 x i16> ) ret <4 x i16> %d } define <7 x i16> @rotl_v7i16_c(<7 x i16> %a) { ; CHECK-SD-LABEL: rotl_v7i16_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: adrp x8, .LCPI98_0 ; CHECK-SD-NEXT: adrp x9, .LCPI98_1 ; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI98_0] ; CHECK-SD-NEXT: ldr q2, [x9, :lo12:.LCPI98_1] ; CHECK-SD-NEXT: ushl v1.8h, v0.8h, v1.8h ; CHECK-SD-NEXT: ushl v0.8h, v0.8h, v2.8h ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotl_v7i16_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: shl v1.8h, v0.8h, #3 ; CHECK-GI-NEXT: usra v1.8h, v0.8h, #13 ; CHECK-GI-NEXT: mov v0.16b, v1.16b ; CHECK-GI-NEXT: ret entry: %d = call <7 x i16> @llvm.fshl(<7 x i16> %a, <7 x i16> %a, <7 x i16> ) ret <7 x i16> %d } define <7 x i16> @rotr_v7i16_c(<7 x i16> %a) { ; CHECK-SD-LABEL: rotr_v7i16_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: adrp x8, .LCPI99_0 ; CHECK-SD-NEXT: adrp x9, .LCPI99_1 ; CHECK-SD-NEXT: ldr q1, [x8, :lo12:.LCPI99_0] ; CHECK-SD-NEXT: ldr q2, [x9, :lo12:.LCPI99_1] ; CHECK-SD-NEXT: ushl v1.8h, v0.8h, v1.8h ; CHECK-SD-NEXT: ushl v0.8h, v0.8h, v2.8h ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotr_v7i16_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: shl v1.8h, v0.8h, #13 ; CHECK-GI-NEXT: usra v1.8h, v0.8h, #3 ; CHECK-GI-NEXT: mov v0.16b, v1.16b ; CHECK-GI-NEXT: ret entry: %d = call <7 x i16> @llvm.fshr(<7 x i16> %a, <7 x i16> %a, <7 x i16> ) ret <7 x i16> %d } define <8 x i16> @rotl_v8i16_c(<8 x i16> %a) { ; CHECK-LABEL: rotl_v8i16_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v1.8h, v0.8h, #3 ; CHECK-NEXT: usra v1.8h, v0.8h, #13 ; CHECK-NEXT: mov v0.16b, v1.16b ; CHECK-NEXT: ret entry: %d = call <8 x i16> @llvm.fshl(<8 x i16> %a, <8 x i16> %a, <8 x i16> ) ret <8 x i16> %d } define <8 x i16> @rotr_v8i16_c(<8 x i16> %a) { ; CHECK-LABEL: rotr_v8i16_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v1.8h, v0.8h, #13 ; CHECK-NEXT: usra v1.8h, v0.8h, #3 ; CHECK-NEXT: mov v0.16b, v1.16b ; CHECK-NEXT: ret entry: %d = call <8 x i16> @llvm.fshr(<8 x i16> %a, <8 x i16> %a, <8 x i16> ) ret <8 x i16> %d } define <16 x i16> @rotl_v16i16_c(<16 x i16> %a) { ; CHECK-LABEL: rotl_v16i16_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v2.8h, v0.8h, #3 ; CHECK-NEXT: shl v3.8h, v1.8h, #3 ; CHECK-NEXT: usra v2.8h, v0.8h, #13 ; CHECK-NEXT: usra v3.8h, v1.8h, #13 ; CHECK-NEXT: mov v0.16b, v2.16b ; CHECK-NEXT: mov v1.16b, v3.16b ; CHECK-NEXT: ret entry: %d = call <16 x i16> @llvm.fshl(<16 x i16> %a, <16 x i16> %a, <16 x i16> ) ret <16 x i16> %d } define <16 x i16> @rotr_v16i16_c(<16 x i16> %a) { ; CHECK-LABEL: rotr_v16i16_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v2.8h, v0.8h, #13 ; CHECK-NEXT: shl v3.8h, v1.8h, #13 ; CHECK-NEXT: usra v2.8h, v0.8h, #3 ; CHECK-NEXT: usra v3.8h, v1.8h, #3 ; CHECK-NEXT: mov v0.16b, v2.16b ; CHECK-NEXT: mov v1.16b, v3.16b ; CHECK-NEXT: ret entry: %d = call <16 x i16> @llvm.fshr(<16 x i16> %a, <16 x i16> %a, <16 x i16> ) ret <16 x i16> %d } define <2 x i32> @rotl_v2i32_c(<2 x i32> %a) { ; CHECK-LABEL: rotl_v2i32_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v1.2s, v0.2s, #3 ; CHECK-NEXT: usra v1.2s, v0.2s, #29 ; CHECK-NEXT: fmov d0, d1 ; CHECK-NEXT: ret entry: %d = call <2 x i32> @llvm.fshl(<2 x i32> %a, <2 x i32> %a, <2 x i32> ) ret <2 x i32> %d } define <2 x i32> @rotr_v2i32_c(<2 x i32> %a) { ; CHECK-LABEL: rotr_v2i32_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v1.2s, v0.2s, #29 ; CHECK-NEXT: usra v1.2s, v0.2s, #3 ; CHECK-NEXT: fmov d0, d1 ; CHECK-NEXT: ret entry: %d = call <2 x i32> @llvm.fshr(<2 x i32> %a, <2 x i32> %a, <2 x i32> ) ret <2 x i32> %d } define <4 x i32> @rotl_v4i32_c(<4 x i32> %a) { ; CHECK-LABEL: rotl_v4i32_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v1.4s, v0.4s, #3 ; CHECK-NEXT: usra v1.4s, v0.4s, #29 ; CHECK-NEXT: mov v0.16b, v1.16b ; CHECK-NEXT: ret entry: %d = call <4 x i32> @llvm.fshl(<4 x i32> %a, <4 x i32> %a, <4 x i32> ) ret <4 x i32> %d } define <4 x i32> @rotr_v4i32_c(<4 x i32> %a) { ; CHECK-LABEL: rotr_v4i32_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v1.4s, v0.4s, #29 ; CHECK-NEXT: usra v1.4s, v0.4s, #3 ; CHECK-NEXT: mov v0.16b, v1.16b ; CHECK-NEXT: ret entry: %d = call <4 x i32> @llvm.fshr(<4 x i32> %a, <4 x i32> %a, <4 x i32> ) ret <4 x i32> %d } define <7 x i32> @rotl_v7i32_c(<7 x i32> %a) { ; CHECK-SD-LABEL: rotl_v7i32_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fmov s0, w0 ; CHECK-SD-NEXT: fmov s1, w4 ; CHECK-SD-NEXT: mov v0.s[1], w1 ; CHECK-SD-NEXT: mov v1.s[1], w5 ; CHECK-SD-NEXT: mov v0.s[2], w2 ; CHECK-SD-NEXT: mov v1.s[2], w6 ; CHECK-SD-NEXT: mov v0.s[3], w3 ; CHECK-SD-NEXT: shl v3.4s, v1.4s, #3 ; CHECK-SD-NEXT: usra v3.4s, v1.4s, #29 ; CHECK-SD-NEXT: shl v2.4s, v0.4s, #3 ; CHECK-SD-NEXT: mov w5, v3.s[1] ; CHECK-SD-NEXT: mov w6, v3.s[2] ; CHECK-SD-NEXT: fmov w4, s3 ; CHECK-SD-NEXT: usra v2.4s, v0.4s, #29 ; CHECK-SD-NEXT: mov w1, v2.s[1] ; CHECK-SD-NEXT: mov w2, v2.s[2] ; CHECK-SD-NEXT: mov w3, v2.s[3] ; CHECK-SD-NEXT: fmov w0, s2 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotl_v7i32_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: fmov s0, w0 ; CHECK-GI-NEXT: fmov s1, w0 ; CHECK-GI-NEXT: fmov s2, w4 ; CHECK-GI-NEXT: fmov s3, w4 ; CHECK-GI-NEXT: mov v0.s[1], w1 ; CHECK-GI-NEXT: mov v1.s[1], w1 ; CHECK-GI-NEXT: mov v2.s[1], w5 ; CHECK-GI-NEXT: mov v3.s[1], w5 ; CHECK-GI-NEXT: mov v0.s[2], w2 ; CHECK-GI-NEXT: mov v1.s[2], w2 ; CHECK-GI-NEXT: mov v2.s[2], w6 ; CHECK-GI-NEXT: mov v3.s[2], w6 ; CHECK-GI-NEXT: mov v0.s[3], w3 ; CHECK-GI-NEXT: mov v1.s[3], w3 ; CHECK-GI-NEXT: shl v2.4s, v2.4s, #3 ; CHECK-GI-NEXT: usra v2.4s, v3.4s, #29 ; CHECK-GI-NEXT: shl v0.4s, v0.4s, #3 ; CHECK-GI-NEXT: mov s5, v2.s[1] ; CHECK-GI-NEXT: mov s6, v2.s[2] ; CHECK-GI-NEXT: fmov w4, s2 ; CHECK-GI-NEXT: usra v0.4s, v1.4s, #29 ; CHECK-GI-NEXT: mov s1, v0.s[1] ; CHECK-GI-NEXT: mov s3, v0.s[2] ; CHECK-GI-NEXT: mov s4, v0.s[3] ; CHECK-GI-NEXT: fmov w0, s0 ; CHECK-GI-NEXT: fmov w5, s5 ; CHECK-GI-NEXT: fmov w6, s6 ; CHECK-GI-NEXT: fmov w1, s1 ; CHECK-GI-NEXT: fmov w2, s3 ; CHECK-GI-NEXT: fmov w3, s4 ; CHECK-GI-NEXT: ret entry: %d = call <7 x i32> @llvm.fshl(<7 x i32> %a, <7 x i32> %a, <7 x i32> ) ret <7 x i32> %d } define <7 x i32> @rotr_v7i32_c(<7 x i32> %a) { ; CHECK-SD-LABEL: rotr_v7i32_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fmov s0, w0 ; CHECK-SD-NEXT: fmov s1, w4 ; CHECK-SD-NEXT: mov v0.s[1], w1 ; CHECK-SD-NEXT: mov v1.s[1], w5 ; CHECK-SD-NEXT: mov v0.s[2], w2 ; CHECK-SD-NEXT: mov v1.s[2], w6 ; CHECK-SD-NEXT: mov v0.s[3], w3 ; CHECK-SD-NEXT: shl v3.4s, v1.4s, #29 ; CHECK-SD-NEXT: usra v3.4s, v1.4s, #3 ; CHECK-SD-NEXT: shl v2.4s, v0.4s, #29 ; CHECK-SD-NEXT: mov w5, v3.s[1] ; CHECK-SD-NEXT: mov w6, v3.s[2] ; CHECK-SD-NEXT: fmov w4, s3 ; CHECK-SD-NEXT: usra v2.4s, v0.4s, #3 ; CHECK-SD-NEXT: mov w1, v2.s[1] ; CHECK-SD-NEXT: mov w2, v2.s[2] ; CHECK-SD-NEXT: mov w3, v2.s[3] ; CHECK-SD-NEXT: fmov w0, s2 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotr_v7i32_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: fmov s0, w0 ; CHECK-GI-NEXT: fmov s1, w0 ; CHECK-GI-NEXT: fmov s2, w4 ; CHECK-GI-NEXT: fmov s3, w4 ; CHECK-GI-NEXT: mov v0.s[1], w1 ; CHECK-GI-NEXT: mov v1.s[1], w1 ; CHECK-GI-NEXT: mov v2.s[1], w5 ; CHECK-GI-NEXT: mov v3.s[1], w5 ; CHECK-GI-NEXT: mov v0.s[2], w2 ; CHECK-GI-NEXT: mov v1.s[2], w2 ; CHECK-GI-NEXT: mov v2.s[2], w6 ; CHECK-GI-NEXT: mov v3.s[2], w6 ; CHECK-GI-NEXT: mov v0.s[3], w3 ; CHECK-GI-NEXT: mov v1.s[3], w3 ; CHECK-GI-NEXT: shl v2.4s, v2.4s, #29 ; CHECK-GI-NEXT: usra v2.4s, v3.4s, #3 ; CHECK-GI-NEXT: shl v0.4s, v0.4s, #29 ; CHECK-GI-NEXT: mov s5, v2.s[1] ; CHECK-GI-NEXT: mov s6, v2.s[2] ; CHECK-GI-NEXT: fmov w4, s2 ; CHECK-GI-NEXT: usra v0.4s, v1.4s, #3 ; CHECK-GI-NEXT: mov s1, v0.s[1] ; CHECK-GI-NEXT: mov s3, v0.s[2] ; CHECK-GI-NEXT: mov s4, v0.s[3] ; CHECK-GI-NEXT: fmov w0, s0 ; CHECK-GI-NEXT: fmov w5, s5 ; CHECK-GI-NEXT: fmov w6, s6 ; CHECK-GI-NEXT: fmov w1, s1 ; CHECK-GI-NEXT: fmov w2, s3 ; CHECK-GI-NEXT: fmov w3, s4 ; CHECK-GI-NEXT: ret entry: %d = call <7 x i32> @llvm.fshr(<7 x i32> %a, <7 x i32> %a, <7 x i32> ) ret <7 x i32> %d } define <8 x i32> @rotl_v8i32_c(<8 x i32> %a) { ; CHECK-LABEL: rotl_v8i32_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v2.4s, v0.4s, #3 ; CHECK-NEXT: shl v3.4s, v1.4s, #3 ; CHECK-NEXT: usra v2.4s, v0.4s, #29 ; CHECK-NEXT: usra v3.4s, v1.4s, #29 ; CHECK-NEXT: mov v0.16b, v2.16b ; CHECK-NEXT: mov v1.16b, v3.16b ; CHECK-NEXT: ret entry: %d = call <8 x i32> @llvm.fshl(<8 x i32> %a, <8 x i32> %a, <8 x i32> ) ret <8 x i32> %d } define <8 x i32> @rotr_v8i32_c(<8 x i32> %a) { ; CHECK-LABEL: rotr_v8i32_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v2.4s, v0.4s, #29 ; CHECK-NEXT: shl v3.4s, v1.4s, #29 ; CHECK-NEXT: usra v2.4s, v0.4s, #3 ; CHECK-NEXT: usra v3.4s, v1.4s, #3 ; CHECK-NEXT: mov v0.16b, v2.16b ; CHECK-NEXT: mov v1.16b, v3.16b ; CHECK-NEXT: ret entry: %d = call <8 x i32> @llvm.fshr(<8 x i32> %a, <8 x i32> %a, <8 x i32> ) ret <8 x i32> %d } define <2 x i64> @rotl_v2i64_c(<2 x i64> %a) { ; CHECK-LABEL: rotl_v2i64_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v1.2d, v0.2d, #3 ; CHECK-NEXT: usra v1.2d, v0.2d, #61 ; CHECK-NEXT: mov v0.16b, v1.16b ; CHECK-NEXT: ret entry: %d = call <2 x i64> @llvm.fshl(<2 x i64> %a, <2 x i64> %a, <2 x i64> ) ret <2 x i64> %d } define <2 x i64> @rotr_v2i64_c(<2 x i64> %a) { ; CHECK-LABEL: rotr_v2i64_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v1.2d, v0.2d, #61 ; CHECK-NEXT: usra v1.2d, v0.2d, #3 ; CHECK-NEXT: mov v0.16b, v1.16b ; CHECK-NEXT: ret entry: %d = call <2 x i64> @llvm.fshr(<2 x i64> %a, <2 x i64> %a, <2 x i64> ) ret <2 x i64> %d } define <4 x i64> @rotl_v4i64_c(<4 x i64> %a) { ; CHECK-LABEL: rotl_v4i64_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v2.2d, v0.2d, #3 ; CHECK-NEXT: shl v3.2d, v1.2d, #3 ; CHECK-NEXT: usra v2.2d, v0.2d, #61 ; CHECK-NEXT: usra v3.2d, v1.2d, #61 ; CHECK-NEXT: mov v0.16b, v2.16b ; CHECK-NEXT: mov v1.16b, v3.16b ; CHECK-NEXT: ret entry: %d = call <4 x i64> @llvm.fshl(<4 x i64> %a, <4 x i64> %a, <4 x i64> ) ret <4 x i64> %d } define <4 x i64> @rotr_v4i64_c(<4 x i64> %a) { ; CHECK-LABEL: rotr_v4i64_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v2.2d, v0.2d, #61 ; CHECK-NEXT: shl v3.2d, v1.2d, #61 ; CHECK-NEXT: usra v2.2d, v0.2d, #3 ; CHECK-NEXT: usra v3.2d, v1.2d, #3 ; CHECK-NEXT: mov v0.16b, v2.16b ; CHECK-NEXT: mov v1.16b, v3.16b ; CHECK-NEXT: ret entry: %d = call <4 x i64> @llvm.fshr(<4 x i64> %a, <4 x i64> %a, <4 x i64> ) ret <4 x i64> %d } define <2 x i128> @rotl_v2i128_c(<2 x i128> %a) { ; CHECK-SD-LABEL: rotl_v2i128_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: extr x8, x2, x3, #61 ; CHECK-SD-NEXT: extr x9, x0, x1, #61 ; CHECK-SD-NEXT: extr x1, x1, x0, #61 ; CHECK-SD-NEXT: extr x3, x3, x2, #61 ; CHECK-SD-NEXT: mov x0, x9 ; CHECK-SD-NEXT: mov x2, x8 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotl_v2i128_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: extr x8, x0, x1, #61 ; CHECK-GI-NEXT: extr x9, x3, x2, #61 ; CHECK-GI-NEXT: extr x1, x1, x0, #61 ; CHECK-GI-NEXT: extr x2, x2, x3, #61 ; CHECK-GI-NEXT: mov x0, x8 ; CHECK-GI-NEXT: mov x3, x9 ; CHECK-GI-NEXT: ret entry: %d = call <2 x i128> @llvm.fshl(<2 x i128> %a, <2 x i128> %a, <2 x i128> ) ret <2 x i128> %d } define <2 x i128> @rotr_v2i128_c(<2 x i128> %a) { ; CHECK-SD-LABEL: rotr_v2i128_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: extr x8, x3, x2, #3 ; CHECK-SD-NEXT: extr x9, x1, x0, #3 ; CHECK-SD-NEXT: extr x1, x0, x1, #3 ; CHECK-SD-NEXT: extr x3, x2, x3, #3 ; CHECK-SD-NEXT: mov x0, x9 ; CHECK-SD-NEXT: mov x2, x8 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: rotr_v2i128_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: extr x8, x1, x0, #3 ; CHECK-GI-NEXT: extr x9, x3, x2, #3 ; CHECK-GI-NEXT: extr x1, x0, x1, #3 ; CHECK-GI-NEXT: extr x3, x2, x3, #3 ; CHECK-GI-NEXT: mov x0, x8 ; CHECK-GI-NEXT: mov x2, x9 ; CHECK-GI-NEXT: ret entry: %d = call <2 x i128> @llvm.fshr(<2 x i128> %a, <2 x i128> %a, <2 x i128> ) ret <2 x i128> %d } define <8 x i8> @fshl_v8i8_c(<8 x i8> %a, <8 x i8> %b) { ; CHECK-LABEL: fshl_v8i8_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v0.8b, v0.8b, #3 ; CHECK-NEXT: usra v0.8b, v1.8b, #5 ; CHECK-NEXT: ret entry: %d = call <8 x i8> @llvm.fshl(<8 x i8> %a, <8 x i8> %b, <8 x i8> ) ret <8 x i8> %d } define <8 x i8> @fshr_v8i8_c(<8 x i8> %a, <8 x i8> %b) { ; CHECK-LABEL: fshr_v8i8_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v0.8b, v0.8b, #5 ; CHECK-NEXT: usra v0.8b, v1.8b, #3 ; CHECK-NEXT: ret entry: %d = call <8 x i8> @llvm.fshr(<8 x i8> %a, <8 x i8> %b, <8 x i8> ) ret <8 x i8> %d } define <16 x i8> @fshl_v16i8_c(<16 x i8> %a, <16 x i8> %b) { ; CHECK-LABEL: fshl_v16i8_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v0.16b, v0.16b, #3 ; CHECK-NEXT: usra v0.16b, v1.16b, #5 ; CHECK-NEXT: ret entry: %d = call <16 x i8> @llvm.fshl(<16 x i8> %a, <16 x i8> %b, <16 x i8> ) ret <16 x i8> %d } define <16 x i8> @fshr_v16i8_c(<16 x i8> %a, <16 x i8> %b) { ; CHECK-LABEL: fshr_v16i8_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v0.16b, v0.16b, #5 ; CHECK-NEXT: usra v0.16b, v1.16b, #3 ; CHECK-NEXT: ret entry: %d = call <16 x i8> @llvm.fshr(<16 x i8> %a, <16 x i8> %b, <16 x i8> ) ret <16 x i8> %d } define <4 x i16> @fshl_v4i16_c(<4 x i16> %a, <4 x i16> %b) { ; CHECK-LABEL: fshl_v4i16_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v0.4h, v0.4h, #3 ; CHECK-NEXT: usra v0.4h, v1.4h, #13 ; CHECK-NEXT: ret entry: %d = call <4 x i16> @llvm.fshl(<4 x i16> %a, <4 x i16> %b, <4 x i16> ) ret <4 x i16> %d } define <4 x i16> @fshr_v4i16_c(<4 x i16> %a, <4 x i16> %b) { ; CHECK-LABEL: fshr_v4i16_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v0.4h, v0.4h, #13 ; CHECK-NEXT: usra v0.4h, v1.4h, #3 ; CHECK-NEXT: ret entry: %d = call <4 x i16> @llvm.fshr(<4 x i16> %a, <4 x i16> %b, <4 x i16> ) ret <4 x i16> %d } define <7 x i16> @fshl_v7i16_c(<7 x i16> %a, <7 x i16> %b) { ; CHECK-SD-LABEL: fshl_v7i16_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: adrp x8, .LCPI124_0 ; CHECK-SD-NEXT: adrp x9, .LCPI124_1 ; CHECK-SD-NEXT: ldr q2, [x8, :lo12:.LCPI124_0] ; CHECK-SD-NEXT: ldr q3, [x9, :lo12:.LCPI124_1] ; CHECK-SD-NEXT: ushl v1.8h, v1.8h, v2.8h ; CHECK-SD-NEXT: ushl v0.8h, v0.8h, v3.8h ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshl_v7i16_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: shl v0.8h, v0.8h, #3 ; CHECK-GI-NEXT: usra v0.8h, v1.8h, #13 ; CHECK-GI-NEXT: ret entry: %d = call <7 x i16> @llvm.fshl(<7 x i16> %a, <7 x i16> %b, <7 x i16> ) ret <7 x i16> %d } define <7 x i16> @fshr_v7i16_c(<7 x i16> %a, <7 x i16> %b) { ; CHECK-SD-LABEL: fshr_v7i16_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: adrp x8, .LCPI125_0 ; CHECK-SD-NEXT: adrp x9, .LCPI125_1 ; CHECK-SD-NEXT: ldr q2, [x8, :lo12:.LCPI125_0] ; CHECK-SD-NEXT: ldr q3, [x9, :lo12:.LCPI125_1] ; CHECK-SD-NEXT: ushl v1.8h, v1.8h, v2.8h ; CHECK-SD-NEXT: ushl v0.8h, v0.8h, v3.8h ; CHECK-SD-NEXT: orr v0.16b, v0.16b, v1.16b ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_v7i16_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: shl v0.8h, v0.8h, #13 ; CHECK-GI-NEXT: usra v0.8h, v1.8h, #3 ; CHECK-GI-NEXT: ret entry: %d = call <7 x i16> @llvm.fshr(<7 x i16> %a, <7 x i16> %b, <7 x i16> ) ret <7 x i16> %d } define <8 x i16> @fshl_v8i16_c(<8 x i16> %a, <8 x i16> %b) { ; CHECK-LABEL: fshl_v8i16_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v0.8h, v0.8h, #3 ; CHECK-NEXT: usra v0.8h, v1.8h, #13 ; CHECK-NEXT: ret entry: %d = call <8 x i16> @llvm.fshl(<8 x i16> %a, <8 x i16> %b, <8 x i16> ) ret <8 x i16> %d } define <8 x i16> @fshr_v8i16_c(<8 x i16> %a, <8 x i16> %b) { ; CHECK-LABEL: fshr_v8i16_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v0.8h, v0.8h, #13 ; CHECK-NEXT: usra v0.8h, v1.8h, #3 ; CHECK-NEXT: ret entry: %d = call <8 x i16> @llvm.fshr(<8 x i16> %a, <8 x i16> %b, <8 x i16> ) ret <8 x i16> %d } define <16 x i16> @fshl_v16i16_c(<16 x i16> %a, <16 x i16> %b) { ; CHECK-SD-LABEL: fshl_v16i16_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: shl v1.8h, v1.8h, #3 ; CHECK-SD-NEXT: shl v0.8h, v0.8h, #3 ; CHECK-SD-NEXT: usra v1.8h, v3.8h, #13 ; CHECK-SD-NEXT: usra v0.8h, v2.8h, #13 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshl_v16i16_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: shl v0.8h, v0.8h, #3 ; CHECK-GI-NEXT: shl v1.8h, v1.8h, #3 ; CHECK-GI-NEXT: usra v0.8h, v2.8h, #13 ; CHECK-GI-NEXT: usra v1.8h, v3.8h, #13 ; CHECK-GI-NEXT: ret entry: %d = call <16 x i16> @llvm.fshl(<16 x i16> %a, <16 x i16> %b, <16 x i16> ) ret <16 x i16> %d } define <16 x i16> @fshr_v16i16_c(<16 x i16> %a, <16 x i16> %b) { ; CHECK-SD-LABEL: fshr_v16i16_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: shl v1.8h, v1.8h, #13 ; CHECK-SD-NEXT: shl v0.8h, v0.8h, #13 ; CHECK-SD-NEXT: usra v1.8h, v3.8h, #3 ; CHECK-SD-NEXT: usra v0.8h, v2.8h, #3 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_v16i16_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: shl v0.8h, v0.8h, #13 ; CHECK-GI-NEXT: shl v1.8h, v1.8h, #13 ; CHECK-GI-NEXT: usra v0.8h, v2.8h, #3 ; CHECK-GI-NEXT: usra v1.8h, v3.8h, #3 ; CHECK-GI-NEXT: ret entry: %d = call <16 x i16> @llvm.fshr(<16 x i16> %a, <16 x i16> %b, <16 x i16> ) ret <16 x i16> %d } define <2 x i32> @fshl_v2i32_c(<2 x i32> %a, <2 x i32> %b) { ; CHECK-LABEL: fshl_v2i32_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v0.2s, v0.2s, #3 ; CHECK-NEXT: usra v0.2s, v1.2s, #29 ; CHECK-NEXT: ret entry: %d = call <2 x i32> @llvm.fshl(<2 x i32> %a, <2 x i32> %b, <2 x i32> ) ret <2 x i32> %d } define <2 x i32> @fshr_v2i32_c(<2 x i32> %a, <2 x i32> %b) { ; CHECK-LABEL: fshr_v2i32_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v0.2s, v0.2s, #29 ; CHECK-NEXT: usra v0.2s, v1.2s, #3 ; CHECK-NEXT: ret entry: %d = call <2 x i32> @llvm.fshr(<2 x i32> %a, <2 x i32> %b, <2 x i32> ) ret <2 x i32> %d } define <4 x i32> @fshl_v4i32_c(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: fshl_v4i32_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v0.4s, v0.4s, #3 ; CHECK-NEXT: usra v0.4s, v1.4s, #29 ; CHECK-NEXT: ret entry: %d = call <4 x i32> @llvm.fshl(<4 x i32> %a, <4 x i32> %b, <4 x i32> ) ret <4 x i32> %d } define <4 x i32> @fshr_v4i32_c(<4 x i32> %a, <4 x i32> %b) { ; CHECK-LABEL: fshr_v4i32_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v0.4s, v0.4s, #29 ; CHECK-NEXT: usra v0.4s, v1.4s, #3 ; CHECK-NEXT: ret entry: %d = call <4 x i32> @llvm.fshr(<4 x i32> %a, <4 x i32> %b, <4 x i32> ) ret <4 x i32> %d } define <7 x i32> @fshl_v7i32_c(<7 x i32> %a, <7 x i32> %b) { ; CHECK-SD-LABEL: fshl_v7i32_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fmov s0, w0 ; CHECK-SD-NEXT: fmov s1, w4 ; CHECK-SD-NEXT: mov x8, sp ; CHECK-SD-NEXT: fmov s2, w7 ; CHECK-SD-NEXT: ldr s3, [sp, #24] ; CHECK-SD-NEXT: add x9, sp, #32 ; CHECK-SD-NEXT: mov v0.s[1], w1 ; CHECK-SD-NEXT: mov v1.s[1], w5 ; CHECK-SD-NEXT: ld1 { v3.s }[1], [x9] ; CHECK-SD-NEXT: ld1 { v2.s }[1], [x8] ; CHECK-SD-NEXT: add x8, sp, #8 ; CHECK-SD-NEXT: add x9, sp, #40 ; CHECK-SD-NEXT: ld1 { v3.s }[2], [x9] ; CHECK-SD-NEXT: mov v0.s[2], w2 ; CHECK-SD-NEXT: mov v1.s[2], w6 ; CHECK-SD-NEXT: ld1 { v2.s }[2], [x8] ; CHECK-SD-NEXT: add x8, sp, #16 ; CHECK-SD-NEXT: ld1 { v2.s }[3], [x8] ; CHECK-SD-NEXT: mov v0.s[3], w3 ; CHECK-SD-NEXT: shl v1.4s, v1.4s, #3 ; CHECK-SD-NEXT: usra v1.4s, v3.4s, #29 ; CHECK-SD-NEXT: shl v0.4s, v0.4s, #3 ; CHECK-SD-NEXT: mov w5, v1.s[1] ; CHECK-SD-NEXT: mov w6, v1.s[2] ; CHECK-SD-NEXT: fmov w4, s1 ; CHECK-SD-NEXT: usra v0.4s, v2.4s, #29 ; CHECK-SD-NEXT: mov w1, v0.s[1] ; CHECK-SD-NEXT: mov w2, v0.s[2] ; CHECK-SD-NEXT: mov w3, v0.s[3] ; CHECK-SD-NEXT: fmov w0, s0 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshl_v7i32_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: fmov s0, w0 ; CHECK-GI-NEXT: fmov s2, w7 ; CHECK-GI-NEXT: ldr s1, [sp] ; CHECK-GI-NEXT: ldr s3, [sp, #8] ; CHECK-GI-NEXT: ldr s4, [sp, #24] ; CHECK-GI-NEXT: ldr s5, [sp, #32] ; CHECK-GI-NEXT: mov v0.s[1], w1 ; CHECK-GI-NEXT: mov v2.s[1], v1.s[0] ; CHECK-GI-NEXT: fmov s1, w4 ; CHECK-GI-NEXT: mov v4.s[1], v5.s[0] ; CHECK-GI-NEXT: ldr s5, [sp, #40] ; CHECK-GI-NEXT: mov v1.s[1], w5 ; CHECK-GI-NEXT: mov v0.s[2], w2 ; CHECK-GI-NEXT: mov v2.s[2], v3.s[0] ; CHECK-GI-NEXT: ldr s3, [sp, #16] ; CHECK-GI-NEXT: mov v4.s[2], v5.s[0] ; CHECK-GI-NEXT: mov v1.s[2], w6 ; CHECK-GI-NEXT: mov v0.s[3], w3 ; CHECK-GI-NEXT: mov v2.s[3], v3.s[0] ; CHECK-GI-NEXT: shl v1.4s, v1.4s, #3 ; CHECK-GI-NEXT: shl v0.4s, v0.4s, #3 ; CHECK-GI-NEXT: usra v1.4s, v4.4s, #29 ; CHECK-GI-NEXT: usra v0.4s, v2.4s, #29 ; CHECK-GI-NEXT: mov s5, v1.s[1] ; CHECK-GI-NEXT: mov s6, v1.s[2] ; CHECK-GI-NEXT: fmov w4, s1 ; CHECK-GI-NEXT: mov s2, v0.s[1] ; CHECK-GI-NEXT: mov s3, v0.s[2] ; CHECK-GI-NEXT: mov s4, v0.s[3] ; CHECK-GI-NEXT: fmov w0, s0 ; CHECK-GI-NEXT: fmov w5, s5 ; CHECK-GI-NEXT: fmov w6, s6 ; CHECK-GI-NEXT: fmov w1, s2 ; CHECK-GI-NEXT: fmov w2, s3 ; CHECK-GI-NEXT: fmov w3, s4 ; CHECK-GI-NEXT: ret entry: %d = call <7 x i32> @llvm.fshl(<7 x i32> %a, <7 x i32> %b, <7 x i32> ) ret <7 x i32> %d } define <7 x i32> @fshr_v7i32_c(<7 x i32> %a, <7 x i32> %b) { ; CHECK-SD-LABEL: fshr_v7i32_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: fmov s0, w0 ; CHECK-SD-NEXT: fmov s1, w4 ; CHECK-SD-NEXT: mov x8, sp ; CHECK-SD-NEXT: fmov s2, w7 ; CHECK-SD-NEXT: ldr s3, [sp, #24] ; CHECK-SD-NEXT: add x9, sp, #32 ; CHECK-SD-NEXT: mov v0.s[1], w1 ; CHECK-SD-NEXT: mov v1.s[1], w5 ; CHECK-SD-NEXT: ld1 { v3.s }[1], [x9] ; CHECK-SD-NEXT: ld1 { v2.s }[1], [x8] ; CHECK-SD-NEXT: add x8, sp, #8 ; CHECK-SD-NEXT: add x9, sp, #40 ; CHECK-SD-NEXT: ld1 { v3.s }[2], [x9] ; CHECK-SD-NEXT: mov v0.s[2], w2 ; CHECK-SD-NEXT: mov v1.s[2], w6 ; CHECK-SD-NEXT: ld1 { v2.s }[2], [x8] ; CHECK-SD-NEXT: add x8, sp, #16 ; CHECK-SD-NEXT: ld1 { v2.s }[3], [x8] ; CHECK-SD-NEXT: mov v0.s[3], w3 ; CHECK-SD-NEXT: shl v1.4s, v1.4s, #29 ; CHECK-SD-NEXT: usra v1.4s, v3.4s, #3 ; CHECK-SD-NEXT: shl v0.4s, v0.4s, #29 ; CHECK-SD-NEXT: mov w5, v1.s[1] ; CHECK-SD-NEXT: mov w6, v1.s[2] ; CHECK-SD-NEXT: fmov w4, s1 ; CHECK-SD-NEXT: usra v0.4s, v2.4s, #3 ; CHECK-SD-NEXT: mov w1, v0.s[1] ; CHECK-SD-NEXT: mov w2, v0.s[2] ; CHECK-SD-NEXT: mov w3, v0.s[3] ; CHECK-SD-NEXT: fmov w0, s0 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_v7i32_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: fmov s0, w0 ; CHECK-GI-NEXT: fmov s2, w7 ; CHECK-GI-NEXT: ldr s1, [sp] ; CHECK-GI-NEXT: ldr s3, [sp, #8] ; CHECK-GI-NEXT: ldr s4, [sp, #24] ; CHECK-GI-NEXT: ldr s5, [sp, #32] ; CHECK-GI-NEXT: mov v0.s[1], w1 ; CHECK-GI-NEXT: mov v2.s[1], v1.s[0] ; CHECK-GI-NEXT: fmov s1, w4 ; CHECK-GI-NEXT: mov v4.s[1], v5.s[0] ; CHECK-GI-NEXT: ldr s5, [sp, #40] ; CHECK-GI-NEXT: mov v1.s[1], w5 ; CHECK-GI-NEXT: mov v0.s[2], w2 ; CHECK-GI-NEXT: mov v2.s[2], v3.s[0] ; CHECK-GI-NEXT: ldr s3, [sp, #16] ; CHECK-GI-NEXT: mov v4.s[2], v5.s[0] ; CHECK-GI-NEXT: mov v1.s[2], w6 ; CHECK-GI-NEXT: mov v0.s[3], w3 ; CHECK-GI-NEXT: mov v2.s[3], v3.s[0] ; CHECK-GI-NEXT: shl v1.4s, v1.4s, #29 ; CHECK-GI-NEXT: shl v0.4s, v0.4s, #29 ; CHECK-GI-NEXT: usra v1.4s, v4.4s, #3 ; CHECK-GI-NEXT: usra v0.4s, v2.4s, #3 ; CHECK-GI-NEXT: mov s5, v1.s[1] ; CHECK-GI-NEXT: mov s6, v1.s[2] ; CHECK-GI-NEXT: fmov w4, s1 ; CHECK-GI-NEXT: mov s2, v0.s[1] ; CHECK-GI-NEXT: mov s3, v0.s[2] ; CHECK-GI-NEXT: mov s4, v0.s[3] ; CHECK-GI-NEXT: fmov w0, s0 ; CHECK-GI-NEXT: fmov w5, s5 ; CHECK-GI-NEXT: fmov w6, s6 ; CHECK-GI-NEXT: fmov w1, s2 ; CHECK-GI-NEXT: fmov w2, s3 ; CHECK-GI-NEXT: fmov w3, s4 ; CHECK-GI-NEXT: ret entry: %d = call <7 x i32> @llvm.fshr(<7 x i32> %a, <7 x i32> %b, <7 x i32> ) ret <7 x i32> %d } define <8 x i32> @fshl_v8i32_c(<8 x i32> %a, <8 x i32> %b) { ; CHECK-SD-LABEL: fshl_v8i32_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: shl v1.4s, v1.4s, #3 ; CHECK-SD-NEXT: shl v0.4s, v0.4s, #3 ; CHECK-SD-NEXT: usra v1.4s, v3.4s, #29 ; CHECK-SD-NEXT: usra v0.4s, v2.4s, #29 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshl_v8i32_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: shl v0.4s, v0.4s, #3 ; CHECK-GI-NEXT: shl v1.4s, v1.4s, #3 ; CHECK-GI-NEXT: usra v0.4s, v2.4s, #29 ; CHECK-GI-NEXT: usra v1.4s, v3.4s, #29 ; CHECK-GI-NEXT: ret entry: %d = call <8 x i32> @llvm.fshl(<8 x i32> %a, <8 x i32> %b, <8 x i32> ) ret <8 x i32> %d } define <8 x i32> @fshr_v8i32_c(<8 x i32> %a, <8 x i32> %b) { ; CHECK-SD-LABEL: fshr_v8i32_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: shl v1.4s, v1.4s, #29 ; CHECK-SD-NEXT: shl v0.4s, v0.4s, #29 ; CHECK-SD-NEXT: usra v1.4s, v3.4s, #3 ; CHECK-SD-NEXT: usra v0.4s, v2.4s, #3 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_v8i32_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: shl v0.4s, v0.4s, #29 ; CHECK-GI-NEXT: shl v1.4s, v1.4s, #29 ; CHECK-GI-NEXT: usra v0.4s, v2.4s, #3 ; CHECK-GI-NEXT: usra v1.4s, v3.4s, #3 ; CHECK-GI-NEXT: ret entry: %d = call <8 x i32> @llvm.fshr(<8 x i32> %a, <8 x i32> %b, <8 x i32> ) ret <8 x i32> %d } define <2 x i64> @fshl_v2i64_c(<2 x i64> %a, <2 x i64> %b) { ; CHECK-LABEL: fshl_v2i64_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v0.2d, v0.2d, #3 ; CHECK-NEXT: usra v0.2d, v1.2d, #61 ; CHECK-NEXT: ret entry: %d = call <2 x i64> @llvm.fshl(<2 x i64> %a, <2 x i64> %b, <2 x i64> ) ret <2 x i64> %d } define <2 x i64> @fshr_v2i64_c(<2 x i64> %a, <2 x i64> %b) { ; CHECK-LABEL: fshr_v2i64_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: shl v0.2d, v0.2d, #61 ; CHECK-NEXT: usra v0.2d, v1.2d, #3 ; CHECK-NEXT: ret entry: %d = call <2 x i64> @llvm.fshr(<2 x i64> %a, <2 x i64> %b, <2 x i64> ) ret <2 x i64> %d } define <4 x i64> @fshl_v4i64_c(<4 x i64> %a, <4 x i64> %b) { ; CHECK-SD-LABEL: fshl_v4i64_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: shl v1.2d, v1.2d, #3 ; CHECK-SD-NEXT: shl v0.2d, v0.2d, #3 ; CHECK-SD-NEXT: usra v1.2d, v3.2d, #61 ; CHECK-SD-NEXT: usra v0.2d, v2.2d, #61 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshl_v4i64_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: shl v0.2d, v0.2d, #3 ; CHECK-GI-NEXT: shl v1.2d, v1.2d, #3 ; CHECK-GI-NEXT: usra v0.2d, v2.2d, #61 ; CHECK-GI-NEXT: usra v1.2d, v3.2d, #61 ; CHECK-GI-NEXT: ret entry: %d = call <4 x i64> @llvm.fshl(<4 x i64> %a, <4 x i64> %b, <4 x i64> ) ret <4 x i64> %d } define <4 x i64> @fshr_v4i64_c(<4 x i64> %a, <4 x i64> %b) { ; CHECK-SD-LABEL: fshr_v4i64_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: shl v1.2d, v1.2d, #61 ; CHECK-SD-NEXT: shl v0.2d, v0.2d, #61 ; CHECK-SD-NEXT: usra v1.2d, v3.2d, #3 ; CHECK-SD-NEXT: usra v0.2d, v2.2d, #3 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshr_v4i64_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: shl v0.2d, v0.2d, #61 ; CHECK-GI-NEXT: shl v1.2d, v1.2d, #61 ; CHECK-GI-NEXT: usra v0.2d, v2.2d, #3 ; CHECK-GI-NEXT: usra v1.2d, v3.2d, #3 ; CHECK-GI-NEXT: ret entry: %d = call <4 x i64> @llvm.fshr(<4 x i64> %a, <4 x i64> %b, <4 x i64> ) ret <4 x i64> %d } define <2 x i128> @fshl_v2i128_c(<2 x i128> %a, <2 x i128> %b) { ; CHECK-SD-LABEL: fshl_v2i128_c: ; CHECK-SD: // %bb.0: // %entry ; CHECK-SD-NEXT: extr x8, x0, x5, #61 ; CHECK-SD-NEXT: extr x9, x2, x7, #61 ; CHECK-SD-NEXT: extr x1, x1, x0, #61 ; CHECK-SD-NEXT: extr x3, x3, x2, #61 ; CHECK-SD-NEXT: mov x0, x8 ; CHECK-SD-NEXT: mov x2, x9 ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshl_v2i128_c: ; CHECK-GI: // %bb.0: // %entry ; CHECK-GI-NEXT: extr x8, x0, x5, #61 ; CHECK-GI-NEXT: extr x1, x1, x0, #61 ; CHECK-GI-NEXT: extr x3, x3, x2, #61 ; CHECK-GI-NEXT: extr x2, x2, x7, #61 ; CHECK-GI-NEXT: mov x0, x8 ; CHECK-GI-NEXT: ret entry: %d = call <2 x i128> @llvm.fshl(<2 x i128> %a, <2 x i128> %b, <2 x i128> ) ret <2 x i128> %d } define <2 x i128> @fshr_v2i128_c(<2 x i128> %a, <2 x i128> %b) { ; CHECK-LABEL: fshr_v2i128_c: ; CHECK: // %bb.0: // %entry ; CHECK-NEXT: extr x8, x5, x4, #3 ; CHECK-NEXT: extr x9, x7, x6, #3 ; CHECK-NEXT: extr x1, x0, x5, #3 ; CHECK-NEXT: extr x3, x2, x7, #3 ; CHECK-NEXT: mov x0, x8 ; CHECK-NEXT: mov x2, x9 ; CHECK-NEXT: ret entry: %d = call <2 x i128> @llvm.fshr(<2 x i128> %a, <2 x i128> %b, <2 x i128> ) ret <2 x i128> %d } define <2 x i64> @fshl_to_rev2i64(<2 x i64> %r) { ; CHECK-SD-LABEL: fshl_to_rev2i64: ; CHECK-SD: // %bb.0: ; CHECK-SD-NEXT: rev64 v0.4s, v0.4s ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshl_to_rev2i64: ; CHECK-GI: // %bb.0: ; CHECK-GI-NEXT: shl v1.2d, v0.2d, #32 ; CHECK-GI-NEXT: usra v1.2d, v0.2d, #32 ; CHECK-GI-NEXT: mov v0.16b, v1.16b ; CHECK-GI-NEXT: ret %or = tail call <2 x i64> @llvm.fshl.v2i64(<2 x i64> %r, <2 x i64> %r, <2 x i64> splat (i64 32)) ret <2 x i64> %or } define <4 x i32> @fshl_to_rev4i32(<4 x i32> %r) { ; CHECK-SD-LABEL: fshl_to_rev4i32: ; CHECK-SD: // %bb.0: ; CHECK-SD-NEXT: rev32 v0.8h, v0.8h ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshl_to_rev4i32: ; CHECK-GI: // %bb.0: ; CHECK-GI-NEXT: shl v1.4s, v0.4s, #16 ; CHECK-GI-NEXT: usra v1.4s, v0.4s, #16 ; CHECK-GI-NEXT: mov v0.16b, v1.16b ; CHECK-GI-NEXT: ret %or = tail call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %r, <4 x i32> %r, <4 x i32> splat (i32 16)) ret <4 x i32> %or } define <2 x i32> @fshl_to_rev2i32(<2 x i32> %r) { ; CHECK-SD-LABEL: fshl_to_rev2i32: ; CHECK-SD: // %bb.0: ; CHECK-SD-NEXT: rev32 v0.4h, v0.4h ; CHECK-SD-NEXT: ret ; ; CHECK-GI-LABEL: fshl_to_rev2i32: ; CHECK-GI: // %bb.0: ; CHECK-GI-NEXT: shl v1.2s, v0.2s, #16 ; CHECK-GI-NEXT: usra v1.2s, v0.2s, #16 ; CHECK-GI-NEXT: fmov d0, d1 ; CHECK-GI-NEXT: ret %or = tail call <2 x i32> @llvm.fshl.v2i32(<2 x i32> %r, <2 x i32> %r, <2 x i32> splat (i32 16)) ret <2 x i32> %or }