derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
1049 lines
39 KiB
C++
1049 lines
39 KiB
C++
//===- X86LegalizerInfo.cpp --------------------------------------*- C++ -*-==//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
/// \file
|
|
/// This file implements the targeting of the Machinelegalizer class for X86.
|
|
/// \todo This should be generated by TableGen.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "X86LegalizerInfo.h"
|
|
#include "X86Subtarget.h"
|
|
#include "X86TargetMachine.h"
|
|
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
|
|
#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
|
|
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
|
|
#include "llvm/CodeGen/MachineConstantPool.h"
|
|
#include "llvm/CodeGen/MachineFrameInfo.h"
|
|
#include "llvm/CodeGen/TargetOpcodes.h"
|
|
#include "llvm/CodeGen/ValueTypes.h"
|
|
#include "llvm/IR/DerivedTypes.h"
|
|
#include "llvm/IR/IntrinsicsX86.h"
|
|
#include "llvm/IR/Type.h"
|
|
|
|
using namespace llvm;
|
|
using namespace TargetOpcode;
|
|
using namespace LegalizeActions;
|
|
using namespace LegalityPredicates;
|
|
|
|
X86LegalizerInfo::X86LegalizerInfo(const X86Subtarget &STI,
|
|
const X86TargetMachine &TM)
|
|
: Subtarget(STI) {
|
|
|
|
bool Is64Bit = Subtarget.is64Bit();
|
|
bool HasCMOV = Subtarget.canUseCMOV();
|
|
bool HasSSE1 = Subtarget.hasSSE1();
|
|
bool HasSSE2 = Subtarget.hasSSE2();
|
|
bool HasSSE41 = Subtarget.hasSSE41();
|
|
bool HasAVX = Subtarget.hasAVX();
|
|
bool HasAVX2 = Subtarget.hasAVX2();
|
|
bool HasAVX512 = Subtarget.hasAVX512();
|
|
bool HasVLX = Subtarget.hasVLX();
|
|
bool HasDQI = Subtarget.hasAVX512() && Subtarget.hasDQI();
|
|
bool HasBWI = Subtarget.hasAVX512() && Subtarget.hasBWI();
|
|
bool UseX87 = !Subtarget.useSoftFloat() && Subtarget.hasX87();
|
|
bool HasPOPCNT = Subtarget.hasPOPCNT();
|
|
bool HasLZCNT = Subtarget.hasLZCNT();
|
|
bool HasBMI = Subtarget.hasBMI();
|
|
|
|
const LLT p0 = LLT::pointer(0, TM.getPointerSizeInBits(0));
|
|
const LLT s1 = LLT::scalar(1);
|
|
const LLT s8 = LLT::scalar(8);
|
|
const LLT s16 = LLT::scalar(16);
|
|
const LLT s32 = LLT::scalar(32);
|
|
const LLT s64 = LLT::scalar(64);
|
|
const LLT s80 = LLT::scalar(80);
|
|
const LLT s128 = LLT::scalar(128);
|
|
const LLT sMaxScalar = Subtarget.is64Bit() ? s64 : s32;
|
|
const LLT v2s32 = LLT::fixed_vector(2, 32);
|
|
const LLT v4s8 = LLT::fixed_vector(4, 8);
|
|
|
|
const LLT v16s8 = LLT::fixed_vector(16, 8);
|
|
const LLT v8s16 = LLT::fixed_vector(8, 16);
|
|
const LLT v4s32 = LLT::fixed_vector(4, 32);
|
|
const LLT v2s64 = LLT::fixed_vector(2, 64);
|
|
const LLT v2p0 = LLT::fixed_vector(2, p0);
|
|
|
|
const LLT v32s8 = LLT::fixed_vector(32, 8);
|
|
const LLT v16s16 = LLT::fixed_vector(16, 16);
|
|
const LLT v8s32 = LLT::fixed_vector(8, 32);
|
|
const LLT v4s64 = LLT::fixed_vector(4, 64);
|
|
const LLT v4p0 = LLT::fixed_vector(4, p0);
|
|
|
|
const LLT v64s8 = LLT::fixed_vector(64, 8);
|
|
const LLT v32s16 = LLT::fixed_vector(32, 16);
|
|
const LLT v16s32 = LLT::fixed_vector(16, 32);
|
|
const LLT v8s64 = LLT::fixed_vector(8, 64);
|
|
|
|
const LLT s8MaxVector = HasAVX512 ? v64s8 : HasAVX ? v32s8 : v16s8;
|
|
const LLT s16MaxVector = HasAVX512 ? v32s16 : HasAVX ? v16s16 : v8s16;
|
|
const LLT s32MaxVector = HasAVX512 ? v16s32 : HasAVX ? v8s32 : v4s32;
|
|
const LLT s64MaxVector = HasAVX512 ? v8s64 : HasAVX ? v4s64 : v2s64;
|
|
|
|
// todo: AVX512 bool vector predicate types
|
|
|
|
// implicit/constants
|
|
// 32/64-bits needs support for s64/s128 to handle cases:
|
|
// s64 = EXTEND (G_IMPLICIT_DEF s32) -> s64 = G_IMPLICIT_DEF
|
|
// s128 = EXTEND (G_IMPLICIT_DEF s32/s64) -> s128 = G_IMPLICIT_DEF
|
|
getActionDefinitionsBuilder(
|
|
{G_IMPLICIT_DEF, G_PHI, G_FREEZE, G_CONSTANT_FOLD_BARRIER})
|
|
.legalFor({p0, s1, s8, s16, s32, s64})
|
|
.legalFor(UseX87, {s80})
|
|
.legalFor(Is64Bit, {s128})
|
|
.legalFor(HasSSE2, {v16s8, v8s16, v4s32, v2s64})
|
|
.legalFor(HasAVX, {v32s8, v16s16, v8s32, v4s64})
|
|
.legalFor(HasAVX512, {v64s8, v32s16, v16s32, v8s64})
|
|
.widenScalarOrEltToNextPow2(0, /*Min=*/8)
|
|
.clampScalarOrElt(0, s8, sMaxScalar)
|
|
.moreElementsToNextPow2(0)
|
|
.clampNumElements(0, v16s8, s8MaxVector)
|
|
.clampNumElements(0, v8s16, s16MaxVector)
|
|
.clampNumElements(0, v4s32, s32MaxVector)
|
|
.clampNumElements(0, v2s64, s64MaxVector)
|
|
.clampMaxNumElements(0, p0,
|
|
Is64Bit ? s64MaxVector.getNumElements()
|
|
: s32MaxVector.getNumElements())
|
|
.scalarizeIf(scalarOrEltWiderThan(0, 64), 0);
|
|
|
|
getActionDefinitionsBuilder(G_CONSTANT)
|
|
.legalFor({p0, s8, s16, s32})
|
|
.legalFor(Is64Bit, {s64})
|
|
.widenScalarToNextPow2(0, /*Min=*/8)
|
|
.clampScalar(0, s8, sMaxScalar);
|
|
|
|
getActionDefinitionsBuilder({G_LROUND, G_LLROUND})
|
|
.widenScalarIf(typeIs(1, s16),
|
|
[=](const LegalityQuery &) {
|
|
return std::pair<unsigned, LLT>(1, s32);
|
|
})
|
|
.libcall();
|
|
|
|
getActionDefinitionsBuilder(
|
|
{G_FCOS, G_FCOSH, G_FACOS, G_FSIN, G_FSINH, G_FASIN, G_FTAN,
|
|
G_FTANH, G_FATAN, G_FATAN2, G_FPOW, G_FEXP, G_FEXP2, G_FEXP10,
|
|
G_FLOG, G_FLOG2, G_FLOG10, G_FPOWI, G_FSINCOS, G_FCEIL, G_FFLOOR})
|
|
.libcall();
|
|
|
|
getActionDefinitionsBuilder(G_FNEG)
|
|
.legalFor(UseX87 && !HasSSE1, {s32})
|
|
.legalFor(UseX87 && !HasSSE2, {s64})
|
|
.legalFor(UseX87, {s80})
|
|
.lower();
|
|
|
|
getActionDefinitionsBuilder(G_FSQRT)
|
|
.legalFor(HasSSE1 || UseX87, {s32})
|
|
.legalFor(HasSSE2 || UseX87, {s64})
|
|
.legalFor(UseX87, {s80});
|
|
|
|
getActionDefinitionsBuilder({G_GET_ROUNDING, G_SET_ROUNDING})
|
|
.customFor({s32});
|
|
|
|
// merge/unmerge
|
|
for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
|
|
unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
|
|
unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
|
|
getActionDefinitionsBuilder(Op)
|
|
.widenScalarToNextPow2(LitTyIdx, /*Min=*/8)
|
|
.widenScalarToNextPow2(BigTyIdx, /*Min=*/16)
|
|
.minScalar(LitTyIdx, s8)
|
|
.minScalar(BigTyIdx, s32)
|
|
.legalIf([=](const LegalityQuery &Q) {
|
|
switch (Q.Types[BigTyIdx].getSizeInBits()) {
|
|
case 16:
|
|
case 32:
|
|
case 64:
|
|
case 128:
|
|
case 256:
|
|
case 512:
|
|
break;
|
|
default:
|
|
return false;
|
|
}
|
|
switch (Q.Types[LitTyIdx].getSizeInBits()) {
|
|
case 8:
|
|
case 16:
|
|
case 32:
|
|
case 64:
|
|
case 128:
|
|
case 256:
|
|
return true;
|
|
default:
|
|
return false;
|
|
}
|
|
});
|
|
}
|
|
|
|
getActionDefinitionsBuilder({G_UMIN, G_UMAX, G_SMIN, G_SMAX})
|
|
.widenScalarToNextPow2(0, /*Min=*/32)
|
|
.lower();
|
|
|
|
// integer addition/subtraction
|
|
getActionDefinitionsBuilder({G_ADD, G_SUB})
|
|
.legalFor({s8, s16, s32})
|
|
.legalFor(Is64Bit, {s64})
|
|
.legalFor(HasSSE2, {v16s8, v8s16, v4s32, v2s64})
|
|
.legalFor(HasAVX2, {v32s8, v16s16, v8s32, v4s64})
|
|
.legalFor(HasAVX512, {v16s32, v8s64})
|
|
.legalFor(HasBWI, {v64s8, v32s16})
|
|
.clampMinNumElements(0, s8, 16)
|
|
.clampMinNumElements(0, s16, 8)
|
|
.clampMinNumElements(0, s32, 4)
|
|
.clampMinNumElements(0, s64, 2)
|
|
.clampMaxNumElements(0, s8, HasBWI ? 64 : (HasAVX2 ? 32 : 16))
|
|
.clampMaxNumElements(0, s16, HasBWI ? 32 : (HasAVX2 ? 16 : 8))
|
|
.clampMaxNumElements(0, s32, HasAVX512 ? 16 : (HasAVX2 ? 8 : 4))
|
|
.clampMaxNumElements(0, s64, HasAVX512 ? 8 : (HasAVX2 ? 4 : 2))
|
|
.widenScalarToNextPow2(0, /*Min=*/32)
|
|
.clampScalar(0, s8, sMaxScalar)
|
|
.scalarize(0);
|
|
|
|
getActionDefinitionsBuilder({G_UADDE, G_UADDO, G_USUBE, G_USUBO})
|
|
.legalFor({{s8, s8}, {s16, s8}, {s32, s8}})
|
|
.legalFor(Is64Bit, {{s64, s8}})
|
|
.widenScalarToNextPow2(0, /*Min=*/32)
|
|
.clampScalar(0, s8, sMaxScalar)
|
|
.clampScalar(1, s8, s8)
|
|
.scalarize(0);
|
|
|
|
// integer multiply
|
|
getActionDefinitionsBuilder(G_MUL)
|
|
.legalFor({s8, s16, s32})
|
|
.legalFor(Is64Bit, {s64})
|
|
.legalFor(HasSSE2, {v8s16})
|
|
.legalFor(HasSSE41, {v4s32})
|
|
.legalFor(HasAVX2, {v16s16, v8s32})
|
|
.legalFor(HasAVX512, {v16s32})
|
|
.legalFor(HasDQI, {v8s64})
|
|
.legalFor(HasDQI && HasVLX, {v2s64, v4s64})
|
|
.legalFor(HasBWI, {v32s16})
|
|
.clampMinNumElements(0, s16, 8)
|
|
.clampMinNumElements(0, s32, 4)
|
|
.clampMinNumElements(0, s64, HasVLX ? 2 : 8)
|
|
.clampMaxNumElements(0, s16, HasBWI ? 32 : (HasAVX2 ? 16 : 8))
|
|
.clampMaxNumElements(0, s32, HasAVX512 ? 16 : (HasAVX2 ? 8 : 4))
|
|
.clampMaxNumElements(0, s64, 8)
|
|
.widenScalarToNextPow2(0, /*Min=*/32)
|
|
.clampScalar(0, s8, sMaxScalar)
|
|
.scalarize(0);
|
|
|
|
getActionDefinitionsBuilder({G_SMULH, G_UMULH})
|
|
.legalFor({s8, s16, s32})
|
|
.legalFor(Is64Bit, {s64})
|
|
.widenScalarToNextPow2(0, /*Min=*/32)
|
|
.clampScalar(0, s8, sMaxScalar)
|
|
.scalarize(0);
|
|
|
|
// integer divisions
|
|
getActionDefinitionsBuilder({G_SDIV, G_SREM, G_UDIV, G_UREM})
|
|
.legalFor({s8, s16, s32})
|
|
.legalFor(Is64Bit, {s64})
|
|
.libcallFor({s64})
|
|
.clampScalar(0, s8, sMaxScalar);
|
|
|
|
// integer shifts
|
|
getActionDefinitionsBuilder({G_SHL, G_LSHR, G_ASHR})
|
|
.legalFor({{s8, s8}, {s16, s8}, {s32, s8}})
|
|
.legalFor(Is64Bit, {{s64, s8}})
|
|
.clampScalar(0, s8, sMaxScalar)
|
|
.clampScalar(1, s8, s8);
|
|
|
|
// integer logic
|
|
getActionDefinitionsBuilder({G_AND, G_OR, G_XOR})
|
|
.legalFor({s8, s16, s32})
|
|
.legalFor(Is64Bit, {s64})
|
|
.legalFor(HasSSE2, {v16s8, v8s16, v4s32, v2s64})
|
|
.legalFor(HasAVX, {v32s8, v16s16, v8s32, v4s64})
|
|
.legalFor(HasAVX512, {v64s8, v32s16, v16s32, v8s64})
|
|
.clampNumElements(0, v16s8, s8MaxVector)
|
|
.clampNumElements(0, v8s16, s16MaxVector)
|
|
.clampNumElements(0, v4s32, s32MaxVector)
|
|
.clampNumElements(0, v2s64, s64MaxVector)
|
|
.widenScalarToNextPow2(0, /*Min=*/32)
|
|
.clampScalar(0, s8, sMaxScalar)
|
|
.scalarize(0);
|
|
|
|
// integer comparison
|
|
const std::initializer_list<LLT> IntTypes32 = {s8, s16, s32, p0};
|
|
const std::initializer_list<LLT> IntTypes64 = {s8, s16, s32, s64, p0};
|
|
|
|
getActionDefinitionsBuilder(G_ICMP)
|
|
.legalForCartesianProduct({s8}, Is64Bit ? IntTypes64 : IntTypes32)
|
|
.clampScalar(0, s8, s8)
|
|
.widenScalarToNextPow2(1, /*Min=*/8)
|
|
.clampScalar(1, s8, sMaxScalar);
|
|
|
|
// bswap
|
|
getActionDefinitionsBuilder(G_BSWAP)
|
|
.legalFor({s32})
|
|
.legalFor(Is64Bit, {s64})
|
|
.widenScalarToNextPow2(0, /*Min=*/32)
|
|
.clampScalar(0, s32, sMaxScalar);
|
|
|
|
// popcount
|
|
getActionDefinitionsBuilder(G_CTPOP)
|
|
.legalFor(HasPOPCNT, {{s16, s16}, {s32, s32}})
|
|
.legalFor(HasPOPCNT && Is64Bit, {{s64, s64}})
|
|
.widenScalarToNextPow2(1, /*Min=*/16)
|
|
.clampScalar(1, s16, sMaxScalar)
|
|
.scalarSameSizeAs(0, 1);
|
|
|
|
// count leading zeros (LZCNT)
|
|
getActionDefinitionsBuilder(G_CTLZ)
|
|
.legalFor(HasLZCNT, {{s16, s16}, {s32, s32}})
|
|
.legalFor(HasLZCNT && Is64Bit, {{s64, s64}})
|
|
.widenScalarToNextPow2(1, /*Min=*/16)
|
|
.clampScalar(1, s16, sMaxScalar)
|
|
.scalarSameSizeAs(0, 1);
|
|
|
|
// count trailing zeros
|
|
getActionDefinitionsBuilder(G_CTTZ_ZERO_POISON)
|
|
.legalFor({{s16, s16}, {s32, s32}})
|
|
.legalFor(Is64Bit, {{s64, s64}})
|
|
.widenScalarToNextPow2(1, /*Min=*/16)
|
|
.clampScalar(1, s16, sMaxScalar)
|
|
.scalarSameSizeAs(0, 1);
|
|
|
|
getActionDefinitionsBuilder(G_CTTZ)
|
|
.legalFor(HasBMI, {{s16, s16}, {s32, s32}})
|
|
.legalFor(HasBMI && Is64Bit, {{s64, s64}})
|
|
.widenScalarToNextPow2(1, /*Min=*/16)
|
|
.clampScalar(1, s16, sMaxScalar)
|
|
.scalarSameSizeAs(0, 1);
|
|
|
|
getActionDefinitionsBuilder(G_BRCOND).legalFor({s1});
|
|
|
|
// pointer handling
|
|
const std::initializer_list<LLT> PtrTypes32 = {s1, s8, s16, s32};
|
|
const std::initializer_list<LLT> PtrTypes64 = {s1, s8, s16, s32, s64};
|
|
|
|
getActionDefinitionsBuilder(G_PTRTOINT)
|
|
.legalForCartesianProduct(Is64Bit ? PtrTypes64 : PtrTypes32, {p0})
|
|
.maxScalar(0, sMaxScalar)
|
|
.widenScalarToNextPow2(0, /*Min*/ 8);
|
|
|
|
getActionDefinitionsBuilder(G_INTTOPTR).legalFor({{p0, sMaxScalar}});
|
|
|
|
getActionDefinitionsBuilder(G_CONSTANT_POOL).legalFor({p0});
|
|
|
|
getActionDefinitionsBuilder(G_PTR_ADD)
|
|
.legalFor({{p0, s32}})
|
|
.legalFor(Is64Bit, {{p0, s64}})
|
|
.widenScalarToNextPow2(1, /*Min*/ 32)
|
|
.clampScalar(1, s32, sMaxScalar);
|
|
|
|
getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({p0});
|
|
|
|
getActionDefinitionsBuilder(G_GLOBAL_VALUE).customFor({p0});
|
|
|
|
// load/store: add more corner cases
|
|
for (unsigned Op : {G_LOAD, G_STORE}) {
|
|
auto &Action = getActionDefinitionsBuilder(Op);
|
|
Action.legalForTypesWithMemDesc({{s8, p0, s8, 1},
|
|
{s16, p0, s16, 1},
|
|
{s32, p0, s32, 1},
|
|
{s80, p0, s80, 1},
|
|
{p0, p0, p0, 1},
|
|
{v4s8, p0, v4s8, 1}});
|
|
if (Is64Bit)
|
|
Action.legalForTypesWithMemDesc(
|
|
{{s64, p0, s64, 1}, {v2s32, p0, v2s32, 1}});
|
|
|
|
if (HasSSE1)
|
|
Action.legalForTypesWithMemDesc({{v4s32, p0, v4s32, 1}});
|
|
if (HasSSE2)
|
|
Action.legalForTypesWithMemDesc({{v16s8, p0, v16s8, 1},
|
|
{v8s16, p0, v8s16, 1},
|
|
{v2s64, p0, v2s64, 1},
|
|
{v2p0, p0, v2p0, 1}});
|
|
if (HasAVX)
|
|
Action.legalForTypesWithMemDesc({{v32s8, p0, v32s8, 1},
|
|
{v16s16, p0, v16s16, 1},
|
|
{v8s32, p0, v8s32, 1},
|
|
{v4s64, p0, v4s64, 1},
|
|
{v4p0, p0, v4p0, 1}});
|
|
if (HasAVX512)
|
|
Action.legalForTypesWithMemDesc({{v64s8, p0, v64s8, 1},
|
|
{v32s16, p0, v32s16, 1},
|
|
{v16s32, p0, v16s32, 1},
|
|
{v8s64, p0, v8s64, 1}});
|
|
|
|
// X86 supports extending loads but not stores for GPRs
|
|
if (Op == G_LOAD) {
|
|
Action.legalForTypesWithMemDesc({{s8, p0, s1, 1},
|
|
{s16, p0, s8, 1},
|
|
{s32, p0, s8, 1},
|
|
{s32, p0, s16, 1}});
|
|
if (Is64Bit)
|
|
Action.legalForTypesWithMemDesc(
|
|
{{s64, p0, s8, 1}, {s64, p0, s16, 1}, {s64, p0, s32, 1}});
|
|
} else {
|
|
Action.customIf([=](const LegalityQuery &Query) {
|
|
return Query.Types[0] != Query.MMODescrs[0].MemoryTy;
|
|
});
|
|
}
|
|
Action.widenScalarToNextPow2(0, /*Min=*/8)
|
|
.clampScalar(0, s8, sMaxScalar)
|
|
.scalarize(0);
|
|
}
|
|
|
|
for (unsigned Op : {G_SEXTLOAD, G_ZEXTLOAD}) {
|
|
auto &Action = getActionDefinitionsBuilder(Op);
|
|
Action.legalForTypesWithMemDesc(
|
|
{{s16, p0, s8, 1}, {s32, p0, s8, 1}, {s32, p0, s16, 1}});
|
|
if (Is64Bit)
|
|
Action.legalForTypesWithMemDesc(
|
|
{{s64, p0, s8, 1}, {s64, p0, s16, 1}, {s64, p0, s32, 1}});
|
|
// TODO - SSE41/AVX2/AVX512F/AVX512BW vector extensions
|
|
}
|
|
|
|
for (unsigned Op : {G_FPEXTLOAD, G_FPTRUNCSTORE}) {
|
|
auto &Action = getActionDefinitionsBuilder(Op);
|
|
Action.legalForTypesWithMemDesc(
|
|
UseX87, {{s80, p0, s32, 1}, {s80, p0, s64, 1}, {s64, p0, s32, 1}});
|
|
}
|
|
|
|
// sext, zext, and anyext
|
|
getActionDefinitionsBuilder(G_ANYEXT)
|
|
.legalFor({s8, s16, s32, s128})
|
|
.legalFor(Is64Bit, {s64})
|
|
.widenScalarToNextPow2(0, /*Min=*/8)
|
|
.clampScalar(0, s8, sMaxScalar)
|
|
.widenScalarToNextPow2(1, /*Min=*/8)
|
|
.clampScalar(1, s8, sMaxScalar)
|
|
.scalarize(0);
|
|
|
|
getActionDefinitionsBuilder({G_SEXT, G_ZEXT})
|
|
.legalFor({s8, s16, s32})
|
|
.legalFor(Is64Bit, {s64})
|
|
.widenScalarToNextPow2(0, /*Min=*/8)
|
|
.clampScalar(0, s8, sMaxScalar)
|
|
.widenScalarToNextPow2(1, /*Min=*/8)
|
|
.clampScalar(1, s8, sMaxScalar)
|
|
.scalarize(0);
|
|
|
|
getActionDefinitionsBuilder(G_SEXT_INREG).lower();
|
|
|
|
// fp constants
|
|
getActionDefinitionsBuilder(G_FCONSTANT)
|
|
.legalFor({s32, s64})
|
|
.legalFor(UseX87, {s80});
|
|
|
|
// fp arithmetic
|
|
getActionDefinitionsBuilder({G_FADD, G_FSUB, G_FMUL, G_FDIV})
|
|
.legalFor({s32, s64})
|
|
.legalFor(HasSSE1, {v4s32})
|
|
.legalFor(HasSSE2, {v2s64})
|
|
.legalFor(HasAVX, {v8s32, v4s64})
|
|
.legalFor(HasAVX512, {v16s32, v8s64})
|
|
.legalFor(UseX87, {s80});
|
|
|
|
getActionDefinitionsBuilder(G_FABS)
|
|
.legalFor(UseX87, {s80})
|
|
.legalFor(UseX87 && !Is64Bit, {s64})
|
|
.lower();
|
|
|
|
// fp comparison
|
|
getActionDefinitionsBuilder(G_FCMP)
|
|
.legalFor(HasSSE1 || UseX87, {s8, s32})
|
|
.legalFor(HasSSE2 || UseX87, {s8, s64})
|
|
.legalFor(UseX87, {s8, s80})
|
|
.clampScalar(0, s8, s8)
|
|
.clampScalar(1, s32, HasSSE2 ? s64 : s32)
|
|
.widenScalarToNextPow2(1);
|
|
|
|
// fp conversions
|
|
getActionDefinitionsBuilder(G_FPEXT)
|
|
.legalFor(HasSSE2, {{s64, s32}})
|
|
.legalFor(HasAVX, {{v4s64, v4s32}})
|
|
.legalFor(HasAVX512, {{v8s64, v8s32}})
|
|
.lowerFor(UseX87, {{s64, s32}, {s80, s32}, {s80, s64}})
|
|
.libcall();
|
|
|
|
getActionDefinitionsBuilder(G_FPTRUNC)
|
|
.legalFor(HasSSE2, {{s32, s64}})
|
|
.legalFor(HasAVX, {{v4s32, v4s64}})
|
|
.legalFor(HasAVX512, {{v8s32, v8s64}})
|
|
.lowerFor(UseX87, {{s32, s64}, {s32, s80}, {s64, s80}});
|
|
|
|
getActionDefinitionsBuilder(G_SITOFP)
|
|
.legalFor(HasSSE1, {{s32, s32}})
|
|
.legalFor(HasSSE1 && Is64Bit, {{s32, s64}})
|
|
.legalFor(HasSSE2, {{s64, s32}})
|
|
.legalFor(HasSSE2 && Is64Bit, {{s64, s64}})
|
|
.clampScalar(1, (UseX87 && !HasSSE1) ? s16 : s32, sMaxScalar)
|
|
.widenScalarToNextPow2(1)
|
|
.customForCartesianProduct(UseX87, {s32, s64, s80}, {s16, s32, s64})
|
|
.clampScalar(0, s32, HasSSE2 ? s64 : s32)
|
|
.widenScalarToNextPow2(0);
|
|
|
|
getActionDefinitionsBuilder(G_FPTOSI)
|
|
.legalFor(HasSSE1, {{s32, s32}})
|
|
.legalFor(HasSSE1 && Is64Bit, {{s64, s32}})
|
|
.legalFor(HasSSE2, {{s32, s64}})
|
|
.legalFor(HasSSE2 && Is64Bit, {{s64, s64}})
|
|
.clampScalar(0, (UseX87 && !HasSSE1) ? s16 : s32, sMaxScalar)
|
|
.widenScalarToNextPow2(0)
|
|
.customForCartesianProduct(UseX87, {s16, s32, s64}, {s32, s64, s80})
|
|
.clampScalar(1, s32, HasSSE2 ? s64 : s32)
|
|
.widenScalarToNextPow2(1);
|
|
|
|
// For G_UITOFP and G_FPTOUI without AVX512, we have to custom legalize types
|
|
// <= s32 manually. Otherwise, in custom handler there is no way to
|
|
// understand whether s32 is an original type and we need to promote it to
|
|
// s64 or s32 is obtained after widening and we shouldn't widen it to s64.
|
|
//
|
|
// For AVX512 we simply widen types as there is direct mapping from opcodes
|
|
// to asm instructions.
|
|
getActionDefinitionsBuilder(G_UITOFP)
|
|
.legalFor(HasAVX512, {{s32, s32}, {s32, s64}, {s64, s32}, {s64, s64}})
|
|
.customIf([=](const LegalityQuery &Query) {
|
|
return !HasAVX512 &&
|
|
((HasSSE1 && typeIs(0, s32)(Query)) ||
|
|
(HasSSE2 && typeIs(0, s64)(Query))) &&
|
|
scalarNarrowerThan(1, Is64Bit ? 64 : 32)(Query);
|
|
})
|
|
.lowerIf([=](const LegalityQuery &Query) {
|
|
// Lower conversions from s64
|
|
return !HasAVX512 &&
|
|
((HasSSE1 && typeIs(0, s32)(Query)) ||
|
|
(HasSSE2 && typeIs(0, s64)(Query))) &&
|
|
(Is64Bit && typeIs(1, s64)(Query));
|
|
})
|
|
.clampScalar(0, s32, HasSSE2 ? s64 : s32)
|
|
.widenScalarToNextPow2(0)
|
|
.clampScalar(1, s32, sMaxScalar)
|
|
.widenScalarToNextPow2(1);
|
|
|
|
getActionDefinitionsBuilder(G_FPTOUI)
|
|
.legalFor(HasAVX512, {{s32, s32}, {s32, s64}, {s64, s32}, {s64, s64}})
|
|
.customIf([=](const LegalityQuery &Query) {
|
|
return !HasAVX512 &&
|
|
((HasSSE1 && typeIs(1, s32)(Query)) ||
|
|
(HasSSE2 && typeIs(1, s64)(Query))) &&
|
|
scalarNarrowerThan(0, Is64Bit ? 64 : 32)(Query);
|
|
})
|
|
// TODO: replace with customized legalization using
|
|
// specifics of cvttsd2si. The selection of this node requires
|
|
// a vector type. Either G_SCALAR_TO_VECTOR is needed or more advanced
|
|
// support of G_BUILD_VECTOR/G_INSERT_VECTOR_ELT is required beforehand.
|
|
.lowerIf([=](const LegalityQuery &Query) {
|
|
return !HasAVX512 &&
|
|
((HasSSE1 && typeIs(1, s32)(Query)) ||
|
|
(HasSSE2 && typeIs(1, s64)(Query))) &&
|
|
(Is64Bit && typeIs(0, s64)(Query));
|
|
})
|
|
.clampScalar(0, s32, sMaxScalar)
|
|
.widenScalarToNextPow2(0)
|
|
.clampScalar(1, s32, HasSSE2 ? s64 : s32)
|
|
.widenScalarToNextPow2(1);
|
|
|
|
// vector ops
|
|
getActionDefinitionsBuilder(G_BUILD_VECTOR)
|
|
.customIf([=](const LegalityQuery &Query) {
|
|
return (HasSSE1 && typeInSet(0, {v4s32})(Query)) ||
|
|
(HasSSE2 && typeInSet(0, {v2s64, v8s16, v16s8})(Query)) ||
|
|
(HasAVX && typeInSet(0, {v4s64, v8s32, v16s16, v32s8})(Query)) ||
|
|
(HasAVX512 && typeInSet(0, {v8s64, v16s32, v32s16, v64s8}));
|
|
})
|
|
.clampNumElements(0, v16s8, s8MaxVector)
|
|
.clampNumElements(0, v8s16, s16MaxVector)
|
|
.clampNumElements(0, v4s32, s32MaxVector)
|
|
.clampNumElements(0, v2s64, s64MaxVector)
|
|
.moreElementsToNextPow2(0);
|
|
|
|
getActionDefinitionsBuilder({G_EXTRACT, G_INSERT})
|
|
.legalIf([=](const LegalityQuery &Query) {
|
|
unsigned SubIdx = Query.Opcode == G_EXTRACT ? 0 : 1;
|
|
unsigned FullIdx = Query.Opcode == G_EXTRACT ? 1 : 0;
|
|
return (HasAVX && typePairInSet(SubIdx, FullIdx,
|
|
{{v16s8, v32s8},
|
|
{v8s16, v16s16},
|
|
{v4s32, v8s32},
|
|
{v2s64, v4s64}})(Query)) ||
|
|
(HasAVX512 && typePairInSet(SubIdx, FullIdx,
|
|
{{v16s8, v64s8},
|
|
{v32s8, v64s8},
|
|
{v8s16, v32s16},
|
|
{v16s16, v32s16},
|
|
{v4s32, v16s32},
|
|
{v8s32, v16s32},
|
|
{v2s64, v8s64},
|
|
{v4s64, v8s64}})(Query));
|
|
});
|
|
|
|
// todo: only permit dst types up to max legal vector register size?
|
|
getActionDefinitionsBuilder(G_CONCAT_VECTORS)
|
|
.legalFor(
|
|
HasSSE1,
|
|
{{v32s8, v16s8}, {v16s16, v8s16}, {v8s32, v4s32}, {v4s64, v2s64}})
|
|
.legalFor(HasAVX, {{v64s8, v16s8},
|
|
{v64s8, v32s8},
|
|
{v32s16, v8s16},
|
|
{v32s16, v16s16},
|
|
{v16s32, v4s32},
|
|
{v16s32, v8s32},
|
|
{v8s64, v2s64},
|
|
{v8s64, v4s64}});
|
|
|
|
// todo: vectors and address spaces
|
|
getActionDefinitionsBuilder(G_SELECT)
|
|
.legalFor({{s16, s32}, {s32, s32}, {p0, s32}})
|
|
.legalFor(!HasCMOV, {{s8, s32}})
|
|
.legalFor(Is64Bit, {{s64, s32}})
|
|
.legalFor(UseX87, {{s80, s32}})
|
|
.clampScalar(1, s32, s32)
|
|
.widenScalarToNextPow2(0, /*Min=*/8)
|
|
.clampScalar(0, HasCMOV ? s16 : s8, sMaxScalar);
|
|
|
|
// memory intrinsics
|
|
getActionDefinitionsBuilder({G_MEMCPY, G_MEMMOVE, G_MEMSET}).libcall();
|
|
|
|
getActionDefinitionsBuilder({G_DYN_STACKALLOC, G_STACKSAVE, G_STACKRESTORE})
|
|
.lower();
|
|
|
|
// fp intrinsics
|
|
// fpclass for i686 is disabled for llvm issue #171992
|
|
getActionDefinitionsBuilder(G_IS_FPCLASS)
|
|
.lowerFor(Is64Bit, {{s1, s32}, {s1, s64}, {s1, s80}});
|
|
|
|
getActionDefinitionsBuilder({G_INTRINSIC_ROUNDEVEN, G_INTRINSIC_TRUNC})
|
|
.scalarize(0)
|
|
.minScalar(0, LLT::scalar(32))
|
|
.libcall();
|
|
|
|
getLegacyLegalizerInfo().computeTables();
|
|
verify(*STI.getInstrInfo());
|
|
}
|
|
|
|
bool X86LegalizerInfo::legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI,
|
|
LostDebugLocObserver &LocObserver) const {
|
|
MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
|
|
MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
|
|
switch (MI.getOpcode()) {
|
|
default:
|
|
// No idea what to do.
|
|
return false;
|
|
case TargetOpcode::G_BUILD_VECTOR:
|
|
return legalizeBuildVector(MI, MRI, Helper);
|
|
case TargetOpcode::G_FPTOUI:
|
|
return legalizeFPTOUI(MI, MRI, Helper);
|
|
case TargetOpcode::G_UITOFP:
|
|
return legalizeUITOFP(MI, MRI, Helper);
|
|
case TargetOpcode::G_STORE:
|
|
return legalizeNarrowingStore(MI, MRI, Helper);
|
|
case TargetOpcode::G_SITOFP:
|
|
return legalizeSITOFP(MI, MRI, Helper);
|
|
case TargetOpcode::G_FPTOSI:
|
|
return legalizeFPTOSI(MI, MRI, Helper);
|
|
case TargetOpcode::G_GET_ROUNDING:
|
|
return legalizeGETROUNDING(MI, MRI, Helper);
|
|
case TargetOpcode::G_SET_ROUNDING:
|
|
return legalizeSETROUNDING(MI, MRI, Helper);
|
|
case TargetOpcode::G_GLOBAL_VALUE:
|
|
return legalizeGLOBAL_VALUE(MI, MRI, Helper);
|
|
}
|
|
llvm_unreachable("expected switch to return");
|
|
}
|
|
|
|
bool X86LegalizerInfo::legalizeSITOFP(MachineInstr &MI,
|
|
MachineRegisterInfo &MRI,
|
|
LegalizerHelper &Helper) const {
|
|
MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
|
|
MachineFunction &MF = *MI.getMF();
|
|
auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
|
|
|
|
assert((SrcTy.getSizeInBits() == 16 || SrcTy.getSizeInBits() == 32 ||
|
|
SrcTy.getSizeInBits() == 64) &&
|
|
"Unexpected source type for SITOFP in X87 mode.");
|
|
|
|
TypeSize MemSize = SrcTy.getSizeInBytes();
|
|
MachinePointerInfo PtrInfo;
|
|
Align Alignmt = Helper.getStackTemporaryAlignment(SrcTy);
|
|
auto SlotPointer = Helper.createStackTemporary(MemSize, Alignmt, PtrInfo);
|
|
MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
|
|
PtrInfo, MachineMemOperand::MOStore, MemSize, Align(MemSize));
|
|
|
|
// Store the integer value on the FPU stack.
|
|
MIRBuilder.buildStore(Src, SlotPointer, *StoreMMO);
|
|
|
|
MachineMemOperand *LoadMMO = MF.getMachineMemOperand(
|
|
PtrInfo, MachineMemOperand::MOLoad, MemSize, Align(MemSize));
|
|
MIRBuilder.buildInstr(X86::G_FILD)
|
|
.addDef(Dst)
|
|
.addUse(SlotPointer.getReg(0))
|
|
.addMemOperand(LoadMMO);
|
|
|
|
MI.eraseFromParent();
|
|
return true;
|
|
}
|
|
|
|
bool X86LegalizerInfo::legalizeFPTOSI(MachineInstr &MI,
|
|
MachineRegisterInfo &MRI,
|
|
LegalizerHelper &Helper) const {
|
|
MachineFunction &MF = *MI.getMF();
|
|
MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
|
|
auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
|
|
|
|
TypeSize MemSize = DstTy.getSizeInBytes();
|
|
MachinePointerInfo PtrInfo;
|
|
Align Alignmt = Helper.getStackTemporaryAlignment(DstTy);
|
|
auto SlotPointer = Helper.createStackTemporary(MemSize, Alignmt, PtrInfo);
|
|
MachineMemOperand *StoreMMO = MF.getMachineMemOperand(
|
|
PtrInfo, MachineMemOperand::MOStore, MemSize, Align(MemSize));
|
|
|
|
MIRBuilder.buildInstr(X86::G_FIST)
|
|
.addUse(Src)
|
|
.addUse(SlotPointer.getReg(0))
|
|
.addMemOperand(StoreMMO);
|
|
|
|
MIRBuilder.buildLoad(Dst, SlotPointer, PtrInfo, Align(MemSize));
|
|
MI.eraseFromParent();
|
|
return true;
|
|
}
|
|
|
|
bool X86LegalizerInfo::legalizeBuildVector(MachineInstr &MI,
|
|
MachineRegisterInfo &MRI,
|
|
LegalizerHelper &Helper) const {
|
|
MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
|
|
const auto &BuildVector = cast<GBuildVector>(MI);
|
|
Register Dst = BuildVector.getReg(0);
|
|
LLT DstTy = MRI.getType(Dst);
|
|
MachineFunction &MF = MIRBuilder.getMF();
|
|
LLVMContext &Ctx = MF.getFunction().getContext();
|
|
uint64_t DstTySize = DstTy.getScalarSizeInBits();
|
|
|
|
SmallVector<Constant *, 4> CstIdxs;
|
|
for (unsigned i = 0; i < BuildVector.getNumSources(); ++i) {
|
|
Register Source = BuildVector.getSourceReg(i);
|
|
|
|
auto ValueAndReg = getIConstantVRegValWithLookThrough(Source, MRI);
|
|
if (ValueAndReg) {
|
|
CstIdxs.emplace_back(ConstantInt::get(Ctx, ValueAndReg->Value));
|
|
continue;
|
|
}
|
|
|
|
auto FPValueAndReg = getFConstantVRegValWithLookThrough(Source, MRI);
|
|
if (FPValueAndReg) {
|
|
CstIdxs.emplace_back(ConstantFP::get(Ctx, FPValueAndReg->Value));
|
|
continue;
|
|
}
|
|
|
|
if (getOpcodeDef<GImplicitDef>(Source, MRI)) {
|
|
CstIdxs.emplace_back(UndefValue::get(Type::getIntNTy(Ctx, DstTySize)));
|
|
continue;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
Constant *ConstVal = ConstantVector::get(CstIdxs);
|
|
|
|
const DataLayout &DL = MIRBuilder.getDataLayout();
|
|
unsigned AddrSpace = DL.getDefaultGlobalsAddressSpace();
|
|
Align Alignment(DL.getABITypeAlign(ConstVal->getType()));
|
|
auto Addr = MIRBuilder.buildConstantPool(
|
|
LLT::pointer(AddrSpace, DL.getPointerSizeInBits(AddrSpace)),
|
|
MF.getConstantPool()->getConstantPoolIndex(ConstVal, Alignment));
|
|
MachineMemOperand *MMO =
|
|
MF.getMachineMemOperand(MachinePointerInfo::getConstantPool(MF),
|
|
MachineMemOperand::MOLoad, DstTy, Alignment);
|
|
|
|
MIRBuilder.buildLoad(Dst, Addr, *MMO);
|
|
MI.eraseFromParent();
|
|
return true;
|
|
}
|
|
|
|
bool X86LegalizerInfo::legalizeFPTOUI(MachineInstr &MI,
|
|
MachineRegisterInfo &MRI,
|
|
LegalizerHelper &Helper) const {
|
|
MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
|
|
auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
|
|
unsigned DstSizeInBits = DstTy.getScalarSizeInBits();
|
|
const LLT s32 = LLT::scalar(32);
|
|
const LLT s64 = LLT::scalar(64);
|
|
|
|
// Simply reuse FPTOSI when it is possible to widen the type
|
|
if (DstSizeInBits <= 32) {
|
|
auto Casted = MIRBuilder.buildFPTOSI(DstTy == s32 ? s64 : s32, Src);
|
|
MIRBuilder.buildTrunc(Dst, Casted);
|
|
MI.eraseFromParent();
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
bool X86LegalizerInfo::legalizeUITOFP(MachineInstr &MI,
|
|
MachineRegisterInfo &MRI,
|
|
LegalizerHelper &Helper) const {
|
|
MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
|
|
auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
|
|
const LLT s32 = LLT::scalar(32);
|
|
const LLT s64 = LLT::scalar(64);
|
|
|
|
// Simply reuse SITOFP when it is possible to widen the type
|
|
if (SrcTy.getSizeInBits() <= 32) {
|
|
auto Ext = MIRBuilder.buildZExt(SrcTy == s32 ? s64 : s32, Src);
|
|
MIRBuilder.buildSITOFP(Dst, Ext);
|
|
MI.eraseFromParent();
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
bool X86LegalizerInfo::legalizeNarrowingStore(MachineInstr &MI,
|
|
MachineRegisterInfo &MRI,
|
|
LegalizerHelper &Helper) const {
|
|
auto &Store = cast<GStore>(MI);
|
|
MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
|
|
MachineMemOperand &MMO = **Store.memoperands_begin();
|
|
MachineFunction &MF = MIRBuilder.getMF();
|
|
LLT ValTy = MRI.getType(Store.getValueReg());
|
|
auto *NewMMO = MF.getMachineMemOperand(&MMO, MMO.getPointerInfo(), ValTy);
|
|
|
|
Helper.Observer.changingInstr(Store);
|
|
Store.setMemRefs(MF, {NewMMO});
|
|
Helper.Observer.changedInstr(Store);
|
|
return true;
|
|
}
|
|
|
|
bool X86LegalizerInfo::legalizeGETROUNDING(MachineInstr &MI,
|
|
MachineRegisterInfo &MRI,
|
|
LegalizerHelper &Helper) const {
|
|
/*
|
|
The rounding mode is in bits 11:10 of FPSR, and has the following
|
|
settings:
|
|
00 Round to nearest
|
|
01 Round to -inf
|
|
10 Round to +inf
|
|
11 Round to 0
|
|
|
|
GET_ROUNDING, on the other hand, expects the following:
|
|
-1 Undefined
|
|
0 Round to 0
|
|
1 Round to nearest
|
|
2 Round to +inf
|
|
3 Round to -inf
|
|
|
|
To perform the conversion, we use a packed lookup table of the four 2-bit
|
|
values that we can index by FPSP[11:10]
|
|
0x2d --> (0b00,10,11,01) --> (0,2,3,1) >> FPSR[11:10]
|
|
|
|
(0x2d >> ((FPSR >> 9) & 6)) & 3
|
|
*/
|
|
|
|
MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
|
|
MachineFunction &MF = MIRBuilder.getMF();
|
|
Register Dst = MI.getOperand(0).getReg();
|
|
LLT DstTy = MRI.getType(Dst);
|
|
const LLT s8 = LLT::scalar(8);
|
|
const LLT s16 = LLT::scalar(16);
|
|
const LLT s32 = LLT::scalar(32);
|
|
|
|
// Save FP Control Word to stack slot
|
|
int MemSize = 2;
|
|
Align Alignment = Align(2);
|
|
MachinePointerInfo PtrInfo;
|
|
auto StackTemp = Helper.createStackTemporary(TypeSize::getFixed(MemSize),
|
|
Alignment, PtrInfo);
|
|
Register StackPtr = StackTemp.getReg(0);
|
|
|
|
auto StoreMMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore,
|
|
MemSize, Alignment);
|
|
|
|
// Store FP Control Word to stack slot using G_FNSTCW16
|
|
MIRBuilder.buildInstr(X86::G_FNSTCW16)
|
|
.addUse(StackPtr)
|
|
.addMemOperand(StoreMMO);
|
|
|
|
// Load FP Control Word from stack slot
|
|
auto LoadMMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad,
|
|
MemSize, Alignment);
|
|
|
|
auto CWD32 =
|
|
MIRBuilder.buildZExt(s32, MIRBuilder.buildLoad(s16, StackPtr, *LoadMMO));
|
|
auto Shifted8 = MIRBuilder.buildTrunc(
|
|
s8, MIRBuilder.buildLShr(s32, CWD32, MIRBuilder.buildConstant(s8, 9)));
|
|
auto Masked32 = MIRBuilder.buildZExt(
|
|
s32, MIRBuilder.buildAnd(s8, Shifted8, MIRBuilder.buildConstant(s8, 6)));
|
|
|
|
// LUT is a packed lookup table (0x2d) used to map the 2-bit x87 FPU rounding
|
|
// mode (from bits 11:10 of the control word) to the values expected by
|
|
// GET_ROUNDING. The mapping is performed by shifting LUT right by the
|
|
// extracted rounding mode and masking the result with 3 to obtain the final
|
|
auto LUT = MIRBuilder.buildConstant(s32, 0x2d);
|
|
auto LUTShifted = MIRBuilder.buildLShr(s32, LUT, Masked32);
|
|
auto RetVal =
|
|
MIRBuilder.buildAnd(s32, LUTShifted, MIRBuilder.buildConstant(s32, 3));
|
|
auto RetValTrunc = MIRBuilder.buildZExtOrTrunc(DstTy, RetVal);
|
|
|
|
MIRBuilder.buildCopy(Dst, RetValTrunc);
|
|
|
|
MI.eraseFromParent();
|
|
return true;
|
|
}
|
|
|
|
bool X86LegalizerInfo::legalizeSETROUNDING(MachineInstr &MI,
|
|
MachineRegisterInfo &MRI,
|
|
LegalizerHelper &Helper) const {
|
|
MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
|
|
MachineFunction &MF = MIRBuilder.getMF();
|
|
Register Src = MI.getOperand(0).getReg();
|
|
const LLT s8 = LLT::scalar(8);
|
|
const LLT s16 = LLT::scalar(16);
|
|
const LLT s32 = LLT::scalar(32);
|
|
|
|
// Allocate stack slot for control word and MXCSR (4 bytes).
|
|
int MemSize = 4;
|
|
Align Alignment = Align(4);
|
|
MachinePointerInfo PtrInfo;
|
|
auto StackTemp = Helper.createStackTemporary(TypeSize::getFixed(MemSize),
|
|
Alignment, PtrInfo);
|
|
Register StackPtr = StackTemp.getReg(0);
|
|
|
|
auto StoreMMO =
|
|
MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore, 2, Align(2));
|
|
MIRBuilder.buildInstr(X86::G_FNSTCW16)
|
|
.addUse(StackPtr)
|
|
.addMemOperand(StoreMMO);
|
|
|
|
auto LoadMMO =
|
|
MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad, 2, Align(2));
|
|
auto CWD16 = MIRBuilder.buildLoad(s16, StackPtr, *LoadMMO);
|
|
|
|
// Clear RM field (bits 11:10)
|
|
auto ClearedCWD =
|
|
MIRBuilder.buildAnd(s16, CWD16, MIRBuilder.buildConstant(s16, 0xf3ff));
|
|
|
|
// Check if Src is a constant
|
|
auto *SrcDef = MRI.getVRegDef(Src);
|
|
Register RMBits;
|
|
Register MXCSRRMBits;
|
|
|
|
if (SrcDef && SrcDef->getOpcode() == TargetOpcode::G_CONSTANT) {
|
|
uint64_t RM = getIConstantFromReg(Src, MRI).getZExtValue();
|
|
int FieldVal = X86::getRoundingModeX86(RM);
|
|
|
|
if (FieldVal == X86::rmInvalid) {
|
|
FieldVal = X86::rmToNearest;
|
|
LLVMContext &C = MF.getFunction().getContext();
|
|
C.diagnose(DiagnosticInfoUnsupported(
|
|
MF.getFunction(), "rounding mode is not supported by X86 hardware",
|
|
DiagnosticLocation(MI.getDebugLoc()), DS_Error));
|
|
return false;
|
|
}
|
|
|
|
FieldVal = FieldVal << 3;
|
|
RMBits = MIRBuilder.buildConstant(s16, FieldVal).getReg(0);
|
|
MXCSRRMBits = MIRBuilder.buildConstant(s32, FieldVal).getReg(0);
|
|
} else {
|
|
// Convert Src (rounding mode) to bits for control word
|
|
// (0xc9 << (2 * Src + 4)) & 0xc00
|
|
auto Src32 = MIRBuilder.buildZExtOrTrunc(s32, Src);
|
|
auto ShiftAmt = MIRBuilder.buildAdd(
|
|
s32, MIRBuilder.buildShl(s32, Src32, MIRBuilder.buildConstant(s32, 1)),
|
|
MIRBuilder.buildConstant(s32, 4));
|
|
auto ShiftAmt8 = MIRBuilder.buildTrunc(s8, ShiftAmt);
|
|
auto Shifted = MIRBuilder.buildShl(s16, MIRBuilder.buildConstant(s16, 0xc9),
|
|
ShiftAmt8);
|
|
RMBits =
|
|
MIRBuilder.buildAnd(s16, Shifted, MIRBuilder.buildConstant(s16, 0xc00))
|
|
.getReg(0);
|
|
|
|
// For non-constant case, we still need to compute MXCSR bits dynamically
|
|
auto RMBits32 = MIRBuilder.buildZExt(s32, RMBits);
|
|
MXCSRRMBits =
|
|
MIRBuilder.buildShl(s32, RMBits32, MIRBuilder.buildConstant(s32, 3))
|
|
.getReg(0);
|
|
}
|
|
// Update rounding mode bits
|
|
auto NewCWD =
|
|
MIRBuilder.buildOr(s16, ClearedCWD, RMBits, MachineInstr::Disjoint);
|
|
|
|
// Store new FP Control Word to stack
|
|
auto StoreNewMMO =
|
|
MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOStore, 2, Align(2));
|
|
MIRBuilder.buildStore(NewCWD, StackPtr, *StoreNewMMO);
|
|
|
|
// Load FP control word from the slot using G_FLDCW16
|
|
auto LoadNewMMO =
|
|
MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad, 2, Align(2));
|
|
MIRBuilder.buildInstr(X86::G_FLDCW16)
|
|
.addUse(StackPtr)
|
|
.addMemOperand(LoadNewMMO);
|
|
|
|
if (Subtarget.hasSSE1()) {
|
|
// Store MXCSR to stack (use STMXCSR)
|
|
auto StoreMXCSRMMO = MF.getMachineMemOperand(
|
|
PtrInfo, MachineMemOperand::MOStore, 4, Align(4));
|
|
MIRBuilder.buildInstr(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)
|
|
.addIntrinsicID(Intrinsic::x86_sse_stmxcsr)
|
|
.addUse(StackPtr)
|
|
.addMemOperand(StoreMXCSRMMO);
|
|
|
|
// Load MXCSR from stack
|
|
auto LoadMXCSRMMO = MF.getMachineMemOperand(
|
|
PtrInfo, MachineMemOperand::MOLoad, 4, Align(4));
|
|
auto MXCSR = MIRBuilder.buildLoad(s32, StackPtr, *LoadMXCSRMMO);
|
|
|
|
// Clear RM field (bits 14:13)
|
|
auto ClearedMXCSR = MIRBuilder.buildAnd(
|
|
s32, MXCSR, MIRBuilder.buildConstant(s32, 0xffff9fff));
|
|
|
|
// Update rounding mode bits
|
|
auto NewMXCSR = MIRBuilder.buildOr(s32, ClearedMXCSR, MXCSRRMBits);
|
|
|
|
// Store new MXCSR to stack
|
|
auto StoreNewMXCSRMMO = MF.getMachineMemOperand(
|
|
PtrInfo, MachineMemOperand::MOStore, 4, Align(4));
|
|
MIRBuilder.buildStore(NewMXCSR, StackPtr, *StoreNewMXCSRMMO);
|
|
|
|
// Load MXCSR from stack (use LDMXCSR)
|
|
auto LoadNewMXCSRMMO = MF.getMachineMemOperand(
|
|
PtrInfo, MachineMemOperand::MOLoad, 4, Align(4));
|
|
MIRBuilder.buildInstr(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS)
|
|
.addIntrinsicID(Intrinsic::x86_sse_ldmxcsr)
|
|
.addUse(StackPtr)
|
|
.addMemOperand(LoadNewMXCSRMMO);
|
|
}
|
|
|
|
MI.eraseFromParent();
|
|
return true;
|
|
}
|
|
|
|
bool X86LegalizerInfo::legalizeGLOBAL_VALUE(MachineInstr &MI,
|
|
MachineRegisterInfo &MRI,
|
|
LegalizerHelper &Helper) const {
|
|
const GlobalValue *GV = MI.getOperand(1).getGlobal();
|
|
Register Dst = MI.getOperand(0).getReg();
|
|
LLT DstTy = MRI.getType(Dst);
|
|
unsigned GVOpFlags = Subtarget.classifyGlobalReference(GV);
|
|
|
|
// For stub references (GOT/PLT), we need G_WRAPPER_RIP + load
|
|
if (isGlobalStubReference(GVOpFlags)) {
|
|
MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
|
|
MachineFunction &MF = MIRBuilder.getMF();
|
|
|
|
Register StubAddr = MRI.createGenericVirtualRegister(DstTy);
|
|
MIRBuilder.buildInstr(X86::G_WRAPPER_RIP)
|
|
.addDef(StubAddr)
|
|
.addGlobalAddress(GV);
|
|
|
|
MachineMemOperand *MMO = MF.getMachineMemOperand(
|
|
MachinePointerInfo::getGOT(MF), MachineMemOperand::MOLoad, DstTy,
|
|
Align(DstTy.getSizeInBytes()));
|
|
MIRBuilder.buildLoad(Dst, StubAddr, *MMO);
|
|
MI.eraseFromParent();
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool X86LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
|
|
MachineInstr &MI) const {
|
|
return true;
|
|
}
|