derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
1114 lines
50 KiB
C++
1114 lines
50 KiB
C++
//===---- CIRGenBuiltinNVPTX.cpp - Emit CIR for NVPTX builtins ------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// This contains code to emit NVPTX Builtin calls.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "CIRGenFunction.h"
|
|
|
|
#include "mlir/IR/Value.h"
|
|
#include "clang/Basic/TargetBuiltins.h"
|
|
#include "clang/CIR/Dialect/IR/CIRDataLayout.h"
|
|
#include "clang/CIR/Dialect/IR/CIRDialect.h"
|
|
|
|
using namespace clang;
|
|
using namespace clang::CIRGen;
|
|
|
|
/// Emit a CIR LLVMIntrinsicCallOp for a unary NVVM intrinsic.
|
|
/// The result type is inferred from the single argument.
|
|
static mlir::Value emitUnaryNVVMIntrinsic(CIRGenFunction &cgf,
|
|
const CallExpr *expr,
|
|
llvm::StringRef intrinsicName) {
|
|
auto &builder = cgf.getBuilder();
|
|
mlir::Value arg = cgf.emitScalarExpr(expr->getArg(0));
|
|
return cir::LLVMIntrinsicCallOp::create(
|
|
builder, cgf.getLoc(expr->getExprLoc()),
|
|
builder.getStringAttr(intrinsicName), arg.getType(), {arg})
|
|
.getResult();
|
|
}
|
|
|
|
std::optional<mlir::Value>
|
|
CIRGenFunction::emitNVPTXBuiltinExpr(unsigned builtinId, const CallExpr *expr) {
|
|
switch (builtinId) {
|
|
case NVPTX::BI__nvvm_atom_add_gen_i:
|
|
case NVPTX::BI__nvvm_atom_add_gen_l:
|
|
case NVPTX::BI__nvvm_atom_add_gen_ll:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_sub_gen_i:
|
|
case NVPTX::BI__nvvm_atom_sub_gen_l:
|
|
case NVPTX::BI__nvvm_atom_sub_gen_ll:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_and_gen_i:
|
|
case NVPTX::BI__nvvm_atom_and_gen_l:
|
|
case NVPTX::BI__nvvm_atom_and_gen_ll:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_or_gen_i:
|
|
case NVPTX::BI__nvvm_atom_or_gen_l:
|
|
case NVPTX::BI__nvvm_atom_or_gen_ll:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_xor_gen_i:
|
|
case NVPTX::BI__nvvm_atom_xor_gen_l:
|
|
case NVPTX::BI__nvvm_atom_xor_gen_ll:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_xchg_gen_i:
|
|
case NVPTX::BI__nvvm_atom_xchg_gen_l:
|
|
case NVPTX::BI__nvvm_atom_xchg_gen_ll:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_max_gen_i:
|
|
case NVPTX::BI__nvvm_atom_max_gen_l:
|
|
case NVPTX::BI__nvvm_atom_max_gen_ll:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_max_gen_ui:
|
|
case NVPTX::BI__nvvm_atom_max_gen_ul:
|
|
case NVPTX::BI__nvvm_atom_max_gen_ull:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_min_gen_i:
|
|
case NVPTX::BI__nvvm_atom_min_gen_l:
|
|
case NVPTX::BI__nvvm_atom_min_gen_ll:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_min_gen_ui:
|
|
case NVPTX::BI__nvvm_atom_min_gen_ul:
|
|
case NVPTX::BI__nvvm_atom_min_gen_ull:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_cas_gen_us:
|
|
case NVPTX::BI__nvvm_atom_cas_gen_i:
|
|
case NVPTX::BI__nvvm_atom_cas_gen_l:
|
|
case NVPTX::BI__nvvm_atom_cas_gen_ll:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
// success flag.
|
|
case NVPTX::BI__nvvm_atom_add_gen_f:
|
|
case NVPTX::BI__nvvm_atom_add_gen_d:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_inc_gen_ui:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_dec_gen_ui:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_ldg_c:
|
|
case NVPTX::BI__nvvm_ldg_sc:
|
|
case NVPTX::BI__nvvm_ldg_c2:
|
|
case NVPTX::BI__nvvm_ldg_sc2:
|
|
case NVPTX::BI__nvvm_ldg_c4:
|
|
case NVPTX::BI__nvvm_ldg_sc4:
|
|
case NVPTX::BI__nvvm_ldg_s:
|
|
case NVPTX::BI__nvvm_ldg_s2:
|
|
case NVPTX::BI__nvvm_ldg_s4:
|
|
case NVPTX::BI__nvvm_ldg_i:
|
|
case NVPTX::BI__nvvm_ldg_i2:
|
|
case NVPTX::BI__nvvm_ldg_i4:
|
|
case NVPTX::BI__nvvm_ldg_l:
|
|
case NVPTX::BI__nvvm_ldg_l2:
|
|
case NVPTX::BI__nvvm_ldg_ll:
|
|
case NVPTX::BI__nvvm_ldg_ll2:
|
|
case NVPTX::BI__nvvm_ldg_uc:
|
|
case NVPTX::BI__nvvm_ldg_uc2:
|
|
case NVPTX::BI__nvvm_ldg_uc4:
|
|
case NVPTX::BI__nvvm_ldg_us:
|
|
case NVPTX::BI__nvvm_ldg_us2:
|
|
case NVPTX::BI__nvvm_ldg_us4:
|
|
case NVPTX::BI__nvvm_ldg_ui:
|
|
case NVPTX::BI__nvvm_ldg_ui2:
|
|
case NVPTX::BI__nvvm_ldg_ui4:
|
|
case NVPTX::BI__nvvm_ldg_ul:
|
|
case NVPTX::BI__nvvm_ldg_ul2:
|
|
case NVPTX::BI__nvvm_ldg_ull:
|
|
case NVPTX::BI__nvvm_ldg_ull2:
|
|
case NVPTX::BI__nvvm_ldg_f:
|
|
case NVPTX::BI__nvvm_ldg_f2:
|
|
case NVPTX::BI__nvvm_ldg_f4:
|
|
case NVPTX::BI__nvvm_ldg_d:
|
|
case NVPTX::BI__nvvm_ldg_d2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_ldu_c:
|
|
case NVPTX::BI__nvvm_ldu_sc:
|
|
case NVPTX::BI__nvvm_ldu_c2:
|
|
case NVPTX::BI__nvvm_ldu_sc2:
|
|
case NVPTX::BI__nvvm_ldu_c4:
|
|
case NVPTX::BI__nvvm_ldu_sc4:
|
|
case NVPTX::BI__nvvm_ldu_s:
|
|
case NVPTX::BI__nvvm_ldu_s2:
|
|
case NVPTX::BI__nvvm_ldu_s4:
|
|
case NVPTX::BI__nvvm_ldu_i:
|
|
case NVPTX::BI__nvvm_ldu_i2:
|
|
case NVPTX::BI__nvvm_ldu_i4:
|
|
case NVPTX::BI__nvvm_ldu_l:
|
|
case NVPTX::BI__nvvm_ldu_l2:
|
|
case NVPTX::BI__nvvm_ldu_ll:
|
|
case NVPTX::BI__nvvm_ldu_ll2:
|
|
case NVPTX::BI__nvvm_ldu_uc:
|
|
case NVPTX::BI__nvvm_ldu_uc2:
|
|
case NVPTX::BI__nvvm_ldu_uc4:
|
|
case NVPTX::BI__nvvm_ldu_us:
|
|
case NVPTX::BI__nvvm_ldu_us2:
|
|
case NVPTX::BI__nvvm_ldu_us4:
|
|
case NVPTX::BI__nvvm_ldu_ui:
|
|
case NVPTX::BI__nvvm_ldu_ui2:
|
|
case NVPTX::BI__nvvm_ldu_ui4:
|
|
case NVPTX::BI__nvvm_ldu_ul:
|
|
case NVPTX::BI__nvvm_ldu_ul2:
|
|
case NVPTX::BI__nvvm_ldu_ull:
|
|
case NVPTX::BI__nvvm_ldu_ull2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_ldu_f:
|
|
case NVPTX::BI__nvvm_ldu_f2:
|
|
case NVPTX::BI__nvvm_ldu_f4:
|
|
case NVPTX::BI__nvvm_ldu_d:
|
|
case NVPTX::BI__nvvm_ldu_d2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_cta_add_gen_i:
|
|
case NVPTX::BI__nvvm_atom_cta_add_gen_l:
|
|
case NVPTX::BI__nvvm_atom_cta_add_gen_ll:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_sys_add_gen_i:
|
|
case NVPTX::BI__nvvm_atom_sys_add_gen_l:
|
|
case NVPTX::BI__nvvm_atom_sys_add_gen_ll:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_cta_add_gen_f:
|
|
case NVPTX::BI__nvvm_atom_cta_add_gen_d:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_sys_add_gen_f:
|
|
case NVPTX::BI__nvvm_atom_sys_add_gen_d:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_cta_xchg_gen_i:
|
|
case NVPTX::BI__nvvm_atom_cta_xchg_gen_l:
|
|
case NVPTX::BI__nvvm_atom_cta_xchg_gen_ll:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_sys_xchg_gen_i:
|
|
case NVPTX::BI__nvvm_atom_sys_xchg_gen_l:
|
|
case NVPTX::BI__nvvm_atom_sys_xchg_gen_ll:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_cta_max_gen_i:
|
|
case NVPTX::BI__nvvm_atom_cta_max_gen_ui:
|
|
case NVPTX::BI__nvvm_atom_cta_max_gen_l:
|
|
case NVPTX::BI__nvvm_atom_cta_max_gen_ul:
|
|
case NVPTX::BI__nvvm_atom_cta_max_gen_ll:
|
|
case NVPTX::BI__nvvm_atom_cta_max_gen_ull:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_sys_max_gen_i:
|
|
case NVPTX::BI__nvvm_atom_sys_max_gen_ui:
|
|
case NVPTX::BI__nvvm_atom_sys_max_gen_l:
|
|
case NVPTX::BI__nvvm_atom_sys_max_gen_ul:
|
|
case NVPTX::BI__nvvm_atom_sys_max_gen_ll:
|
|
case NVPTX::BI__nvvm_atom_sys_max_gen_ull:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_cta_min_gen_i:
|
|
case NVPTX::BI__nvvm_atom_cta_min_gen_ui:
|
|
case NVPTX::BI__nvvm_atom_cta_min_gen_l:
|
|
case NVPTX::BI__nvvm_atom_cta_min_gen_ul:
|
|
case NVPTX::BI__nvvm_atom_cta_min_gen_ll:
|
|
case NVPTX::BI__nvvm_atom_cta_min_gen_ull:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_sys_min_gen_i:
|
|
case NVPTX::BI__nvvm_atom_sys_min_gen_ui:
|
|
case NVPTX::BI__nvvm_atom_sys_min_gen_l:
|
|
case NVPTX::BI__nvvm_atom_sys_min_gen_ul:
|
|
case NVPTX::BI__nvvm_atom_sys_min_gen_ll:
|
|
case NVPTX::BI__nvvm_atom_sys_min_gen_ull:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_cta_inc_gen_ui:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_cta_dec_gen_ui:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_sys_inc_gen_ui:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_sys_dec_gen_ui:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_cta_and_gen_i:
|
|
case NVPTX::BI__nvvm_atom_cta_and_gen_l:
|
|
case NVPTX::BI__nvvm_atom_cta_and_gen_ll:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_sys_and_gen_i:
|
|
case NVPTX::BI__nvvm_atom_sys_and_gen_l:
|
|
case NVPTX::BI__nvvm_atom_sys_and_gen_ll:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_cta_or_gen_i:
|
|
case NVPTX::BI__nvvm_atom_cta_or_gen_l:
|
|
case NVPTX::BI__nvvm_atom_cta_or_gen_ll:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_sys_or_gen_i:
|
|
case NVPTX::BI__nvvm_atom_sys_or_gen_l:
|
|
case NVPTX::BI__nvvm_atom_sys_or_gen_ll:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_cta_xor_gen_i:
|
|
case NVPTX::BI__nvvm_atom_cta_xor_gen_l:
|
|
case NVPTX::BI__nvvm_atom_cta_xor_gen_ll:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_sys_xor_gen_i:
|
|
case NVPTX::BI__nvvm_atom_sys_xor_gen_l:
|
|
case NVPTX::BI__nvvm_atom_sys_xor_gen_ll:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_cta_cas_gen_us:
|
|
case NVPTX::BI__nvvm_atom_cta_cas_gen_i:
|
|
case NVPTX::BI__nvvm_atom_cta_cas_gen_l:
|
|
case NVPTX::BI__nvvm_atom_cta_cas_gen_ll:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_atom_sys_cas_gen_us:
|
|
case NVPTX::BI__nvvm_atom_sys_cas_gen_i:
|
|
case NVPTX::BI__nvvm_atom_sys_cas_gen_l:
|
|
case NVPTX::BI__nvvm_atom_sys_cas_gen_ll:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_match_all_sync_i32p:
|
|
case NVPTX::BI__nvvm_match_all_sync_i64p:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
// FP MMA loads
|
|
case NVPTX::BI__hmma_m16n16k16_ld_a:
|
|
case NVPTX::BI__hmma_m16n16k16_ld_b:
|
|
case NVPTX::BI__hmma_m16n16k16_ld_c_f16:
|
|
case NVPTX::BI__hmma_m16n16k16_ld_c_f32:
|
|
case NVPTX::BI__hmma_m32n8k16_ld_a:
|
|
case NVPTX::BI__hmma_m32n8k16_ld_b:
|
|
case NVPTX::BI__hmma_m32n8k16_ld_c_f16:
|
|
case NVPTX::BI__hmma_m32n8k16_ld_c_f32:
|
|
case NVPTX::BI__hmma_m8n32k16_ld_a:
|
|
case NVPTX::BI__hmma_m8n32k16_ld_b:
|
|
case NVPTX::BI__hmma_m8n32k16_ld_c_f16:
|
|
case NVPTX::BI__hmma_m8n32k16_ld_c_f32:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__imma_m16n16k16_ld_a_s8:
|
|
case NVPTX::BI__imma_m16n16k16_ld_a_u8:
|
|
case NVPTX::BI__imma_m16n16k16_ld_b_s8:
|
|
case NVPTX::BI__imma_m16n16k16_ld_b_u8:
|
|
case NVPTX::BI__imma_m16n16k16_ld_c:
|
|
case NVPTX::BI__imma_m32n8k16_ld_a_s8:
|
|
case NVPTX::BI__imma_m32n8k16_ld_a_u8:
|
|
case NVPTX::BI__imma_m32n8k16_ld_b_s8:
|
|
case NVPTX::BI__imma_m32n8k16_ld_b_u8:
|
|
case NVPTX::BI__imma_m32n8k16_ld_c:
|
|
case NVPTX::BI__imma_m8n32k16_ld_a_s8:
|
|
case NVPTX::BI__imma_m8n32k16_ld_a_u8:
|
|
case NVPTX::BI__imma_m8n32k16_ld_b_s8:
|
|
case NVPTX::BI__imma_m8n32k16_ld_b_u8:
|
|
case NVPTX::BI__imma_m8n32k16_ld_c:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__imma_m8n8k32_ld_a_s4:
|
|
case NVPTX::BI__imma_m8n8k32_ld_a_u4:
|
|
case NVPTX::BI__imma_m8n8k32_ld_b_s4:
|
|
case NVPTX::BI__imma_m8n8k32_ld_b_u4:
|
|
case NVPTX::BI__imma_m8n8k32_ld_c:
|
|
case NVPTX::BI__bmma_m8n8k128_ld_a_b1:
|
|
case NVPTX::BI__bmma_m8n8k128_ld_b_b1:
|
|
case NVPTX::BI__bmma_m8n8k128_ld_c:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__dmma_m8n8k4_ld_a:
|
|
case NVPTX::BI__dmma_m8n8k4_ld_b:
|
|
case NVPTX::BI__dmma_m8n8k4_ld_c:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__mma_bf16_m16n16k16_ld_a:
|
|
case NVPTX::BI__mma_bf16_m16n16k16_ld_b:
|
|
case NVPTX::BI__mma_bf16_m8n32k16_ld_a:
|
|
case NVPTX::BI__mma_bf16_m8n32k16_ld_b:
|
|
case NVPTX::BI__mma_bf16_m32n8k16_ld_a:
|
|
case NVPTX::BI__mma_bf16_m32n8k16_ld_b:
|
|
case NVPTX::BI__mma_tf32_m16n16k8_ld_a:
|
|
case NVPTX::BI__mma_tf32_m16n16k8_ld_b:
|
|
case NVPTX::BI__mma_tf32_m16n16k8_ld_c:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__hmma_m16n16k16_st_c_f16:
|
|
case NVPTX::BI__hmma_m16n16k16_st_c_f32:
|
|
case NVPTX::BI__hmma_m32n8k16_st_c_f16:
|
|
case NVPTX::BI__hmma_m32n8k16_st_c_f32:
|
|
case NVPTX::BI__hmma_m8n32k16_st_c_f16:
|
|
case NVPTX::BI__hmma_m8n32k16_st_c_f32:
|
|
case NVPTX::BI__imma_m16n16k16_st_c_i32:
|
|
case NVPTX::BI__imma_m32n8k16_st_c_i32:
|
|
case NVPTX::BI__imma_m8n32k16_st_c_i32:
|
|
case NVPTX::BI__imma_m8n8k32_st_c_i32:
|
|
case NVPTX::BI__bmma_m8n8k128_st_c_i32:
|
|
case NVPTX::BI__dmma_m8n8k4_st_c_f64:
|
|
case NVPTX::BI__mma_m16n16k8_st_c_f32:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
// BI__hmma_m16n16k16_mma_<Dtype><CType>(d, a, b, c, layout, satf) -->
|
|
// Intrinsic::nvvm_wmma_m16n16k16_mma_sync<layout A,B><DType><CType><Satf>
|
|
case NVPTX::BI__hmma_m16n16k16_mma_f16f16:
|
|
case NVPTX::BI__hmma_m16n16k16_mma_f32f16:
|
|
case NVPTX::BI__hmma_m16n16k16_mma_f32f32:
|
|
case NVPTX::BI__hmma_m16n16k16_mma_f16f32:
|
|
case NVPTX::BI__hmma_m32n8k16_mma_f16f16:
|
|
case NVPTX::BI__hmma_m32n8k16_mma_f32f16:
|
|
case NVPTX::BI__hmma_m32n8k16_mma_f32f32:
|
|
case NVPTX::BI__hmma_m32n8k16_mma_f16f32:
|
|
case NVPTX::BI__hmma_m8n32k16_mma_f16f16:
|
|
case NVPTX::BI__hmma_m8n32k16_mma_f32f16:
|
|
case NVPTX::BI__hmma_m8n32k16_mma_f32f32:
|
|
case NVPTX::BI__hmma_m8n32k16_mma_f16f32:
|
|
case NVPTX::BI__imma_m16n16k16_mma_s8:
|
|
case NVPTX::BI__imma_m16n16k16_mma_u8:
|
|
case NVPTX::BI__imma_m32n8k16_mma_s8:
|
|
case NVPTX::BI__imma_m32n8k16_mma_u8:
|
|
case NVPTX::BI__imma_m8n32k16_mma_s8:
|
|
case NVPTX::BI__imma_m8n32k16_mma_u8:
|
|
case NVPTX::BI__imma_m8n8k32_mma_s4:
|
|
case NVPTX::BI__imma_m8n8k32_mma_u4:
|
|
case NVPTX::BI__bmma_m8n8k128_mma_xor_popc_b1:
|
|
case NVPTX::BI__bmma_m8n8k128_mma_and_popc_b1:
|
|
case NVPTX::BI__dmma_m8n8k4_mma_f64:
|
|
case NVPTX::BI__mma_bf16_m16n16k16_mma_f32:
|
|
case NVPTX::BI__mma_bf16_m8n32k16_mma_f32:
|
|
case NVPTX::BI__mma_bf16_m32n8k16_mma_f32:
|
|
case NVPTX::BI__mma_tf32_m16n16k8_mma_f32:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
// The following builtins require half type support
|
|
case NVPTX::BI__nvvm_ex2_approx_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_ex2_approx_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_ff2f16x2_rn:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_ff2f16x2_rn_relu:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_ff2f16x2_rz:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_ff2f16x2_rz_relu:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fma_rn_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fma_rn_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fma_rn_ftz_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fma_rn_ftz_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fma_rn_ftz_relu_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fma_rn_ftz_sat_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fma_rn_relu_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fma_rn_relu_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fma_rn_sat_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fma_rn_sat_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fma_rn_oob_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fma_rn_oob_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fma_rn_oob_bf16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fma_rn_oob_bf16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fma_rn_oob_relu_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fma_rn_oob_relu_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fma_rn_oob_relu_bf16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fma_rn_oob_relu_bf16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmax_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmax_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmax_ftz_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmax_ftz_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmax_ftz_nan_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmax_ftz_nan_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmax_ftz_nan_xorsign_abs_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmax_ftz_xorsign_abs_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmax_nan_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmax_nan_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmax_nan_xorsign_abs_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmax_xorsign_abs_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmax_xorsign_abs_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmin_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmin_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmin_ftz_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmin_ftz_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmin_ftz_nan_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmin_ftz_nan_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmin_ftz_nan_xorsign_abs_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmin_ftz_xorsign_abs_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmin_nan_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmin_nan_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmin_nan_xorsign_abs_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmin_xorsign_abs_f16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fmin_xorsign_abs_f16x2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_fabs_f:
|
|
case NVPTX::BI__nvvm_abs_bf16:
|
|
case NVPTX::BI__nvvm_abs_bf16x2:
|
|
case NVPTX::BI__nvvm_fabs_f16:
|
|
case NVPTX::BI__nvvm_fabs_f16x2:
|
|
return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.fabs");
|
|
case NVPTX::BI__nvvm_fabs_ftz_f:
|
|
case NVPTX::BI__nvvm_fabs_ftz_f16:
|
|
case NVPTX::BI__nvvm_fabs_ftz_f16x2:
|
|
return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.fabs.ftz");
|
|
case NVPTX::BI__nvvm_fabs_d:
|
|
return emitUnaryNVVMIntrinsic(*this, expr, "fabs");
|
|
case NVPTX::BI__nvvm_ex2_approx_d:
|
|
case NVPTX::BI__nvvm_ex2_approx_f:
|
|
return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.ex2.approx");
|
|
case NVPTX::BI__nvvm_ex2_approx_ftz_f:
|
|
return emitUnaryNVVMIntrinsic(*this, expr, "nvvm.ex2.approx.ftz");
|
|
case NVPTX::BI__nvvm_ldg_h:
|
|
case NVPTX::BI__nvvm_ldg_h2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_ldu_h:
|
|
case NVPTX::BI__nvvm_ldu_h2:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_cp_async_ca_shared_global_4:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_cp_async_ca_shared_global_8:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_cp_async_ca_shared_global_16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_cp_async_cg_shared_global_16:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_x:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_y:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_z:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_read_ptx_sreg_clusterid_w:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_x:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_y:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_z:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_read_ptx_sreg_nclusterid_w:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_x:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_y:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_z:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctaid_w:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_x:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_y:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_z:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctaid_w:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_read_ptx_sreg_cluster_ctarank:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_read_ptx_sreg_cluster_nctarank:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_is_explicit_cluster:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_isspacep_shared_cluster:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_mapa:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_mapa_shared_cluster:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_getctarank:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_getctarank_shared_cluster:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_barrier_cluster_arrive:
|
|
return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()),
|
|
"nvvm.barrier.cluster.arrive",
|
|
builder.getVoidTy());
|
|
case NVPTX::BI__nvvm_barrier_cluster_arrive_relaxed:
|
|
return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()),
|
|
"nvvm.barrier.cluster.arrive.relaxed",
|
|
builder.getVoidTy());
|
|
case NVPTX::BI__nvvm_barrier_cluster_wait:
|
|
return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()),
|
|
"nvvm.barrier.cluster.wait",
|
|
builder.getVoidTy());
|
|
case NVPTX::BI__nvvm_fence_sc_cluster:
|
|
return builder.emitIntrinsicCallOp(getLoc(expr->getExprLoc()),
|
|
"nvvm.fence.sc.cluster",
|
|
builder.getVoidTy());
|
|
case NVPTX::BI__nvvm_bar_sync:
|
|
return builder.emitIntrinsicCallOp(
|
|
getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.aligned.all",
|
|
builder.getVoidTy(), mlir::ValueRange{emitScalarExpr(expr->getArg(0))});
|
|
case NVPTX::BI__syncthreads:
|
|
return builder.emitIntrinsicCallOp(
|
|
getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.aligned.all",
|
|
builder.getVoidTy(),
|
|
mlir::ValueRange{builder.getConstInt(getLoc(expr->getExprLoc()),
|
|
builder.getSInt32Ty(), 0)});
|
|
case NVPTX::BI__nvvm_barrier_sync:
|
|
return builder.emitIntrinsicCallOp(
|
|
getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.all",
|
|
builder.getVoidTy(), mlir::ValueRange{emitScalarExpr(expr->getArg(0))});
|
|
case NVPTX::BI__nvvm_barrier_sync_cnt:
|
|
return builder.emitIntrinsicCallOp(
|
|
getLoc(expr->getExprLoc()), "nvvm.barrier.cta.sync.count",
|
|
builder.getVoidTy(),
|
|
mlir::ValueRange{emitScalarExpr(expr->getArg(0)),
|
|
emitScalarExpr(expr->getArg(1))});
|
|
case NVPTX::BI__nvvm_bar0_and:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_bar0_or:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
case NVPTX::BI__nvvm_bar0_popc:
|
|
cgm.errorNYI(expr->getSourceRange(),
|
|
std::string("unimplemented NVPTX builtin call: ") +
|
|
getContext().BuiltinInfo.getName(builtinId));
|
|
return mlir::Value{};
|
|
|
|
default:
|
|
return std::nullopt;
|
|
}
|
|
}
|
|
|
|
// vprintf takes two args: A format string, and a pointer to a buffer containing
|
|
// the varargs.
|
|
//
|
|
// For example, the call
|
|
//
|
|
// printf("format string", arg1, arg2, arg3);
|
|
//
|
|
// is converted into something resembling
|
|
//
|
|
// struct Tmp {
|
|
// Arg1 a1;
|
|
// Arg2 a2;
|
|
// Arg3 a3;
|
|
// };
|
|
// char* buf = alloca(sizeof(Tmp));
|
|
// *(Tmp*)buf = {a1, a2, a3};
|
|
// vprintf("format string", buf);
|
|
//
|
|
// `buf` is aligned to the max of {alignof(Arg1), ...}. Furthermore, each of
|
|
// the args is itself aligned to its preferred alignment.
|
|
//
|
|
// Note that by the time this function runs, the arguments have already
|
|
// undergone the standard C vararg promotion (short -> int, float -> double
|
|
// etc). In this function we pack the arguments into the buffer described above.
|
|
static mlir::Value packArgsIntoNVPTXFormatBuffer(CIRGenFunction &cgf,
|
|
const CallArgList &args,
|
|
mlir::Location loc) {
|
|
const cir::CIRDataLayout dataLayout = cgf.cgm.getDataLayout();
|
|
CIRGenBuilderTy &builder = cgf.getBuilder();
|
|
|
|
if (args.size() <= 1)
|
|
// If there are no arguments other than the format string,
|
|
// pass a nullptr to vprintf.
|
|
return builder.getNullPtr(builder.getVoidPtrTy(), loc);
|
|
|
|
llvm::SmallVector<mlir::Type> argTypes;
|
|
for (const auto &arg : llvm::drop_begin(args))
|
|
argTypes.push_back(arg.getKnownRValue().getValue().getType());
|
|
|
|
// We can directly store the arguments into a struct, and the alignment
|
|
// would automatically be correct. That's because vprintf does not
|
|
// accept aggregates.
|
|
mlir::Type allocaTy = builder.getAnonRecordTy(argTypes);
|
|
auto allocaAlign = clang::CharUnits::fromQuantity(
|
|
dataLayout.getABITypeAlign(allocaTy).value());
|
|
Address allocaAddr =
|
|
cgf.createTempAlloca(allocaTy, allocaAlign, loc, "printf_args");
|
|
mlir::Value alloca = allocaAddr.getPointer();
|
|
|
|
for (auto [i, arg] : llvm::enumerate(llvm::drop_begin(args))) {
|
|
mlir::Value member = builder.createGetMember(
|
|
loc, cir::PointerType::get(argTypes[i]), alloca, /*name=*/"",
|
|
/*index=*/i);
|
|
auto abiAlign = clang::CharUnits::fromQuantity(
|
|
dataLayout.getABITypeAlign(argTypes[i]).value());
|
|
cir::StoreOp::create(builder, loc, arg.getKnownRValue().getValue(), member,
|
|
/*is_volatile=*/false,
|
|
builder.getAlignmentAttr(abiAlign),
|
|
/*sync_scope=*/cir::SyncScopeKindAttr{},
|
|
/*mem_order=*/cir::MemOrderAttr{});
|
|
}
|
|
|
|
return builder.createBitcast(alloca, builder.getVoidPtrTy());
|
|
}
|
|
|
|
mlir::Value
|
|
CIRGenFunction::emitNVPTXDevicePrintfCallExpr(const CallExpr *expr) {
|
|
assert(cgm.getTriple().isNVPTX());
|
|
assert(expr->getBuiltinCallee() == Builtin::BIprintf ||
|
|
expr->getBuiltinCallee() == Builtin::BI__builtin_printf);
|
|
assert(expr->getNumArgs() >= 1); // printf always has at least one arg.
|
|
CallArgList args;
|
|
emitCallArgs(args,
|
|
expr->getDirectCallee()->getType()->getAs<FunctionProtoType>(),
|
|
expr->arguments(), expr->getDirectCallee());
|
|
|
|
mlir::Location loc = getLoc(expr->getBeginLoc());
|
|
|
|
// We don't know how to emit non-scalar varargs.
|
|
bool hasNonScalar =
|
|
llvm::any_of(llvm::drop_begin(args), [&](const CallArg &a) {
|
|
return a.hasLValue() || !a.getKnownRValue().isScalar();
|
|
});
|
|
if (hasNonScalar) {
|
|
cgm.errorUnsupported(expr, "non-scalar args to printf");
|
|
return builder.getConstInt(loc, builder.getSInt32Ty(), 0);
|
|
}
|
|
|
|
mlir::Value packedData = packArgsIntoNVPTXFormatBuffer(*this, args, loc);
|
|
|
|
// int vprintf(char *format, void *packedData);
|
|
auto vprintf = cgm.createRuntimeFunction(
|
|
cir::FuncType::get(
|
|
{cir::PointerType::get(builder.getSInt8Ty()), builder.getVoidPtrTy()},
|
|
builder.getSInt32Ty()),
|
|
"vprintf");
|
|
auto formatString = args[0].getKnownRValue().getValue();
|
|
return builder
|
|
.createCallOp(loc, vprintf, mlir::ValueRange{formatString, packedData})
|
|
.getResult();
|
|
}
|