// Copyright Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT #include #include #include #include #include #include namespace rocRoller::Expression::EvaluateDetail { template struct UnaryEvaluatorVisitor { UnaryExpr expr; using TheEvaluator = OperationEvaluatorVisitor; template requires CCanEvaluateUnary CommandArgumentValue operator()(ARG const& arg) const { auto evaluator = static_cast(this); return evaluator->evaluate(arg); } template requires(!CCanEvaluateUnary) CommandArgumentValue operator()(ARG const& arg) const { if constexpr(CHasTypeInfo) { Throw("Incompatible type ", TypeInfo::Name(), " for expression ", typeName()); } else { Throw("Incompatible type ", ShowValue(ARG()), typeName(), " for expression ", typeName()); } } CommandArgumentValue call(CommandArgumentValue const& arg) const { return std::visit(*this, arg); } }; template <> struct OperationEvaluatorVisitor : public UnaryEvaluatorVisitor { uint32_t evaluate(uint32_t const& arg) const { assertNonNullPointer(arg); if(arg == 0) return std::numeric_limits::max() / 2; // When the divisor is 1, this constant has no use, so return 0. if(arg == 1) return 0; auto magic = libdivide::libdivide_u32_branchfree_gen(arg); return magic.magic; } int32_t evaluate(int32_t const& arg) const { assertNonNullPointer(arg); if(arg == 0) return std::numeric_limits::max() / 2; auto magic = libdivide::libdivide_s32_branchfree_gen(arg); return magic.magic; } int64_t evaluate(int64_t const& arg) const { assertNonNullPointer(arg); if(arg == 0) return std::numeric_limits::max() / 2; auto magic = libdivide::libdivide_s64_branchfree_gen(arg); return magic.magic; } }; template <> struct OperationEvaluatorVisitor { Convert exp; template CommandArgumentValue call(CommandArgumentValue const& arg) { constexpr auto IdxType = static_cast(Idx); if constexpr(IdxType == DataType::None) { Throw("Cannot convert to None type."); return 0; } else if constexpr(IdxType == DataType::Count) { Throw("Cannot convert to Count type."); return 0; } else { using ResultType = typename EnumTypeInfo::Type; if(exp.destinationType == IdxType) { if constexpr(!CCommandArgumentValue) { Throw("Cannot convert to ", friendlyTypeName()); return 0; } else { auto theVisitor = [](auto value) -> ResultType { using ExistingType = std::decay_t; if constexpr(CCanStaticCastTo) { return static_cast(value); } else { Throw("Cannot cast ", friendlyTypeName(), " to ", friendlyTypeName()); return ResultType{}; } }; return std::visit(theVisitor, arg); } } else { return call(arg); } } } }; template <> struct OperationEvaluatorVisitor { Reinterpret exp; template CommandArgumentValue reinterpret(FromType const& value, DataType targetDataType) { constexpr auto IdxType = static_cast(Idx); if constexpr(IdxType == DataType::None || IdxType == DataType::Count) { Throw("Unsupported reinterpret to type: ", toString(targetDataType)); return 0; } else { using ToType = typename EnumTypeInfo::Type; if(targetDataType == IdxType) { AssertFatal(std::is_trivially_copyable_v, "FromType must be trivially copyable"); AssertFatal(std::is_trivially_copyable_v, "ToType must be trivially copyable"); if constexpr(!CCommandArgumentValue< ToType> || sizeof(ToType) != sizeof(FromType)) { Throw("Cannot reinterpret to ", friendlyTypeName(), " from ", friendlyTypeName()); return 0; } else { return std::bit_cast(value); } } else { return reinterpret(value, targetDataType); } } } CommandArgumentValue call(CommandArgumentValue const& arg) { return std::visit( [this](auto const& val) -> CommandArgumentValue { using FromType = std::decay_t; return reinterpret(val, exp.destinationType); }, arg); } }; template CommandArgumentValue reinterpretTruncateValue(FromType const& value, DataType targetDataType, std::endian endianness = std::endian::native) { constexpr auto IdxType = static_cast(Idx); AssertFatal( endianness == std::endian::little || endianness == std::endian::big, "Unsupported or mixed endianness: only pure little- or big-endian are supported."); if constexpr(IdxType == DataType::None || IdxType == DataType::Count) { Throw("Unsupported reinterpretTruncate to type: ", toString(targetDataType)); return 0; } else { using ToType = typename EnumTypeInfo::Type; AssertFatal(std::is_trivially_copyable_v, "FromType must be trivially copyable"); AssertFatal(std::is_trivially_copyable_v, "ToType must be trivially copyable"); if(targetDataType == IdxType) { if constexpr(!CCommandArgumentValue) { Throw("Cannot reinterpret to ", friendlyTypeName()); return 0; } else { if constexpr(sizeof(ToType) == sizeof(FromType)) { return std::bit_cast(value); } // Truncate else if constexpr(sizeof(ToType) < sizeof(FromType)) { constexpr std::size_t N = sizeof(ToType); // Source and destination as bytes const auto src_bytes = std::bit_cast>(value); std::array dst_bytes{}; dst_bytes.fill(std::byte{0}); if(endianness == std::endian::little) { // Keep least-significant bytes: at low addresses. std::copy_n(src_bytes.data(), N, dst_bytes.data()); } else { // Big-endian: least-significant bytes live at high addresses. std::copy_n( src_bytes.data() + (sizeof(FromType) - N), N, dst_bytes.data()); } return std::bit_cast(dst_bytes); } else { Throw("Cannot truncate to wider type ", friendlyTypeName(), " from ", friendlyTypeName()); return 0; } } } else { return reinterpretTruncateValue( value, targetDataType, endianness); } } } CommandArgumentValue reinterpretTruncateValue(CommandArgumentValue const& value, DataType targetDataType, std::endian endianness) { return std::visit( [targetDataType, endianness](auto const& val) -> CommandArgumentValue { using FromType = std::decay_t; return reinterpretTruncateValue(val, targetDataType, endianness); }, value); } template <> struct OperationEvaluatorVisitor { BitFieldExtract expr; template requires(CIntegral) CommandArgumentValue operator()(ARG const& arg) const { auto argBits = resultVariableType(arg).getElementSize() * 8u; AssertFatal(argBits >= expr.offset + expr.width, "BitFieldExtract out of bounds: offset={} + width={} > argBits={}", expr.offset, expr.width, argBits); using UnsignedARG = std::make_unsigned_t; UnsignedARG unsignedArg = static_cast(arg); UnsignedARG mask = 0; if(argBits == this->expr.width) { mask = std::numeric_limits::max(); } else { mask = (static_cast(1) << this->expr.width) - 1; } UnsignedARG result = (unsignedArg >> this->expr.offset) & mask; // Sign extend if needed if constexpr(std::is_signed_v) { bool signBit = (result >> (this->expr.width - 1)) & 1; if(signBit) { UnsignedARG signExtensionMask = ~mask; result |= signExtensionMask; } } AssertFatal(expr.width <= DataTypeInfo::Get(expr.outputDataType).elementBits, fmt::format("BitFieldExtract: width {} exceeds output type size {} bits", expr.width, DataTypeInfo::Get(expr.outputDataType).elementBits)); return reinterpretTruncateValue(static_cast(result), expr.outputDataType); } CommandArgumentValue operator()(Raw32 const& arg) const { return call(arg.value); } template CommandArgumentValue operator()(ARG const&) const { Throw("BitFieldExtract: unsupported argument type ", ShowValue(expr)); } CommandArgumentValue call(CommandArgumentValue const& arg) const { return std::visit(*this, arg); } }; template <> struct OperationEvaluatorVisitor : public UnaryEvaluatorVisitor { int evaluate(uint32_t const& arg) const { assertNonNullPointer(arg); if(arg == 0) return 0; // When the divisor is 1, we set the MSB of MagicShift to 1 so we can detect this case // by checking the MSB. if(arg == 1) return 1 << 31; auto magic = libdivide::libdivide_u32_branchfree_gen(arg); return magic.more & libdivide::LIBDIVIDE_32_SHIFT_MASK; } }; template <> struct OperationEvaluatorVisitor : public UnaryEvaluatorVisitor { static_assert(libdivide::LIBDIVIDE_32_SHIFT_MASK == 31, "magicNumberDivision assumes this is true."); static_assert(libdivide::LIBDIVIDE_64_SHIFT_MASK == 63, "magicNumberDivision assumes this is true."); static_assert(libdivide::LIBDIVIDE_NEGATIVE_DIVISOR == 1 << 7, "magicNumberDivision assumes this is true."); uint32_t evaluate(int32_t const& arg) const { assertNonNullPointer(arg); if(arg == 0) return 0; auto magic = libdivide::libdivide_s32_branchfree_gen(arg); return static_cast(magic.more); } uint32_t evaluate(int64_t const& arg) const { assertNonNullPointer(arg); if(arg == 0) return 0; auto magic = libdivide::libdivide_s64_branchfree_gen(arg); return static_cast(magic.more); } }; template <> struct OperationEvaluatorVisitor : public UnaryEvaluatorVisitor { template requires(CIntegral or std::is_same_v) constexpr T evaluate(T const& arg) const { return ~arg; } }; template <> struct OperationEvaluatorVisitor : public UnaryEvaluatorVisitor { template constexpr T evaluate(T const& arg) const { return std::exp2(arg); } }; template <> struct OperationEvaluatorVisitor : public UnaryEvaluatorVisitor { template constexpr T evaluate(T const& arg) const { return std::exp(arg); } }; template <> struct OperationEvaluatorVisitor : public UnaryEvaluatorVisitor { template requires(std::floating_point || std::signed_integral) constexpr T evaluate(T const& arg) const { return -arg; } }; template <> struct OperationEvaluatorVisitor : public UnaryEvaluatorVisitor { template constexpr bool evaluate(T const& arg) const { return !arg; } }; template <> struct OperationEvaluatorVisitor : public UnaryEvaluatorVisitor { template requires(!std::same_as && std::unsigned_integral) constexpr T evaluate(T const& arg) const { return LFSRRandomNumberGenerator(arg); } }; template <> struct OperationEvaluatorVisitor : public UnaryEvaluatorVisitor { template constexpr T evaluate(T arg) const { return arg; } }; struct ToBoolVisitor { template constexpr bool operator()(T const& val) { Throw("Invalid bool type: ", typeName()); return 0; } template constexpr bool operator()(T const& val) { return val != 0; } bool call(CommandArgumentValue const& val) { return std::visit(*this, val); } }; template __attribute__((noinline)) CommandArgumentValue evaluateOp(T const& op, CommandArgumentValue const& arg) { OperationEvaluatorVisitor visitor{op}; return visitor.call(arg); } #define INSTANTIATE_UNARY_OP(Op) \ template CommandArgumentValue evaluateOp(Op const& op, CommandArgumentValue const& arg) INSTANTIATE_UNARY_OP(MagicMultiple); INSTANTIATE_UNARY_OP(MagicShifts); INSTANTIATE_UNARY_OP(MagicShiftAndSign); INSTANTIATE_UNARY_OP(Negate); INSTANTIATE_UNARY_OP(BitwiseNegate); INSTANTIATE_UNARY_OP(LogicalNot); INSTANTIATE_UNARY_OP(Exponential2); INSTANTIATE_UNARY_OP(Exponential); INSTANTIATE_UNARY_OP(RandomNumber); INSTANTIATE_UNARY_OP(ToScalar); INSTANTIATE_UNARY_OP(BitFieldExtract); INSTANTIATE_UNARY_OP(Convert); INSTANTIATE_UNARY_OP(Reinterpret); }