derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
284 lines
11 KiB
C++
284 lines
11 KiB
C++
// Copyright Advanced Micro Devices, Inc., or its affiliates.
|
|
// SPDX-License-Identifier: MIT
|
|
|
|
#include <rocRoller/AssemblyKernel.hpp>
|
|
#include <rocRoller/Context.hpp>
|
|
#include <rocRoller/Expression.hpp>
|
|
#include <rocRoller/ExpressionTransformations.hpp>
|
|
#include <rocRoller/KernelOptions_detail.hpp>
|
|
#include <rocRoller/Operations/Command.hpp>
|
|
|
|
#include "CustomMatchers.hpp"
|
|
#include "CustomSections.hpp"
|
|
#include "ExpressionMatchers.hpp"
|
|
#include "TestContext.hpp"
|
|
#include "TestKernels.hpp"
|
|
|
|
#include <common/TestValues.hpp>
|
|
|
|
#include <catch2/catch_test_macros.hpp>
|
|
|
|
namespace FastDivisionTest
|
|
{
|
|
auto getMagicMultiple(auto x)
|
|
{
|
|
using namespace rocRoller;
|
|
namespace Ex = Expression;
|
|
auto expr = magicMultiple(Ex::literal(x));
|
|
|
|
return Ex::literal(evaluate(expr));
|
|
}
|
|
|
|
TEST_CASE("FastDivision ExpressionTransformation works for constant expressions.",
|
|
"[division][expression][expression-transformation]")
|
|
{
|
|
using namespace rocRoller;
|
|
namespace Ex = Expression;
|
|
|
|
auto context = TestContext::ForDefaultTarget({{.minLaunchTimeExpressionComplexity = 49}});
|
|
|
|
auto command = std::make_shared<Command>();
|
|
|
|
auto aTag = command->allocateTag();
|
|
auto a = command
|
|
->allocateArgument(
|
|
{DataType::Int32, PointerType::Value}, aTag, ArgumentType::Value)
|
|
->expression();
|
|
|
|
auto b = Register::Value::Placeholder(
|
|
context.get(), Register::Type::Vector, DataType::Int32, 1)
|
|
->expression();
|
|
|
|
auto argsBefore = context->kernel()->arguments().size();
|
|
|
|
auto expr = a / Ex::literal(8u);
|
|
auto expr_fast = fastDivision(expr, context.get());
|
|
CHECK_THAT(expr_fast, IdenticalTo(a >> Ex::literal(3u)));
|
|
|
|
expr = a / Ex::literal(8);
|
|
expr_fast = fastDivision(expr, context.get());
|
|
CHECK_THAT(expr_fast,
|
|
IdenticalTo((a + Ex::logicalShiftR((a >> Ex::literal(31u)), Ex::literal(29u))
|
|
>> Ex::literal(3))));
|
|
|
|
expr = b / Ex::literal(7u);
|
|
expr_fast = fastDivision(expr, context.get());
|
|
|
|
{
|
|
auto mulHigh = multiplyHigh(b, getMagicMultiple(7u));
|
|
CHECK_THAT(
|
|
expr_fast,
|
|
EquivalentTo((((b - mulHigh) >> Ex::literal(1u)) + mulHigh) >> Ex::literal(2)));
|
|
}
|
|
|
|
expr = b / Ex::literal(1);
|
|
expr_fast = fastDivision(expr, context.get());
|
|
CHECK_THAT(expr_fast, EquivalentTo(b));
|
|
|
|
expr = b / Ex::literal(-5);
|
|
expr_fast = fastDivision(expr, context.get());
|
|
|
|
{
|
|
auto mulPlusB = multiplyHigh(b, getMagicMultiple(-5)) + b;
|
|
CHECK_THAT(expr_fast,
|
|
EquivalentTo((((mulPlusB + ((mulPlusB >> Ex::literal(31)) & Ex::literal(4)))
|
|
>> Ex::literal(2u))
|
|
^ Ex::literal(-1))
|
|
+ Ex::literal(1)));
|
|
}
|
|
|
|
expr = a / Ex::literal(8u);
|
|
expr_fast = fastDivision(expr, context.get());
|
|
CHECK_THAT(expr_fast, EquivalentTo(a >> Ex::literal(3u)));
|
|
|
|
expr = a / Ex::literal(128u);
|
|
expr_fast = fastDivision(expr, context.get());
|
|
CHECK_THAT(expr_fast, EquivalentTo(a >> Ex::literal(7u)));
|
|
|
|
expr = a / (Ex::literal(43u) + Ex::literal(85u));
|
|
expr_fast = fastDivision(expr, context.get());
|
|
CHECK_THAT(expr_fast, EquivalentTo(a >> Ex::literal(7u)));
|
|
|
|
auto argsAfter = context->kernel()->arguments().size();
|
|
CHECK(argsBefore == argsAfter);
|
|
}
|
|
|
|
TEST_CASE("FastDivision ExpressionTransformation works for argument expressions.",
|
|
"[division][expression][expression-transformation]")
|
|
{
|
|
using namespace rocRoller;
|
|
namespace Ex = Expression;
|
|
|
|
auto context = TestContext::ForDefaultTarget({{.minLaunchTimeExpressionComplexity = 49}});
|
|
|
|
auto command = std::make_shared<rocRoller::Command>();
|
|
|
|
auto reg = Register::Value::Placeholder(
|
|
context.get(), Register::Type::Vector, DataType::Int32, 1);
|
|
auto a = reg->expression();
|
|
|
|
auto reg2 = Register::Value::Placeholder(
|
|
context.get(), Register::Type::Vector, DataType::UInt32, 1);
|
|
auto a_unsigned = reg2->expression();
|
|
|
|
auto bSignedTag = command->allocateTag();
|
|
auto b_signed = command
|
|
->allocateArgument({DataType::Int32, PointerType::Value},
|
|
bSignedTag,
|
|
ArgumentType::Value)
|
|
->expression();
|
|
|
|
auto bUnsignedTag = command->allocateTag();
|
|
auto b_unsigned = command
|
|
->allocateArgument({DataType::UInt32, PointerType::Value},
|
|
bUnsignedTag,
|
|
ArgumentType::Value)
|
|
->expression();
|
|
|
|
auto expr = a / b_signed;
|
|
auto expr_fast = fastDivision(expr, context.get());
|
|
|
|
{
|
|
auto [mul, shift, sign, _] = getMagicDivisionParams(b_signed, context.get());
|
|
|
|
auto mulPlusA = a + multiplyHigh(a, mul);
|
|
|
|
CHECK_THAT(expr_fast,
|
|
EquivalentTo(
|
|
((((mulPlusA
|
|
+ ((mulPlusA >> Ex::literal(31))
|
|
& ((Ex::literal(1) << shift)
|
|
+ Ex::conditional(
|
|
mul == Ex::literal(0), Ex::literal(-1), Ex::literal(0)))))
|
|
>> shift)
|
|
^ sign)
|
|
- sign)));
|
|
}
|
|
|
|
expr = a_unsigned / b_unsigned;
|
|
expr_fast = fastDivision(expr, context.get());
|
|
|
|
{
|
|
auto [mul, shift, sign, shiftMSB] = getMagicDivisionParams(b_unsigned, context.get());
|
|
|
|
auto mulHigh = multiplyHigh(a_unsigned, mul);
|
|
|
|
auto t = ((a_unsigned - mulHigh >> Ex::literal(1u)) + mulHigh) >> shift;
|
|
|
|
auto result = Ex::conditional(shiftMSB == Ex::literal(0u), t, a_unsigned);
|
|
|
|
CHECK_THAT(expr_fast, EquivalentTo(result));
|
|
}
|
|
}
|
|
|
|
TEST_CASE("FastDivision ExpressionTransformation works for constant modulo expressions.",
|
|
"[division][expression][expression-transformation]")
|
|
{
|
|
using namespace rocRoller;
|
|
namespace Ex = Expression;
|
|
|
|
auto context = TestContext::ForDefaultTarget({{.minLaunchTimeExpressionComplexity = 49}});
|
|
|
|
auto command = std::make_shared<Command>();
|
|
|
|
auto regPtr = Register::Value::Placeholder(
|
|
context.get(), Register::Type::Vector, DataType::UInt32, 1);
|
|
auto reg = regPtr->expression();
|
|
|
|
auto aTag = command->allocateTag();
|
|
auto a = command
|
|
->allocateArgument(
|
|
{DataType::Int32, PointerType::Value}, aTag, ArgumentType::Value)
|
|
->expression();
|
|
|
|
a = context->kernel()->addArgument({"arg", DataType::Int32, DataDirection::ReadOnly, a});
|
|
|
|
auto argsBefore = context->kernel()->arguments();
|
|
|
|
auto expr = a % Ex::literal(8u);
|
|
auto expr_fast = fastDivision(expr, context.get());
|
|
CHECK_THAT(expr_fast, EquivalentTo(a & Ex::literal(7u)));
|
|
|
|
expr = a % Ex::literal(8);
|
|
expr_fast = fastDivision(expr, context.get());
|
|
CHECK_THAT(expr_fast,
|
|
EquivalentTo(a
|
|
- ((a + logicalShiftR((a >> Ex::literal(31u)), Ex::literal(29u)))
|
|
& Ex::literal(-8))));
|
|
|
|
expr = a % Ex::literal(7u);
|
|
expr_fast = fastDivision(expr, context.get());
|
|
CHECK_THAT(
|
|
expr_fast,
|
|
EquivalentTo(a - (fastDivision(a / Ex::literal(7u), context.get()) * Ex::literal(7u))));
|
|
|
|
expr = a % Ex::literal(1);
|
|
expr_fast = fastDivision(expr, context.get());
|
|
CHECK_THAT(expr_fast, EquivalentTo(Ex::literal(0)));
|
|
|
|
expr = a % Ex::literal(1u);
|
|
expr_fast = fastDivision(expr, context.get());
|
|
CHECK_THAT(expr_fast, EquivalentTo(Ex::literal(0u)));
|
|
|
|
expr = a % Ex::literal(-5);
|
|
expr_fast = fastDivision(expr, context.get());
|
|
CHECK_THAT(
|
|
expr_fast,
|
|
EquivalentTo(a - (fastDivision(a / Ex::literal(-5), context.get()) * Ex::literal(-5))));
|
|
|
|
expr = a % Ex::literal(8u);
|
|
expr_fast = fastDivision(expr, context.get());
|
|
CHECK_THAT(expr_fast, EquivalentTo(a & Ex::literal(7u)));
|
|
|
|
expr = a % Ex::literal(128u);
|
|
expr_fast = fastDivision(expr, context.get());
|
|
CHECK_THAT(expr_fast, EquivalentTo(a & Ex::literal(127u)));
|
|
|
|
auto argsAfter = context->kernel()->arguments();
|
|
CHECK(argsBefore == argsAfter);
|
|
}
|
|
|
|
TEST_CASE("FastDivision ExpressionTransformation works for modulo of argument expressions.",
|
|
"[division][expression][expression-transformation]")
|
|
{
|
|
using namespace rocRoller;
|
|
namespace Ex = Expression;
|
|
|
|
auto context = TestContext::ForDefaultTarget({{.minLaunchTimeExpressionComplexity = 49}});
|
|
|
|
auto command = std::make_shared<Command>();
|
|
|
|
auto reg = Register::Value::Placeholder(
|
|
context.get(), Register::Type::Vector, DataType::Int32, 1);
|
|
auto a = reg->expression();
|
|
|
|
auto reg2 = Register::Value::Placeholder(
|
|
context.get(), Register::Type::Vector, DataType::UInt32, 1);
|
|
auto a_unsigned = reg2->expression();
|
|
|
|
auto bSignedTag = command->allocateTag();
|
|
auto b_signed = command
|
|
->allocateArgument({DataType::Int32, PointerType::Value},
|
|
bSignedTag,
|
|
ArgumentType::Value)
|
|
->expression();
|
|
|
|
auto bUnsignedTag = command->allocateTag();
|
|
auto b_unsigned = command
|
|
->allocateArgument({DataType::UInt32, PointerType::Value},
|
|
bUnsignedTag,
|
|
ArgumentType::Value)
|
|
->expression();
|
|
|
|
auto expr = a % b_signed;
|
|
auto expr_fast = fastDivision(expr, context.get());
|
|
auto fastDiv = fastDivision(a / b_signed, context.get());
|
|
CHECK_THAT(expr_fast, EquivalentTo(a - (fastDiv * b_signed)));
|
|
|
|
expr = a_unsigned % b_unsigned;
|
|
expr_fast = fastDivision(expr, context.get());
|
|
fastDiv = fastDivision(a_unsigned / b_unsigned, context.get());
|
|
CHECK_THAT(expr_fast, EquivalentTo(a_unsigned - (fastDiv * b_unsigned)));
|
|
}
|
|
}
|