// Copyright Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT #ifdef ROCROLLER_USE_HIP #include #include #endif /* ROCROLLER_USE_HIP */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "GPUContextFixture.hpp" #include "GenericContextFixture.hpp" #include "SourceMatcher.hpp" #include "Utilities.hpp" using namespace rocRoller; using namespace rocRoller::KernelGraph; using namespace rocRoller::KernelGraph::CoordinateGraph; using namespace rocRoller::KernelGraph::ControlGraph; using ::testing::HasSubstr; namespace KernelGraphTest { class KernelGraphTestGPU : public CurrentGPUContextFixture { public: Expression::FastArithmetic fastArith{m_context}; void SetUp() override { Settings::getInstance()->set(Settings::SaveAssembly, true); CurrentGPUContextFixture::SetUp(); fastArith = Expression::FastArithmetic(m_context); } void GPU_SAXPBY(bool reload); }; class KernelGraphTest : public GenericContextFixture { public: Expression::FastArithmetic fastArith{m_context}; void SetUp() override { GenericContextFixture::SetUp(); fastArith = Expression::FastArithmetic(m_context); } }; // TODO: Add transforms and tests for VectorAdd with: 1. ForLoop // and 2. ForLoop+Unroll. // // The tests should also make sure the lowering fails if the // WorkitemCount is missing. // // See KernelGraphTestGPULoopSize :: MissingWorkitemCount // KernelGraphTestGPULoopSize :: TestForLoop TEST_F(KernelGraphTest, BasicTranslateLinear) { auto example = rocRollerTest::Graphs::VectorAddNegSquare(); auto kgraph0 = example.getKernelGraph(); auto commandParams = example.getCommandParameters(); auto bottom = kgraph0.coordinates.roots().to(); EXPECT_EQ(bottom.size(), 2); for(auto const& id : bottom) { EXPECT_TRUE(kgraph0.coordinates.get(id).has_value()); } auto top = kgraph0.coordinates.leaves().to(); EXPECT_EQ(top.size(), 1); for(auto const& id : top) { EXPECT_TRUE(kgraph0.coordinates.get(id).has_value()); } auto visitor = rocRoller::KernelGraph::BaseGraphVisitor(m_context); auto kgraphC = rewrite(kgraph0, visitor); std::string expectedC = R".( digraph { "coord1"[label="User{NA}(1)"]; "coord2"[label="User{NA}(2)"]; "coord3"[label="SubDimension{0, CommandArgument(Tensor_0_size_0)I64}(3)"]; "coord4"[label="Split(4)",shape=box]; "coord5"[label="Linear{CommandArgument(Tensor_0_size_0)I64}(5)"]; "coord6"[label="Flatten(6)",shape=box]; "coord7"[label="DataFlow(7)",shape=box]; "coord8"[label="SubDimension{0, CommandArgument(Tensor_2_size_0)I64}(8)"]; "coord9"[label="Split(9)",shape=box]; "coord10"[label="Linear{CommandArgument(Tensor_2_size_0)I64}(10)"]; "coord11"[label="Flatten(11)",shape=box]; "coord12"[label="DataFlow(12)",shape=box]; "coord13"[label="Linear{NA}(13)"]; "coord14"[label="DataFlow(14)",shape=box]; "coord15"[label="Linear{NA}(15)"]; "coord16"[label="DataFlow(16)",shape=box]; "coord17"[label="Linear{NA}(17)"]; "coord18"[label="DataFlow(18)",shape=box]; "coord19"[label="SubDimension{0, CommandArgument(Tensor_8_size_0)I64}(19)"]; "coord20"[label="Split(20)",shape=box]; "coord21"[label="User{NA}(21)"]; "coord22"[label="Join(22)",shape=box]; "coord23"[label="DataFlow(23)",shape=box]; "coord1" -> "coord4" "coord1" -> "coord7" "coord2" -> "coord9" "coord2" -> "coord12" "coord3" -> "coord6" "coord4" -> "coord3" "coord5" -> "coord14" "coord6" -> "coord5" "coord7" -> "coord5" "coord8" -> "coord11" "coord9" -> "coord8" "coord10" -> "coord14" "coord11" -> "coord10" "coord12" -> "coord10" "coord13" -> "coord16" "coord13" -> "coord18" "coord14" -> "coord13" "coord15" -> "coord18" "coord16" -> "coord15" "coord17" -> "coord20" "coord17" -> "coord23" "coord18" -> "coord17" "coord19" -> "coord22" "coord20" -> "coord19" "coord22" -> "coord21" "coord23" -> "coord21" { rank=same "coord5"->"coord10"[style=invis] rankdir=LR } { rank=same "coord13"->"coord15"[style=invis] rankdir=LR } subgraph clusterCF {label = "Control Graph"; "cntrl1"[label="Kernel(1)"]; "cntrl2"[label="LoadLinear Value: Int32(2)"]; "cntrl3"[label="Body(3)",shape=box]; "cntrl4"[label="LoadLinear Value: Int32(4)"]; "cntrl5"[label="Body(5)",shape=box]; "cntrl6"[label="Assign VGPR Add(DataFlowTag(5)NA, DataFlowTag(10)NA)NA(6)"]; "cntrl7"[label="Sequence(7)",shape=box]; "cntrl8"[label="Sequence(8)",shape=box]; "cntrl9"[label="Assign VGPR Negate(DataFlowTag(13)NA)NA(9)"]; "cntrl10"[label="Sequence(10)",shape=box]; "cntrl11"[label="Assign VGPR Multiply(DataFlowTag(13)NA, DataFlowTag(15)NA)NA(11)"]; "cntrl12"[label="Sequence(12)",shape=box]; "cntrl13"[label="Sequence(13)",shape=box]; "cntrl14"[label="StoreLinear(14)"]; "cntrl15"[label="Sequence(15)",shape=box]; "cntrl1" -> "cntrl3" "cntrl1" -> "cntrl5" "cntrl2" -> "cntrl7" "cntrl3" -> "cntrl2" "cntrl4" -> "cntrl8" "cntrl5" -> "cntrl4" "cntrl6" -> "cntrl10" "cntrl6" -> "cntrl12" "cntrl7" -> "cntrl6" "cntrl8" -> "cntrl6" "cntrl9" -> "cntrl13" "cntrl10" -> "cntrl9" "cntrl11" -> "cntrl15" "cntrl12" -> "cntrl11" "cntrl13" -> "cntrl11" "cntrl15" -> "cntrl14" } "coord1" -> "to_cntrl_1_2" "to_cntrl_1_2"[label="2->1: User: (0)", shape=cds] "cntrl2" -> "to_coord_2_1" "to_coord_2_1"[label="2->1: User: (0)", shape=cds] "coord5" -> "to_cntrl_5_2" "to_cntrl_5_2"[label="2->5: Linear: (0)", shape=cds] "cntrl2" -> "to_coord_2_5" "to_coord_2_5"[label="2->5: Linear: (0)", shape=cds] "coord2" -> "to_cntrl_2_4" "to_cntrl_2_4"[label="4->2: User: (0)", shape=cds] "cntrl4" -> "to_coord_4_2" "to_coord_4_2"[label="4->2: User: (0)", shape=cds] "coord10" -> "to_cntrl_10_4" "to_cntrl_10_4"[label="4->10: Linear: (0)", shape=cds] "cntrl4" -> "to_coord_4_10" "to_coord_4_10"[label="4->10: Linear: (0)", shape=cds] "coord13" -> "to_cntrl_13_6" "to_cntrl_13_6"[label="6->13: DEST", shape=cds] "cntrl6" -> "to_coord_6_13" "to_coord_6_13"[label="6->13: DEST", shape=cds] "coord15" -> "to_cntrl_15_9" "to_cntrl_15_9"[label="9->15: DEST", shape=cds] "cntrl9" -> "to_coord_9_15" "to_coord_9_15"[label="9->15: DEST", shape=cds] "coord17" -> "to_cntrl_17_11" "to_cntrl_17_11"[label="11->17: DEST", shape=cds] "cntrl11" -> "to_coord_11_17" "to_coord_11_17"[label="11->17: DEST", shape=cds] "coord17" -> "to_cntrl_17_14" "to_cntrl_17_14"[label="14->17: Linear: (0)", shape=cds] "cntrl14" -> "to_coord_14_17" "to_coord_14_17"[label="14->17: Linear: (0)", shape=cds] "coord21" -> "to_cntrl_21_14" "to_cntrl_21_14"[label="14->21: User: (0)", shape=cds] "cntrl14" -> "to_coord_14_21" "to_coord_14_21"[label="14->21: User: (0)", shape=cds] })."; EXPECT_EQ(NormalizedSource(expectedC), NormalizedSource(kgraphC.toDOT(true))); std::string expected0 = R".( digraph { "coord1"[label="User{NA}(1)"]; "coord2"[label="SubDimension{0, CommandArgument(Tensor_0_size_0)I64}(2)"]; "coord3"[label="Split(3)",shape=box]; "coord4"[label="Linear{CommandArgument(Tensor_0_size_0)I64}(4)"]; "coord5"[label="Flatten(5)",shape=box]; "coord6"[label="DataFlow(6)",shape=box]; "coord7"[label="User{NA}(7)"]; "coord8"[label="SubDimension{0, CommandArgument(Tensor_2_size_0)I64}(8)"]; "coord9"[label="Split(9)",shape=box]; "coord10"[label="Linear{CommandArgument(Tensor_2_size_0)I64}(10)"]; "coord11"[label="Flatten(11)",shape=box]; "coord12"[label="DataFlow(12)",shape=box]; "coord13"[label="Linear{NA}(13)"]; "coord14"[label="DataFlow(14)",shape=box]; "coord15"[label="Linear{NA}(15)"]; "coord16"[label="DataFlow(16)",shape=box]; "coord17"[label="Linear{NA}(17)"]; "coord18"[label="DataFlow(18)",shape=box]; "coord19"[label="SubDimension{0, CommandArgument(Tensor_8_size_0)I64}(19)"]; "coord20"[label="User{NA}(20)"]; "coord21"[label="Split(21)",shape=box]; "coord22"[label="Join(22)",shape=box]; "coord23"[label="DataFlow(23)",shape=box]; "coord1" -> "coord3" "coord1" -> "coord6" "coord2" -> "coord5" "coord3" -> "coord2" "coord4" -> "coord14" "coord5" -> "coord4" "coord6" -> "coord4" "coord7" -> "coord9" "coord7" -> "coord12" "coord8" -> "coord11" "coord9" -> "coord8" "coord10" -> "coord14" "coord11" -> "coord10" "coord12" -> "coord10" "coord13" -> "coord16" "coord13" -> "coord18" "coord14" -> "coord13" "coord15" -> "coord18" "coord16" -> "coord15" "coord17" -> "coord21" "coord17" -> "coord23" "coord18" -> "coord17" "coord19" -> "coord22" "coord21" -> "coord19" "coord22" -> "coord20" "coord23" -> "coord20" { rank=same "coord4"->"coord10"[style=invis] rankdir=LR } { rank=same "coord13"->"coord15"[style=invis] rankdir=LR } subgraph clusterCF {label = "Control Graph"; "cntrl1"[label="Kernel(1)"]; "cntrl2"[label="LoadLinear Value: Int32(2)"]; "cntrl3"[label="Body(3)",shape=box]; "cntrl4"[label="LoadLinear Value: Int32(4)"]; "cntrl5"[label="Body(5)",shape=box]; "cntrl6"[label="Assign VGPR Add(DataFlowTag(4)NA, DataFlowTag(10)NA)NA(6)"]; "cntrl7"[label="Sequence(7)",shape=box]; "cntrl8"[label="Sequence(8)",shape=box]; "cntrl9"[label="Assign VGPR Negate(DataFlowTag(13)NA)NA(9)"]; "cntrl10"[label="Sequence(10)",shape=box]; "cntrl11"[label="Assign VGPR Multiply(DataFlowTag(13)NA, DataFlowTag(15)NA)NA(11)"]; "cntrl12"[label="Sequence(12)",shape=box]; "cntrl13"[label="Sequence(13)",shape=box]; "cntrl14"[label="StoreLinear(14)"]; "cntrl15"[label="Sequence(15)",shape=box]; "cntrl1" -> "cntrl3" "cntrl1" -> "cntrl5" "cntrl2" -> "cntrl7" "cntrl3" -> "cntrl2" "cntrl4" -> "cntrl8" "cntrl5" -> "cntrl4" "cntrl6" -> "cntrl10" "cntrl6" -> "cntrl12" "cntrl7" -> "cntrl6" "cntrl8" -> "cntrl6" "cntrl9" -> "cntrl13" "cntrl10" -> "cntrl9" "cntrl11" -> "cntrl15" "cntrl12" -> "cntrl11" "cntrl13" -> "cntrl11" "cntrl15" -> "cntrl14" } "coord1" -> "to_cntrl_1_2" "to_cntrl_1_2"[label="2->1: User: (0)", shape=cds] "cntrl2" -> "to_coord_2_1" "to_coord_2_1"[label="2->1: User: (0)", shape=cds] "coord4" -> "to_cntrl_4_2" "to_cntrl_4_2"[label="2->4: Linear: (0)", shape=cds] "cntrl2" -> "to_coord_2_4" "to_coord_2_4"[label="2->4: Linear: (0)", shape=cds] "coord7" -> "to_cntrl_7_4" "to_cntrl_7_4"[label="4->7: User: (0)", shape=cds] "cntrl4" -> "to_coord_4_7" "to_coord_4_7"[label="4->7: User: (0)", shape=cds] "coord10" -> "to_cntrl_10_4" "to_cntrl_10_4"[label="4->10: Linear: (0)", shape=cds] "cntrl4" -> "to_coord_4_10" "to_coord_4_10"[label="4->10: Linear: (0)", shape=cds] "coord13" -> "to_cntrl_13_6" "to_cntrl_13_6"[label="6->13: DEST", shape=cds] "cntrl6" -> "to_coord_6_13" "to_coord_6_13"[label="6->13: DEST", shape=cds] "coord15" -> "to_cntrl_15_9" "to_cntrl_15_9"[label="9->15: DEST", shape=cds] "cntrl9" -> "to_coord_9_15" "to_coord_9_15"[label="9->15: DEST", shape=cds] "coord17" -> "to_cntrl_17_11" "to_cntrl_17_11"[label="11->17: DEST", shape=cds] "cntrl11" -> "to_coord_11_17" "to_coord_11_17"[label="11->17: DEST", shape=cds] "coord17" -> "to_cntrl_17_14" "to_cntrl_17_14"[label="14->17: Linear: (0)", shape=cds] "cntrl14" -> "to_coord_14_17" "to_coord_14_17"[label="14->17: Linear: (0)", shape=cds] "coord20" -> "to_cntrl_20_14" "to_cntrl_20_14"[label="14->20: User: (0)", shape=cds] "cntrl14" -> "to_coord_14_20" "to_coord_14_20"[label="14->20: User: (0)", shape=cds] })."; EXPECT_EQ(NormalizedSource(expected0), NormalizedSource(kgraph0.toDOT(true))); std::string expected1 = R".( digraph { "coord1"[label="User{Multiply(CommandArgument(Tensor_0_stride_0)I64, CommandArgument(Tensor_0_size_0)I64)I64}(1)"]; "coord2"[label="User{Multiply(CommandArgument(Tensor_2_stride_0)I64, CommandArgument(Tensor_2_size_0)I64)I64}(2)"]; "coord3"[label="SubDimension{0, CommandArgument(Tensor_0_size_0)I64}(3)"]; "coord4"[label="Split(4)",shape=box]; "coord5"[label="Linear{CommandArgument(Tensor_0_size_0)I64}(5)"]; "coord6"[label="Flatten(6)",shape=box]; "coord7"[label="SubDimension{0, CommandArgument(Tensor_2_size_0)I64}(7)"]; "coord8"[label="Split(8)",shape=box]; "coord9"[label="Linear{CommandArgument(Tensor_2_size_0)I64}(9)"]; "coord10"[label="Flatten(10)",shape=box]; "coord11"[label="Linear{NA}(11)"]; "coord12"[label="SubDimension{0, CommandArgument(Tensor_8_size_0)I64}(12)"]; "coord13"[label="Split(13)",shape=box]; "coord14"[label="User{Multiply(CommandArgument(Tensor_8_stride_0)I64, CommandArgument(Tensor_8_size_0)I64)I64}(14)"]; "coord15"[label="Join(15)",shape=box]; "coord16"[label="VGPR{NA}(16)"]; "coord17"[label="Workgroup{0, Divide(Subtract(Add(Multiply(CommandArgument(Tensor_0_stride_0)I64, CommandArgument(Tensor_0_size_0)I64)I64, 64:U32)I64, 1:U32)I64, 64:U32)I64}(17)"]; "coord18"[label="Workitem{0, 64:U32}(18)"]; "coord19"[label="Tile(19)",shape=box]; "coord20"[label="Forget(20)",shape=box]; "coord21"[label="DataFlow(21)",shape=box]; "coord22"[label="VGPR{NA}(22)"]; "coord23"[label="Workgroup{0, Divide(Subtract(Add(Multiply(CommandArgument(Tensor_2_stride_0)I64, CommandArgument(Tensor_2_size_0)I64)I64, 64:U32)I64, 1:U32)I64, 64:U32)I64}(23)"]; "coord24"[label="Workitem{0, 64:U32}(24)"]; "coord25"[label="Tile(25)",shape=box]; "coord26"[label="Forget(26)",shape=box]; "coord27"[label="DataFlow(27)",shape=box]; "coord28"[label="VGPR{NA}(28)"]; "coord29"[label="DataFlow(29)",shape=box]; "coord30"[label="VGPR{NA}(30)"]; "coord31"[label="DataFlow(31)",shape=box]; "coord32"[label="VGPR{NA}(32)"]; "coord33"[label="DataFlow(33)",shape=box]; "coord34"[label="Workgroup{0, Divide(Subtract(Add(Multiply(CommandArgument(Tensor_8_stride_0)I64, CommandArgument(Tensor_8_size_0)I64)I64, 64:U32)I64, 1:U32)I64, 64:U32)I64}(34)"]; "coord35"[label="Workitem{0, 64:U32}(35)"]; "coord36"[label="Inherit(36)",shape=box]; "coord37"[label="Flatten(37)",shape=box]; "coord38"[label="DataFlow(38)",shape=box]; "coord1" -> "coord4" "coord1" -> "coord21" "coord2" -> "coord8" "coord2" -> "coord27" "coord3" -> "coord6" "coord4" -> "coord3" "coord5" -> "coord19" "coord6" -> "coord5" "coord7" -> "coord10" "coord8" -> "coord7" "coord9" -> "coord25" "coord10" -> "coord9" "coord11" -> "coord13" "coord12" -> "coord15" "coord13" -> "coord12" "coord15" -> "coord14" "coord16" -> "coord29" "coord17" -> "coord20" "coord18" -> "coord20" "coord19" -> "coord17" "coord19" -> "coord18" "coord20" -> "coord16" "coord21" -> "coord16" "coord22" -> "coord29" "coord23" -> "coord26" "coord24" -> "coord26" "coord25" -> "coord23" "coord25" -> "coord24" "coord26" -> "coord22" "coord27" -> "coord22" "coord28" -> "coord31" "coord28" -> "coord33" "coord29" -> "coord28" "coord30" -> "coord33" "coord31" -> "coord30" "coord32" -> "coord36" "coord32" -> "coord38" "coord33" -> "coord32" "coord34" -> "coord37" "coord35" -> "coord37" "coord36" -> "coord34" "coord36" -> "coord35" "coord37" -> "coord11" "coord38" -> "coord14" { rank=same "coord17"->"coord18"[style=invis] rankdir=LR } { rank=same "coord17"->"coord18"[style=invis] rankdir=LR } { rank=same "coord23"->"coord24"[style=invis] rankdir=LR } { rank=same "coord23"->"coord24"[style=invis] rankdir=LR } { rank=same "coord16"->"coord22"[style=invis] rankdir=LR } { rank=same "coord28"->"coord30"[style=invis] rankdir=LR } { rank=same "coord34"->"coord35"[style=invis] rankdir=LR } { rank=same "coord34"->"coord35"[style=invis] rankdir=LR } subgraph clusterCF {label = "Control Graph"; "cntrl1"[label="Kernel(1)"]; "cntrl2"[label="LoadVGPR Value: Int32(2)"]; "cntrl3"[label="Body(3)",shape=box]; "cntrl4"[label="LoadVGPR Value: Int32(4)"]; "cntrl5"[label="Body(5)",shape=box]; "cntrl6"[label="Assign VGPR Add(DataFlowTag(16)NA, DataFlowTag(22)NA)NA(6)"]; "cntrl7"[label="Sequence(7)",shape=box]; "cntrl8"[label="Sequence(8)",shape=box]; "cntrl9"[label="Assign VGPR Negate(DataFlowTag(28)NA)NA(9)"]; "cntrl10"[label="Sequence(10)",shape=box]; "cntrl11"[label="Assign VGPR Multiply(DataFlowTag(28)NA, DataFlowTag(30)NA)NA(11)"]; "cntrl12"[label="Sequence(12)",shape=box]; "cntrl13"[label="Sequence(13)",shape=box]; "cntrl14"[label="StoreVGPR(14)"]; "cntrl15"[label="Sequence(15)",shape=box]; "cntrl1" -> "cntrl3" "cntrl1" -> "cntrl5" "cntrl2" -> "cntrl7" "cntrl3" -> "cntrl2" "cntrl4" -> "cntrl8" "cntrl5" -> "cntrl4" "cntrl6" -> "cntrl10" "cntrl6" -> "cntrl12" "cntrl7" -> "cntrl6" "cntrl8" -> "cntrl6" "cntrl9" -> "cntrl13" "cntrl10" -> "cntrl9" "cntrl11" -> "cntrl15" "cntrl12" -> "cntrl11" "cntrl13" -> "cntrl11" "cntrl15" -> "cntrl14" } "coord1" -> "to_cntrl_1_2" "to_cntrl_1_2"[label="2->1: User: (0)", shape=cds] "cntrl2" -> "to_coord_2_1" "to_coord_2_1"[label="2->1: User: (0)", shape=cds] "coord16" -> "to_cntrl_16_2" "to_cntrl_16_2"[label="2->16: VGPR: (0)", shape=cds] "cntrl2" -> "to_coord_2_16" "to_coord_2_16"[label="2->16: VGPR: (0)", shape=cds] "coord2" -> "to_cntrl_2_4" "to_cntrl_2_4"[label="4->2: User: (0)", shape=cds] "cntrl4" -> "to_coord_4_2" "to_coord_4_2"[label="4->2: User: (0)", shape=cds] "coord22" -> "to_cntrl_22_4" "to_cntrl_22_4"[label="4->22: VGPR: (0)", shape=cds] "cntrl4" -> "to_coord_4_22" "to_coord_4_22"[label="4->22: VGPR: (0)", shape=cds] "coord28" -> "to_cntrl_28_6" "to_cntrl_28_6"[label="6->28: DEST", shape=cds] "cntrl6" -> "to_coord_6_28" "to_coord_6_28"[label="6->28: DEST", shape=cds] "coord30" -> "to_cntrl_30_9" "to_cntrl_30_9"[label="9->30: DEST", shape=cds] "cntrl9" -> "to_coord_9_30" "to_coord_9_30"[label="9->30: DEST", shape=cds] "coord32" -> "to_cntrl_32_11" "to_cntrl_32_11"[label="11->32: DEST", shape=cds] "cntrl11" -> "to_coord_11_32" "to_coord_11_32"[label="11->32: DEST", shape=cds] "coord14" -> "to_cntrl_14_14" "to_cntrl_14_14"[label="14->14: User: (0)", shape=cds] "cntrl14" -> "to_coord_14_14" "to_coord_14_14"[label="14->14: User: (0)", shape=cds] "coord32" -> "to_cntrl_32_14" "to_cntrl_32_14"[label="14->32: VGPR: (0)", shape=cds] "cntrl14" -> "to_coord_14_32" "to_coord_14_32"[label="14->32: VGPR: (0)", shape=cds] })."; auto one = Expression::literal(1u); m_context->kernel()->setWorkgroupSize({64, 1, 1}); m_context->kernel()->setWorkitemCount({one, one, one}); auto lowerLinearTransform = std::make_shared(m_context); auto updateParametersTransform = std::make_shared(commandParams); auto kgraph1 = kgraph0.transform(updateParametersTransform); auto kgraph2 = kgraph1.transform(lowerLinearTransform); EXPECT_EQ(NormalizedSource(expected1), NormalizedSource(kgraph2.toDOT(true))); std::string expected2 = R".( digraph { "coord1"[label="User{Multiply(CommandArgument(Tensor_0_stride_0)I64, CommandArgument(Tensor_0_size_0)I64)I64}(1)"]; "coord2"[label="User{Multiply(CommandArgument(Tensor_2_stride_0)I64, CommandArgument(Tensor_2_size_0)I64)I64}(2)"]; "coord3"[label="SubDimension{0, CommandArgument(Tensor_0_size_0)I64}(3)"]; "coord4"[label="Split(4)",shape=box]; "coord5"[label="Linear{CommandArgument(Tensor_0_size_0)I64}(5)"]; "coord6"[label="Flatten(6)",shape=box]; "coord7"[label="Workgroup{0, Divide(Subtract(Add(Multiply(CommandArgument(Tensor_0_stride_0)I64, CommandArgument(Tensor_0_size_0)I64)I64, 64:U32)I64, 1:U32)I64, 64:U32)I64}(7)"]; "coord8"[label="Workitem{0, 64:U32}(8)"]; "coord9"[label="Tile(9)",shape=box]; "coord10"[label="Linear{16:I}(10)"]; "coord11"[label="ForLoop{16:I}(11)"]; "coord12"[label="DataFlow(12)",shape=box]; "coord13"[label="VGPR{NA}(13)"]; "coord14"[label="Forget(14)",shape=box]; "coord15"[label="DataFlow(15)",shape=box]; "coord16"[label="SubDimension{0, CommandArgument(Tensor_2_size_0)I64}(16)"]; "coord17"[label="Split(17)",shape=box]; "coord18"[label="Linear{CommandArgument(Tensor_2_size_0)I64}(18)"]; "coord19"[label="Flatten(19)",shape=box]; "coord20"[label="Workgroup{0, Divide(Subtract(Add(Multiply(CommandArgument(Tensor_2_stride_0)I64, CommandArgument(Tensor_2_size_0)I64)I64, 64:U32)I64, 1:U32)I64, 64:U32)I64}(20)"]; "coord21"[label="Workitem{0, 64:U32}(21)"]; "coord22"[label="Tile(22)",shape=box]; "coord23"[label="ForLoop{16:I}(23)"]; "coord24"[label="DataFlow(24)",shape=box]; "coord25"[label="VGPR{NA}(25)"]; "coord26"[label="Forget(26)",shape=box]; "coord27"[label="DataFlow(27)",shape=box]; "coord28"[label="VGPR{NA}(28)"]; "coord29"[label="DataFlow(29)",shape=box]; "coord30"[label="VGPR{NA}(30)"]; "coord31"[label="DataFlow(31)",shape=box]; "coord32"[label="VGPR{NA}(32)"]; "coord33"[label="DataFlow(33)",shape=box]; "coord34"[label="Workgroup{0, Divide(Subtract(Add(Multiply(CommandArgument(Tensor_8_stride_0)I64, CommandArgument(Tensor_8_size_0)I64)I64, 64:U32)I64, 1:U32)I64, 64:U32)I64}(34)"]; "coord35"[label="Workitem{0, 64:U32}(35)"]; "coord36"[label="Inherit(36)",shape=box]; "coord37"[label="ForLoop{16:I}(37)"]; "coord38"[label="DataFlow(38)",shape=box]; "coord39"[label="Linear{NA}(39)"]; "coord40"[label="Flatten(40)",shape=box]; "coord41"[label="SubDimension{0, CommandArgument(Tensor_8_size_0)I64}(41)"]; "coord42"[label="Split(42)",shape=box]; "coord43"[label="User{Multiply(CommandArgument(Tensor_8_stride_0)I64, CommandArgument(Tensor_8_size_0)I64)I64}(43)"]; "coord44"[label="Join(44)",shape=box]; "coord45"[label="DataFlow(45)",shape=box]; "coord1" -> "coord4" "coord1" -> "coord15" "coord2" -> "coord17" "coord2" -> "coord27" "coord3" -> "coord6" "coord4" -> "coord3" "coord5" -> "coord9" "coord6" -> "coord5" "coord7" -> "coord14" "coord8" -> "coord14" "coord9" -> "coord11" "coord9" -> "coord7" "coord9" -> "coord8" "coord10" -> "coord12" "coord10" -> "coord24" "coord10" -> "coord38" "coord11" -> "coord14" "coord12" -> "coord11" "coord13" -> "coord29" "coord14" -> "coord13" "coord15" -> "coord13" "coord16" -> "coord19" "coord17" -> "coord16" "coord18" -> "coord22" "coord19" -> "coord18" "coord20" -> "coord26" "coord21" -> "coord26" "coord22" -> "coord23" "coord22" -> "coord20" "coord22" -> "coord21" "coord23" -> "coord26" "coord24" -> "coord23" "coord25" -> "coord29" "coord26" -> "coord25" "coord27" -> "coord25" "coord28" -> "coord31" "coord28" -> "coord33" "coord29" -> "coord28" "coord30" -> "coord33" "coord31" -> "coord30" "coord32" -> "coord36" "coord32" -> "coord45" "coord33" -> "coord32" "coord34" -> "coord40" "coord35" -> "coord40" "coord36" -> "coord37" "coord36" -> "coord34" "coord36" -> "coord35" "coord37" -> "coord40" "coord38" -> "coord37" "coord39" -> "coord42" "coord40" -> "coord39" "coord41" -> "coord44" "coord42" -> "coord41" "coord44" -> "coord43" "coord45" -> "coord43" { rank=same "coord11"->"coord7"->"coord8"[style=invis] rankdir=LR } { rank=same "coord11"->"coord7"->"coord8"[style=invis] rankdir=LR } { rank=same "coord23"->"coord20"->"coord21"[style=invis] rankdir=LR } { rank=same "coord23"->"coord20"->"coord21"[style=invis] rankdir=LR } { rank=same "coord13"->"coord25"[style=invis] rankdir=LR } { rank=same "coord28"->"coord30"[style=invis] rankdir=LR } { rank=same "coord37"->"coord34"->"coord35"[style=invis] rankdir=LR } { rank=same "coord37"->"coord34"->"coord35"[style=invis] rankdir=LR } subgraph clusterCF {label = "Control Graph"; "cntrl1"[label="Kernel(1)"]; "cntrl2"[label="ForLoopOp : LessThan(DataFlowTag(10)I, 16:I)BL(2)"]; "cntrl3"[label="Body(3)",shape=box]; "cntrl4"[label="Assign SGPR 0:I(4)"]; "cntrl5"[label="Initialize(5)",shape=box]; "cntrl6"[label="Assign SGPR Add(DataFlowTag(10)I, 1:I)I(6)"]; "cntrl7"[label="ForLoopIncrement(7)",shape=box]; "cntrl8"[label="LoadVGPR Value: Int32(8)"]; "cntrl9"[label="Body(9)",shape=box]; "cntrl10"[label="LoadVGPR Value: Int32(10)"]; "cntrl11"[label="Body(11)",shape=box]; "cntrl12"[label="Assign VGPR Add(DataFlowTag(13)NA, DataFlowTag(25)NA)NA(12)"]; "cntrl13"[label="Sequence(13)",shape=box]; "cntrl14"[label="Sequence(14)",shape=box]; "cntrl15"[label="Assign VGPR Negate(DataFlowTag(28)NA)NA(15)"]; "cntrl16"[label="Sequence(16)",shape=box]; "cntrl17"[label="Assign VGPR Multiply(DataFlowTag(28)NA, DataFlowTag(30)NA)NA(17)"]; "cntrl18"[label="Sequence(18)",shape=box]; "cntrl19"[label="Sequence(19)",shape=box]; "cntrl20"[label="StoreVGPR(20)"]; "cntrl21"[label="Sequence(21)",shape=box]; "cntrl1" -> "cntrl3" "cntrl2" -> "cntrl5" "cntrl2" -> "cntrl7" "cntrl2" -> "cntrl9" "cntrl2" -> "cntrl11" "cntrl3" -> "cntrl2" "cntrl5" -> "cntrl4" "cntrl7" -> "cntrl6" "cntrl8" -> "cntrl13" "cntrl9" -> "cntrl8" "cntrl10" -> "cntrl14" "cntrl11" -> "cntrl10" "cntrl12" -> "cntrl16" "cntrl12" -> "cntrl18" "cntrl13" -> "cntrl12" "cntrl14" -> "cntrl12" "cntrl15" -> "cntrl19" "cntrl16" -> "cntrl15" "cntrl17" -> "cntrl21" "cntrl18" -> "cntrl17" "cntrl19" -> "cntrl17" "cntrl21" -> "cntrl20" } "coord10" -> "to_cntrl_10_2" "to_cntrl_10_2"[label="2->10: ForLoop: (0)", shape=cds] "cntrl2" -> "to_coord_2_10" "to_coord_2_10"[label="2->10: ForLoop: (0)", shape=cds] "coord10" -> "to_cntrl_10_4" "to_cntrl_10_4"[label="4->10: DEST", shape=cds] "cntrl4" -> "to_coord_4_10" "to_coord_4_10"[label="4->10: DEST", shape=cds] "coord10" -> "to_cntrl_10_6" "to_cntrl_10_6"[label="6->10: DEST", shape=cds] "cntrl6" -> "to_coord_6_10" "to_coord_6_10"[label="6->10: DEST", shape=cds] "coord1" -> "to_cntrl_1_8" "to_cntrl_1_8"[label="8->1: User: (0)", shape=cds] "cntrl8" -> "to_coord_8_1" "to_coord_8_1"[label="8->1: User: (0)", shape=cds] "coord13" -> "to_cntrl_13_8" "to_cntrl_13_8"[label="8->13: VGPR: (0)", shape=cds] "cntrl8" -> "to_coord_8_13" "to_coord_8_13"[label="8->13: VGPR: (0)", shape=cds] "coord2" -> "to_cntrl_2_10" "to_cntrl_2_10"[label="10->2: User: (0)", shape=cds] "cntrl10" -> "to_coord_10_2" "to_coord_10_2"[label="10->2: User: (0)", shape=cds] "coord25" -> "to_cntrl_25_10" "to_cntrl_25_10"[label="10->25: VGPR: (0)", shape=cds] "cntrl10" -> "to_coord_10_25" "to_coord_10_25"[label="10->25: VGPR: (0)", shape=cds] "coord28" -> "to_cntrl_28_12" "to_cntrl_28_12"[label="12->28: DEST", shape=cds] "cntrl12" -> "to_coord_12_28" "to_coord_12_28"[label="12->28: DEST", shape=cds] "coord30" -> "to_cntrl_30_15" "to_cntrl_30_15"[label="15->30: DEST", shape=cds] "cntrl15" -> "to_coord_15_30" "to_coord_15_30"[label="15->30: DEST", shape=cds] "coord32" -> "to_cntrl_32_17" "to_cntrl_32_17"[label="17->32: DEST", shape=cds] "cntrl17" -> "to_coord_17_32" "to_coord_17_32"[label="17->32: DEST", shape=cds] "coord32" -> "to_cntrl_32_20" "to_cntrl_32_20"[label="20->32: VGPR: (0)", shape=cds] "cntrl20" -> "to_coord_20_32" "to_coord_20_32"[label="20->32: VGPR: (0)", shape=cds] "coord43" -> "to_cntrl_43_20" "to_cntrl_43_20"[label="20->43: User: (0)", shape=cds] "cntrl20" -> "to_coord_20_43" "to_coord_20_43"[label="20->43: User: (0)", shape=cds] })."; int loopSize = 16; auto loopSizeExpr = Expression::literal(loopSize); auto lowerLinerLoopTransform = std::make_shared(loopSizeExpr, m_context); auto kgraph3 = kgraph2.transform(lowerLinerLoopTransform); EXPECT_EQ(NormalizedSource(expected2), NormalizedSource(kgraph3.toDOT(true))); } TEST_F(KernelGraphTest, BasicTranslateScalar) { auto example = rocRollerTest::Graphs::VectorAddNegSquare(true); auto kgraph0 = example.getKernelGraph(); auto bottom = kgraph0.coordinates.roots().to(); EXPECT_EQ(bottom.size(), 2); for(auto const& id : bottom) { EXPECT_TRUE(kgraph0.coordinates.get(id).has_value()); } std::string expected0 = R".( digraph { "coord1"[label="User{NA}(1)"]; "coord2"[label="VGPR{NA}(2)"]; "coord3"[label="DataFlow(3)",shape=box]; "coord4"[label="User{NA}(4)"]; "coord5"[label="VGPR{NA}(5)"]; "coord6"[label="DataFlow(6)",shape=box]; "coord7"[label="VGPR{NA}(7)"]; "coord8"[label="DataFlow(8)",shape=box]; "coord9"[label="VGPR{NA}(9)"]; "coord10"[label="DataFlow(10)",shape=box]; "coord11"[label="VGPR{NA}(11)"]; "coord12"[label="DataFlow(12)",shape=box]; "coord1" -> "coord3" "coord2" -> "coord8" "coord3" -> "coord2" "coord4" -> "coord6" "coord5" -> "coord8" "coord6" -> "coord5" "coord7" -> "coord10" "coord7" -> "coord12" "coord8" -> "coord7" "coord9" -> "coord12" "coord10" -> "coord9" "coord12" -> "coord11" { rank=same "coord2"->"coord5"[style=invis] rankdir=LR } { rank=same "coord7"->"coord9"[style=invis] rankdir=LR } subgraph clusterCF {label = "Control Graph"; "cntrl1"[label="Kernel(1)"]; "cntrl2"[label="LoadVGPR Value: Int32(2)"]; "cntrl3"[label="Body(3)",shape=box]; "cntrl4"[label="LoadVGPR Value: Int32(4)"]; "cntrl5"[label="Body(5)",shape=box]; "cntrl6"[label="Assign VGPR Add(DataFlowTag(2)NA, DataFlowTag(5)NA)NA(6)"]; "cntrl7"[label="Sequence(7)",shape=box]; "cntrl8"[label="Sequence(8)",shape=box]; "cntrl9"[label="Assign VGPR Negate(DataFlowTag(7)NA)NA(9)"]; "cntrl10"[label="Sequence(10)",shape=box]; "cntrl11"[label="Assign VGPR Multiply(DataFlowTag(7)NA, DataFlowTag(9)NA)NA(11)"]; "cntrl12"[label="Sequence(12)",shape=box]; "cntrl13"[label="Sequence(13)",shape=box]; "cntrl1" -> "cntrl3" "cntrl1" -> "cntrl5" "cntrl2" -> "cntrl7" "cntrl3" -> "cntrl2" "cntrl4" -> "cntrl8" "cntrl5" -> "cntrl4" "cntrl6" -> "cntrl10" "cntrl6" -> "cntrl12" "cntrl7" -> "cntrl6" "cntrl8" -> "cntrl6" "cntrl9" -> "cntrl13" "cntrl10" -> "cntrl9" "cntrl12" -> "cntrl11" "cntrl13" -> "cntrl11" } "coord1" -> "to_cntrl_1_2" "to_cntrl_1_2"[label="2->1: User: (0)", shape=cds] "cntrl2" -> "to_coord_2_1" "to_coord_2_1"[label="2->1: User: (0)", shape=cds] "coord2" -> "to_cntrl_2_2" "to_cntrl_2_2"[label="2->2: VGPR: (0)", shape=cds] "cntrl2" -> "to_coord_2_2" "to_coord_2_2"[label="2->2: VGPR: (0)", shape=cds] "coord4" -> "to_cntrl_4_4" "to_cntrl_4_4"[label="4->4: User: (0)", shape=cds] "cntrl4" -> "to_coord_4_4" "to_coord_4_4"[label="4->4: User: (0)", shape=cds] "coord5" -> "to_cntrl_5_4" "to_cntrl_5_4"[label="4->5: VGPR: (0)", shape=cds] "cntrl4" -> "to_coord_4_5" "to_coord_4_5"[label="4->5: VGPR: (0)", shape=cds] "coord7" -> "to_cntrl_7_6" "to_cntrl_7_6"[label="6->7: DEST", shape=cds] "cntrl6" -> "to_coord_6_7" "to_coord_6_7"[label="6->7: DEST", shape=cds] "coord9" -> "to_cntrl_9_9" "to_cntrl_9_9"[label="9->9: DEST", shape=cds] "cntrl9" -> "to_coord_9_9" "to_coord_9_9"[label="9->9: DEST", shape=cds] "coord11" -> "to_cntrl_11_11" "to_cntrl_11_11"[label="11->11: DEST", shape=cds] "cntrl11" -> "to_coord_11_11" "to_coord_11_11"[label="11->11: DEST", shape=cds] })."; EXPECT_EQ(NormalizedSource(expected0), NormalizedSource(kgraph0.toDOT(true))); } TEST_F(KernelGraphTest, TranslateMatrixMultiply) { auto example = rocRollerTest::Graphs::MatrixMultiply(DataType::Int32); auto kgraph0 = example.getKernelGraph(); auto bottom = kgraph0.coordinates.roots().to(); EXPECT_EQ(bottom.size(), 2); for(auto const& id : bottom) { EXPECT_TRUE(kgraph0.coordinates.get(id).has_value()); } auto top = kgraph0.coordinates.leaves().to(); EXPECT_EQ(top.size(), 1); for(auto const& id : top) { EXPECT_TRUE(kgraph0.coordinates.get(id).has_value()); } std::string expected0 = R".( digraph { "coord1"[label="User{NA}(1)"]; "coord2"[label="SubDimension{0, CommandArgument(Tensor_0_size_0)I64}(2)"]; "coord3"[label="SubDimension{1, CommandArgument(Tensor_0_size_1)I64}(3)"]; "coord4"[label="MacroTile{NA}(2/None/None){}-()(4)"]; "coord5"[label="Split(5)",shape=box]; "coord6"[label="ConstructMacroTile(6)",shape=box]; "coord7"[label="DataFlow(7)",shape=box]; "coord8"[label="User{NA}(8)"]; "coord9"[label="SubDimension{0, CommandArgument(Tensor_2_size_0)I64}(9)"]; "coord10"[label="SubDimension{1, CommandArgument(Tensor_2_size_1)I64}(10)"]; "coord11"[label="MacroTile{NA}(2/None/None){}-()(11)"]; "coord12"[label="Split(12)",shape=box]; "coord13"[label="ConstructMacroTile(13)",shape=box]; "coord14"[label="DataFlow(14)",shape=box]; "coord15"[label="MacroTile{NA}(0/None/None){}-()(15)"]; "coord16"[label="DataFlow(16)",shape=box]; "coord17"[label="User{NA}(17)"]; "coord18"[label="SubDimension{0, CommandArgument(Tensor_5_size_0)I64}(18)"]; "coord19"[label="SubDimension{1, CommandArgument(Tensor_5_size_1)I64}(19)"]; "coord20"[label="DestructMacroTile(20)",shape=box]; "coord21"[label="Join(21)",shape=box]; "coord22"[label="DataFlow(22)",shape=box]; "coord1" -> "coord5" "coord1" -> "coord7" "coord2" -> "coord6" "coord3" -> "coord6" "coord4" -> "coord16" "coord5" -> "coord2" "coord5" -> "coord3" "coord6" -> "coord4" "coord7" -> "coord4" "coord8" -> "coord12" "coord8" -> "coord14" "coord9" -> "coord13" "coord10" -> "coord13" "coord11" -> "coord16" "coord12" -> "coord9" "coord12" -> "coord10" "coord13" -> "coord11" "coord14" -> "coord11" "coord15" -> "coord20" "coord15" -> "coord22" "coord16" -> "coord15" "coord18" -> "coord21" "coord19" -> "coord21" "coord20" -> "coord18" "coord20" -> "coord19" "coord21" -> "coord17" "coord22" -> "coord17" { rank=same "coord2"->"coord3"[style=invis] rankdir=LR } { rank=same "coord2"->"coord3"[style=invis] rankdir=LR } { rank=same "coord9"->"coord10"[style=invis] rankdir=LR } { rank=same "coord9"->"coord10"[style=invis] rankdir=LR } { rank=same "coord4"->"coord11"[style=invis] rankdir=LR } { rank=same "coord18"->"coord19"[style=invis] rankdir=LR } { rank=same "coord18"->"coord19"[style=invis] rankdir=LR } subgraph clusterCF {label = "Control Graph"; "cntrl1"[label="Kernel(1)"]; "cntrl2"[label="LoadTiled Value: Int32(2)"]; "cntrl3"[label="Body(3)",shape=box]; "cntrl4"[label="LoadTiled Value: Int32(4)"]; "cntrl5"[label="Body(5)",shape=box]; "cntrl6"[label="TensorContraction(6)"]; "cntrl7"[label="Sequence(7)",shape=box]; "cntrl8"[label="Sequence(8)",shape=box]; "cntrl9"[label="StoreTiled Value: Int32(9)"]; "cntrl10"[label="Sequence(10)",shape=box]; "cntrl1" -> "cntrl3" "cntrl1" -> "cntrl5" "cntrl2" -> "cntrl7" "cntrl3" -> "cntrl2" "cntrl4" -> "cntrl8" "cntrl5" -> "cntrl4" "cntrl6" -> "cntrl10" "cntrl7" -> "cntrl6" "cntrl8" -> "cntrl6" "cntrl10" -> "cntrl9" } "coord1" -> "to_cntrl_1_2" "to_cntrl_1_2"[label="2->1: User: (0)", shape=cds] "cntrl2" -> "to_coord_2_1" "to_coord_2_1"[label="2->1: User: (0)", shape=cds] "coord4" -> "to_cntrl_4_2" "to_cntrl_4_2"[label="2->4: MacroTile: (0)", shape=cds] "cntrl2" -> "to_coord_2_4" "to_coord_2_4"[label="2->4: MacroTile: (0)", shape=cds] "coord8" -> "to_cntrl_8_4" "to_cntrl_8_4"[label="4->8: User: (0)", shape=cds] "cntrl4" -> "to_coord_4_8" "to_coord_4_8"[label="4->8: User: (0)", shape=cds] "coord11" -> "to_cntrl_11_4" "to_cntrl_11_4"[label="4->11: MacroTile: (0)", shape=cds] "cntrl4" -> "to_coord_4_11" "to_coord_4_11"[label="4->11: MacroTile: (0)", shape=cds] "coord4" -> "to_cntrl_4_6" "to_cntrl_4_6"[label="6->4: LHS", shape=cds] "cntrl6" -> "to_coord_6_4" "to_coord_6_4"[label="6->4: LHS", shape=cds] "coord11" -> "to_cntrl_11_6" "to_cntrl_11_6"[label="6->11: RHS", shape=cds] "cntrl6" -> "to_coord_6_11" "to_coord_6_11"[label="6->11: RHS", shape=cds] "coord15" -> "to_cntrl_15_6" "to_cntrl_15_6"[label="6->15: DEST", shape=cds] "cntrl6" -> "to_coord_6_15" "to_coord_6_15"[label="6->15: DEST", shape=cds] "coord15" -> "to_cntrl_15_9" "to_cntrl_15_9"[label="9->15: MacroTile: (0)", shape=cds] "cntrl9" -> "to_coord_9_15" "to_coord_9_15"[label="9->15: MacroTile: (0)", shape=cds] "coord17" -> "to_cntrl_17_9" "to_cntrl_17_9"[label="9->17: User: (0)", shape=cds] "cntrl9" -> "to_coord_9_17" "to_coord_9_17"[label="9->17: User: (0)", shape=cds] })."; EXPECT_EQ(NormalizedSource(expected0), NormalizedSource(kgraph0.toDOT(true))); } TEST_F(KernelGraphTest, TranslateUnscaledMatrixMultiply) { auto example = rocRollerTest::Graphs::MatrixMultiply(DataType::FP8, DataType::FP6, DataType::Float, Operations::ScaleMode::None, Operations::ScaleMode::None); auto expectedCommand = R".( Tensor.FP8.d2 0, (base=&0, sizes={&8 &16 }, strides={&24 &32 }) T_LOAD_TILED 1 Source 0 Tensor.FP6.d2 2, (base=&40, sizes={&48 &56 }, strides={&64 &72 }) T_LOAD_TILED 3 Source 2 T_Mul 1 3 Value: Float Tensor.Float.d2 5, (base=&80, sizes={&88 &96 }, strides={&104 &112 }) T_STORE_TILED 6 Source 4 Dest 5 )."; EXPECT_EQ(NormalizedSource(expectedCommand), NormalizedSource(example.getCommand()->toString())); auto kgraph0 = example.getKernelGraph(); auto roots = kgraph0.coordinates.roots().to(); EXPECT_EQ(roots.size(), 2); for(auto const& id : roots) { EXPECT_TRUE(std::holds_alternative( std::get(kgraph0.coordinates.getElement(id)))); } auto leaves = kgraph0.coordinates.leaves().to(); ASSERT_EQ(leaves.size(), 1); for(auto const& id : leaves) { EXPECT_TRUE(std::holds_alternative( std::get(kgraph0.coordinates.getElement(id)))); } auto reachableFromLeaf = kgraph0.coordinates.depthFirstVisit(leaves[0], Graph::Direction::Upstream) .to(); for(auto id : roots) { EXPECT_TRUE(reachableFromLeaf.contains(id)) << ShowValue(id) << reachableFromLeaf; } // There should be 4 LoadTiled nodes with direct sequence edges to the single TensorContraction node. // These should be all of the LoadTiled nodes in the graph. auto is_tc = kgraph0.control.isElemType(); auto tc_id = only(kgraph0.control.findElements(is_tc)); ASSERT_TRUE(tc_id.has_value()); auto is_loadTiled = kgraph0.control.isElemType(); auto loadTiled_ids = kgraph0.control.findElements(is_loadTiled).to(); auto tc_parent_ids = kgraph0.control.parentNodes(*tc_id).to(); EXPECT_EQ(loadTiled_ids, tc_parent_ids); EXPECT_EQ(2, loadTiled_ids.size()); } TEST_F(KernelGraphTest, TranslateScaledMatrixMultiply) { auto example = rocRollerTest::Graphs::MatrixMultiply(DataType::FP8, DataType::FP6, DataType::Float, Operations::ScaleMode::Separate, Operations::ScaleMode::Separate); auto expectedCommand = R".( Tensor.FP8.d2 0, (base=&0, sizes={&8 &16 }, strides={&24 &32 }) T_LOAD_TILED 1 Source 0 Tensor.E8M0.d2 2, (base=&40, sizes={&48 &56 }, strides={&64 &72 }) T_LOAD_TILED 3 Source 2 BlockScale(Separate, {1, 32}): Data: 1, Scale: 3 Tensor.FP6.d2 5, (base=&80, sizes={&88 &96 }, strides={&104 &112 }) T_LOAD_TILED 6 Source 5 Tensor.E8M0.d2 7, (base=&120, sizes={&128 &136 }, strides={&144 &152 }) T_LOAD_TILED 8 Source 7 BlockScale(Separate, {32, 1}): Data: 6, Scale: 8 T_Mul 4 9 Value: Float Tensor.Float.d2 11, (base=&160, sizes={&168 &176 }, strides={&184 &192 }) T_STORE_TILED 12 Source 10 Dest 11 )."; EXPECT_EQ(NormalizedSource(expectedCommand), NormalizedSource(example.getCommand()->toString())); auto kgraph0 = example.getKernelGraph(); auto roots = kgraph0.coordinates.roots().to(); EXPECT_EQ(roots.size(), 4); for(auto const& id : roots) { EXPECT_TRUE(std::holds_alternative( std::get(kgraph0.coordinates.getElement(id)))); } auto leaves = kgraph0.coordinates.leaves().to(); ASSERT_EQ(leaves.size(), 1); for(auto const& id : leaves) { EXPECT_TRUE(std::holds_alternative( std::get(kgraph0.coordinates.getElement(id)))); } auto reachableFromLeaf = kgraph0.coordinates.depthFirstVisit(leaves[0], Graph::Direction::Upstream) .to(); for(auto id : roots) { EXPECT_TRUE(reachableFromLeaf.contains(id)) << ShowValue(id) << reachableFromLeaf; } // There should be 4 LoadTiled nodes with direct sequence edges to the single TensorContraction node. // These should be all of the LoadTiled nodes in the graph. auto is_tc = kgraph0.control.isElemType(); auto tc_id = only(kgraph0.control.findElements(is_tc)); ASSERT_TRUE(tc_id.has_value()); auto is_loadTiled = kgraph0.control.isElemType(); auto loadTiled_ids = kgraph0.control.findElements(is_loadTiled).to(); auto tc_parent_ids = kgraph0.control.parentNodes(*tc_id).to(); EXPECT_EQ(loadTiled_ids, tc_parent_ids); EXPECT_EQ(4, loadTiled_ids.size()); } TEST_F(KernelGraphTest, LowerTensor) { auto example = rocRollerTest::Graphs::GEMM(DataType::Float); int macK = 16; int waveK = 8; example.setTileSize(128, 256, macK); example.setMFMA(32, 32, waveK, 1); example.setUseLDS(true, false, false); auto kgraph0 = example.getKernelGraph(); auto params = example.getCommandParameters(); auto updateParametersTransform = std::make_shared(params); auto addLDSTransform = std::make_shared(params, m_context); auto lowerTileTransform = std::make_shared(params, m_context); auto lowerTensorContractionTransform = std::make_shared(params, m_context); auto unrollLoopsTransform = std::make_shared(params, m_context); auto fuseLoopsTransform = std::make_shared(); auto removeDuplicatesTransform = std::make_shared(); auto cleanLoopsTransform = std::make_shared(); auto updateWavefrontParametersTransform = std::make_shared(params); auto assignIndexExprsTransform = std::make_shared(m_context, example.getCommand()); kgraph0 = kgraph0.transform(updateParametersTransform); auto kgraph1 = kgraph0.transform(addLDSTransform); kgraph1 = kgraph1.transform(lowerTileTransform); kgraph1 = kgraph1.transform(lowerTensorContractionTransform); // Verify the number of Multiply nodes in the graph after lowerTile auto multiplyNodes = kgraph1.control.getNodes().to(); EXPECT_EQ(multiplyNodes.size(), macK / waveK); // Verify number of loads auto loads = kgraph0.control.getNodes().to(); EXPECT_EQ(loads.size(), 3); // A, B, C loads = kgraph1.control.getNodes().to(); EXPECT_EQ(loads.size(), 4); // 1 for A, 2 for B (no LDS), 1 for C loads = kgraph1.control.getNodes().to(); EXPECT_EQ(loads.size(), 2); // 2 for A auto forLoops = kgraph1.control.getNodes().to(); EXPECT_EQ(forLoops.size(), 5); // main: X, Y, K; epilogue: X, Y auto kgraphUnrolled = kgraph1.transform(unrollLoopsTransform); // Verify that loops have been unrolled auto unrolledForLoops = kgraphUnrolled.control.getNodes().to(); EXPECT_EQ(unrolledForLoops.size(), 14); auto kgraphFused = kgraphUnrolled.transform(fuseLoopsTransform); kgraphFused = kgraphFused.transform(removeDuplicatesTransform); // Verify that loops have been fused auto fusedForLoops = kgraphFused.control.getNodes().to(); EXPECT_EQ(fusedForLoops.size(), 5); auto fusedLoads = kgraphFused.control.getNodes().to(); EXPECT_EQ(fusedLoads.size(), 17); // Verify that single iteration loops have been removed. auto kgraphClean = kgraphFused.transform(cleanLoopsTransform); auto cleanedForLoops = kgraphClean.control.getNodes().to(); EXPECT_EQ(cleanedForLoops.size(), 1); // Verify that there is only a single StoreLDSTile node per K loop auto unrolled_kgraph_lds = kgraphUnrolled.transform(addLDSTransform); auto unrolledStoreLDS = unrolled_kgraph_lds.control.getNodes().to(); auto kloops = kgraphUnrolled.control.getNodes() .filter([&](int node) { auto loop = kgraphUnrolled.control.getNode(node); return loop.loopName == KLOOP; }) .to(); EXPECT_EQ(unrolledStoreLDS.size(), kloops.size()); // Verify number of Assigns: A loads; A LDS loads; B loads; C load; D kgraph1 = kgraph1.transform(updateWavefrontParametersTransform); kgraph1 = kgraph1.transform(assignIndexExprsTransform); auto indexAssigns = kgraph1.control.getNodes().to(); EXPECT_EQ(indexAssigns.size(), 44); // Verify number of deallocated dimensions. They may be merged into fewer deallocate nodes. auto addDeallocate = std::make_shared(); auto kgraph2 = kgraph1.transform(addDeallocate); { std::set deallocatedDims; auto deallocates = kgraph2.control.getNodes(); for(auto deallocate : deallocates) { auto connections = kgraph2.mapper.getConnections(deallocate); for(auto const& c : connections) { EXPECT_THAT(deallocatedDims, ::testing::Not(::testing::Contains(c.coordinate))); deallocatedDims.insert(c.coordinate); } } EXPECT_EQ(deallocatedDims.size(), 61); } auto storeLDS = kgraphUnrolled.control.getNodes().to(); EXPECT_EQ(storeLDS.size(), 8); auto fusedStoreLDS = kgraphFused.control.getNodes().to(); EXPECT_EQ(fusedStoreLDS.size(), 1); // Verify number of Assigns after unroll/fuse/lds unrolled_kgraph_lds = unrolled_kgraph_lds.transform(updateWavefrontParametersTransform); unrolled_kgraph_lds = unrolled_kgraph_lds.transform(assignIndexExprsTransform); indexAssigns = unrolled_kgraph_lds.control.getNodes().to(); EXPECT_EQ(indexAssigns.size(), 248); // Verify number of deallocated dimensions. They may be merged into fewer deallocate nodes. unrolled_kgraph_lds = unrolled_kgraph_lds.transform(addDeallocate); { std::set deallocatedDims; auto deallocates = unrolled_kgraph_lds.control.getNodes(); for(auto deallocate : deallocates) { auto connections = unrolled_kgraph_lds.mapper.getConnections(deallocate); for(auto const& c : connections) { EXPECT_THAT(deallocatedDims, ::testing::Not(::testing::Contains(c.coordinate))); deallocatedDims.insert(c.coordinate); } } EXPECT_EQ(deallocatedDims.size(), 303); } } TEST_F(KernelGraphTest, InlineIncrement) { auto example = rocRollerTest::Graphs::GEMM(DataType::Float); example.setTileSize(128, 256, 8); example.setMFMA(32, 32, 2, 1); example.setUseLDS(true, true, true); auto kgraph = example.getKernelGraph(); auto params = example.getCommandParameters(); auto updateParametersTransform = std::make_shared(params); auto addLDSTransform = std::make_shared(params, m_context); auto lowerLinearTransform = std::make_shared(m_context); auto lowerTileTransform = std::make_shared(params, m_context); auto lowerTensorContractionTransform = std::make_shared(params, m_context); auto unrollLoopsTransform = std::make_shared(params, m_context); auto cleanLoopsTransform = std::make_shared(); auto updateWavefrontParametersTransform = std::make_shared(params); auto assignIndexExprsTransform = std::make_shared(m_context, example.getCommand()); auto inlineInrecrementsTransform = std::make_shared(); kgraph = kgraph.transform(updateParametersTransform); kgraph = kgraph.transform(addLDSTransform); kgraph = kgraph.transform(lowerLinearTransform); kgraph = kgraph.transform(lowerTileTransform); kgraph = kgraph.transform(lowerTensorContractionTransform); // Usual lowering, should be able to inline everything. auto kgraph1 = kgraph.transform(unrollLoopsTransform); kgraph1 = kgraph1.transform(updateWavefrontParametersTransform); kgraph1 = kgraph1.transform(cleanLoopsTransform); kgraph1 = kgraph1.transform(assignIndexExprsTransform); auto pre1 = kgraph1.control.getEdges().to(); kgraph1 = kgraph1.transform(inlineInrecrementsTransform); auto post1 = kgraph1.control.getEdges().to(); EXPECT_TRUE(pre1.size() > 0); EXPECT_TRUE(post1.empty()); } TEST_F(KernelGraphTest, TileAdd) { auto example = rocRollerTest::Graphs::TileDoubleAdd(); example.setTileSize(16, 8); example.setSubTileSize(4, 2); auto params = example.getCommandParameters(512, 512); auto kgraph0 = example.getKernelGraph(); auto updateParametersTransform = std::make_shared(params); kgraph0 = kgraph0.transform(updateParametersTransform); std::string expected0 = R".( digraph { "coord1"[label="User{Multiply(CommandArgument(Tensor_0_stride_0)I64, CommandArgument(Tensor_0_size_0)I64)I64}(1)"]; "coord2"[label="SubDimension{0, CommandArgument(Tensor_0_size_0)I64}(2)"]; "coord3"[label="SubDimension{1, CommandArgument(Tensor_0_size_1)I64}(3)"]; "coord4"[label="MacroTile{NA}(2/LDS/ROW_MAJOR){16,8}-(4,2)(4)"]; "coord5"[label="Split(5)",shape=box]; "coord6"[label="ConstructMacroTile(6)",shape=box]; "coord7"[label="DataFlow(7)",shape=box]; "coord8"[label="User{Multiply(CommandArgument(Tensor_2_stride_0)I64, CommandArgument(Tensor_2_size_0)I64)I64}(8)"]; "coord9"[label="SubDimension{0, CommandArgument(Tensor_2_size_0)I64}(9)"]; "coord10"[label="SubDimension{1, CommandArgument(Tensor_2_size_1)I64}(10)"]; "coord11"[label="MacroTile{NA}(2/VGPR/ROW_MAJOR){16,8}-(4,2)(11)"]; "coord12"[label="Split(12)",shape=box]; "coord13"[label="ConstructMacroTile(13)",shape=box]; "coord14"[label="DataFlow(14)",shape=box]; "coord15"[label="MacroTile{NA}(2/VGPR/ROW_MAJOR){16,8}-(4,2)(15)"]; "coord16"[label="DataFlow(16)",shape=box]; "coord17"[label="MacroTile{NA}(2/VGPR/ROW_MAJOR){16,8}-(4,2)(17)"]; "coord18"[label="DataFlow(18)",shape=box]; "coord19"[label="MacroTile{NA}(2/VGPR/ROW_MAJOR){16,8}-(4,2)(19)"]; "coord20"[label="DataFlow(20)",shape=box]; "coord21"[label="User{Multiply(CommandArgument(Tensor_8_stride_0)I64, CommandArgument(Tensor_8_size_0)I64)I64}(21)"]; "coord22"[label="SubDimension{0, CommandArgument(Tensor_8_size_0)I64}(22)"]; "coord23"[label="SubDimension{1, CommandArgument(Tensor_8_size_1)I64}(23)"]; "coord24"[label="DestructMacroTile(24)",shape=box]; "coord25"[label="Join(25)",shape=box]; "coord26"[label="DataFlow(26)",shape=box]; "coord1" -> "coord5" "coord1" -> "coord7" "coord2" -> "coord6" "coord3" -> "coord6" "coord4" -> "coord16" "coord5" -> "coord2" "coord5" -> "coord3" "coord6" -> "coord4" "coord7" -> "coord4" "coord8" -> "coord12" "coord8" -> "coord14" "coord9" -> "coord13" "coord10" -> "coord13" "coord11" -> "coord18" "coord12" -> "coord9" "coord12" -> "coord10" "coord13" -> "coord11" "coord14" -> "coord11" "coord15" -> "coord20" "coord16" -> "coord15" "coord17" -> "coord20" "coord18" -> "coord17" "coord19" -> "coord24" "coord19" -> "coord26" "coord20" -> "coord19" "coord22" -> "coord25" "coord23" -> "coord25" "coord24" -> "coord22" "coord24" -> "coord23" "coord25" -> "coord21" "coord26" -> "coord21" { rank=same "coord2"->"coord3"[style=invis] rankdir=LR } { rank=same "coord2"->"coord3"[style=invis] rankdir=LR } { rank=same "coord9"->"coord10"[style=invis] rankdir=LR } { rank=same "coord9"->"coord10"[style=invis] rankdir=LR } { rank=same "coord15"->"coord17"[style=invis] rankdir=LR } { rank=same "coord22"->"coord23"[style=invis] rankdir=LR } { rank=same "coord22"->"coord23"[style=invis] rankdir=LR } subgraph clusterCF {label = "Control Graph"; "cntrl1"[label="Kernel(1)"]; "cntrl2"[label="LoadTiled Value: Int32(2)"]; "cntrl3"[label="Body(3)",shape=box]; "cntrl4"[label="LoadTiled Value: Int32(4)"]; "cntrl5"[label="Body(5)",shape=box]; "cntrl6"[label="Assign VGPR Add(DataFlowTag(4)NA, DataFlowTag(4)NA)NA(6)"]; "cntrl7"[label="Sequence(7)",shape=box]; "cntrl8"[label="Sequence(8)",shape=box]; "cntrl9"[label="Assign VGPR Add(DataFlowTag(11)NA, DataFlowTag(11)NA)NA(9)"]; "cntrl10"[label="Sequence(10)",shape=box]; "cntrl11"[label="Sequence(11)",shape=box]; "cntrl12"[label="Assign VGPR Add(DataFlowTag(15)NA, DataFlowTag(17)NA)NA(12)"]; "cntrl13"[label="Sequence(13)",shape=box]; "cntrl14"[label="Sequence(14)",shape=box]; "cntrl15"[label="StoreTiled Value: Int32(15)"]; "cntrl16"[label="Sequence(16)",shape=box]; "cntrl1" -> "cntrl3" "cntrl1" -> "cntrl5" "cntrl2" -> "cntrl7" "cntrl2" -> "cntrl8" "cntrl3" -> "cntrl2" "cntrl4" -> "cntrl10" "cntrl4" -> "cntrl11" "cntrl5" -> "cntrl4" "cntrl6" -> "cntrl13" "cntrl7" -> "cntrl6" "cntrl8" -> "cntrl6" "cntrl9" -> "cntrl14" "cntrl10" -> "cntrl9" "cntrl11" -> "cntrl9" "cntrl12" -> "cntrl16" "cntrl13" -> "cntrl12" "cntrl14" -> "cntrl12" "cntrl16" -> "cntrl15" } "coord1" -> "to_cntrl_1_2" "to_cntrl_1_2"[label="2->1: User: (0)", shape=cds] "cntrl2" -> "to_coord_2_1" "to_coord_2_1"[label="2->1: User: (0)", shape=cds] "coord4" -> "to_cntrl_4_2" "to_cntrl_4_2"[label="2->4: MacroTile: (0)", shape=cds] "cntrl2" -> "to_coord_2_4" "to_coord_2_4"[label="2->4: MacroTile: (0)", shape=cds] "coord8" -> "to_cntrl_8_4" "to_cntrl_8_4"[label="4->8: User: (0)", shape=cds] "cntrl4" -> "to_coord_4_8" "to_coord_4_8"[label="4->8: User: (0)", shape=cds] "coord11" -> "to_cntrl_11_4" "to_cntrl_11_4"[label="4->11: MacroTile: (0)", shape=cds] "cntrl4" -> "to_coord_4_11" "to_coord_4_11"[label="4->11: MacroTile: (0)", shape=cds] "coord15" -> "to_cntrl_15_6" "to_cntrl_15_6"[label="6->15: DEST", shape=cds] "cntrl6" -> "to_coord_6_15" "to_coord_6_15"[label="6->15: DEST", shape=cds] "coord17" -> "to_cntrl_17_9" "to_cntrl_17_9"[label="9->17: DEST", shape=cds] "cntrl9" -> "to_coord_9_17" "to_coord_9_17"[label="9->17: DEST", shape=cds] "coord19" -> "to_cntrl_19_12" "to_cntrl_19_12"[label="12->19: DEST", shape=cds] "cntrl12" -> "to_coord_12_19" "to_coord_12_19"[label="12->19: DEST", shape=cds] "coord19" -> "to_cntrl_19_15" "to_cntrl_19_15"[label="15->19: MacroTile: (0)", shape=cds] "cntrl15" -> "to_coord_15_19" "to_coord_15_19"[label="15->19: MacroTile: (0)", shape=cds] "coord21" -> "to_cntrl_21_15" "to_cntrl_21_15"[label="15->21: User: (0)", shape=cds] "cntrl15" -> "to_coord_15_21" "to_coord_15_21"[label="15->21: User: (0)", shape=cds] })."; EXPECT_EQ(NormalizedSource(expected0), NormalizedSource(kgraph0.toDOT(true))); auto addLDSTransform = std::make_shared(params, m_context); auto lowerTileTransform = std::make_shared(params, m_context); auto assignIndexExprsTransform = std::make_shared(m_context, example.getCommand()); auto kgraph1 = kgraph0.transform(addLDSTransform); kgraph1 = kgraph1.transform(lowerTileTransform); kgraph1 = kgraph1.transform(assignIndexExprsTransform); namespace CG = rocRoller::KernelGraph::ControlGraph; ASSERT_EQ(kgraph1.control.getNodes().to().size(), 2); ASSERT_EQ(kgraph1.control.getNodes().to().size(), 1); ASSERT_EQ(kgraph1.control.getNodes().to().size(), 1); } TEST_F(KernelGraphTest, Translate02) { auto example = rocRollerTest::Graphs::VectorAddNegSquare(); auto command = example.getCommand(); auto one = Expression::literal(1); m_context->kernel()->setWorkgroupSize({64, 1, 1}); m_context->kernel()->setWorkitemCount({one, one, one}); auto lowerLinearTransform = std::make_shared(m_context); auto kgraph0 = translate(command); auto kgraph1 = kgraph0.transform(lowerLinearTransform); auto user0 = 1; auto subdim0 = 3; auto block0 = 17; auto thread0 = 18; // given block id and thread id, compute regular (user) index for first (0) dataflow array auto block_id = Expression::literal(2); auto thread_id = Expression::literal(33); auto exprs = kgraph1.coordinates.reverse({block_id, thread_id}, {user0}, {block0, thread0}); auto sexpr = Expression::toString(exprs[0]); EXPECT_EQ(sexpr, "{Split: Multiply({Tile: Add(Multiply(2:I, 64:U32)U32, 33:I)U32}, " "CommandArgument(Tensor_0_stride_0)I64)I64}"); auto stride = getStride(kgraph1.coordinates.getNode(subdim0)); auto fa = fastArith(stride); exprs = kgraph1.coordinates.reverse({block_id, thread_id}, {user0}, {block0, thread0}); sexpr = Expression::toString(fastArith(exprs[0])); EXPECT_EQ(sexpr, "{Split: Multiply(161:U32, Tensor_0_stride_0_0:I64)I64}"); } #if 0 TEST_F(KernelGraphTestGPU, GPU_Translate03) { TIMER(t_total, "Translate03"); TIMER(t_gpu, "Translate03::GPU"); TIMER(t_hip, "Translate03::HIP"); TIC(t_total); auto command = std::make_shared(); Operations::T_Load_Linear load_A(DataType::Int32, 1, 0); command->addOperation(std::make_shared(std::move(load_A))); Operations::T_Load_Linear load_B(DataType::Int32, 1, 2); command->addOperation(std::make_shared(std::move(load_B))); Operations::T_Execute execute; execute.addXOp(std::make_shared(Operations::E_Add(3, 2, 0))); execute.addXOp(std::make_shared(Operations::E_Mul(5, 3, 0))); command->addOperation(std::make_shared(std::move(execute))); Operations::T_Store_Linear store_C(1, 5); command->addOperation(std::make_shared(std::move(store_C))); CommandKernel commandKernel(command, "Translate03"); auto kgraph2 = commandKernel.getKernelGraph(); auto kernelNode = kgraph2.control.getRootOperation(); { auto expected = getTag(kernelNode); auto outputs = kgraph2.control.getOutputs(getTag(kernelNode)); EXPECT_EQ(2, outputs.size()); auto outputs2 = kgraph2.control.getOutputs( getTag(kernelNode)); EXPECT_EQ(0, outputs2.size()); auto outputs3 = kgraph2.control.getOutputs(getTag(kernelNode), Body{}); auto outputTags3 = kgraph2.control.getOutputTags(getTag(kernelNode), Body{}); EXPECT_EQ(outputs3.size(), outputTags3.size()); for(size_t i = 0; i < outputs3.size(); i++) { EXPECT_EQ(getTag(outputs3[i]), outputTags3[i]); } EXPECT_EQ(getTag(outputs[0]), getTag(outputs3[0])); auto inputs1 = kgraph2.control.getInputs(getTag(outputs.at(0))); ASSERT_EQ(1, inputs1.size()); auto actual1 = getTag(inputs1.at(0)); EXPECT_EQ(actual1, expected); auto inputs2 = kgraph2.control.getInputs(getTag(outputs.at(0)), Body{}); ASSERT_EQ(1, inputs2.size()); auto inputTags2 = kgraph2.control.getInputTags(getTag(outputs.at(0)), Body{}); EXPECT_EQ(inputs2.size(), inputTags2.size()); for(size_t i = 0; i < inputs2.size(); i++) { EXPECT_EQ(getTag(inputs2[i]), inputTags2[i]); } auto actual2 = getTag(inputs2.at(0)); EXPECT_EQ(actual1, actual2); auto inputs3 = kgraph2.control.getInputs( getTag(outputs.at(0))); EXPECT_EQ(0, inputs3.size()); auto inputs4 = kgraph2.control.getInputs( getTag(outputs.at(0))); ASSERT_EQ(0, inputs4.size()); auto inputs5 = kgraph2.control.getInputs( getTag(outputs.at(0))); ASSERT_EQ(0, inputs5.size()); } { std::ostringstream msg; msg << kgraph2.control; std::ostringstream msg2; kgraph2.control.toDOT(msg2, "krn"); EXPECT_EQ(msg.str(), msg2.str()); } TIC(t_hip); size_t nx = 64; RandomGenerator random(17629u); auto a = random.vector(nx, -100, 100); auto b = random.vector(nx, -100, 100); auto user0 = make_shared_device(a); auto user2 = make_shared_device(b); auto user4 = make_shared_device(nx); std::vector r(nx), x(nx); TOC(t_hip); KernelArguments runtimeArgs; runtimeArgs.append("user0", user0.get()); runtimeArgs.append("user1", nx); runtimeArgs.append("user2", nx); runtimeArgs.append("user3", (size_t)1); runtimeArgs.append("user4", user2.get()); runtimeArgs.append("user5", nx); runtimeArgs.append("user6", nx); runtimeArgs.append("user7", (size_t)1); runtimeArgs.append("user8", user4.get()); runtimeArgs.append("user9", nx); runtimeArgs.append("user10", (size_t)1); TIC(t_gpu); commandKernel.launchKernel(runtimeArgs.runtimeArguments()); TOC(t_gpu); TIC(t_hip); ASSERT_THAT(hipMemcpy(r.data(), user4.get(), nx * sizeof(int), hipMemcpyDefault), HasHipSuccess(0)); TOC(t_hip); // reference solution for(size_t i = 0; i < nx; ++i) x[i] = a[i] * (a[i] + b[i]); double rnorm = relativeNorm(r, x); ASSERT_LT(rnorm, 1.e-12); TIC(t_hip); user0.reset(); user2.reset(); user4.reset(); TOC(t_hip); TOC(t_total); std::cout << TimerPool::summary() << std::endl; std::cout << TimerPool::CSV() << std::endl; } #endif void KernelGraphTestGPU::GPU_SAXPBY(bool reload) { RandomGenerator random(1263u); size_t nx = 64; auto a = random.vector(nx, -100, 100); auto b = random.vector(nx, -100, 100); auto d_a = make_shared_device(a); auto d_b = make_shared_device(b); auto d_c = make_shared_device(nx); auto d_alpha = make_shared_device(); int alpha = 22; int beta = 33; ASSERT_THAT(hipMemcpy(d_alpha.get(), &alpha, 1 * sizeof(int), hipMemcpyDefault), HasHipSuccess(0)); auto example = rocRollerTest::Graphs::VectorAdd(true); auto command = example.getCommand(); auto runtimeArgs = example.getRuntimeArguments(nx, d_alpha.get(), beta, d_a.get(), d_b.get(), d_c.get()); CommandKernel commandKernel(command, testKernelName()); commandKernel.setContext(m_context); commandKernel.generateKernel(); commandKernel.launchKernel(runtimeArgs.runtimeArguments()); // launch again, using saved assembly auto assemblyFileName = m_context->assemblyFileName(); if(reload) { commandKernel.loadKernelFromAssembly(assemblyFileName, testKernelName()); commandKernel.launchKernel(runtimeArgs.runtimeArguments()); } std::vector r(nx); ASSERT_THAT(hipMemcpy(r.data(), d_c.get(), nx * sizeof(int), hipMemcpyDefault), HasHipSuccess(0)); // reference solution double rnorm = relativeNormL2(r, example.referenceSolution(alpha, beta, a, b)); ASSERT_LT(rnorm, 1.e-12); if(reload) { // load, using bad kernel name EXPECT_THROW(commandKernel.loadKernelFromAssembly(assemblyFileName, "SAXPBY_BAD"), FatalError); // load, using non-existant file EXPECT_THROW( commandKernel.loadKernelFromAssembly(assemblyFileName + "_bad", testKernelName()), FatalError); std::filesystem::remove(assemblyFileName); } } TEST_F(KernelGraphTestGPU, GPU_SAXPBY) { GPU_SAXPBY(false); } TEST_F(KernelGraphTestGPU, GPU_SAXPBYDebug) { // Make sure Debug mode doesn't introduce bad pointer // references in observers auto settings = Settings::getInstance(); settings->set(Settings::LogLvl, LogLevel::Debug); GPU_SAXPBY(false); settings->reset(); } TEST_F(KernelGraphTestGPU, GPU_SAXPBYLoadAssembly) { GPU_SAXPBY(true); } TEST_F(KernelGraphTestGPU, GPU_LeakyRelu) { auto command = std::make_shared(); constexpr auto dataType = DataType::Float; auto xTensorTag = command->addOperation(rocRoller::Operations::Tensor(1, dataType)); auto xLoadTag = command->addOperation(rocRoller::Operations::T_Load_Linear(xTensorTag)); auto alphaScalarTag = command->addOperation( rocRoller::Operations::Scalar({dataType, PointerType::PointerGlobal})); auto alphaLoadTag = command->addOperation(rocRoller::Operations::T_Load_Scalar(alphaScalarTag)); auto zeroLiteralTag = command->addOperation(rocRoller::Operations::Literal(0.0f)); auto execute = rocRoller::Operations::T_Execute(command->getNextTag()); auto condTag = execute.addXOp(rocRoller::Operations::E_GreaterThan(xLoadTag, zeroLiteralTag)); auto productTag = execute.addXOp(rocRoller::Operations::E_Mul(xLoadTag, alphaLoadTag)); auto reluTag = execute.addXOp(rocRoller::Operations::E_Conditional(condTag, xLoadTag, productTag)); command->addOperation(std::move(execute)); auto reluTensorTag = command->addOperation(rocRoller::Operations::Tensor(1, dataType)); command->addOperation(rocRoller::Operations::T_Store_Linear(reluTag, reluTensorTag)); CommandKernel commandKernel(command, "LeakyRelu"); commandKernel.setContext(m_context); commandKernel.generateKernel(); size_t nx = 64; float alpha = 0.9; RandomGenerator random(135679u); auto a = random.vector(nx, -5, 5); auto d_a = make_shared_device(a); auto d_b = make_shared_device(nx); auto d_alpha = make_shared_device(); std::vector r(nx), x(nx); ASSERT_THAT(hipMemcpy(d_alpha.get(), &alpha, 1 * sizeof(float), hipMemcpyDefault), HasHipSuccess(0)); CommandArguments commandArgs = command->createArguments(); commandArgs.setArgument(xTensorTag, ArgumentType::Value, d_a.get()); commandArgs.setArgument(xTensorTag, ArgumentType::Size, 0, nx); commandArgs.setArgument(xTensorTag, ArgumentType::Stride, 0, (size_t)1); commandArgs.setArgument(alphaScalarTag, ArgumentType::Value, d_alpha.get()); commandArgs.setArgument(reluTensorTag, ArgumentType::Value, d_b.get()); commandArgs.setArgument(reluTensorTag, ArgumentType::Size, 0, nx); commandArgs.setArgument(reluTensorTag, ArgumentType::Stride, 0, (size_t)1); commandKernel.launchKernel(commandArgs.runtimeArguments()); ASSERT_THAT(hipMemcpy(r.data(), d_b.get(), nx * sizeof(float), hipMemcpyDefault), HasHipSuccess(0)); // reference solution for(size_t i = 0; i < nx; ++i) { x[i] = a[i] > 0 ? a[i] : alpha * a[i]; } double rnorm = relativeNormL2(r, x); ASSERT_LT(rnorm, 1.e-12); } TEST_F(KernelGraphTestGPU, GPU_LinearCopy) { auto command = std::make_shared(); Operations::Tensor tensor_A(1, DataType::Int32); auto tagTensorA = command->addOperation(std::make_shared(std::move(tensor_A))); Operations::Tensor tensor_B(1, DataType::Int32); auto tagTensorB = command->addOperation(std::make_shared(std::move(tensor_B))); Operations::T_Load_Linear load_A(tagTensorA); auto tagLoadStore = command->addOperation(std::make_shared(std::move(load_A))); Operations::T_Store_Linear store_B(tagLoadStore, tagTensorB); command->addOperation(std::make_shared(std::move(store_B))); CommandKernel commandKernel(command, "LinearCopy"); commandKernel.setContext(m_context); commandKernel.generateKernel(); size_t nx = 64; RandomGenerator random(135679u); auto a = random.vector(nx, -100, 100); auto d_a = make_shared_device(a); auto d_b = make_shared_device(nx); std::vector r(nx), x(nx); CommandArguments commandArgs = command->createArguments(); commandArgs.setArgument(tagTensorA, ArgumentType::Value, d_a.get()); commandArgs.setArgument(tagTensorA, ArgumentType::Size, 0, nx); commandArgs.setArgument(tagTensorA, ArgumentType::Stride, 0, (size_t)1); commandArgs.setArgument(tagTensorB, ArgumentType::Value, d_b.get()); commandArgs.setArgument(tagTensorB, ArgumentType::Size, 0, nx); commandArgs.setArgument(tagTensorB, ArgumentType::Stride, 0, (size_t)1); commandKernel.launchKernel(commandArgs.runtimeArguments()); ASSERT_THAT(hipMemcpy(r.data(), d_b.get(), nx * sizeof(int), hipMemcpyDefault), HasHipSuccess(0)); // reference solution for(size_t i = 0; i < nx; ++i) x[i] = a[i]; double rnorm = relativeNormL2(r, x); ASSERT_LT(rnorm, 1.e-12); } template void CopyStrideOverride(CommandKernelPtr& commandKernel, bool override = false) { auto example = rocRollerTest::Graphs::TileCopy(); example.setTileSize(16, 4); example.setSubTileSize(4, 2); if(override) { example.setLiteralStrides({(size_t)0, (size_t)1}); } size_t nx = 256; size_t ny = 128; RandomGenerator random(193674u); auto ax = static_cast(-100.); auto ay = static_cast(100.); auto a = random.vector(nx * ny, ax, ay); std::vector r(nx * ny, 0.); auto d_a = make_shared_device(a); auto d_b = make_shared_device(nx * ny); auto command = example.getCommand(); auto params = example.getCommandParameters(nx, ny); auto launch = example.getCommandLaunchParameters(nx, ny); auto runtimeArgs = example.getRuntimeArguments(nx, ny, d_a.get(), d_b.get()); std::string colName = (override) ? "ColOverride" : ""; std::string kernelName = "TensorTileCopy" + colName + TypeInfo::Name(); commandKernel = std::make_shared(command, kernelName); commandKernel->setContext(Context::ForDefaultHipDevice(kernelName)); commandKernel->setCommandParameters(params); commandKernel->generateKernel(); commandKernel->setLaunchParameters(launch); commandKernel->launchKernel(runtimeArgs.runtimeArguments()); ASSERT_THAT(hipMemcpy(r.data(), d_b.get(), nx * ny * sizeof(T), hipMemcpyDefault), HasHipSuccess(0)); auto ref = example.referenceSolution(a); double rnorm = relativeNormL2(r, ref); std::ostringstream msg; for(int i = 0; i < r.size(); i++) { msg << i << " " << r[i] << " | " << ref[i]; if(r[i] != ref[i]) msg << " * "; msg << std::endl; } ASSERT_LT(rnorm, 1.e-12) << msg.str(); } TEST_F(KernelGraphTestGPU, GPU_TensorTileCopy) { CommandKernelPtr commandKernel; CopyStrideOverride(commandKernel); } TEST_F(KernelGraphTestGPU, GPU_TensorTileCopyColStrideHalf) { CommandKernelPtr commandKernel; CopyStrideOverride(commandKernel, true); auto instructions = NormalizedSourceLines(commandKernel->getInstructions(), false); int numRead = 0; int numWrite = 0; for(auto const& instruction : instructions) { if(instruction.starts_with("buffer_load_dword ")) { numRead++; } else if(instruction.starts_with("buffer_store_dword ")) { numWrite++; } } EXPECT_EQ(numRead, 4); EXPECT_EQ(numWrite, 4); } TEST_F(KernelGraphTestGPU, GPU_TensorTileCopyColStrideFloat) { CommandKernelPtr commandKernel; CopyStrideOverride(commandKernel, true); auto instructions = NormalizedSourceLines(commandKernel->getInstructions(), false); int numRead = 0; int numWrite = 0; for(auto const& instruction : instructions) { if(instruction.starts_with("buffer_load_dwordx2")) { numRead++; } else if(instruction.starts_with("buffer_store_dwordx2")) { numWrite++; } } EXPECT_EQ(numRead, 4); EXPECT_EQ(numWrite, 4); } TEST_F(KernelGraphTestGPU, GPU_TensorTileCopyColStrideDouble) { CommandKernelPtr commandKernel; CopyStrideOverride(commandKernel, true); auto instructions = NormalizedSourceLines(commandKernel->getInstructions(), false); int numRead = 0; int numWrite = 0; for(auto const& instruction : instructions) { if(instruction.starts_with("buffer_load_dwordx4")) { numRead++; } else if(instruction.starts_with("buffer_store_dwordx4")) { numWrite++; } } EXPECT_EQ(numRead, 4); EXPECT_EQ(numWrite, 4); } TEST_F(KernelGraphTestGPU, GPU_TensorTileAdd) { size_t nx = 256; // tensor size x size_t ny = 512; // tensor size y RandomGenerator random(129674u); auto a = random.vector(nx * ny, -100, 100); auto b = random.vector(nx * ny, -100, 100); auto r = random.vector(nx * ny, -100, 100); auto d_a = make_shared_device(a); auto d_b = make_shared_device(b); auto d_c = make_shared_device(nx * ny); auto example = rocRollerTest::Graphs::TileDoubleAdd(); example.setTileSize(8, 64); example.setSubTileSize(2, 8); auto command = example.getCommand(); auto runtimeArgs = example.getRuntimeArguments(nx, ny, d_a.get(), d_b.get(), d_c.get()); auto params = example.getCommandParameters(nx, ny); auto launch = example.getCommandLaunchParameters(nx, ny); CommandKernel commandKernel(command, "TensorTileAdd"); commandKernel.setContext(m_context); commandKernel.setCommandParameters(params); commandKernel.generateKernel(); commandKernel.setLaunchParameters(launch); commandKernel.launchKernel(runtimeArgs.runtimeArguments()); ASSERT_THAT(hipMemcpy(r.data(), d_c.get(), nx * ny * sizeof(int), hipMemcpyDefault), HasHipSuccess(0)); // reference solution double rnorm = relativeNormL2(r, example.referenceSolution(a, b)); ASSERT_LT(rnorm, 1.e-12); } TEST_F(KernelGraphTest, CleanArguments) { auto example = rocRollerTest::Graphs::VectorAddNegSquare(); auto command = example.getCommand(); int workGroupSize = 64; m_context->kernel()->setKernelDimensions(1); m_context->kernel()->setWorkgroupSize({64, 1, 1}); auto cleanArgumentsTransform = std::make_shared(m_context, command); auto kgraph = translate(command); auto beforePredicates = {HasSubstr("SubDimension{0, CommandArgument(Tensor_0_size_0)I64}"), HasSubstr("SubDimension{0, CommandArgument(Tensor_2_size_0)I64}"), HasSubstr("Linear{CommandArgument(Tensor_0_size_0)I64}"), HasSubstr("Linear{CommandArgument(Tensor_2_size_0)I64}")}; // Note that these searches do not include the close braces ("}"). This is because the // argument name will have a number appended which is subject to change // (Load_Linear_0_size_0 might become Load_Linear_0_size_0_2). auto afterPredicates = { HasSubstr("SubDimension{0, Tensor_0_size_0"), HasSubstr("SubDimension{0, Tensor_2_size_0"), HasSubstr("Linear{Tensor_0_size_0"), HasSubstr("Linear{Tensor_2_size_0"), }; { auto dot = kgraph.toDOT(); for(auto const& pred : beforePredicates) EXPECT_THAT(dot, pred); for(auto const& pred : afterPredicates) EXPECT_THAT(dot, Not(pred)); } kgraph = kgraph.transform(cleanArgumentsTransform); { auto dot = kgraph.toDOT(); for(auto const& pred : beforePredicates) EXPECT_THAT(dot, Not(pred)); for(auto const& pred : afterPredicates) EXPECT_THAT(dot, pred); } } TEST_F(KernelGraphTest, Basic) { auto kgraph = rocRoller::KernelGraph::KernelGraph(); // Control Graph int kernel_index = kgraph.control.addElement(Kernel()); int loadA_index = kgraph.control.addElement(LoadLinear(DataType::Float)); int loadB_index = kgraph.control.addElement(LoadLinear(DataType::Float)); int body1_index = kgraph.control.addElement(Body(), {kernel_index}, {loadA_index}); int body2_index = kgraph.control.addElement(Body(), {kernel_index}, {loadB_index}); int op1_index = kgraph.control.addElement(Assign{Register::Type::Vector, Expression::literal(5)}); int sequence1_index = kgraph.control.addElement(Sequence(), {loadA_index}, {op1_index}); int sequence2_index = kgraph.control.addElement(Sequence(), {loadB_index}, {op1_index}); int op2_index = kgraph.control.addElement(Assign{Register::Type::Vector, Expression::literal(7)}); int sequence3_index = kgraph.control.addElement(Sequence(), {op1_index}, {op2_index}); int op3_index = kgraph.control.addElement(Assign{Register::Type::Vector, Expression::literal(9)}); int sequence4_index = kgraph.control.addElement(Sequence(), {op1_index}, {op3_index}); int sequence5_index = kgraph.control.addElement(Sequence(), {op2_index}, {op3_index}); int storeC_index = kgraph.control.addElement(StoreLinear()); int sequence6_index = kgraph.control.addElement(Sequence(), {op3_index}, {storeC_index}); // Coordinate Graph int u1_index = kgraph.coordinates.addElement(User()); int sd1_index = kgraph.coordinates.addElement(SubDimension()); int split1_index = kgraph.coordinates.addElement(Split(), {u1_index}, {sd1_index}); int linear1_index = kgraph.coordinates.addElement(Linear()); int flatten1_index = kgraph.coordinates.addElement(Flatten(), {sd1_index}, {linear1_index}); int dataflow1_index = kgraph.coordinates.addElement(DataFlow(), {u1_index}, {linear1_index}); int buffer1_index = kgraph.coordinates.addElement( rocRoller::KernelGraph::CoordinateGraph::Buffer(), {u1_index}, {linear1_index}); int u2_index = kgraph.coordinates.addElement(User()); int sd2_index = kgraph.coordinates.addElement(SubDimension()); int split2_index = kgraph.coordinates.addElement(Split(), {u2_index}, {sd2_index}); int linear2_index = kgraph.coordinates.addElement(Linear()); int flatten2_index = kgraph.coordinates.addElement(Flatten(), {sd2_index}, {linear2_index}); int dataflow2_index = kgraph.coordinates.addElement(DataFlow(), {u2_index}, {linear2_index}); int linear3_index = kgraph.coordinates.addElement(Linear()); int dataflow3_index = kgraph.coordinates.addElement( DataFlow(), {linear1_index, linear2_index}, {linear3_index}); int linear4_index = kgraph.coordinates.addElement(Linear()); int dataflow4_index = kgraph.coordinates.addElement(DataFlow(), {linear3_index}, {linear4_index}); int linear5i_index = kgraph.coordinates.addElement(Linear()); int dataflow5_index = kgraph.coordinates.addElement( DataFlow(), {linear3_index, linear4_index}, {linear5i_index}); int linear5o_index = kgraph.coordinates.addElement(Linear()); int makeoutput1_index = kgraph.coordinates.addElement(MakeOutput(), {linear5i_index}, {linear5o_index}); int sd5o_index = kgraph.coordinates.addElement(SubDimension(0)); int split3_index = kgraph.coordinates.addElement(Split(), {linear5o_index}, {sd5o_index}); int u5o_index = kgraph.coordinates.addElement(User({}, "")); int join1_index = kgraph.coordinates.addElement(Join(), {sd5o_index}, {u5o_index}); int dataflow6_index = kgraph.coordinates.addElement(DataFlow(), {linear5i_index}, {u5o_index}); auto yamlData = toYAML(kgraph); auto graph2 = rocRoller::KernelGraph::fromYAML(yamlData); auto yaml2 = toYAML(graph2); EXPECT_EQ(yamlData, yaml2); std::string expected = R".( digraph { "coord1"[label="User{NA}(1)"]; "coord2"[label="SubDimension{0, NA}(2)"]; "coord3"[label="Split(3)",shape=box]; "coord4"[label="Linear{NA}(4)"]; "coord5"[label="Flatten(5)",shape=box]; "coord6"[label="DataFlow(6)",shape=box]; "coord7"[label="Buffer(7)",shape=box]; "coord8"[label="User{NA}(8)"]; "coord9"[label="SubDimension{0, NA}(9)"]; "coord10"[label="Split(10)",shape=box]; "coord11"[label="Linear{NA}(11)"]; "coord12"[label="Flatten(12)",shape=box]; "coord13"[label="DataFlow(13)",shape=box]; "coord14"[label="Linear{NA}(14)"]; "coord15"[label="DataFlow(15)",shape=box]; "coord16"[label="Linear{NA}(16)"]; "coord17"[label="DataFlow(17)",shape=box]; "coord18"[label="Linear{NA}(18)"]; "coord19"[label="DataFlow(19)",shape=box]; "coord20"[label="Linear{NA}(20)"]; "coord21"[label="MakeOutput(21)",shape=box]; "coord22"[label="SubDimension{0, NA}(22)"]; "coord23"[label="Split(23)",shape=box]; "coord24"[label="User{NA}(24)"]; "coord25"[label="Join(25)",shape=box]; "coord26"[label="DataFlow(26)",shape=box]; "coord1" -> "coord3" "coord1" -> "coord6" "coord1" -> "coord7" "coord2" -> "coord5" "coord3" -> "coord2" "coord4" -> "coord15" "coord5" -> "coord4" "coord6" -> "coord4" "coord7" -> "coord4" "coord8" -> "coord10" "coord8" -> "coord13" "coord9" -> "coord12" "coord10" -> "coord9" "coord11" -> "coord15" "coord12" -> "coord11" "coord13" -> "coord11" "coord14" -> "coord17" "coord14" -> "coord19" "coord15" -> "coord14" "coord16" -> "coord19" "coord17" -> "coord16" "coord18" -> "coord21" "coord18" -> "coord26" "coord19" -> "coord18" "coord20" -> "coord23" "coord21" -> "coord20" "coord22" -> "coord25" "coord23" -> "coord22" "coord25" -> "coord24" "coord26" -> "coord24" { rank=same "coord4"->"coord11"[style=invis] rankdir=LR } { rank=same "coord14"->"coord16"[style=invis] rankdir=LR } subgraph clusterCF {label = "Control Graph"; "cntrl1"[label="Kernel(1)"]; "cntrl2"[label="LoadLinear Value: Float(2)"]; "cntrl3"[label="LoadLinear Value: Float(3)"]; "cntrl4"[label="Body(4)",shape=box]; "cntrl5"[label="Body(5)",shape=box]; "cntrl6"[label="Assign VGPR 5:I(6)"]; "cntrl7"[label="Sequence(7)",shape=box]; "cntrl8"[label="Sequence(8)",shape=box]; "cntrl9"[label="Assign VGPR 7:I(9)"]; "cntrl10"[label="Sequence(10)",shape=box]; "cntrl11"[label="Assign VGPR 9:I(11)"]; "cntrl12"[label="Sequence(12)",shape=box]; "cntrl13"[label="Sequence(13)",shape=box]; "cntrl14"[label="StoreLinear(14)"]; "cntrl15"[label="Sequence(15)",shape=box]; "cntrl1" -> "cntrl4" "cntrl1" -> "cntrl5" "cntrl2" -> "cntrl7" "cntrl3" -> "cntrl8" "cntrl4" -> "cntrl2" "cntrl5" -> "cntrl3" "cntrl6" -> "cntrl10" "cntrl6" -> "cntrl12" "cntrl7" -> "cntrl6" "cntrl8" -> "cntrl6" "cntrl9" -> "cntrl13" "cntrl10" -> "cntrl9" "cntrl11" -> "cntrl15" "cntrl12" -> "cntrl11" "cntrl13" -> "cntrl11" "cntrl15" -> "cntrl14" } } )."; EXPECT_EQ(NormalizedSource(expected), NormalizedSource(kgraph.toDOT())); } TEST_F(KernelGraphTest, UpdateParamsTMul) { auto command = std::make_shared(); auto tagTensorA = command->addOperation(rocRoller::Operations::Tensor(2, DataType::Float)); // A auto tagLoadA = command->addOperation(rocRoller::Operations::T_Load_Tiled(tagTensorA)); auto tagTensorB = command->addOperation(rocRoller::Operations::Tensor(2, DataType::Float)); // B auto tagLoadB = command->addOperation(rocRoller::Operations::T_Load_Tiled(tagTensorB)); auto tagStoreD = command->addOperation(rocRoller::Operations::T_Mul(tagLoadA, tagLoadB)); // D = A * B auto tagTensorD = command->addOperation(rocRoller::Operations::Tensor(2, DataType::Float)); // D command->addOperation(rocRoller::Operations::T_Store_Tiled(tagStoreD, tagTensorD)); auto kgraph0 = translate(command); std::string expected0 = R".( digraph { "coord1"[label="User{NA}(1)"]; "coord2"[label="SubDimension{0, CommandArgument(Tensor_0_size_0)I64}(2)"]; "coord3"[label="SubDimension{1, CommandArgument(Tensor_0_size_1)I64}(3)"]; "coord4"[label="MacroTile{NA}(2/None/None){}-()(4)"]; "coord5"[label="Split(5)",shape=box]; "coord6"[label="ConstructMacroTile(6)",shape=box]; "coord7"[label="DataFlow(7)",shape=box]; "coord8"[label="User{NA}(8)"]; "coord9"[label="SubDimension{0, CommandArgument(Tensor_2_size_0)I64}(9)"]; "coord10"[label="SubDimension{1, CommandArgument(Tensor_2_size_1)I64}(10)"]; "coord11"[label="MacroTile{NA}(2/None/None){}-()(11)"]; "coord12"[label="Split(12)",shape=box]; "coord13"[label="ConstructMacroTile(13)",shape=box]; "coord14"[label="DataFlow(14)",shape=box]; "coord15"[label="MacroTile{NA}(0/None/None){}-()(15)"]; "coord16"[label="DataFlow(16)",shape=box]; "coord17"[label="User{NA}(17)"]; "coord18"[label="SubDimension{0, CommandArgument(Tensor_5_size_0)I64}(18)"]; "coord19"[label="SubDimension{1, CommandArgument(Tensor_5_size_1)I64}(19)"]; "coord20"[label="DestructMacroTile(20)",shape=box]; "coord21"[label="Join(21)",shape=box]; "coord22"[label="DataFlow(22)",shape=box]; "coord1" -> "coord5" "coord1" -> "coord7" "coord2" -> "coord6" "coord3" -> "coord6" "coord4" -> "coord16" "coord5" -> "coord2" "coord5" -> "coord3" "coord6" -> "coord4" "coord7" -> "coord4" "coord8" -> "coord12" "coord8" -> "coord14" "coord9" -> "coord13" "coord10" -> "coord13" "coord11" -> "coord16" "coord12" -> "coord9" "coord12" -> "coord10" "coord13" -> "coord11" "coord14" -> "coord11" "coord15" -> "coord20" "coord15" -> "coord22" "coord16" -> "coord15" "coord18" -> "coord21" "coord19" -> "coord21" "coord20" -> "coord18" "coord20" -> "coord19" "coord21" -> "coord17" "coord22" -> "coord17" { rank=same "coord2"->"coord3"[style=invis] rankdir=LR } { rank=same "coord2"->"coord3"[style=invis] rankdir=LR } { rank=same "coord9"->"coord10"[style=invis] rankdir=LR } { rank=same "coord9"->"coord10"[style=invis] rankdir=LR } { rank=same "coord4"->"coord11"[style=invis] rankdir=LR } { rank=same "coord18"->"coord19"[style=invis] rankdir=LR } { rank=same "coord18"->"coord19"[style=invis] rankdir=LR } subgraph clusterCF {label = "Control Graph"; "cntrl1"[label="Kernel(1)"]; "cntrl2"[label="LoadTiled Value: Float(2)"]; "cntrl3"[label="Body(3)",shape=box]; "cntrl4"[label="LoadTiled Value: Float(4)"]; "cntrl5"[label="Body(5)",shape=box]; "cntrl6"[label="TensorContraction(6)"]; "cntrl7"[label="Sequence(7)",shape=box]; "cntrl8"[label="Sequence(8)",shape=box]; "cntrl9"[label="StoreTiled Value: Float(9)"]; "cntrl10"[label="Sequence(10)",shape=box]; "cntrl1" -> "cntrl3" "cntrl1" -> "cntrl5" "cntrl2" -> "cntrl7" "cntrl3" -> "cntrl2" "cntrl4" -> "cntrl8" "cntrl5" -> "cntrl4" "cntrl6" -> "cntrl10" "cntrl7" -> "cntrl6" "cntrl8" -> "cntrl6" "cntrl10" -> "cntrl9" } } )."; EXPECT_EQ(NormalizedSource(expected0), NormalizedSource(kgraph0.toDOT())); // macro tile sizes int mac_m = 64; int mac_n = 64; int mac_k = 64; auto macTileA = MacroTile({mac_m, mac_k}, MemoryType::VGPR); // A auto macTileB = MacroTile({mac_k, mac_n}, MemoryType::VGPR); // B auto params = std::make_shared(); params->setDimensionInfo(tagLoadA, macTileA); params->setDimensionInfo(tagLoadB, macTileB); auto updateParametersTransform = std::make_shared(params); kgraph0 = kgraph0.transform(updateParametersTransform); std::string expected1 = R".( digraph { "coord1"[label="User{Multiply(CommandArgument(Tensor_0_stride_1)I64, CommandArgument(Tensor_0_size_1)I64)I64}(1)"]; "coord2"[label="SubDimension{0, CommandArgument(Tensor_0_size_0)I64}(2)"]; "coord3"[label="SubDimension{1, CommandArgument(Tensor_0_size_1)I64}(3)"]; "coord4"[label="MacroTile{NA}(2/VGPR/None){64,64}-()(4)"]; "coord5"[label="Split(5)",shape=box]; "coord6"[label="ConstructMacroTile(6)",shape=box]; "coord7"[label="DataFlow(7)",shape=box]; "coord8"[label="User{Multiply(CommandArgument(Tensor_2_stride_1)I64, CommandArgument(Tensor_2_size_1)I64)I64}(8)"]; "coord9"[label="SubDimension{0, CommandArgument(Tensor_2_size_0)I64}(9)"]; "coord10"[label="SubDimension{1, CommandArgument(Tensor_2_size_1)I64}(10)"]; "coord11"[label="MacroTile{NA}(2/VGPR/None){64,64}-()(11)"]; "coord12"[label="Split(12)",shape=box]; "coord13"[label="ConstructMacroTile(13)",shape=box]; "coord14"[label="DataFlow(14)",shape=box]; "coord15"[label="MacroTile{NA}(2/WAVE/MATRIX_ACCUMULATOR){64,64}-()(15)"]; "coord16"[label="DataFlow(16)",shape=box]; "coord17"[label="User{Multiply(CommandArgument(Tensor_5_stride_1)I64, CommandArgument(Tensor_5_size_1)I64)I64}(17)"]; "coord18"[label="SubDimension{0, CommandArgument(Tensor_5_size_0)I64}(18)"]; "coord19"[label="SubDimension{1, CommandArgument(Tensor_5_size_1)I64}(19)"]; "coord20"[label="DestructMacroTile(20)",shape=box]; "coord21"[label="Join(21)",shape=box]; "coord22"[label="DataFlow(22)",shape=box]; "coord1" -> "coord5" "coord1" -> "coord7" "coord2" -> "coord6" "coord3" -> "coord6" "coord4" -> "coord16" "coord5" -> "coord2" "coord5" -> "coord3" "coord6" -> "coord4" "coord7" -> "coord4" "coord8" -> "coord12" "coord8" -> "coord14" "coord9" -> "coord13" "coord10" -> "coord13" "coord11" -> "coord16" "coord12" -> "coord9" "coord12" -> "coord10" "coord13" -> "coord11" "coord14" -> "coord11" "coord15" -> "coord20" "coord15" -> "coord22" "coord16" -> "coord15" "coord18" -> "coord21" "coord19" -> "coord21" "coord20" -> "coord18" "coord20" -> "coord19" "coord21" -> "coord17" "coord22" -> "coord17" { rank=same "coord2"->"coord3"[style=invis] rankdir=LR } { rank=same "coord2"->"coord3"[style=invis] rankdir=LR } { rank=same "coord9"->"coord10"[style=invis] rankdir=LR } { rank=same "coord9"->"coord10"[style=invis] rankdir=LR } { rank=same "coord4"->"coord11"[style=invis] rankdir=LR } { rank=same "coord18"->"coord19"[style=invis] rankdir=LR } { rank=same "coord18"->"coord19"[style=invis] rankdir=LR } subgraph clusterCF {label = "Control Graph"; "cntrl1"[label="Kernel(1)"]; "cntrl2"[label="LoadTiled Value: Float(2)"]; "cntrl3"[label="Body(3)",shape=box]; "cntrl4"[label="LoadTiled Value: Float(4)"]; "cntrl5"[label="Body(5)",shape=box]; "cntrl6"[label="TensorContraction(6)"]; "cntrl7"[label="Sequence(7)",shape=box]; "cntrl8"[label="Sequence(8)",shape=box]; "cntrl9"[label="StoreTiled Value: Float(9)"]; "cntrl10"[label="Sequence(10)",shape=box]; "cntrl1" -> "cntrl3" "cntrl1" -> "cntrl5" "cntrl2" -> "cntrl7" "cntrl3" -> "cntrl2" "cntrl4" -> "cntrl8" "cntrl5" -> "cntrl4" "cntrl6" -> "cntrl10" "cntrl7" -> "cntrl6" "cntrl8" -> "cntrl6" "cntrl10" -> "cntrl9" } } )."; EXPECT_EQ(NormalizedSource(expected1), NormalizedSource(kgraph0.toDOT())); } TEST_F(KernelGraphTestGPU, GPU_Conditional) { rocRoller::KernelGraph::KernelGraph kgraph; m_context->kernel()->setKernelDimensions(1); m_context->kernel()->setWorkgroupSize({64, 1, 1}); auto kernel = kgraph.control.addElement(Kernel()); auto unit = Expression::literal(1); auto zero = Expression::literal(0); auto test = m_context->kernel()->addArgument({"foo", DataType::Int32}); auto destReg = kgraph.coordinates.addElement(Linear()); Expression::DataFlowTag destRegTag{destReg, Register::Type::Vector, DataType::Int32}; auto beforeConditionalAssign = kgraph.control.addElement(Assign{Register::Type::Vector, Expression::literal(0)}); kgraph.control.addElement(Body(), {kernel}, {beforeConditionalAssign}); auto conditional = kgraph.control.addElement( ConditionalOp{test < unit, ConditionalMode::Branch, "Test Conditional"}); kgraph.control.addElement(Sequence(), {beforeConditionalAssign}, {conditional}); auto trueOp = kgraph.control.addElement(Assign{Register::Type::Vector, unit}); auto trueBody = kgraph.control.addElement(Body(), {conditional}, {trueOp}); auto falseOp = kgraph.control.addElement(Assign{Register::Type::Vector, zero}); auto falseBody = kgraph.control.addElement(Else(), {conditional}, {falseOp}); kgraph.mapper.connect(beforeConditionalAssign, destReg, NaryArgument::DEST); kgraph.mapper.connect(trueOp, destReg, NaryArgument::DEST); kgraph.mapper.connect(falseOp, destReg, NaryArgument::DEST); kgraph = kgraph.transform(std::make_shared()); m_context->schedule(m_context->kernel()->preamble()); m_context->schedule(m_context->kernel()->prolog()); m_context->schedule(rocRoller::KernelGraph::generate(kgraph, m_context->kernel())); EXPECT_THAT(output(), testing::HasSubstr("s_cmp_lt_i32 s3, 1")); EXPECT_THAT(output(), testing::HasSubstr("s_cbranch_scc0")); //Branch for False EXPECT_THAT(output(), testing::HasSubstr("s_branch")); //Branch after True if(m_context->targetArchitecture().HasCapability(GPUCapability::HasVGPRIndexing)) { int idx = Register::RegisterAllocatorDetail::ReservedRegionSize(); EXPECT_THAT(output(), testing::HasSubstr(fmt::format("v_mov_b32 v{}, 1", idx))); //True Body EXPECT_THAT(output(), testing::HasSubstr(fmt::format("v_mov_b32 v{}, 0", idx))); //False Body } else { EXPECT_THAT(output(), testing::HasSubstr(fmt::format("v_mov_b32 v1, 1"))); //True Body EXPECT_THAT(output(), testing::HasSubstr(fmt::format("v_mov_b32 v1, 0"))); //False Body } } TEST_F(KernelGraphTestGPU, GPU_ConditionalExecute) { rocRoller::KernelGraph::KernelGraph kgraph; std::vector testValues = {22, 66}; auto zero = Expression::literal(0u); auto one = Expression::literal(1u); auto two = Expression::literal(2u); auto three = Expression::literal(3u); auto k = m_context->kernel(); k->addArgument( {"result", {DataType::Int32, PointerType::PointerGlobal}, DataDirection::WriteOnly}); k->setKernelDimensions(1); k->setWorkitemCount({three, one, one}); k->setWorkgroupSize({1, 1, 1}); k->setDynamicSharedMemBytes(zero); m_context->schedule(k->preamble()); m_context->schedule(k->prolog()); // global result auto user = kgraph.coordinates.addElement(User({}, "result")); auto wg = kgraph.coordinates.addElement(Workgroup()); kgraph.coordinates.addElement(PassThrough(), {wg}, {user}); // result auto dstVGPR = kgraph.coordinates.addElement(VGPR()); // set result to testValues[0] auto assignTrueBranch1 = kgraph.control.addElement( Assign{Register::Type::Vector, Expression::literal(testValues[0])}); kgraph.mapper.connect(assignTrueBranch1, dstVGPR, NaryArgument::DEST); auto assignTrueBranch2 = kgraph.control.addElement( Assign{Register::Type::Vector, Expression::literal(testValues[0])}); kgraph.mapper.connect(assignTrueBranch2, dstVGPR, NaryArgument::DEST); // set result to testValues[1] auto assignFalseBranch = kgraph.control.addElement( Assign{Register::Type::Vector, Expression::literal(testValues[1])}); kgraph.mapper.connect(assignFalseBranch, dstVGPR, NaryArgument::DEST); auto workgroupExpr = k->workgroupIndex().at(0)->expression(); auto firstConditional = kgraph.control.addElement( ConditionalOp{workgroupExpr < one, ConditionalMode::Branch, "First Conditional"}); auto secondConditional = kgraph.control.addElement( ConditionalOp{(workgroupExpr > one) && (workgroupExpr <= two), ConditionalMode::Branch, "Second Conditional"}); auto storeIndex = kgraph.control.addElement(StoreVGPR()); kgraph.mapper.connect(storeIndex, user); kgraph.mapper.connect(storeIndex, dstVGPR); auto kernel = kgraph.control.addElement(Kernel()); kgraph.control.addElement(Body(), {kernel}, {firstConditional}); kgraph.control.addElement(Body(), {firstConditional}, {assignTrueBranch1}); kgraph.control.addElement(Else(), {firstConditional}, {secondConditional}); kgraph.control.addElement(Body(), {secondConditional}, {assignTrueBranch2}); kgraph.control.addElement(Else(), {secondConditional}, {assignFalseBranch}); kgraph.control.addElement(Sequence(), {firstConditional}, {storeIndex}); kgraph = kgraph.transform(std::make_shared()); m_context->schedule(rocRoller::KernelGraph::generate(kgraph, m_context->kernel())); m_context->schedule(k->postamble()); m_context->schedule(k->amdgpu_metadata()); if(isLocalDevice()) { auto d_result = make_shared_device(3); KernelArguments kargs; kargs.append("result", d_result.get()); KernelInvocation kinv; kinv.workitemCount = {3, 1, 1}; kinv.workgroupSize = {1, 1, 1}; auto executableKernel = m_context->instructions()->getExecutableKernel(); executableKernel->executeKernel(kargs, kinv); std::vector result(3); ASSERT_THAT( hipMemcpy( result.data(), d_result.get(), result.size() * sizeof(int), hipMemcpyDefault), HasHipSuccess(0)); EXPECT_EQ(result[0], testValues[0]); EXPECT_EQ(result[1], testValues[1]); EXPECT_EQ(result[2], testValues[0]); } else { std::vector assembledKernel = m_context->instructions()->assemble(); EXPECT_GT(assembledKernel.size(), 0); } } TEST_F(KernelGraphTestGPU, GPU_DoWhileExecute) { rocRoller::KernelGraph::KernelGraph kgraph; auto zero = Expression::literal(0u); auto one = Expression::literal(1u); auto three = Expression::literal(3u); auto k = m_context->kernel(); k->addArgument( {"result", {DataType::Int32, PointerType::PointerGlobal}, DataDirection::WriteOnly}); k->setKernelDimensions(1); k->setWorkitemCount({three, one, one}); k->setWorkgroupSize({1, 1, 1}); k->setDynamicSharedMemBytes(zero); m_context->schedule(k->preamble()); m_context->schedule(k->prolog()); // global result auto user = kgraph.coordinates.addElement(User({}, "result")); auto wg = kgraph.coordinates.addElement(Workgroup()); kgraph.coordinates.addElement(PassThrough(), {wg}, {user}); // result auto dstVGPR = kgraph.coordinates.addElement(VGPR()); auto dfa = std::make_shared( Expression::DataFlowTag{dstVGPR, Register::Type::Vector, DataType::UInt32}); auto assignVGPR = kgraph.control.addElement(Assign{Register::Type::Vector, zero}); kgraph.mapper.connect(assignVGPR, dstVGPR, NaryArgument::DEST); auto assignBody = kgraph.control.addElement(Assign{Register::Type::Vector, dfa + one}); kgraph.mapper.connect(assignBody, dstVGPR, NaryArgument::DEST); auto workgroupExpr = k->workgroupIndex().at(0)->expression(); auto condVGPR = kgraph.coordinates.addElement(VGPR()); auto condDFT = std::make_shared( Expression::DataFlowTag{condVGPR, Register::Type::Vector, DataType::UInt32}); auto assignCond = kgraph.control.addElement(Assign{Register::Type::Vector, workgroupExpr}); kgraph.mapper.connect(assignCond, condVGPR, NaryArgument::DEST); auto doWhile = kgraph.control.addElement(DoWhileOp{dfa < condDFT, "Test DoWhile"}); auto storeIndex = kgraph.control.addElement(StoreVGPR()); kgraph.mapper.connect(storeIndex, user); kgraph.mapper.connect(storeIndex, dstVGPR); auto kernel = kgraph.control.addElement(Kernel()); kgraph.control.addElement(Body(), {kernel}, {assignVGPR}); kgraph.control.addElement(Body(), {kernel}, {assignCond}); kgraph.control.addElement(Sequence(), {assignCond}, {doWhile}); kgraph.control.addElement(Sequence(), {assignVGPR}, {doWhile}); kgraph.control.addElement(Body(), {doWhile}, {assignBody}); kgraph.control.addElement(Sequence(), {doWhile}, {storeIndex}); kgraph = kgraph.transform(std::make_shared()); m_context->schedule(rocRoller::KernelGraph::generate(kgraph, m_context->kernel())); m_context->schedule(k->postamble()); m_context->schedule(k->amdgpu_metadata()); if(isLocalDevice()) { auto d_result = make_shared_device(3); KernelArguments kargs; kargs.append("result", d_result.get()); KernelInvocation kinv; kinv.workitemCount = {3, 1, 1}; kinv.workgroupSize = {1, 1, 1}; auto executableKernel = m_context->instructions()->getExecutableKernel(); executableKernel->executeKernel(kargs, kinv); std::vector result(3); ASSERT_THAT( hipMemcpy( result.data(), d_result.get(), result.size() * sizeof(int), hipMemcpyDefault), HasHipSuccess(0)); EXPECT_EQ(result[0], 1); EXPECT_EQ(result[1], 1); EXPECT_EQ(result[2], 2); } else { std::vector assembledKernel = m_context->instructions()->assemble(); EXPECT_GT(assembledKernel.size(), 0); } } TEST_F(KernelGraphTest, WaitZero) { rocRoller::KernelGraph::KernelGraph kgraph; auto kernel = kgraph.control.addElement(Kernel()); auto wait = kgraph.control.addElement(WaitZero()); kgraph.control.addElement(Body(), {kernel}, {wait}); kgraph = kgraph.transform(std::make_shared()); m_context->schedule(rocRoller::KernelGraph::generate(kgraph, m_context->kernel())); EXPECT_THAT(output(), testing::HasSubstr("s_waitcnt")); EXPECT_THAT(output(), testing::HasSubstr("vmcnt(0)")); EXPECT_THAT(output(), testing::HasSubstr("lgkmcnt(0)")); if(m_context->targetArchitecture().HasCapability(GPUCapability::HasExpcnt)) { EXPECT_THAT(output(), testing::HasSubstr("expcnt(0)")); } } TEST_F(KernelGraphTest, ReindexConditionalOpExpression) { rocRoller::KernelGraph::KernelGraph kgraph; auto unit = Expression::literal(1); auto kernel = kgraph.control.addElement(Kernel()); auto loadA = kgraph.control.addElement(LoadVGPR(DataType::Int32, true)); kgraph.control.addElement(Body(), {kernel}, {loadA}); auto user0 = kgraph.coordinates.addElement(User({}, "user0")); auto vgprA = kgraph.coordinates.addElement(VGPR()); kgraph.coordinates.addElement(DataFlow(), {user0}, {vgprA}); kgraph.mapper.connect(loadA, vgprA); auto exprA = std::make_shared( Expression::DataFlowTag{vgprA, Register::Type::Scalar, DataType::Int32}); auto conditional = kgraph.control.addElement( ConditionalOp{exprA > unit, ConditionalMode::Branch, "conditional"}); kgraph.control.addElement(Sequence(), {loadA}, {conditional}); auto loadB = kgraph.control.addElement(LoadVGPR(DataType::Int32, true)); kgraph.control.addElement(Body(), {kernel}, {loadB}); auto vgprB = kgraph.coordinates.addElement(VGPR()); kgraph.coordinates.addElement(DataFlow(), {user0}, {vgprB}); kgraph.mapper.connect(loadB, vgprB); kgraph.control.addElement(Sequence(), {loadB}, {conditional}); GraphReindexer reindexer; reindexer.coordinates.emplace(vgprA, vgprB); reindexExpressions(kgraph, conditional, reindexer); auto condition = kgraph.control.get(conditional)->condition; auto lhs = std::get(*condition).lhs; auto tag = std::get(*lhs).tag; EXPECT_EQ(tag, vgprB); } TEST_F(KernelGraphTest, ReindexAssertOpExpression) { rocRoller::KernelGraph::KernelGraph kgraph; auto unit = Expression::literal(1); auto kernel = kgraph.control.addElement(Kernel()); auto loadA = kgraph.control.addElement(LoadVGPR(DataType::Int32, true)); kgraph.control.addElement(Body(), {kernel}, {loadA}); auto user0 = kgraph.coordinates.addElement(User(Operations::OperationTag(0), "user0")); auto vgprA = kgraph.coordinates.addElement(VGPR()); kgraph.coordinates.addElement(DataFlow(), {user0}, {vgprA}); kgraph.mapper.connect(loadA, vgprA); auto exprA = std::make_shared( Expression::DataFlowTag{vgprA, Register::Type::Scalar, DataType::Int32}); auto assertOp = kgraph.control.addElement(AssertOp{"assert", exprA > unit}); kgraph.control.addElement(Sequence(), {loadA}, {assertOp}); auto loadB = kgraph.control.addElement(LoadVGPR(DataType::Int32, true)); kgraph.control.addElement(Body(), {kernel}, {loadB}); auto vgprB = kgraph.coordinates.addElement(VGPR()); kgraph.coordinates.addElement(DataFlow(), {user0}, {vgprB}); kgraph.mapper.connect(loadB, vgprB); kgraph.control.addElement(Sequence(), {loadB}, {assertOp}); GraphReindexer reindexer; reindexer.coordinates.emplace(vgprA, vgprB); reindexExpressions(kgraph, assertOp, reindexer); auto condition = kgraph.control.get(assertOp)->condition; auto lhs = std::get(*condition).lhs; auto tag = std::get(*lhs).tag; EXPECT_EQ(tag, vgprB); } TEST_F(KernelGraphTest, Transformer) { auto example = rocRollerTest::Graphs::GEMM(DataType::Float); int macK = 16; int waveK = 8; example.setTileSize(128, 256, macK); example.setMFMA(32, 32, waveK, 1); example.setUseLDS(true, false, false); auto kgraph0 = example.getKernelGraph(); auto params = example.getCommandParameters(); auto updateParametersTransform = std::make_shared(params); auto addLDSTransform = std::make_shared(params, m_context); auto lowerTileTransform = std::make_shared(params, m_context); auto lowerTensorContractionTransform = std::make_shared(params, m_context); auto unrollLoopsTransform = std::make_shared(params, m_context); auto fuseLoopsTransform = std::make_shared(); auto removeDuplicatesTransform = std::make_shared(); auto cleanLoopsTransform = std::make_shared(); auto assignIndexExprsTransform = std::make_shared(m_context, example.getCommand()); kgraph0 = kgraph0.transform(updateParametersTransform); auto kgraph1 = kgraph0.transform(addLDSTransform); kgraph1 = kgraph1.transform(lowerTileTransform); kgraph1 = kgraph1.transform(lowerTensorContractionTransform); // // Build transformer one by one // std::unordered_map transformers; for(auto op : kgraph1.control.getNodes()) transformers.emplace(op, kgraph1.buildTransformer(op)); // // Build all transformers at once // kgraph1.buildAllTransformers(); // // The resulting transformers should be identical // for(auto op : kgraph1.control.getNodes()) { auto const& expected = transformers.at(op).getIndexes(); auto const& actual = kgraph1.buildTransformer(op).getIndexes(); ASSERT_EQ(expected.size(), actual.size()); for(auto const& [dim, expr] : expected) { auto it = actual.find(dim); ASSERT_NE(it, actual.end()); EXPECT_TRUE(Expression::identical(expr, it->second)); } } } TEST_F(KernelGraphTest, RemoveSetCoordinate) { auto kgraph = rocRoller::KernelGraph::KernelGraph(); int kernel = kgraph.control.addElement(Kernel()); int nop1 = kgraph.control.addElement(NOP()); int nop2 = kgraph.control.addElement(NOP()); int nop3 = kgraph.control.addElement(NOP()); int nop4 = kgraph.control.addElement(NOP()); int nop5 = kgraph.control.addElement(NOP()); int nop6 = kgraph.control.addElement(NOP()); auto one = Expression::literal(1u); int sc1 = kgraph.control.addElement(SetCoordinate(one)); int sc2 = kgraph.control.addElement(SetCoordinate(one)); int sc3 = kgraph.control.addElement(SetCoordinate(one)); int sc4 = kgraph.control.addElement(SetCoordinate(one)); int sc5 = kgraph.control.addElement(SetCoordinate(one)); int sc6 = kgraph.control.addElement(SetCoordinate(one)); int dim = kgraph.coordinates.addElement(Adhoc()); kgraph.mapper.connect(sc1, dim); kgraph.mapper.connect(sc2, dim); kgraph.mapper.connect(sc3, dim); kgraph.mapper.connect(sc4, dim); kgraph.mapper.connect(sc5, dim); kgraph.mapper.connect(sc6, dim); // Original: // // Kernel // | // |[body] // v // nop1 // | // |[seq] // v // nop2 -------------- // | | // |[body] |[seq] // v v // sc1 sc2 ------------- // | | | // |[seq] |[body] |[body] // v v v // sc3 nop3 sc4 ------------------ // |------------ | | // | | |[seq] |[seq] // |[seq] |[seq] v v // v v nop4 nop5 // sc5 sc6 // | // |[body] // v // nop6 // kgraph.control.addElement(Body(), {kernel}, {nop1}); kgraph.control.addElement(Sequence(), {nop1}, {nop2}); kgraph.control.addElement(Body(), {nop2}, {sc1}); kgraph.control.addElement(Sequence(), {sc1}, {sc3}); kgraph.control.addElement(Sequence(), {sc3}, {sc5}); kgraph.control.addElement(Sequence(), {sc3}, {sc6}); kgraph.control.addElement(Body(), {sc6}, {nop6}); kgraph.control.addElement(Sequence(), {nop2}, {sc2}); kgraph.control.addElement(Body(), {sc2}, {nop3}); kgraph.control.addElement(Body(), {sc2}, {sc4}); kgraph.control.addElement(Sequence(), {sc4}, {nop4}); kgraph.control.addElement(Sequence(), {sc4}, {nop5}); auto removeSetCoordinate = std::make_shared(); auto kg2 = kgraph.transform(removeSetCoordinate); // After: // // Kernel // | // |[body] // v // nop1 // | // |[seq] // v // nop2 ------------------------------------- // | | | | // |[body] |[seq] |[seq] |[seq] // v v v v // nop6 nop3 nop4 nop5 // std::string expected = R".( digraph { "1"[label="Kernel(1)"]; "2"[label="NOP(2)"]; "3"[label="NOP(3)"]; "4"[label="NOP(4)"]; "5"[label="NOP(5)"]; "6"[label="NOP(6)"]; "7"[label="NOP(7)"]; "14"[label="Body(14)",shape=box]; "15"[label="Sequence(15)",shape=box]; "30"[label="Sequence(30)",shape=box]; "31"[label="Sequence(31)",shape=box]; "32"[label="Sequence(32)",shape=box]; "33"[label="Body(33)",shape=box]; "1" -> "14" "2" -> "15" "3" -> "30" "3" -> "31" "3" -> "32" "3" -> "33" "14" -> "2" "15" -> "3" "30" -> "4" "31" -> "5" "32" -> "6" "33" -> "7" })."; EXPECT_EQ(NormalizedSource(expected), NormalizedSource(kg2.control.toDOT())); } TEST_F(KernelGraphTest, StreamKTwoTileDPFirst) { auto kgraph = rocRoller::KernelGraph::KernelGraph(); uint numTileM = 57; uint numTileN = 57; uint numTileK = 57; uint numWGs = 128; auto kernel = kgraph.control.addElement(Kernel()); auto [forKCoord, forKOp] = rangeFor(kgraph, Expression::literal(numTileK), rocRoller::KLOOP); auto user = kgraph.coordinates.addElement(User({}, "result")); auto tileM = kgraph.coordinates.addElement( MacroTileNumber(0, Expression::literal(numTileM), nullptr)); auto tileN = kgraph.coordinates.addElement( MacroTileNumber(1, Expression::literal(numTileN), nullptr)); auto tileK = kgraph.coordinates.addElement( MacroTileNumber(-1, Expression::literal(numTileK), nullptr)); kgraph.coordinates.addElement(PassThrough(), {forKCoord}, {tileK}); kgraph.coordinates.addElement(Flatten(), {tileM, tileN, tileK}, {user}); auto dstVGPR = kgraph.coordinates.addElement(VGPR()); auto wgDim = kgraph.coordinates.addElement(Workgroup(0)); auto wgExpr = std::make_shared( Expression::DataFlowTag{wgDim, Register::Type::Vector, DataType::UInt32}); auto assignWGNumber = kgraph.control.addElement(Assign{Register::Type::Vector, wgExpr}); kgraph.mapper.connect(assignWGNumber, dstVGPR, NaryArgument::DEST); kgraph.coordinates.addElement(PassThrough(), {user}, {dstVGPR}); auto storeOp = kgraph.control.addElement(StoreVGPR()); kgraph.mapper.connect(storeOp, user); kgraph.mapper.connect(storeOp, dstVGPR); auto preWaitOp = kgraph.control.addElement(WaitZero()); auto loopWaitOp = kgraph.control.addElement(WaitZero()); kgraph.control.addElement(Body(), {kernel}, {preWaitOp}); kgraph.control.addElement(Sequence(), {preWaitOp}, {forKOp}); kgraph.control.addElement(Body(), {forKOp}, {assignWGNumber}); kgraph.control.addElement(Sequence(), {assignWGNumber}, {storeOp}); kgraph.control.addElement(Sequence(), {storeOp}, {loopWaitOp}); CommandParametersPtr params = std::make_shared(); params->loopOverOutputTilesDimensions = {0, 1}; // Helper to check loop order in a given kernel graph auto checkStreamKLoopOrder = [&](rocRoller::KernelGraph::KernelGraph& kg, const std::string& firstLoop, const std::string& secondLoop) { int body = -1, sequence = -1; for(auto const scope : filter(kg.control.isElemType(), kg.control.depthFirstVisit(kg.control.roots().only().value()))) { auto bodies = kg.control.getOutputNodeIndices(scope).to(); auto sequences = kg.control.getOutputNodeIndices(scope).to(); if(bodies.size() + sequences.size() == 1) continue; AssertFatal( bodies.size() == 1 && sequences.size() == 1, "Expected one body and one sequence in scope, found {} bodies and {} sequences", bodies.size(), sequences.size()); body = *bodies.begin(); sequence = *sequences.begin(); break; } // Check for first loop through the Body edge for(auto const loop : filter(kg.control.isElemType(), kg.control.depthFirstVisit(body))) { auto forloop = kg.control.get(loop).value(); EXPECT_EQ(forloop.loopName, firstLoop); break; } // Check for second loop through the Sequence edge for(auto const loop : filter(kg.control.isElemType(), kg.control.depthFirstVisit(sequence))) { auto forloop = kg.control.get(loop).value(); EXPECT_EQ(forloop.loopName, secondLoop); break; } }; // For streamKTwoTile, the SK loop is first params->streamK = StreamKMode::TwoTile; auto kgraphTwoTile = kgraph.transform(std::make_shared( m_context, params, rocRoller::KLOOP, rocRoller::KLOOP, Expression::literal(numWGs))); if(m_context->kernelOptions()->removeSetCoordinate) kgraphTwoTile = kgraphTwoTile.transform(std::make_shared()); checkStreamKLoopOrder(kgraphTwoTile, "SKStreamTileLoop", "DPStreamTileLoop"); m_context->kernel()->resetArguments(); // For streamKTwoTileDPFirst, the DP loop is first params->streamK = StreamKMode::TwoTileDPFirst; auto kgraphTwoTileDPFirst = kgraph.transform(std::make_shared( m_context, params, rocRoller::KLOOP, rocRoller::KLOOP, Expression::literal(numWGs))); if(m_context->kernelOptions()->removeSetCoordinate) kgraphTwoTileDPFirst = kgraphTwoTileDPFirst.transform(std::make_shared()); checkStreamKLoopOrder(kgraphTwoTileDPFirst, "DPStreamTileLoop", "SKStreamTileLoop"); } }