/******************************************************************************* * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. * Modifications Copyright (c) 2019-2025, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * * Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * Neither the name of the NVIDIA CORPORATION nor the * names of its contributors may be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE * ARE DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * ******************************************************************************/ #pragma once #include #if defined(_CCCL_IMPLICIT_SYSTEM_HEADER_GCC) # pragma GCC system_header #elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_CLANG) # pragma clang system_header #elif defined(_CCCL_IMPLICIT_SYSTEM_HEADER_MSVC) # pragma system_header #endif // no system header #if THRUST_DEVICE_COMPILER == THRUST_DEVICE_COMPILER_HIP # include # include # include # include # include # include # include # include # include # include // rocprim include # include # include THRUST_NAMESPACE_BEGIN namespace hip_rocprim { namespace __extrema { template struct arg_min_f { Predicate predicate; using pair_type = tuple; THRUST_HOST_DEVICE arg_min_f(Predicate p) : predicate(p) {} pair_type THRUST_DEVICE operator()(pair_type const& lhs, pair_type const& rhs) { InputType const& rhs_value = get<0>(rhs); InputType const& lhs_value = get<0>(lhs); IndexType const& rhs_key = get<1>(rhs); IndexType const& lhs_key = get<1>(lhs); // check values first if (predicate(lhs_value, rhs_value)) { return lhs; } else if (predicate(rhs_value, lhs_value)) { return rhs; } // values are equivalent, prefer smaller index if (lhs_key < rhs_key) { return lhs; } else { return rhs; } } }; // struct arg_min_f template struct arg_max_f { Predicate predicate; using pair_type = tuple; THRUST_HOST_DEVICE arg_max_f(Predicate p) : predicate(p) {} pair_type THRUST_DEVICE operator()(pair_type const& lhs, pair_type const& rhs) { InputType const& rhs_value = get<0>(rhs); InputType const& lhs_value = get<0>(lhs); IndexType const& rhs_key = get<1>(rhs); IndexType const& lhs_key = get<1>(lhs); // check values first if (predicate(lhs_value, rhs_value)) { return rhs; } else if (predicate(rhs_value, lhs_value)) { return lhs; } // values are equivalent, prefer smaller index if (lhs_key < rhs_key) { return lhs; } else { return rhs; } } }; // struct arg_max_f template struct arg_minmax_f { Predicate predicate; using pair_type = tuple; using two_pairs_type = tuple; using arg_min_t = arg_min_f; using arg_max_t = arg_max_f; THRUST_HOST_DEVICE arg_minmax_f(Predicate p) : predicate(p) {} two_pairs_type THRUST_DEVICE operator()(two_pairs_type const& lhs, two_pairs_type const& rhs) { pair_type const& rhs_min = get<0>(rhs); pair_type const& lhs_min = get<0>(lhs); pair_type const& rhs_max = get<1>(rhs); pair_type const& lhs_max = get<1>(lhs); auto result = thrust::make_tuple(arg_min_t(predicate)(lhs_min, rhs_min), arg_max_t(predicate)(lhs_max, rhs_max)); return result; } struct duplicate_tuple { THRUST_DEVICE two_pairs_type operator()(pair_type const& t) { return thrust::make_tuple(t, t); } }; }; // struct arg_minmax_f // this is an init-less reduce, needed for min/max-element functionality // this will avoid copying the first value from device->host template THRUST_RUNTIME_FUNCTION T extrema(execution_policy& policy, InputIt first, Size num_items, BinaryOp binary_op, T*) { using namespace thrust::system::hip_rocprim::temp_storage; if (num_items == 0) { hip_rocprim::throw_on_error(hipErrorInvalidValue, "extrema number of items is zero"); } size_t temp_storage_bytes = 0; hipStream_t stream = hip_rocprim::stream(policy); bool debug_sync = THRUST_HIP_DEBUG_SYNC_FLAG; // Determine temporary device storage requirements. hip_rocprim::throw_on_error( rocprim::reduce( nullptr, temp_storage_bytes, first, static_cast(nullptr), num_items, binary_op, stream, debug_sync), "extrema failed on 1st step"); size_t storage_size; void* ptr = nullptr; void* temp_stor = nullptr; T* d_result; auto l_part = make_linear_partition(make_partition(&temp_stor, temp_storage_bytes), ptr_aligned_array(&d_result, 1)); // Calculate storage_size including alignment hip_rocprim::throw_on_error(partition(ptr, storage_size, l_part)); // Allocate temporary storage. thrust::detail::temporary_array tmp(policy, storage_size); ptr = static_cast(tmp.data().get()); // Create pointers with alignment hip_rocprim::throw_on_error(partition(ptr, storage_size, l_part)); hip_rocprim::throw_on_error( rocprim::reduce(ptr, temp_storage_bytes, first, d_result, num_items, binary_op, stream, debug_sync), "extrema failed on 2nd step"); T result = hip_rocprim::get_value(policy, d_result); return result; } template