derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
313 lines
20 KiB
Python
313 lines
20 KiB
Python
# Copyright Advanced Micro Devices, Inc.
|
|
# SPDX-License-Identifier: MIT
|
|
|
|
skip_tests = {
|
|
"gfx950": {
|
|
"cuda": {
|
|
"test_autocast_torch_bf16",
|
|
"test_autocast_torch_fp16",
|
|
}
|
|
},
|
|
"common": {
|
|
"autograd": [
|
|
# Stream comparison mismatch on ROCm (non-default stream vs default stream)
|
|
# AssertionError: <torch.cuda.Stream ...> != <torch.cuda.Stream cuda_stream=0x0>
|
|
# Seems to fails on Linux and Windows across torch versions and all tested GPUs.
|
|
"test_side_stream_backward_overlap",
|
|
],
|
|
"cuda": [
|
|
# RuntimeError: Error building extension 'dummy_allocator'
|
|
# Skipped across all PyTorch versions; the hipblas.h include error
|
|
# persists in the ROCm SDK environment.
|
|
"test_mempool_empty_cache_inactive",
|
|
# TestCudaAllocator - FileNotFoundError: flamegraph.pl missing in CI
|
|
"test_memory_snapshot",
|
|
"test_memory_plots",
|
|
# HIP_VISIBLE_DEVICES and CUDA_VISIBLE_DEVICES not working
|
|
# to restrict visibility of devices
|
|
# AssertionError: String comparison failed: '8, 1' != '8, 8'
|
|
"test_device_count_not_cached_pre_init",
|
|
# empty_stats() in test_cuda.py does not match stats returned
|
|
# Returned is:
|
|
# OrderedDict({'allocated_bytes.allocated': 0, 'allocated_bytes.current': 0, 'allocated_bytes.freed': 0,
|
|
# 'allocated_bytes.peak': 0, 'allocation.allocated': 0, 'allocation.current': 0, 'allocation.freed': 0,
|
|
# 'allocation.peak': 0, 'host_alloc_time.avg': 0, 'host_alloc_time.count': 0, 'host_alloc_time.max': 0,
|
|
# 'host_alloc_time.min': 0, 'host_alloc_time.total': 0, 'host_free_time.avg': 0, 'host_free_time.count': 0,
|
|
# 'host_free_time.max': 0, 'host_free_time.min': 0, 'host_free_time.total': 0, 'num_host_alloc': 0,
|
|
# 'num_host_free': 0, 'reserved_bytes.allocated': 0, 'reserved_bytes.current': 0, 'reserved_bytes.freed': 0,
|
|
# 'reserved_bytes.peak': 0, 'segment.allocated': 0, 'segment.current': 0, 'segment.freed': 0, 'segment.peak': 0})
|
|
"test_host_memory_stats",
|
|
# THIS IS AN OLD ERROR
|
|
# In file included from /home/tester/.cache/torch_extensions/py312_cpu/dummy_allocator/main_hip.cpp:5:
|
|
# /home/tester/TheRock/.venv/lib/python3.12/site-packages/torch/include/ATen/hip/Exceptions.h:4:10: fatal error: hipblas/hipblas.h: No such file or directory
|
|
# 4 | #include <hipblas/hipblas.h>
|
|
# | ^~~~~~~~~~~~~~~~~~~
|
|
# compilation terminated.
|
|
# NEW ERROR
|
|
# RuntimeError: Error building extension 'dummy_allocator'
|
|
"test_mempool_with_allocator",
|
|
# RuntimeError: Error building extension 'dummy_allocator_v3'
|
|
"test_tensor_delete_after_allocator_delete",
|
|
# RuntimeError: Error building extension 'dummy_allocator'
|
|
"test_deleted_mempool_not_used_on_oom",
|
|
# Same hipblas.h compilation error as test_mempool_with_allocator.
|
|
# See https://github.com/pytorch/pytorch/pull/173330
|
|
"test_mempool_expandable",
|
|
# Change detector test (Cublaslt vs Cublas depending on gcn_arch and torch version)
|
|
# Always skip as this test is very basic and needs manual intervention for new architectures
|
|
# See
|
|
# * https://github.com/ROCm/pytorch/pull/2742
|
|
# * https://github.com/ROCm/pytorch/pull/2873
|
|
"test_preferred_blas_library_settings",
|
|
# Python 3.14: PEP 649 changed __annotations__ behavior
|
|
# AttributeError: 'Model' object has no attribute '__annotations__'
|
|
# https://github.com/ROCm/TheRock/issues/2985
|
|
"test_autocast_cat_jit",
|
|
# what(): HIP error: operation not permitted when stream is capturing
|
|
# Search for `hipErrorStreamCaptureUnsupported' in https://docs.nvidia.com/cuda/cuda-runtime-api/group__HIPRT__TYPES.html for more information.
|
|
# HIP kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
|
|
# For debugging consider passing AMD_SERIALIZE_KERNEL=3
|
|
# Compile with `TORCH_USE_HIP_DSA` to enable device-side assertions.
|
|
#
|
|
# Exception raised from ~CUDAGraph at /__w/TheRock/TheRock/external-builds/pytorch/pytorch/aten/src/ATen/hip/HIPGraph.cpp:320 (most recent call first):
|
|
# frame #0: c10::Error::Error(c10::SourceLocation, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >) + 0x80 (0x7f2316f1bdf0 in /home/tester/TheRock/.venv/lib/python3.12/site-packages/torch/lib/libc10.so)
|
|
"test_graph_make_graphed_callables_parameterless_nograd_module_without_amp_allow_unused_input",
|
|
"test_graph_make_graphed_callables_parameterless_nograd_module_without_amp_not_allow_unused_input",
|
|
# ----------------
|
|
# maybe failing
|
|
# ----------------
|
|
# "test_hip_device_count"
|
|
# "test_nvtx"
|
|
# ----------------
|
|
#
|
|
# Multi-processing error in py3.14 - https://github.com/ROCm/TheRock/issues/4197
|
|
"test_is_pinned_no_context",
|
|
],
|
|
"nn": [
|
|
# external-builds/pytorch/pytorch/test/test_nn.py::TestNN::test_RNN_dropout_state MIOpen(HIP): Error [Compile] 'hiprtcCompileProgram(prog.get(), c_options.size(), c_options.data())' MIOpenDropoutHIP.cpp: HIPRTC_ERROR_COMPILATION (6)
|
|
# MIOpen(HIP): Error [BuildHip] HIPRTC status = HIPRTC_ERROR_COMPILATION (6), source file: MIOpenDropoutHIP.cpp
|
|
# MIOpen(HIP): Warning [BuildHip] In file included from /tmp/comgr-01c423/input/MIOpenDropoutHIP.cpp:32:
|
|
# /tmp/comgr-01c423/include/miopen_rocrand.hpp:45:10: fatal error: 'rocrand/rocrand_xorwow.h' file not found
|
|
# 45 | #include <rocrand/rocrand_xorwow.h>
|
|
# | ^~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
# 1 error generated when compiling for gfx942.
|
|
# MIOpen Error: /therock/src/rocm-libraries/projects/miopen/src/hipoc/hipoc_program.cpp:299: Code object build failed. Source: MIOpenDropoutHIP.cpp
|
|
"test_RNN_dropout_state",
|
|
# AssertionError: "Input and parameter tensors are not at the same device" does not match "Expected all tensors
|
|
# to be on the same device, but got weight is on cpu, different from other tensors on cuda:0 (when checking
|
|
# argument in method wrapper_CUDA__miopen_rnn)"
|
|
"test_rnn_check_device",
|
|
],
|
|
"torch": [
|
|
# FLAKY!! AssertionError: 'tensor([2.3000+4.j, 7.0000+6.j])' != 'tensor([2.30000+4.j, 7.00000+6.j])'
|
|
# (Note: this will also skip "test_print" in all other test modules)
|
|
"test_print",
|
|
# Python 3.14: PEP 649 changed storage deallocation behavior
|
|
# AssertionError: False is not true
|
|
# https://github.com/ROCm/TheRock/issues/2985
|
|
"test_storage_dealloc_subclass_resurrected",
|
|
"test_storage_dealloc_subclass_zombie",
|
|
# torch._dynamo.exc.BackendCompilerFailed: backend='aot_eager' raised:
|
|
# TypeError: 'CustomDecompTable' object is not a mapping
|
|
"test_fx_memory_profiler_augmentation",
|
|
],
|
|
"unary_ufuncs": [
|
|
# ----------------
|
|
# maybe failing
|
|
# ----------------
|
|
# this passed on gfx942
|
|
# "test_reference_numerics_large__refs_nn_functional_mish_cuda_float16",
|
|
# "test_reference_numerics_large_nn_functional_mish_cuda_float16",
|
|
# ----------------
|
|
# AttributeError: 'NoneType' object has no attribute 'dtype'
|
|
# it is all due to the same reason "expected" being None
|
|
# in def _test_reference_numerics(self, dtype, op, tensors, equal_nan=True):
|
|
# actual = op(t, **torch_kwargs)
|
|
# expected = op.ref(a, **numpy_kwargs)
|
|
# print("torch_kwargs", torch_kwargs, "t", t, "actual", actual)
|
|
# print("numpy_kwargs", numpy_kwargs, "a", a, "expected", expected)
|
|
# output:
|
|
# torch_kwargs {} t tensor([inf, inf, inf, -inf, -inf, -inf, nan, nan, nan], device='cuda:0') actual tensor([0., 0., 0., 0., 0., 0., nan, nan, nan], device='cuda:0')
|
|
# numpy_kwargs {} a [ inf inf inf -inf -inf -inf nan nan nan] expected None
|
|
"test_reference_numerics_extremal__refs_special_spherical_bessel_j0_cuda_float32",
|
|
"test_reference_numerics_extremal__refs_special_spherical_bessel_j0_cuda_float64",
|
|
"test_reference_numerics_extremal_special_airy_ai_cuda_float32",
|
|
"test_reference_numerics_extremal_special_airy_ai_cuda_float64",
|
|
"test_reference_numerics_extremal_special_spherical_bessel_j0_cuda_float32",
|
|
"test_reference_numerics_extremal_special_spherical_bessel_j0_cuda_float64",
|
|
"test_reference_numerics_large__refs_special_spherical_bessel_j0_cuda_float32",
|
|
"test_reference_numerics_large__refs_special_spherical_bessel_j0_cuda_float64",
|
|
"test_reference_numerics_large__refs_special_spherical_bessel_j0_cuda_int16",
|
|
"test_reference_numerics_large__refs_special_spherical_bessel_j0_cuda_int32",
|
|
"test_reference_numerics_large__refs_special_spherical_bessel_j0_cuda_int64",
|
|
"test_reference_numerics_large_special_spherical_bessel_j0_cuda_float32",
|
|
"test_reference_numerics_large_special_spherical_bessel_j0_cuda_float64",
|
|
"test_reference_numerics_large_special_spherical_bessel_j0_cuda_int16",
|
|
"test_reference_numerics_large_special_spherical_bessel_j0_cuda_int32",
|
|
"test_reference_numerics_large_special_spherical_bessel_j0_cuda_int64",
|
|
"test_reference_numerics_normal__refs_special_spherical_bessel_j0_cuda_bool",
|
|
"test_reference_numerics_normal__refs_special_spherical_bessel_j0_cuda_float32",
|
|
"test_reference_numerics_normal__refs_special_spherical_bessel_j0_cuda_float64",
|
|
"test_reference_numerics_normal__refs_special_spherical_bessel_j0_cuda_int16",
|
|
"test_reference_numerics_normal__refs_special_spherical_bessel_j0_cuda_int32",
|
|
"test_reference_numerics_normal__refs_special_spherical_bessel_j0_cuda_int64",
|
|
"test_reference_numerics_normal__refs_special_spherical_bessel_j0_cuda_int8",
|
|
"test_reference_numerics_normal__refs_special_spherical_bessel_j0_cuda_uint8",
|
|
"test_reference_numerics_normal_special_airy_ai_cuda_bool",
|
|
"test_reference_numerics_normal_special_airy_ai_cuda_float32",
|
|
"test_reference_numerics_normal_special_airy_ai_cuda_float64",
|
|
"test_reference_numerics_normal_special_airy_ai_cuda_int16",
|
|
"test_reference_numerics_normal_special_airy_ai_cuda_int32",
|
|
"test_reference_numerics_normal_special_airy_ai_cuda_int64",
|
|
"test_reference_numerics_normal_special_airy_ai_cuda_int8",
|
|
"test_reference_numerics_normal_special_airy_ai_cuda_uint8",
|
|
"test_reference_numerics_normal_special_spherical_bessel_j0_cuda_bool",
|
|
"test_reference_numerics_normal_special_spherical_bessel_j0_cuda_float32",
|
|
"test_reference_numerics_normal_special_spherical_bessel_j0_cuda_float64",
|
|
"test_reference_numerics_normal_special_spherical_bessel_j0_cuda_int16",
|
|
"test_reference_numerics_normal_special_spherical_bessel_j0_cuda_int32",
|
|
"test_reference_numerics_normal_special_spherical_bessel_j0_cuda_int64",
|
|
"test_reference_numerics_normal_special_spherical_bessel_j0_cuda_int8",
|
|
"test_reference_numerics_normal_special_spherical_bessel_j0_cuda_uint8",
|
|
"test_reference_numerics_small__refs_special_spherical_bessel_j0_cuda_float32",
|
|
"test_reference_numerics_small__refs_special_spherical_bessel_j0_cuda_float64",
|
|
"test_reference_numerics_small__refs_special_spherical_bessel_j0_cuda_int16",
|
|
"test_reference_numerics_small__refs_special_spherical_bessel_j0_cuda_int32",
|
|
"test_reference_numerics_small__refs_special_spherical_bessel_j0_cuda_int64",
|
|
"test_reference_numerics_small__refs_special_spherical_bessel_j0_cuda_int8",
|
|
"test_reference_numerics_small__refs_special_spherical_bessel_j0_cuda_uint8",
|
|
"test_reference_numerics_small_special_airy_ai_cuda_float32",
|
|
"test_reference_numerics_small_special_airy_ai_cuda_float64",
|
|
"test_reference_numerics_small_special_airy_ai_cuda_int16",
|
|
"test_reference_numerics_small_special_airy_ai_cuda_int32",
|
|
"test_reference_numerics_small_special_airy_ai_cuda_int64",
|
|
"test_reference_numerics_small_special_airy_ai_cuda_int8",
|
|
"test_reference_numerics_small_special_airy_ai_cuda_uint8",
|
|
"test_reference_numerics_small_special_spherical_bessel_j0_cuda_float32",
|
|
"test_reference_numerics_small_special_spherical_bessel_j0_cuda_float64",
|
|
"test_reference_numerics_small_special_spherical_bessel_j0_cuda_int16",
|
|
"test_reference_numerics_small_special_spherical_bessel_j0_cuda_int32",
|
|
"test_reference_numerics_small_special_spherical_bessel_j0_cuda_int64",
|
|
"test_reference_numerics_small_special_spherical_bessel_j0_cuda_int8",
|
|
"test_reference_numerics_small_special_spherical_bessel_j0_cuda_uint8",
|
|
],
|
|
},
|
|
# Special notes for Windows:
|
|
# * Some tests hang and *must* be skipped for testing to complete.
|
|
# That is likely related to processes not terminating on their own:
|
|
# https://github.com/ROCm/TheRock/issues/999. Note that even if
|
|
# _test cases_ themselves terminate, the parent process still
|
|
# hangs though. In run_pytorch_tests.py we exit with `os.kill()` to
|
|
# force termination.
|
|
# * Linux has substantial testing on datacenter GPUs while Windows support
|
|
# is newer and skews towards consumer GPUs with lower specs. We disable
|
|
# some tests that are resource intensive or otherwise degrade CI
|
|
# stability. We are also conservative in disabling some tests for all
|
|
# pytorch versions and all GPUs. Perfect is the enemy of the good and
|
|
# we would rather run a subset of tests with high confidence than run
|
|
# all tests with low confidence.
|
|
"windows": {
|
|
"autograd": [
|
|
# JIT compilation without MSVC installed then device mismatch:
|
|
# We should fix the test to fail/skip more gracefully.
|
|
# Error checking compiler version for cl: [WinError 2] The system cannot find the file specified
|
|
# AssertionError: Object comparison failed: <torch.cuda.Stream device=cuda:0 cuda_stream=0x1fa05550b50> != <torch.cuda.Stream device=cuda:0 cuda_stream=0x0>
|
|
"test_consumer_to_single_producer_case_2_correctness",
|
|
# Failures on Python 3.11 (not 3.12 or 3.13) with
|
|
# Windows fatal exception: code 0xc0000374
|
|
# Possibly due to use of `torch.jit.script`?
|
|
"test_fork_join_in_middle",
|
|
# This test JIT compiles and fails if MSVC is not installed on Windows:
|
|
# We should fix the test to fail/skip more gracefully.
|
|
# subprocess.CalledProcessError: Command '['where', 'cl']' returned non-zero exit status 1.
|
|
"test_multi_grad_all_hooks",
|
|
# This is/was also failing on gfx942 linux, see the per-release skip test files.
|
|
# AssertionError: "Simulate error" does not match "grad can be implicitly created only for scalar outputs"
|
|
"test_reentrant_parent_error_on_cpu_cuda",
|
|
],
|
|
"binary_ufuncs": [
|
|
# Failures on Python 3.11 (not 3.12 or 3.13) with
|
|
# Windows fatal exception: code 0xc0000374
|
|
# Possibly due to use of `torch.jit.script`?
|
|
"test_div_and_floordiv_script_vs_python_cuda",
|
|
"test_idiv_and_ifloordiv_vs_python_cuda",
|
|
],
|
|
"cuda": [
|
|
# RuntimeError: miopenStatusUnknownError
|
|
"test_autocast_rnn",
|
|
# On some test runners
|
|
# AssertionError: False is not true
|
|
# self.assertTrue(abs(check_workspace_size(a) - default_workspace_size) < 524288)
|
|
"test_cublas_workspace_explicit_allocation",
|
|
# For Python 3.11, this fails with
|
|
# torch.AcceleratorError: HIP error: operation not permitted when stream is capturing
|
|
"test_cuda_graph_tensor_item_not_allowed",
|
|
# *** Test hang (see above) ***
|
|
"test_graph_error",
|
|
# This test conflicts with how our test script and runners are
|
|
# configured.
|
|
"test_hip_device_count",
|
|
# Multi-processing forking code that may not work on Windows?
|
|
"test_is_pinned_no_context",
|
|
# Bug, needs triage:
|
|
# AssertionError: Scalars are not equal!
|
|
# Expected 0 but got 2173342911312.
|
|
"test_streams",
|
|
# Device-side assert() does not propagate to the host on Windows ROCm:
|
|
# the KMD has no trap handler, so the faulted queue never reports an
|
|
# error and torch.cuda.synchronize() hangs until the CI job timeout.
|
|
# These tests deliberately trigger a device-side assert and await it
|
|
# with no subprocess timeout, so they hang rather than fail.
|
|
# Re-enable once the Windows ROCm driver propagates device-side
|
|
# faults to the runtime.
|
|
# See https://github.com/ROCm/TheRock/issues/5565
|
|
"test_fixed_cuda_assert_async",
|
|
"test_index_out_of_bounds_exception_cuda",
|
|
# Same device-side-assert-propagation issue as the two tests above:
|
|
# spawns a subprocess that feeds invalid probabilities (negative,
|
|
# inf, nan) to torch.multinomial, calls torch.cuda.synchronize(), and
|
|
# asserts the device-side assert surfaces in stderr. On Windows ROCm
|
|
# the fault never propagates, so it hangs/fails instead.
|
|
# See https://github.com/ROCm/TheRock/issues/5565
|
|
"test_multinomial_invalid_probs_cuda",
|
|
],
|
|
"nn": [
|
|
# Hangs on some Windows ROCm runners until the job hits the 6h limit.
|
|
# https://github.com/ROCm/TheRock/issues/5565
|
|
"test_cross_entropy_loss_2d_out_of_bounds_class_index",
|
|
# RuntimeError: miopenStatusUnknownError
|
|
"test_cudnn_weight_format",
|
|
"test_rnn_retain_variables_cuda_float16",
|
|
"test_rnn_retain_variables_cuda_float32",
|
|
"test_variable_sequence_cuda_float16",
|
|
"test_variable_sequence_cuda_float32",
|
|
# Convs are failing numerics on some test machines.
|
|
# Possibly fixed by https://github.com/ROCm/rocm-libraries/commit/7338a0b71f43c41c3882e976ef591cb9adcd64d0
|
|
# At least [gfx1151, torch 2.9], possibly others.
|
|
# Mismatched elements: 4 / 4 (100.0%)
|
|
# Mismatched elements: 96 / 96 (100.0%)
|
|
# TODO: try re-enabling these tests after that commit is included in releases
|
|
"test_Conv", # Broad pattern - this matches 10s of test cases.
|
|
# AssertionError: False is not true
|
|
# self.assertTrue(torch.allclose(x.grad.cpu(), xx.grad, rtol=rtol, atol=1e-3))
|
|
"test_warp_softmax_64bit_indexing_cuda_float16",
|
|
],
|
|
"torch": [
|
|
# Large test that isn't very CI-friendly (takes 1-180 seconds depending on runner and torch version)
|
|
"test_conv_transposed_large_cuda",
|
|
# *** Test hang (see above) ***
|
|
# The callstack for this one points to _fill_mem_eff_dropout_mask, so it may be related to aotriton?
|
|
"test_cublas_config_nondeterministic_alert_cuda",
|
|
# Large test that isn't very CI-friendly (takes ~2 seconds, possibly hanging)
|
|
"test_memory_format_operators_cuda"
|
|
# Flaky tests hanging on some gfx1151 machines...
|
|
# Maybe memory pressure? Tests use some large tensors:
|
|
# v = torch.FloatTensor([64000., 32., 64000.])
|
|
# Move to gfx1151-specific skip list? Check if passing on Linux.
|
|
# We could also skip all test_grad_*.
|
|
"test_grad_scale_will_not_overflow_cuda",
|
|
"test_grad_scaling_unscale_sparse_cuda_float32",
|
|
],
|
|
},
|
|
}
|