/** * Copyright (c) NVIDIA CORPORATION & AFFILIATES, 2001-2015. ALL RIGHTS RESERVED. * * See file LICENSE for terms. */ /* * Copyright (c) 2014-2015 UT-Battelle, LLC. All rights reserved. * Copyright (c) 2014-2020 NVIDIA corporation & affiliates. All rights reserved. * Copyright (C) 2014-2015 The University of Houston System. All rights reserved. * Copyright (C) 2015 The University of Tennessee and The University * of Tennessee Research Foundation. All rights reserved. * Copyright (C) 2016-2020 ARM Ltd. All rights reserved. * Copyright (c) 2016 Los Alamos National Security, LLC. All rights reserved. * Copyright (C) 2016-2020 Advanced Micro Devices, Inc. All rights reserved. * Copyright (C) 2019 UChicago Argonne, LLC. All rights reserved. * Copyright (C) 2020 Huawei Technologies Co., Ltd. All rights reserved. * Copyright (C) 2016-2020 Stony Brook University. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. Neither the name of the copyright holder nor the names of its * contributors may be used to endorse or promote products derived from * this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /** * Sample taken from: https://github.com/openucx/ucx/blob/master/test/mpi/shmem_pingpong.c */ #include #include #include #include #include #include #include #define GLOBAL_DATA_SIZE 1024 static int show_result(const struct timeval* tv_prev, const struct timeval* tv_curr, long iters, size_t msg_size, int force) { double elapsed; elapsed = (tv_curr->tv_sec + tv_curr->tv_usec * 1e-6) - (tv_prev->tv_sec + tv_prev->tv_usec * 1e-6); if(((elapsed >= 1.0) || force) && (shmem_my_pe() == 0)) { printf("%ld iterations, %lu bytes, latency: %.3f usec\n", iters, msg_size, elapsed * 1e6 / iters / 2.0); return 1; } return 0; } static void run_pingpong(char* mem, size_t msg_size, long num_iters, int use_wait, int do_quiet, int use_flag) { struct timeval tv_prev, tv_curr; int my_pe, dst_pe; volatile int* rsn; char* msg; int* ssn; int sn; long i, prev_i; msg = malloc(msg_size); if(msg == NULL) { return; } memset(msg, 0, msg_size); gettimeofday(&tv_prev, NULL); prev_i = 0; my_pe = shmem_my_pe(); dst_pe = 1 - my_pe; rsn = (int*) &mem[msg_size - sizeof(int)]; ssn = (int*) &msg[msg_size - sizeof(int)]; for(i = 0; i < num_iters; ++i) { sn = i & 127; *ssn = sn; if(my_pe == 0) { shmem_putmem(mem, msg, msg_size, dst_pe); if(do_quiet) { shmem_quiet(); } } if(use_wait) { shmem_int_wait_until(rsn, SHMEM_CMP_EQ, sn); } else { while(*rsn != sn) ; } if(my_pe == 1) { if(use_flag) { shmem_putmem(mem, msg, msg_size - sizeof(int), dst_pe); shmem_fence(); shmem_int_put((int*) rsn, ssn, 1, dst_pe); } else { shmem_putmem(mem, msg, msg_size, dst_pe); } if(do_quiet) { shmem_quiet(); } } if((i % 1000) == 0) { gettimeofday(&tv_curr, NULL); if(show_result(&tv_prev, &tv_curr, i - prev_i, msg_size, 0)) { prev_i = i; tv_prev = tv_curr; } } } gettimeofday(&tv_curr, NULL); show_result(&tv_prev, &tv_curr, num_iters - prev_i, msg_size, 1); free(msg); } static void usage() { printf("Usage: shmem-pingpong [options]\n"); printf("\n"); printf("Options are:\n"); printf(" -n Specify number of iterations to run (default: 10000).\n"); printf(" -s Specify message size (default: 4 bytes).\n"); printf(" -w Wait for data using shmem_wait_until() (default: poll on " "memory).\n"); printf(" -f Send data and flag separately with shmem_fence() " "in-between.\n"); printf(" -g Use global data (default: heap).\n"); printf(" -q call shmem_quiet() after every shmem_put().\n"); printf("\n"); } int main(int argc, char** argv) { static char global_buffer[GLOBAL_DATA_SIZE]; int use_wait, use_global, do_quiet, use_flag; size_t msg_size; long num_iters; int my_pe; char* mem; int c; start_pes(0); my_pe = shmem_my_pe(); if(shmem_n_pes() != 2) { fprintf(stderr, "This test requires exactly 2 processes\n"); return -1; } num_iters = 10000; use_global = 0; use_wait = 0; do_quiet = 0; use_flag = 0; msg_size = 8; while((c = getopt(argc, argv, "n:s:wgqfh")) != -1) { switch(c) { case 'n': num_iters = atol(optarg); if(num_iters == 0) { num_iters = LONG_MAX; } break; case 'w': use_wait = 1; break; case 'g': use_global = 1; break; case 'q': do_quiet = 1; break; case 'f': use_flag = 1; break; case 's': msg_size = atol(optarg); break; case 'h': default: if(my_pe == 0) { usage(); } return 0; } } if(msg_size < sizeof(int)) { fprintf(stderr, "message size must be at least %lu\n", sizeof(int)); return -1; } if(use_global) { if(msg_size <= GLOBAL_DATA_SIZE) { mem = global_buffer; } else { fprintf(stderr, "global data can be used only up to %lu bytes\n", (size_t) GLOBAL_DATA_SIZE); return -1; } } else { mem = shmalloc(msg_size); } memset(mem, 0xff, msg_size); shmem_barrier_all(); run_pingpong(mem, msg_size, num_iters, use_wait, do_quiet, use_flag); shmem_barrier_all(); if(!use_global) { shfree(mem); } shmem_finalize(); return 0; }