diff --git a/.gitignore b/.gitignore index a6eece7b97b..86ab1392aec 100644 --- a/.gitignore +++ b/.gitignore @@ -276,6 +276,8 @@ ompi/test/general/proc ompi/test/general/request ompi/test/general/seq_tracker ompi/test/general/win +ompi/test/general/win_notify +ompi/test/general/win_notify_unsupported ompi/test/monitoring/monitoring_test ompi/test/monitoring/check_monitoring diff --git a/ompi/attribute/attribute_predefined.c b/ompi/attribute/attribute_predefined.c index 3bc1849dc52..781bdbb70dd 100644 --- a/ompi/attribute/attribute_predefined.c +++ b/ompi/attribute/attribute_predefined.c @@ -143,6 +143,9 @@ int ompi_attr_create_predefined_keyvals(void) OMPI_SUCCESS != (rc = create_win(MPI_WIN_CREATE_FLAVOR)) || OMPI_SUCCESS != (rc = create_win(MPI_WIN_MODEL)) || OMPI_SUCCESS != (rc = create_comm(MPI_FT, false)) || /* not #if conditional on OPAL_ENABLE_FT_MPI for ABI */ + OMPI_SUCCESS != (rc = create_win(MPI_WIN_NOTIFICATION_NUM_SB)) || + OMPI_SUCCESS != (rc = create_win(MPI_WIN_NOTIFICATION_NUM_UB)) || + OMPI_SUCCESS != (rc = create_win(MPI_WIN_NOTIFICATION_VALUE_UB)) || 0) { ret = rc; } @@ -227,7 +230,10 @@ int ompi_attr_free_predefined(void) OMPI_SUCCESS != (rc = free_win(MPI_WIN_SIZE)) || OMPI_SUCCESS != (rc = free_win(MPI_WIN_DISP_UNIT)) || OMPI_SUCCESS != (rc = free_win(MPI_WIN_CREATE_FLAVOR)) || - OMPI_SUCCESS != (rc = free_win(MPI_WIN_MODEL))) { + OMPI_SUCCESS != (rc = free_win(MPI_WIN_MODEL)) || + OMPI_SUCCESS != (rc = free_win(MPI_WIN_NOTIFICATION_NUM_SB)) || + OMPI_SUCCESS != (rc = free_win(MPI_WIN_NOTIFICATION_NUM_UB)) || + OMPI_SUCCESS != (rc = free_win(MPI_WIN_NOTIFICATION_VALUE_UB))) { ret = rc; } diff --git a/ompi/errhandler/errcode.c b/ompi/errhandler/errcode.c index 631f1483b12..d04b6f9069c 100644 --- a/ompi/errhandler/errcode.c +++ b/ompi/errhandler/errcode.c @@ -131,6 +131,7 @@ static ompi_mpi_errcode_t ompi_err_value_too_large; static ompi_mpi_errcode_t ompi_err_errhandler; static ompi_mpi_errcode_t ompi_t_err_not_accessible; static ompi_mpi_errcode_t ompi_t_err_not_supported; +static ompi_mpi_errcode_t ompi_err_rma_notification; static void ompi_mpi_errcode_construct(ompi_mpi_errcode_t* errcode); static void ompi_mpi_errcode_destruct(ompi_mpi_errcode_t* errcode); @@ -252,6 +253,7 @@ int ompi_mpi_errcode_init (void) CONSTRUCT_ERRCODE( ompi_err_errhandler, MPI_ERR_ERRHANDLER, "MPI_ERR_ERRHANDLER: Invalid error handler handle" ); CONSTRUCT_ERRCODE( ompi_t_err_not_accessible, MPI_T_ERR_NOT_ACCESSIBLE, "MPI_T_ERR_NOT_ACCESSIBLE: Requested functionality is not accessible" ); CONSTRUCT_ERRCODE( ompi_t_err_not_supported, MPI_T_ERR_NOT_SUPPORTED, "MPI_T_ERR_NOT_SUPPORTED: Requested functionality not supported" ); + CONSTRUCT_ERRCODE( ompi_err_rma_notification, MPI_ERR_RMA_NOTIFICATION, "MPI_ERR_RMA_NOTIFICATION: Invalid notification index passed to MPI call" ); /* Per MPI-3 p353:27-32, MPI_LASTUSEDCODE must be >= MPI_ERR_LASTCODE. So just start it as == MPI_ERR_LASTCODE. */ @@ -373,6 +375,7 @@ int ompi_mpi_errcode_finalize (void) OBJ_DESTRUCT(&ompi_err_errhandler); OBJ_DESTRUCT(&ompi_t_err_not_accessible); OBJ_DESTRUCT(&ompi_t_err_not_supported); + OBJ_DESTRUCT(&ompi_err_rma_notification); OBJ_DESTRUCT(&ompi_mpi_errcodes); ompi_mpi_errcode_lastpredefined = 0; opal_mutex_unlock(&errcode_lock); diff --git a/ompi/include/mpi.h.in b/ompi/include/mpi.h.in index 43e9006257e..59b80615828 100644 --- a/ompi/include/mpi.h.in +++ b/ompi/include/mpi.h.in @@ -674,6 +674,12 @@ enum { /* MPI-4 */ MPI_FT, /* used by OPAL_ENABLE_FT_MPI */ + + /* MPI-5.1 used in notified RMA */ + MPI_WIN_NOTIFICATION_NUM_SB, + MPI_WIN_NOTIFICATION_NUM_UB, + MPI_WIN_NOTIFICATION_VALUE_UB, + MPI_ATTR_PREDEFINED_KEY_MAX, }; @@ -767,6 +773,7 @@ enum { #define MPI_ERR_ERRHANDLER 80 #define MPI_T_ERR_NOT_ACCESSIBLE 81 #define MPI_T_ERR_NOT_SUPPORTED 82 +#define MPI_ERR_RMA_NOTIFICATION 83 /* Per MPI-3 p349 47, MPI_ERR_LASTCODE must be >= the last predefined MPI_ERR_ code. Set the last code to allow some room for adding @@ -1462,6 +1469,14 @@ OMPI_DECLSPEC int MPI_Accumulate(const void *origin_addr, int origin_count, MPI OMPI_DECLSPEC int MPI_Accumulate_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win); +OMPI_DECLSPEC int MPI_Accumulate_notify(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, MPI_Op op, int notification_idx, + MPI_Win win); +OMPI_DECLSPEC int MPI_Accumulate_notify_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, MPI_Op op, int notification_idx, + MPI_Win win); OMPI_DECLSPEC int MPI_Add_error_class(int *errorclass); OMPI_DECLSPEC int MPI_Add_error_code(int errorclass, int *errorcode); OMPI_DECLSPEC int MPI_Add_error_string(int errorcode, const char *string); @@ -1921,6 +1936,14 @@ OMPI_DECLSPEC int MPI_Get_c(void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype, MPI_Win win); +OMPI_DECLSPEC int MPI_Get_notify(void *origin_addr, int origin_count, + MPI_Datatype origin_datatype, int target_rank, + MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); +OMPI_DECLSPEC int MPI_Get_notify_c(void *origin_addr, MPI_Count origin_count, + MPI_Datatype origin_datatype, int target_rank, + MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); OMPI_DECLSPEC int MPI_Get_accumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, void *result_addr, int result_count, MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp, int target_count, @@ -1929,6 +1952,16 @@ OMPI_DECLSPEC int MPI_Get_accumulate_c(const void *origin_addr, MPI_Count origi void *result_addr, MPI_Count result_count, MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win); +OMPI_DECLSPEC int MPI_Get_accumulate_notify(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, + void *result_addr, int result_count, MPI_Datatype result_datatype, + int target_rank, MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, MPI_Op op, int notification_idx, + MPI_Win win); +OMPI_DECLSPEC int MPI_Get_accumulate_notify_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, + void *result_addr, MPI_Count result_count, MPI_Datatype result_datatype, + int target_rank, MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, MPI_Op op, int notification_idx, + MPI_Win win); OMPI_DECLSPEC int MPI_Get_library_version(char *version, int *resultlen); OMPI_DECLSPEC int MPI_Get_processor_name(char *name, int *resultlen); OMPI_DECLSPEC int MPI_Get_version(int *version, int *subversion); @@ -2184,6 +2217,12 @@ OMPI_DECLSPEC int MPI_Put(const void *origin_addr, int origin_count, MPI_Dataty OMPI_DECLSPEC int MPI_Put_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype, MPI_Win win); +OMPI_DECLSPEC int MPI_Put_notify(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); +OMPI_DECLSPEC int MPI_Put_notify_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); OMPI_DECLSPEC int MPI_Query_thread(int *provided); OMPI_DECLSPEC int MPI_Raccumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, int target_count, @@ -2191,6 +2230,14 @@ OMPI_DECLSPEC int MPI_Raccumulate(const void *origin_addr, int origin_count, MP OMPI_DECLSPEC int MPI_Raccumulate_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win, MPI_Request *request); +OMPI_DECLSPEC int MPI_Raccumulate_notify(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, MPI_Op op, int notification_idx, + MPI_Win win, MPI_Request *request); +OMPI_DECLSPEC int MPI_Raccumulate_notify_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, MPI_Op op, int notification_idx, + MPI_Win win, MPI_Request *request); OMPI_DECLSPEC int MPI_Recv_init(void *buf, int count, MPI_Datatype datatype, int source, int tag, MPI_Comm comm, MPI_Request *request); OMPI_DECLSPEC int MPI_Recv_init_c(void *buf, MPI_Count count, MPI_Datatype datatype, int source, @@ -2269,6 +2316,14 @@ OMPI_DECLSPEC int MPI_Rget(void *origin_addr, int origin_count, MPI_Datatype or OMPI_DECLSPEC int MPI_Rget_c(void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype, MPI_Win win, MPI_Request *request); +OMPI_DECLSPEC int MPI_Rget_notify(void *origin_addr, int origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, int notification_idx, + MPI_Win win, MPI_Request *request); +OMPI_DECLSPEC int MPI_Rget_notify_c(void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, int notification_idx, + MPI_Win win, MPI_Request *request); OMPI_DECLSPEC int MPI_Rget_accumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, void *result_addr, int result_count, MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp, int target_count, @@ -2279,12 +2334,30 @@ OMPI_DECLSPEC int MPI_Rget_accumulate_c(const void *origin_addr, MPI_Count orig int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win, MPI_Request *request); +OMPI_DECLSPEC int MPI_Rget_accumulate_notify(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, + void *result_addr, int result_count, MPI_Datatype result_datatype, + int target_rank, MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, MPI_Op op, int notification_idx, + MPI_Win win, MPI_Request *request); +OMPI_DECLSPEC int MPI_Rget_accumulate_notify_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, + void *result_addr, MPI_Count result_count, MPI_Datatype result_datatype, + int target_rank, MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, MPI_Op op, int notification_idx, + MPI_Win win, MPI_Request *request); OMPI_DECLSPEC int MPI_Rput(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, int target_cout, MPI_Datatype target_datatype, MPI_Win win, MPI_Request *request); OMPI_DECLSPEC int MPI_Rput_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, MPI_Count target_cout, MPI_Datatype target_datatype, MPI_Win win, MPI_Request *request); +OMPI_DECLSPEC int MPI_Rput_notify(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, int notification_idx, + MPI_Win win, MPI_Request *request); +OMPI_DECLSPEC int MPI_Rput_notify_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, int notification_idx, + MPI_Win win, MPI_Request *request); OMPI_DECLSPEC int MPI_Rsend(const void *ibuf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm); OMPI_DECLSPEC int MPI_Rsend_c(const void *ibuf, MPI_Count count, MPI_Datatype datatype, int dest, @@ -2608,6 +2681,10 @@ OMPI_DECLSPEC int MPI_Win_get_errhandler(MPI_Win win, MPI_Errhandler *errhandle OMPI_DECLSPEC int MPI_Win_get_group(MPI_Win win, MPI_Group *group); OMPI_DECLSPEC int MPI_Win_get_info(MPI_Win win, MPI_Info *info_used); OMPI_DECLSPEC int MPI_Win_get_name(MPI_Win win, char *win_name, int *resultlen); +OMPI_DECLSPEC int MPI_Win_get_notify_value(MPI_Win win, int notification_idx, MPI_Count *value); +OMPI_DECLSPEC int MPI_Win_reset_notify_value(MPI_Win win, int notification_idx, MPI_Count *value); +OMPI_DECLSPEC int MPI_Win_get_num_notify(MPI_Win win, int target_rank, int *num_notifications); +OMPI_DECLSPEC int MPI_Win_set_num_notify(MPI_Win win, MPI_Info info, int num_notifications); OMPI_DECLSPEC int MPI_Win_lock(int lock_type, int rank, int mpi_assert, MPI_Win win); OMPI_DECLSPEC int MPI_Win_lock_all(int mpi_assert, MPI_Win win); OMPI_DECLSPEC int MPI_Win_post(MPI_Group group, int mpi_assert, MPI_Win win); @@ -2636,6 +2713,14 @@ OMPI_DECLSPEC int PMPI_Accumulate(const void *origin_addr, int origin_count, MP OMPI_DECLSPEC int PMPI_Accumulate_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win); +OMPI_DECLSPEC int PMPI_Accumulate_notify(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, MPI_Op op, int notification_idx, + MPI_Win win); +OMPI_DECLSPEC int PMPI_Accumulate_notify_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, MPI_Op op, int notification_idx, + MPI_Win win); OMPI_DECLSPEC int PMPI_Add_error_class(int *errorclass); OMPI_DECLSPEC int PMPI_Add_error_code(int errorclass, int *errorcode); OMPI_DECLSPEC int PMPI_Add_error_string(int errorcode, const char *string); @@ -3095,6 +3180,14 @@ OMPI_DECLSPEC int PMPI_Get_c(void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype, MPI_Win win); +OMPI_DECLSPEC int PMPI_Get_notify(void *origin_addr, int origin_count, + MPI_Datatype origin_datatype, int target_rank, + MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); +OMPI_DECLSPEC int PMPI_Get_notify_c(void *origin_addr, MPI_Count origin_count, + MPI_Datatype origin_datatype, int target_rank, + MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); OMPI_DECLSPEC int PMPI_Get_accumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, void *result_addr, int result_count, MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp, int target_count, @@ -3103,6 +3196,16 @@ OMPI_DECLSPEC int PMPI_Get_accumulate_c(const void *origin_addr, MPI_Count orig void *result_addr, MPI_Count result_count, MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win); +OMPI_DECLSPEC int PMPI_Get_accumulate_notify(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, + void *result_addr, int result_count, MPI_Datatype result_datatype, + int target_rank, MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, MPI_Op op, int notification_idx, + MPI_Win win); +OMPI_DECLSPEC int PMPI_Get_accumulate_notify_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, + void *result_addr, MPI_Count result_count, MPI_Datatype result_datatype, + int target_rank, MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, MPI_Op op, int notification_idx, + MPI_Win win); OMPI_DECLSPEC int PMPI_Get_library_version(char *version, int *resultlen); OMPI_DECLSPEC int PMPI_Get_processor_name(char *name, int *resultlen); OMPI_DECLSPEC int PMPI_Get_version(int *version, int *subversion); @@ -3358,6 +3461,12 @@ OMPI_DECLSPEC int PMPI_Put(const void *origin_addr, int origin_count, MPI_Datat OMPI_DECLSPEC int PMPI_Put_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype, MPI_Win win); +OMPI_DECLSPEC int PMPI_Put_notify(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); +OMPI_DECLSPEC int PMPI_Put_notify_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, int notification_idx, MPI_Win win); OMPI_DECLSPEC int PMPI_Query_thread(int *provided); OMPI_DECLSPEC int PMPI_Raccumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, int target_count, @@ -3365,6 +3474,14 @@ OMPI_DECLSPEC int PMPI_Raccumulate(const void *origin_addr, int origin_count, M OMPI_DECLSPEC int PMPI_Raccumulate_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win, MPI_Request *request); +OMPI_DECLSPEC int PMPI_Raccumulate_notify(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, MPI_Op op, int notification_idx, + MPI_Win win, MPI_Request *request); +OMPI_DECLSPEC int PMPI_Raccumulate_notify_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, MPI_Op op, int notification_idx, + MPI_Win win, MPI_Request *request); OMPI_DECLSPEC int PMPI_Recv_init(void *buf, int count, MPI_Datatype datatype, int source, int tag, MPI_Comm comm, MPI_Request *request); OMPI_DECLSPEC int PMPI_Recv_init_c(void *buf, MPI_Count count, MPI_Datatype datatype, int source, @@ -3443,6 +3560,14 @@ OMPI_DECLSPEC int PMPI_Rget(void *origin_addr, int origin_count, MPI_Datatype o OMPI_DECLSPEC int PMPI_Rget_c(void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype, MPI_Win win, MPI_Request *request); +OMPI_DECLSPEC int PMPI_Rget_notify(void *origin_addr, int origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, int notification_idx, + MPI_Win win, MPI_Request *request); +OMPI_DECLSPEC int PMPI_Rget_notify_c(void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, int notification_idx, + MPI_Win win, MPI_Request *request); OMPI_DECLSPEC int PMPI_Rget_accumulate(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, void *result_addr, int result_count, MPI_Datatype result_datatype, int target_rank, MPI_Aint target_disp, int target_count, @@ -3453,12 +3578,30 @@ OMPI_DECLSPEC int PMPI_Rget_accumulate_c(const void *origin_addr, MPI_Count ori int target_rank, MPI_Aint target_disp, MPI_Count target_count, MPI_Datatype target_datatype, MPI_Op op, MPI_Win win, MPI_Request *request); +OMPI_DECLSPEC int PMPI_Rget_accumulate_notify(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, + void *result_addr, int result_count, MPI_Datatype result_datatype, + int target_rank, MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, MPI_Op op, int notification_idx, + MPI_Win win, MPI_Request *request); +OMPI_DECLSPEC int PMPI_Rget_accumulate_notify_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, + void *result_addr, MPI_Count result_count, MPI_Datatype result_datatype, + int target_rank, MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, MPI_Op op, int notification_idx, + MPI_Win win, MPI_Request *request); OMPI_DECLSPEC int PMPI_Rput(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, int target_cout, MPI_Datatype target_datatype, MPI_Win win, MPI_Request *request); OMPI_DECLSPEC int PMPI_Rput_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, int target_rank, MPI_Aint target_disp, MPI_Count target_cout, MPI_Datatype target_datatype, MPI_Win win, MPI_Request *request); +OMPI_DECLSPEC int PMPI_Rput_notify(const void *origin_addr, int origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, int target_count, + MPI_Datatype target_datatype, int notification_idx, + MPI_Win win, MPI_Request *request); +OMPI_DECLSPEC int PMPI_Rput_notify_c(const void *origin_addr, MPI_Count origin_count, MPI_Datatype origin_datatype, + int target_rank, MPI_Aint target_disp, MPI_Count target_count, + MPI_Datatype target_datatype, int notification_idx, + MPI_Win win, MPI_Request *request); OMPI_DECLSPEC int PMPI_Rsend(const void *ibuf, int count, MPI_Datatype datatype, int dest, int tag, MPI_Comm comm); OMPI_DECLSPEC int PMPI_Rsend_c(const void *ibuf, MPI_Count count, MPI_Datatype datatype, int dest, @@ -3782,6 +3925,10 @@ OMPI_DECLSPEC int PMPI_Win_get_errhandler(MPI_Win win, MPI_Errhandler *errhandl OMPI_DECLSPEC int PMPI_Win_get_group(MPI_Win win, MPI_Group *group); OMPI_DECLSPEC int PMPI_Win_get_info(MPI_Win win, MPI_Info *info_used); OMPI_DECLSPEC int PMPI_Win_get_name(MPI_Win win, char *win_name, int *resultlen); +OMPI_DECLSPEC int PMPI_Win_get_notify_value(MPI_Win win, int notification_idx, MPI_Count *value); +OMPI_DECLSPEC int PMPI_Win_reset_notify_value(MPI_Win win, int notification_idx, MPI_Count *value); +OMPI_DECLSPEC int PMPI_Win_get_num_notify(MPI_Win win, int target_rank, int *num_notifications); +OMPI_DECLSPEC int PMPI_Win_set_num_notify(MPI_Win win, MPI_Info info, int num_notifications); OMPI_DECLSPEC int PMPI_Win_lock(int lock_type, int rank, int mpi_assert, MPI_Win win); OMPI_DECLSPEC int PMPI_Win_lock_all(int mpi_assert, MPI_Win win); OMPI_DECLSPEC int PMPI_Win_post(MPI_Group group, int mpi_assert, MPI_Win win); diff --git a/ompi/include/mpif-values.py b/ompi/include/mpif-values.py index 7cd50d7e9e4..bfaca964abf 100755 --- a/ompi/include/mpif-values.py +++ b/ompi/include/mpif-values.py @@ -195,6 +195,9 @@ 'MPI_WIN_CREATE_FLAVOR': 10, 'MPI_WIN_MODEL': 11, 'MPI_FT': 12, + 'MPI_WIN_NOTIFICATION_NUM_SB': 13, + 'MPI_WIN_NOTIFICATION_NUM_UB': 14, + 'MPI_WIN_NOTIFICATION_VALUE_UB': 15, 'MPI_WIN_FLAVOR_CREATE': 1, 'MPI_WIN_FLAVOR_ALLOCATE': 2, 'MPI_WIN_FLAVOR_DYNAMIC': 3, @@ -304,6 +307,7 @@ 'MPI_ERR_ERRHANDLER': 80, 'MPI_T_ERR_NOT_ACCESSIBLE': 81, 'MPI_T_ERR_NOT_SUPPORTED': 82, + 'MPI_ERR_RMA_NOTIFICATION': 83, 'MPI_ERR_LASTCODE': 92, 'MPI_IDENT': 0, 'MPI_CONGRUENT': 1, diff --git a/ompi/mca/osc/osc.h b/ompi/mca/osc/osc.h index 39063ef0914..10748625db6 100644 --- a/ompi/mca/osc/osc.h +++ b/ompi/mca/osc/osc.h @@ -216,6 +216,15 @@ typedef int (*ompi_osc_base_module_put_fn_t)(const void *origin_addr, struct ompi_datatype_t *target_dt, struct ompi_win_t *win); +typedef int (*ompi_osc_base_module_put_notify_fn_t)(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win); typedef int (*ompi_osc_base_module_get_fn_t)(void *origin_addr, size_t origin_count, @@ -226,6 +235,36 @@ typedef int (*ompi_osc_base_module_get_fn_t)(void *origin_addr, struct ompi_datatype_t *target_dt, struct ompi_win_t *win); +typedef int (*ompi_osc_base_module_get_notify_fn_t)(void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win); + +typedef int (*ompi_osc_base_module_win_get_notify_value_fn_t)(struct ompi_win_t *win, + int notify, + OMPI_MPI_COUNT_TYPE *value); + +typedef int (*ompi_osc_base_module_win_reset_notify_value_fn_t)(struct ompi_win_t *win, + int notify, + OMPI_MPI_COUNT_TYPE *value); + +typedef int (*ompi_osc_base_module_win_set_num_notify_fn_t)(struct ompi_win_t *win, + struct opal_info_t *info, + int num_notifications); + +typedef int (*ompi_osc_base_module_win_get_num_notify_fn_t)(struct ompi_win_t *win, + int target_rank, + int *num_notifications); + +typedef int (*ompi_osc_base_module_win_get_notify_bounds_fn_t)(struct ompi_win_t *win, + int *num_sb, + int *num_ub, + OMPI_MPI_COUNT_TYPE *value_ub); typedef int (*ompi_osc_base_module_accumulate_fn_t)(const void *origin_addr, size_t origin_count, @@ -237,6 +276,17 @@ typedef int (*ompi_osc_base_module_accumulate_fn_t)(const void *origin_addr, struct ompi_op_t *op, struct ompi_win_t *win); +typedef int (*ompi_osc_base_module_accumulate_notify_fn_t)(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + struct ompi_op_t *op, + int notify, + struct ompi_win_t *win); + typedef int (*ompi_osc_base_module_compare_and_swap_fn_t)(const void *origin_addr, const void *compare_addr, void *result_addr, @@ -266,6 +316,20 @@ typedef int (*ompi_osc_base_module_get_accumulate_fn_t)(const void *origin_addr, struct ompi_op_t *op, struct ompi_win_t *win); +typedef int (*ompi_osc_base_module_get_accumulate_notify_fn_t)(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_datatype, + void *result_addr, + size_t result_count, + struct ompi_datatype_t *result_datatype, + int target_rank, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_datatype, + struct ompi_op_t *op, + int notify, + struct ompi_win_t *win); + typedef int (*ompi_osc_base_module_rput_fn_t)(const void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, @@ -276,6 +340,17 @@ typedef int (*ompi_osc_base_module_rput_fn_t)(const void *origin_addr, struct ompi_win_t *win, struct ompi_request_t **request); +typedef int (*ompi_osc_base_module_rput_notify_fn_t)(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **request); + typedef int (*ompi_osc_base_module_rget_fn_t)(void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, @@ -286,6 +361,16 @@ typedef int (*ompi_osc_base_module_rget_fn_t)(void *origin_addr, struct ompi_win_t *win, struct ompi_request_t **request); +typedef int (*ompi_osc_base_module_rget_notify_fn_t)(void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **request); typedef int (*ompi_osc_base_module_raccumulate_fn_t)(const void *origin_addr, size_t origin_count, @@ -298,6 +383,18 @@ typedef int (*ompi_osc_base_module_raccumulate_fn_t)(const void *origin_addr, struct ompi_win_t *win, struct ompi_request_t **request); +typedef int (*ompi_osc_base_module_raccumulate_notify_fn_t)(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + struct ompi_op_t *op, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **request); + typedef int (*ompi_osc_base_module_rget_accumulate_fn_t)(const void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_datatype, @@ -312,6 +409,21 @@ typedef int (*ompi_osc_base_module_rget_accumulate_fn_t)(const void *origin_addr struct ompi_win_t *win, struct ompi_request_t **request); +typedef int (*ompi_osc_base_module_rget_accumulate_notify_fn_t)(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_datatype, + void *result_addr, + size_t result_count, + struct ompi_datatype_t *result_datatype, + int target_rank, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_datatype, + struct ompi_op_t *op, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **request); + typedef int (*ompi_osc_base_module_fence_fn_t)(int mpi_assert, struct ompi_win_t *win); @@ -407,6 +519,19 @@ struct ompi_osc_base_module_4_0_0_t { ompi_osc_base_module_flush_all_fn_t osc_flush_all; ompi_osc_base_module_flush_local_fn_t osc_flush_local; ompi_osc_base_module_flush_local_all_fn_t osc_flush_local_all; + ompi_osc_base_module_put_notify_fn_t osc_put_notify; + ompi_osc_base_module_get_notify_fn_t osc_get_notify; + ompi_osc_base_module_win_get_notify_value_fn_t osc_win_get_notify_value; + ompi_osc_base_module_win_reset_notify_value_fn_t osc_win_reset_notify_value; + ompi_osc_base_module_win_set_num_notify_fn_t osc_win_set_num_notify; + ompi_osc_base_module_win_get_num_notify_fn_t osc_win_get_num_notify; + ompi_osc_base_module_win_get_notify_bounds_fn_t osc_win_get_notify_bounds; + ompi_osc_base_module_rput_notify_fn_t osc_rput_notify; + ompi_osc_base_module_rget_notify_fn_t osc_rget_notify; + ompi_osc_base_module_accumulate_notify_fn_t osc_accumulate_notify; + ompi_osc_base_module_get_accumulate_notify_fn_t osc_get_accumulate_notify; + ompi_osc_base_module_raccumulate_notify_fn_t osc_raccumulate_notify; + ompi_osc_base_module_rget_accumulate_notify_fn_t osc_rget_accumulate_notify; }; typedef struct ompi_osc_base_module_4_0_0_t ompi_osc_base_module_4_0_0_t; typedef ompi_osc_base_module_4_0_0_t ompi_osc_base_module_t; diff --git a/ompi/mca/osc/sm/osc_sm.h b/ompi/mca/osc/sm/osc_sm.h index 363d3429a63..c4c38536246 100644 --- a/ompi/mca/osc/sm/osc_sm.h +++ b/ompi/mca/osc/sm/osc_sm.h @@ -23,6 +23,12 @@ typedef opal_atomic_uint64_t osc_sm_post_atomic_type_t; #define OSC_SM_POST_BITS 6 #define OSC_SM_POST_MASK 0x3f +/* Per-rank notification counter capacity reserved inline in the main shared + * segment at window creation, and the value reported as + * MPI_WIN_NOTIFICATION_NUM_SB -- the number of counters osc/sm supports without + * any further allocation. */ +#define OSC_SM_DEFAULT_NOTIFY_COUNTERS 16 + /* data shared across all peers */ struct ompi_osc_sm_global_state_t { int use_barrier_for_fence; @@ -47,6 +53,9 @@ struct ompi_osc_sm_node_state_t { opal_atomic_int32_t complete_count; ompi_osc_sm_lock_t lock; opal_atomic_lock_t accumulate_lock; + uint32_t notify_counter_count; + uint32_t notify_counter_capacity; + uint64_t notify_counter_offset; }; typedef struct ompi_osc_sm_node_state_t ompi_osc_sm_node_state_t; @@ -57,6 +66,10 @@ struct ompi_osc_sm_component_t { unsigned int priority; char *backing_directory; + + /** Notification counters reserved per MPI process at window creation when + * the window's info gives no mpi_assert_max_num_notify hint */ + unsigned int num_notify_counters; }; typedef struct ompi_osc_sm_component_t ompi_osc_sm_component_t; OMPI_DECLSPEC extern ompi_osc_sm_component_t mca_osc_sm_component; @@ -80,6 +93,12 @@ struct ompi_osc_sm_module_t { void **bases; ptrdiff_t *disp_units; + opal_atomic_int64_t **notify_bases; + opal_shmem_ds_t notify_seg_ds; + void *notify_segment_base; + unsigned int notify_max_assert; + + ompi_group_t *start_group; ompi_group_t *post_group; @@ -105,6 +124,9 @@ int ompi_osc_sm_detach(struct ompi_win_t *win, const void *base); int ompi_osc_sm_free(struct ompi_win_t *win); +void ompi_osc_sm_refresh_notify_bases(ompi_osc_sm_module_t *module); + + int ompi_osc_sm_put(const void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, @@ -114,6 +136,16 @@ int ompi_osc_sm_put(const void *origin_addr, struct ompi_datatype_t *target_dt, struct ompi_win_t *win); + int ompi_osc_sm_put_notify(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win); + int ompi_osc_sm_get(void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, @@ -123,6 +155,37 @@ int ompi_osc_sm_get(void *origin_addr, struct ompi_datatype_t *target_dt, struct ompi_win_t *win); +int ompi_osc_sm_get_notify(void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win); + +int ompi_osc_sm_win_get_notify_value(struct ompi_win_t *win, + int notify, + OMPI_MPI_COUNT_TYPE *value); + +int ompi_osc_sm_win_reset_notify_value(struct ompi_win_t *win, + int notify, + OMPI_MPI_COUNT_TYPE *value); + +int ompi_osc_sm_win_set_num_notify(struct ompi_win_t *win, + struct opal_info_t *info, + int num_notifications); + +int ompi_osc_sm_win_get_num_notify(struct ompi_win_t *win, + int target_rank, + int *num_notifications); + +int ompi_osc_sm_win_get_notify_bounds(struct ompi_win_t *win, + int *num_sb, + int *num_ub, + OMPI_MPI_COUNT_TYPE *value_ub); + int ompi_osc_sm_accumulate(const void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, @@ -133,6 +196,17 @@ int ompi_osc_sm_accumulate(const void *origin_addr, struct ompi_op_t *op, struct ompi_win_t *win); +int ompi_osc_sm_accumulate_notify(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + struct ompi_op_t *op, + int notify, + struct ompi_win_t *win); + int ompi_osc_sm_compare_and_swap(const void *origin_addr, const void *compare_addr, void *result_addr, @@ -162,6 +236,20 @@ int ompi_osc_sm_get_accumulate(const void *origin_addr, struct ompi_op_t *op, struct ompi_win_t *win); +int ompi_osc_sm_get_accumulate_notify(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_datatype, + void *result_addr, + size_t result_count, + struct ompi_datatype_t *result_datatype, + int target_rank, + MPI_Aint target_disp, + size_t target_count, + struct ompi_datatype_t *target_datatype, + struct ompi_op_t *op, + int notify, + struct ompi_win_t *win); + int ompi_osc_sm_rput(const void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, @@ -172,6 +260,17 @@ int ompi_osc_sm_rput(const void *origin_addr, struct ompi_win_t *win, struct ompi_request_t **request); +int ompi_osc_sm_rput_notify(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **request); + int ompi_osc_sm_rget(void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, @@ -182,6 +281,17 @@ int ompi_osc_sm_rget(void *origin_addr, struct ompi_win_t *win, struct ompi_request_t **request); +int ompi_osc_sm_rget_notify(void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **request); + int ompi_osc_sm_raccumulate(const void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, @@ -193,6 +303,18 @@ int ompi_osc_sm_raccumulate(const void *origin_addr, struct ompi_win_t *win, struct ompi_request_t **request); +int ompi_osc_sm_raccumulate_notify(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + struct ompi_op_t *op, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **request); + int ompi_osc_sm_rget_accumulate(const void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_datatype, @@ -207,6 +329,21 @@ int ompi_osc_sm_rget_accumulate(const void *origin_addr, struct ompi_win_t *win, struct ompi_request_t **request); +int ompi_osc_sm_rget_accumulate_notify(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_datatype, + void *result_addr, + size_t result_count, + struct ompi_datatype_t *result_datatype, + int target_rank, + MPI_Aint target_disp, + size_t target_count, + struct ompi_datatype_t *target_datatype, + struct ompi_op_t *op, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **request); + int ompi_osc_sm_fence(int mpi_assert, struct ompi_win_t *win); int ompi_osc_sm_start(struct ompi_group_t *group, diff --git a/ompi/mca/osc/sm/osc_sm_comm.c b/ompi/mca/osc/sm/osc_sm_comm.c index bbd5873bf96..9838d4d94a8 100644 --- a/ompi/mca/osc/sm/osc_sm_comm.c +++ b/ompi/mca/osc/sm/osc_sm_comm.c @@ -17,9 +17,347 @@ #include "ompi/mca/osc/osc.h" #include "ompi/mca/osc/base/base.h" #include "ompi/mca/osc/base/osc_base_obj_convert.h" +#include "ompi/communicator/communicator.h" +#include "opal/align.h" +#include "opal/util/printf.h" +#include "opal/util/sys_limits.h" + +#include +#include #include "osc_sm.h" +static inline opal_atomic_int64_t * +osc_sm_target_notify_base(ompi_osc_sm_module_t *module, int target) +{ + return module->notify_bases[target]; +} + +static inline int +osc_sm_check_notify_idx(ompi_osc_sm_module_t *module, int target, int notify) +{ + if (notify < 0 || (uint32_t) notify >= module->node_states[target].notify_counter_count) { + return MPI_ERR_RMA_NOTIFICATION; + } + + return OMPI_SUCCESS; +} + +static inline void +osc_sm_notify_accumulate_done(ompi_osc_sm_module_t *module, int target, int notify) +{ + opal_atomic_mb(); + opal_atomic_add(&osc_sm_target_notify_base(module, target)[notify], 1); +} + +int +ompi_osc_sm_win_get_notify_value(struct ompi_win_t *win, + int notify, + OMPI_MPI_COUNT_TYPE *value) +{ + ompi_osc_sm_module_t *module = (ompi_osc_sm_module_t *) win->w_osc_module; + int rank = ompi_comm_rank(module->comm); + int ret; + + ret = osc_sm_check_notify_idx(module, rank, notify); + if (OMPI_SUCCESS != ret) { + return ret; + } + + *value = (OMPI_MPI_COUNT_TYPE) osc_sm_target_notify_base(module, rank)[notify]; + opal_atomic_rmb(); + + return OMPI_SUCCESS; +} + +int +ompi_osc_sm_win_reset_notify_value(struct ompi_win_t *win, + int notify, + OMPI_MPI_COUNT_TYPE *value) +{ + ompi_osc_sm_module_t *module = (ompi_osc_sm_module_t *) win->w_osc_module; + int rank = ompi_comm_rank(module->comm); + int ret; + + ret = osc_sm_check_notify_idx(module, rank, notify); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* Atomically swap the counter to 0 and return the previous value. */ + *value = (OMPI_MPI_COUNT_TYPE) opal_atomic_swap_64( + &osc_sm_target_notify_base(module, rank)[notify], 0); + opal_atomic_rmb(); + + return OMPI_SUCCESS; +} + +/* Move every rank's notification counters into a newly created shared segment + * sized for the capacities in new_caps. Collective over the window's + * communicator; every MPI process must call it with an identical new_caps. + */ +static int +osc_sm_grow_notify_counters(ompi_osc_sm_module_t *module, const unsigned long *new_caps, + unsigned long new_count) +{ + int comm_size = ompi_comm_size(module->comm); + int rank = ompi_comm_rank(module->comm); + opal_shmem_ds_t new_seg_ds; + opal_shmem_ds_t old_seg_ds = module->notify_seg_ds; + void *old_segment_base = module->notify_segment_base; + void *new_base; + unsigned long total_counters = 0; + size_t seg_size; + char *data_file; + int ret, i, status; + + for (i = 0 ; i < comm_size ; ++i) { + total_counters += new_caps[i]; + } + seg_size = total_counters * sizeof(int64_t); + seg_size += OPAL_ALIGN_PAD_AMOUNT(seg_size, opal_getpagesize()); + + memset(&new_seg_ds, 0, sizeof(new_seg_ds)); + + if (0 == rank) { + ret = opal_asprintf(&data_file, "%s" OPAL_PATH_SEP "osc_sm_notify.%s.%x.%d.%s.%lu", + mca_osc_sm_component.backing_directory, ompi_process_info.nodename, + OMPI_PROC_MY_NAME->jobid, (int) OMPI_PROC_MY_NAME->vpid, + ompi_comm_print_cid(module->comm), total_counters); + if (ret > 0) { + (void) opal_shmem_segment_create(&new_seg_ds, data_file, seg_size); + free(data_file); + } + } + + ret = module->comm->c_coll->coll_bcast(&new_seg_ds, sizeof(new_seg_ds), MPI_BYTE, 0, + module->comm, + module->comm->c_coll->coll_bcast_module); + if (OMPI_SUCCESS != ret) { + return ret; + } + + if ('\0' == new_seg_ds.seg_name[0]) { + return MPI_ERR_NO_MEM; + } + + new_base = opal_shmem_segment_attach(&new_seg_ds); + + /* Attach can fail at some MPI processes and not others. Agree on the + * outcome before touching any shared state */ + status = (NULL == new_base) ? 1 : 0; + ret = module->comm->c_coll->coll_allreduce(MPI_IN_PLACE, &status, 1, MPI_INT, MPI_MAX, + module->comm, + module->comm->c_coll->coll_allreduce_module); + if (OMPI_SUCCESS != ret) { + return ret; + } + + if (0 != status) { + if (NULL != new_base) { + opal_shmem_segment_detach(&new_seg_ds); + } + return MPI_ERR_NO_MEM; + } + + module->notify_segment_base = new_base; + module->notify_seg_ds = new_seg_ds; + + /* Republish the layout. Every MPI process computes the same offsets from + * the same new_caps, so these stores are identical everywhere; node_states + * lives in the main segment, which does not move. */ + total_counters = 0; + for (i = 0 ; i < comm_size ; ++i) { + module->node_states[i].notify_counter_capacity = (uint32_t) new_caps[i]; + module->node_states[i].notify_counter_offset = total_counters * sizeof(int64_t); + total_counters += new_caps[i]; + } + + ompi_osc_sm_refresh_notify_bases(module); + + memset((void *) module->notify_bases[rank], 0, + module->node_states[rank].notify_counter_capacity * sizeof(int64_t)); + + module->node_states[rank].notify_counter_count = (uint32_t) new_count; + opal_atomic_wmb(); + + /* Everyone has opened the new segment and published their layout. Both + * facts are needed before we continue: the unlink below removes the name + * that the attach above resolves, and an origin returning from this call + * may immediately validate a notification index against our count. */ + ret = module->comm->c_coll->coll_barrier(module->comm, + module->comm->c_coll->coll_barrier_module); + if (OMPI_SUCCESS != ret) { + return ret; + } + + if (0 == rank) { + opal_shmem_unlink(&module->notify_seg_ds); + } + + /* Dropping the old mapping is purely local -- munmap in one MPI process + * does not disturb any other process's view -- so it needs no + * synchronization of its own. On the first growth there is nothing to + * drop: the counters were inline in the main segment, which stays mapped + * for the lifetime of the window. */ + if (NULL != old_segment_base) { + opal_shmem_segment_detach(&old_seg_ds); + } + + return OMPI_SUCCESS; +} + +int +ompi_osc_sm_win_set_num_notify(struct ompi_win_t *win, + struct opal_info_t *info, + int num_notifications) +{ + ompi_osc_sm_module_t *module = (ompi_osc_sm_module_t *) win->w_osc_module; + int comm_size = ompi_comm_size(module->comm); + int rank = ompi_comm_rank(module->comm); + unsigned long requested = (unsigned long) num_notifications; + unsigned long *new_caps; + bool grow = false, bad; + int ret, i; + + /* "mpi_assert_same_num_notifications" would let us skip the allgather below + * and derive the layout from our own num_notifications. Not taken up yet; + * the allgather is one collective on a procedure that is already + * synchronizing and collective. */ + (void) info; + + /* num_notifications is a local argument -- MPI-5.1 12.6.1 allows it to + * differ between MPI processes -- but this is a synchronizing collective. + * A rank that rejected its own value and returned here would leave every + * other rank blocked in the allgather below, turning an erroneous argument + * into a hang. So the validity rides through the collective as a sentinel + * and all ranks fail together. A multi-process window defers the decision; + * a single-process one has nobody to agree with and can answer now. */ + bad = (num_notifications < 0) + || (0 != module->notify_max_assert && + requested > (unsigned long) module->notify_max_assert); + + if (bad && 1 == comm_size) { + return MPI_ERR_ARG; + } + + if (bad) { + /* Leave our counters alone: the call is going to fail on every rank, so + * the window must come out of it unchanged rather than half-reset. */ + requested = ULONG_MAX; + goto agree; + } + + memset((void *) module->notify_bases[rank], 0, + module->node_states[rank].notify_counter_capacity * sizeof(int64_t)); + module->node_states[rank].notify_counter_count = + (requested > module->node_states[rank].notify_counter_capacity) + ? module->node_states[rank].notify_counter_capacity + : (uint32_t) requested; + opal_atomic_wmb(); + + if (1 == comm_size) { + /* No shared segment for a single-process window; the counters are a + * plain allocation, so growing them is a plain reallocation and none of + * the collective machinery below applies. */ + if (requested > module->node_states[0].notify_counter_capacity) { + void *grown = calloc(requested, sizeof(int64_t)); + if (NULL == grown) { + return MPI_ERR_NO_MEM; + } + free((void *) module->notify_bases[0]); + module->notify_bases[0] = (opal_atomic_int64_t *) grown; + module->node_states[0].notify_counter_capacity = (uint32_t) requested; + module->node_states[0].notify_counter_count = (uint32_t) requested; + } + return OMPI_SUCCESS; + } + +agree: + new_caps = malloc(sizeof(*new_caps) * comm_size); + if (NULL == new_caps) { + return OMPI_ERR_TEMP_OUT_OF_RESOURCE; + } + + ret = module->comm->c_coll->coll_allgather(&requested, 1, MPI_UNSIGNED_LONG, + new_caps, 1, MPI_UNSIGNED_LONG, + module->comm, + module->comm->c_coll->coll_allgather_module); + if (OMPI_SUCCESS != ret) { + free(new_caps); + return ret; + } + + for (i = 0 ; i < comm_size ; ++i) { + if (ULONG_MAX == new_caps[i]) { + /* Some rank supplied an invalid count. Every rank sees the same + * gathered array, so they all report the same error and none of + * them reconfigures. */ + free(new_caps); + return MPI_ERR_ARG; + } + } + + for (i = 0 ; i < comm_size ; ++i) { + if (new_caps[i] > module->node_states[i].notify_counter_capacity) { + grow = true; + } else { + /* Never shrink: a rank that lowered its count keeps the space it + * already has, so that only genuine growth costs a reallocation. */ + new_caps[i] = module->node_states[i].notify_counter_capacity; + } + } + + if (grow) { + ret = osc_sm_grow_notify_counters(module, new_caps, requested); + free(new_caps); + return ret; + } + + free(new_caps); + + return OMPI_SUCCESS; +} + +int +ompi_osc_sm_win_get_num_notify(struct ompi_win_t *win, + int target_rank, + int *num_notifications) +{ + ompi_osc_sm_module_t *module = (ompi_osc_sm_module_t *) win->w_osc_module; + + if (target_rank < 0 || target_rank >= ompi_comm_size(module->comm)) { + return MPI_ERR_RANK; + } + + *num_notifications = (int) module->node_states[target_rank].notify_counter_count; + + return OMPI_SUCCESS; +} + +int +ompi_osc_sm_win_get_notify_bounds(struct ompi_win_t *win, + int *num_sb, + int *num_ub, + OMPI_MPI_COUNT_TYPE *value_ub) +{ + ompi_osc_sm_module_t *module = (ompi_osc_sm_module_t *) win->w_osc_module; + + if (0 != module->notify_max_assert) { + *num_sb = (int) module->notify_max_assert; + *num_ub = (int) module->notify_max_assert; + } else { + *num_sb = (int) mca_osc_sm_component.num_notify_counters; + *num_ub = INT_MAX; + } + + /* Counters are int64_t and only ever incremented by one per notified + * operation, so the representable maximum is the real bound. */ + *value_ub = (OMPI_MPI_COUNT_TYPE) INT64_MAX; + + return OMPI_SUCCESS; +} + int ompi_osc_sm_rput(const void *origin_addr, size_t origin_count, @@ -59,6 +397,56 @@ ompi_osc_sm_rput(const void *origin_addr, return OMPI_SUCCESS; } +int +ompi_osc_sm_rput_notify(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **ompi_req) +{ + int ret; + ompi_osc_sm_module_t *module = + (ompi_osc_sm_module_t*) win->w_osc_module; + void *remote_address; + + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "rput_notify: 0x%lx, %zu, %s, %d, %d, %zu, %s, %d, 0x%lx", + (unsigned long) origin_addr, origin_count, + origin_dt->name, target, (int) target_disp, + target_count, target_dt->name, + notify, + (unsigned long) win)); + + ret = osc_sm_check_notify_idx(module, target, notify); + if (OMPI_SUCCESS != ret) { + return ret; + } + + remote_address = ((char*) (module->bases[target])) + module->disp_units[target] * target_disp; + + ret = ompi_datatype_sndrcv((void *)origin_addr, origin_count, origin_dt, + remote_address, target_count, target_dt); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* Release ordering: the data must be visible at the target before the + * notification is */ + opal_atomic_wmb(); + opal_atomic_add(&osc_sm_target_notify_base(module, target)[notify], 1); + + /* the only valid field of RMA request status is the MPI_ERROR field. + * ompi_request_empty has status MPI_SUCCESS and indicates the request is + * complete. */ + *ompi_req = &ompi_request_empty; + + return OMPI_SUCCESS; +} int ompi_osc_sm_rget(void *origin_addr, @@ -99,6 +487,54 @@ ompi_osc_sm_rget(void *origin_addr, return OMPI_SUCCESS; } +int +ompi_osc_sm_rget_notify(void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **ompi_req) +{ + int ret; + ompi_osc_sm_module_t *module = + (ompi_osc_sm_module_t*) win->w_osc_module; + void *remote_address; + + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "rget_notify: 0x%lx, %zu, %s, %d, %d, %zu, %s, %d, 0x%lx", + (unsigned long) origin_addr, origin_count, + origin_dt->name, target, (int) target_disp, + target_count, target_dt->name, + notify, + (unsigned long) win)); + + ret = osc_sm_check_notify_idx(module, target, notify); + if (OMPI_SUCCESS != ret) { + return ret; + } + + remote_address = ((char*) (module->bases[target])) + module->disp_units[target] * target_disp; + + ret = ompi_datatype_sndrcv(remote_address, target_count, target_dt, + origin_addr, origin_count, origin_dt); + if (OMPI_SUCCESS != ret) { + return ret; + } + + opal_atomic_mb(); + opal_atomic_add(&osc_sm_target_notify_base(module, target)[notify], 1); + + /* the only valid field of RMA request status is the MPI_ERROR field. + * ompi_request_empty has status MPI_SUCCESS and indicates the request is + * complete. */ + *ompi_req = &ompi_request_empty; + + return OMPI_SUCCESS; +} int ompi_osc_sm_raccumulate(const void *origin_addr, @@ -147,6 +583,67 @@ ompi_osc_sm_raccumulate(const void *origin_addr, } +int +ompi_osc_sm_raccumulate_notify(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + struct ompi_op_t *op, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **ompi_req) +{ + int ret; + ompi_osc_sm_module_t *module = + (ompi_osc_sm_module_t*) win->w_osc_module; + void *remote_address; + + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "raccumulate_notify: 0x%lx, %zu, %s, %d, %d, %zu, %s, %s, %d, 0x%lx", + (unsigned long) origin_addr, origin_count, + origin_dt->name, target, (int) target_disp, + target_count, target_dt->name, + op->o_name, notify, + (unsigned long) win)); + + ret = osc_sm_check_notify_idx(module, target, notify); + if (OMPI_SUCCESS != ret) { + return ret; + } + + remote_address = ((char*) (module->bases[target])) + module->disp_units[target] * target_disp; + + opal_atomic_lock(&module->node_states[target].accumulate_lock); + if (op == &ompi_mpi_op_replace.op) { + ret = ompi_datatype_sndrcv((void *)origin_addr, origin_count, origin_dt, + remote_address, target_count, target_dt); + } else { + ret = ompi_osc_base_sndrcv_op(origin_addr, origin_count, origin_dt, + remote_address, target_count, target_dt, + op); + } + opal_atomic_unlock(&module->node_states[target].accumulate_lock); + + /* Only notify once the accumulate actually happened -- a counter bumped for + * an operation that failed would tell the target that data it never + * received is ready. */ + if (OMPI_SUCCESS != ret) { + return ret; + } + + osc_sm_notify_accumulate_done(module, target, notify); + + /* the only valid field of RMA request status is the MPI_ERROR field. + * ompi_request_empty has status MPI_SUCCESS and indicates the request is + * complete. */ + *ompi_req = &ompi_request_empty; + + return ret; +} + int ompi_osc_sm_rget_accumulate(const void *origin_addr, @@ -205,6 +702,78 @@ ompi_osc_sm_rget_accumulate(const void *origin_addr, } +int +ompi_osc_sm_rget_accumulate_notify(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + void *result_addr, + size_t result_count, + struct ompi_datatype_t *result_dt, + int target, + MPI_Aint target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + struct ompi_op_t *op, + int notify, + struct ompi_win_t *win, + struct ompi_request_t **ompi_req) +{ + int ret; + ompi_osc_sm_module_t *module = + (ompi_osc_sm_module_t*) win->w_osc_module; + void *remote_address; + + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "rget_accumulate_notify: 0x%lx, %zu, %s, %d, %d, %zu, %s, %s, %d, 0x%lx", + (unsigned long) origin_addr, origin_count, + origin_dt->name, target, (int) target_disp, + target_count, target_dt->name, + op->o_name, notify, + (unsigned long) win)); + + ret = osc_sm_check_notify_idx(module, target, notify); + if (OMPI_SUCCESS != ret) { + return ret; + } + + remote_address = ((char*) (module->bases[target])) + module->disp_units[target] * target_disp; + + opal_atomic_lock(&module->node_states[target].accumulate_lock); + + ret = ompi_datatype_sndrcv(remote_address, target_count, target_dt, + result_addr, result_count, result_dt); + if (OMPI_SUCCESS != ret || op == &ompi_mpi_op_no_op.op) goto done; + + if (op == &ompi_mpi_op_replace.op) { + ret = ompi_datatype_sndrcv((void *)origin_addr, origin_count, origin_dt, + remote_address, target_count, target_dt); + } else { + ret = ompi_osc_base_sndrcv_op(origin_addr, origin_count, origin_dt, + remote_address, target_count, target_dt, + op); + } + + done: + opal_atomic_unlock(&module->node_states[target].accumulate_lock); + + /* Only notify once the operation actually happened. MPI_NO_OP is not a + * failure: the target window was still read into the result buffer, which is + * an access the notification is required to cover. */ + if (OMPI_SUCCESS != ret) { + return ret; + } + + osc_sm_notify_accumulate_done(module, target, notify); + + /* the only valid field of RMA request status is the MPI_ERROR field. + * ompi_request_empty has status MPI_SUCCESS and indicates the request is + * complete. */ + *ompi_req = &ompi_request_empty; + + return ret; +} + + int ompi_osc_sm_put(const void *origin_addr, size_t origin_count, @@ -236,6 +805,52 @@ ompi_osc_sm_put(const void *origin_addr, } +int +ompi_osc_sm_put_notify(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win) +{ + int ret; + ompi_osc_sm_module_t *module = + (ompi_osc_sm_module_t*) win->w_osc_module; + void *remote_address; + + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "put_notify: 0x%lx, %zu, %s, %d, %d, %zu, %s, %d, 0x%lx", + (unsigned long) origin_addr, origin_count, + origin_dt->name, target, (int) target_disp, + target_count, target_dt->name, + notify, + (unsigned long) win)); + + ret = osc_sm_check_notify_idx(module, target, notify); + if (OMPI_SUCCESS != ret) { + return ret; + } + + remote_address = ((char*) (module->bases[target])) + module->disp_units[target] * target_disp; + + ret = ompi_datatype_sndrcv((void *)origin_addr, origin_count, origin_dt, + remote_address, target_count, target_dt); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* (MPI-5.1 section 12.3, "The notification counter will be + * updated at the target only after the completion of the data movement + * operation at the target"). */ + opal_atomic_wmb(); + opal_atomic_add(&osc_sm_target_notify_base(module, target)[notify], 1); + + return ret; +} + int ompi_osc_sm_get(void *origin_addr, size_t origin_count, @@ -267,6 +882,50 @@ ompi_osc_sm_get(void *origin_addr, } +int +ompi_osc_sm_get_notify(void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, + struct ompi_win_t *win) +{ + int ret; + ompi_osc_sm_module_t *module = + (ompi_osc_sm_module_t*) win->w_osc_module; + void *remote_address; + + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "get: 0x%lx, %zu, %s, %d, %d, %zu, %s, 0x%lx", + (unsigned long) origin_addr, origin_count, + origin_dt->name, target, (int) target_disp, + target_count, target_dt->name, + (unsigned long) win)); + + ret = osc_sm_check_notify_idx(module, target, notify); + if (OMPI_SUCCESS != ret) { + return ret; + } + + remote_address = ((char*) (module->bases[target])) + module->disp_units[target] * target_disp; + + ret = ompi_datatype_sndrcv(remote_address, target_count, target_dt, + origin_addr, origin_count, origin_dt); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* Full barrier, not opal_atomic_rmb(): see ompi_osc_sm_rget_notify(). */ + opal_atomic_mb(); + opal_atomic_add(&osc_sm_target_notify_base(module, target)[notify], 1); + + return ret; +} + + int ompi_osc_sm_accumulate(const void *origin_addr, size_t origin_count, @@ -308,6 +967,62 @@ ompi_osc_sm_accumulate(const void *origin_addr, } +int +ompi_osc_sm_accumulate_notify(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, + ptrdiff_t target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + struct ompi_op_t *op, + int notify, + struct ompi_win_t *win) +{ + int ret; + ompi_osc_sm_module_t *module = + (ompi_osc_sm_module_t*) win->w_osc_module; + void *remote_address; + + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "accumulate_notify: 0x%lx, %zu, %s, %d, %d, %zu, %s, %s, %d, 0x%lx", + (unsigned long) origin_addr, origin_count, + origin_dt->name, target, (int) target_disp, + target_count, target_dt->name, + op->o_name, notify, + (unsigned long) win)); + + ret = osc_sm_check_notify_idx(module, target, notify); + if (OMPI_SUCCESS != ret) { + return ret; + } + + remote_address = ((char*) (module->bases[target])) + module->disp_units[target] * target_disp; + + opal_atomic_lock(&module->node_states[target].accumulate_lock); + if (op == &ompi_mpi_op_replace.op) { + ret = ompi_datatype_sndrcv((void *)origin_addr, origin_count, origin_dt, + remote_address, target_count, target_dt); + } else { + ret = ompi_osc_base_sndrcv_op(origin_addr, origin_count, origin_dt, + remote_address, target_count, target_dt, + op); + } + opal_atomic_unlock(&module->node_states[target].accumulate_lock); + + /* Only notify once the accumulate actually happened -- a counter bumped for + * an operation that failed would tell the target that data it never + * received is ready. */ + if (OMPI_SUCCESS != ret) { + return ret; + } + + osc_sm_notify_accumulate_done(module, target, notify); + + return ret; +} + + int ompi_osc_sm_get_accumulate(const void *origin_addr, size_t origin_count, @@ -359,6 +1074,72 @@ ompi_osc_sm_get_accumulate(const void *origin_addr, } +int +ompi_osc_sm_get_accumulate_notify(const void *origin_addr, + size_t origin_count, + struct ompi_datatype_t *origin_dt, + void *result_addr, + size_t result_count, + struct ompi_datatype_t *result_dt, + int target, + MPI_Aint target_disp, + size_t target_count, + struct ompi_datatype_t *target_dt, + struct ompi_op_t *op, + int notify, + struct ompi_win_t *win) +{ + int ret; + ompi_osc_sm_module_t *module = + (ompi_osc_sm_module_t*) win->w_osc_module; + void *remote_address; + + OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, + "get_accumulate_notify: 0x%lx, %zu, %s, %d, %d, %zu, %s, %s, %d, 0x%lx", + (unsigned long) origin_addr, origin_count, + origin_dt->name, target, (int) target_disp, + target_count, target_dt->name, + op->o_name, notify, + (unsigned long) win)); + + ret = osc_sm_check_notify_idx(module, target, notify); + if (OMPI_SUCCESS != ret) { + return ret; + } + + remote_address = ((char*) (module->bases[target])) + module->disp_units[target] * target_disp; + + opal_atomic_lock(&module->node_states[target].accumulate_lock); + + ret = ompi_datatype_sndrcv(remote_address, target_count, target_dt, + result_addr, result_count, result_dt); + if (OMPI_SUCCESS != ret || op == &ompi_mpi_op_no_op.op) goto done; + + if (op == &ompi_mpi_op_replace.op) { + ret = ompi_datatype_sndrcv((void *)origin_addr, origin_count, origin_dt, + remote_address, target_count, target_dt); + } else { + ret = ompi_osc_base_sndrcv_op(origin_addr, origin_count, origin_dt, + remote_address, target_count, target_dt, + op); + } + + done: + opal_atomic_unlock(&module->node_states[target].accumulate_lock); + + /* Only notify once the operation actually happened. MPI_NO_OP is not a + * failure: the target window was still read into the result buffer, which is + * an access the notification is required to cover. */ + if (OMPI_SUCCESS != ret) { + return ret; + } + + osc_sm_notify_accumulate_done(module, target, notify); + + return ret; +} + + int ompi_osc_sm_compare_and_swap(const void *origin_addr, const void *compare_addr, @@ -437,5 +1218,5 @@ ompi_osc_sm_fetch_and_op(const void *origin_addr, done: opal_atomic_unlock(&module->node_states[target].accumulate_lock); - return OMPI_SUCCESS;; + return OMPI_SUCCESS; } diff --git a/ompi/mca/osc/sm/osc_sm_component.c b/ompi/mca/osc/sm/osc_sm_component.c index ad2b3cae25a..055fb460ee7 100644 --- a/ompi/mca/osc/sm/osc_sm_component.c +++ b/ompi/mca/osc/sm/osc_sm_component.c @@ -31,7 +31,9 @@ #include "ompi/request/request.h" #include "opal/util/sys_limits.h" #include "opal/align.h" +#include "opal/util/info.h" #include "opal/util/printf.h" +#include "opal/class/opal_cstring.h" #include "opal/mca/mpool/base/base.h" #include "osc_sm.h" @@ -79,16 +81,29 @@ ompi_osc_sm_module_t ompi_osc_sm_module_template = { .osc_free = ompi_osc_sm_free, .osc_put = ompi_osc_sm_put, + .osc_put_notify = ompi_osc_sm_put_notify, .osc_get = ompi_osc_sm_get, + .osc_get_notify = ompi_osc_sm_get_notify, + .osc_win_get_notify_value = ompi_osc_sm_win_get_notify_value, + .osc_win_reset_notify_value = ompi_osc_sm_win_reset_notify_value, + .osc_win_set_num_notify = ompi_osc_sm_win_set_num_notify, + .osc_win_get_num_notify = ompi_osc_sm_win_get_num_notify, + .osc_win_get_notify_bounds = ompi_osc_sm_win_get_notify_bounds, .osc_accumulate = ompi_osc_sm_accumulate, + .osc_accumulate_notify = ompi_osc_sm_accumulate_notify, .osc_compare_and_swap = ompi_osc_sm_compare_and_swap, .osc_fetch_and_op = ompi_osc_sm_fetch_and_op, .osc_get_accumulate = ompi_osc_sm_get_accumulate, + .osc_get_accumulate_notify = ompi_osc_sm_get_accumulate_notify, .osc_rput = ompi_osc_sm_rput, + .osc_rput_notify = ompi_osc_sm_rput_notify, .osc_rget = ompi_osc_sm_rget, + .osc_rget_notify = ompi_osc_sm_rget_notify, .osc_raccumulate = ompi_osc_sm_raccumulate, + .osc_raccumulate_notify = ompi_osc_sm_raccumulate_notify, .osc_rget_accumulate = ompi_osc_sm_rget_accumulate, + .osc_rget_accumulate_notify = ompi_osc_sm_rget_accumulate_notify, .osc_fence = ompi_osc_sm_fence, @@ -138,9 +153,83 @@ static int component_register (void) &mca_osc_sm_component.priority); free(description_str); + mca_osc_sm_component.num_notify_counters = OSC_SM_DEFAULT_NOTIFY_COUNTERS; + opal_asprintf(&description_str, + "Number of RMA notification counters reserved per MPI process " + "in the shared memory segment of each window. Windows whose " + "info gives an mpi_assert_max_num_notify value use that " + "instead. MPI_Win_set_num_notify may exceed this value, at " + "the cost of allocating a new shared segment (default: %u)", + mca_osc_sm_component.num_notify_counters); + (void) mca_base_component_var_register(&mca_osc_sm_component.super.osc_version, + "num_notify_counters", description_str, + MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0, + OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_GROUP, + &mca_osc_sm_component.num_notify_counters); + free(description_str); + return OPAL_SUCCESS; } + +/* Read the mpi_assert_max_num_notify info key (MPI-5.1 section 12.2). */ +static int osc_sm_reserved_notify_counters(opal_info_t *info, unsigned int *assert_value, + unsigned int *reserved) +{ + opal_cstring_t *value_string; + int flag = 0, value = 0; + + *assert_value = 0; + *reserved = mca_osc_sm_component.num_notify_counters; + + if (NULL == info) { + return OMPI_SUCCESS; + } + + if (OMPI_SUCCESS != opal_info_get(info, "mpi_assert_max_num_notify", + &value_string, &flag) || !flag) { + return OMPI_SUCCESS; + } + + if (OPAL_SUCCESS != opal_cstring_to_int(value_string, &value)) { + OBJ_RELEASE(value_string); + return MPI_ERR_INFO; + } + OBJ_RELEASE(value_string); + + /* A negative value is a malformed key rather than "no assertion"; only 0 + * carries the "assume nothing" meaning. */ + if (value < 0) { + return MPI_ERR_INFO; + } + + if (0 != value) { + *assert_value = (unsigned int) value; + *reserved = (unsigned int) value; + } + + return OMPI_SUCCESS; +} + + +void +ompi_osc_sm_refresh_notify_bases(ompi_osc_sm_module_t *module) +{ + int comm_size = ompi_comm_size(module->comm); + char *base; + int i; + + /* Once the counters have been grown they live in their own segment; + * before that they sit inline in the main one. */ + base = (NULL != module->notify_segment_base) ? (char *) module->notify_segment_base + : (char *) module->segment_base; + + for (i = 0 ; i < comm_size ; ++i) { + module->notify_bases[i] = (opal_atomic_int64_t *) + (base + module->node_states[i].notify_counter_offset); + } +} + static int component_open(void) { @@ -201,6 +290,7 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis bool unlink_needed = false; int ret = OMPI_ERROR; size_t memory_alignment = OPAL_ALIGN_MIN; + unsigned int notify_assert = 0, notify_reserved = 0; assert(MPI_WIN_FLAVOR_SHARED == flavor || MPI_WIN_FLAVOR_ALLOCATE == flavor); @@ -231,6 +321,16 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis module->flavor = flavor; + /* How many notification counters to reserve per MPI process. Read before + * the segment is sized, since the reservation is part of its layout. */ + ret = osc_sm_reserved_notify_counters(info, ¬ify_assert, ¬ify_reserved); + if (OMPI_SUCCESS != ret) goto error; + module->notify_max_assert = notify_assert; + module->notify_segment_base = NULL; + + module->notify_bases = calloc(comm_size, sizeof(module->notify_bases[0])); + if (NULL == module->notify_bases) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; + /* create the segment */ if (1 == comm_size) { module->segment_base = NULL; @@ -251,13 +351,22 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis module->posts = calloc (1, sizeof(module->posts[0]) + sizeof (module->posts[0][0])); if (NULL == module->posts) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; module->posts[0] = (osc_sm_post_atomic_type_t *) (module->posts + 1); + + /* Notification counters for the single process case. */ + module->notify_bases[0] = calloc(notify_reserved, sizeof(int64_t)); + if (NULL == module->notify_bases[0]) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; + module->node_states[0].notify_counter_capacity = notify_reserved; + module->node_states[0].notify_counter_count = notify_reserved; + module->node_states[0].notify_counter_offset = 0; } else { - unsigned long total, *rbuf; + unsigned long total, total_counters, gather_values[2], *rbuf; int i, flag; size_t pagesize; size_t state_size; size_t posts_size, post_size = (comm_size + OSC_SM_POST_MASK) / (OSC_SM_POST_MASK + 1); + size_t notify_counters_size; size_t data_base_size; + opal_atomic_int64_t *notify_counters_base; opal_output_verbose(MCA_BASE_VERBOSE_DEBUG, ompi_osc_base_framework.framework_output, "allocating shared memory region of size %ld\n", (long) size); @@ -265,7 +374,7 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis /* get the pagesize */ pagesize = opal_getpagesize(); - rbuf = malloc(sizeof(unsigned long) * comm_size); + rbuf = malloc(sizeof(unsigned long) * comm_size * 2 ); if (NULL == rbuf) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; /* Note that the alloc_shared_noncontig info key only has @@ -289,9 +398,10 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis "allocating window using contiguous strategy"); } - total = size; - ret = module->comm->c_coll->coll_allgather(&total, 1, MPI_UNSIGNED_LONG, - rbuf, 1, MPI_UNSIGNED_LONG, + gather_values[0] = size; + gather_values[1] = notify_reserved; + ret = module->comm->c_coll->coll_allgather(gather_values, 2, MPI_UNSIGNED_LONG, + rbuf, 2, MPI_UNSIGNED_LONG, module->comm, module->comm->c_coll->coll_allgather_module); if (OMPI_SUCCESS != ret) { @@ -300,8 +410,10 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis } total = 0; + total_counters = 0; for (i = 0 ; i < comm_size ; ++i) { - total += rbuf[i]; + total += rbuf[2 * i]; + total_counters += rbuf[2 * i + 1]; if (module->noncontig) { total += OPAL_ALIGN_PAD_AMOUNT(total, pagesize); } @@ -312,7 +424,9 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis state_size += OPAL_ALIGN_PAD_AMOUNT(state_size, 64); posts_size = comm_size * post_size * sizeof (module->posts[0][0]); posts_size += OPAL_ALIGN_PAD_AMOUNT(posts_size, 64); - data_base_size = state_size + posts_size; + notify_counters_size = total_counters * sizeof(uint64_t); + notify_counters_size += OPAL_ALIGN_PAD_AMOUNT(notify_counters_size, 64); + data_base_size = state_size + posts_size + notify_counters_size; data_base_size += OPAL_ALIGN_PAD_AMOUNT(data_base_size, pagesize); if (0 == ompi_comm_rank (module->comm)) { char *data_file; @@ -373,15 +487,26 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis module->global_state = (ompi_osc_sm_global_state_t *) (module->posts[0] + comm_size * post_size); module->node_states = (ompi_osc_sm_node_state_t *) (module->global_state + 1); - for (i = 0, total = data_base_size ; i < comm_size ; ++i) { + /* set up notify counters in shared memory after node_states */ + notify_counters_base = (opal_atomic_int64_t *) ((char *)(module->node_states + comm_size) + + OPAL_ALIGN_PAD_AMOUNT((uintptr_t)(module->node_states + comm_size), 64)); + + for (i = 0, total = data_base_size, total_counters = 0 ; i < comm_size ; ++i) { if (i > 0) { module->posts[i] = module->posts[i - 1] + post_size; } - module->sizes[i] = rbuf[i]; + module->node_states[i].notify_counter_capacity = (uint32_t) rbuf[2 * i + 1]; + module->node_states[i].notify_counter_count = (uint32_t) rbuf[2 * i + 1]; + module->node_states[i].notify_counter_offset = + (uint64_t) ((char *) (notify_counters_base + total_counters) - + (char *) module->segment_base); + total_counters += rbuf[2 * i + 1]; + + module->sizes[i] = rbuf[2 * i]; if (module->sizes[i] || !module->noncontig) { module->bases[i] = ((char *) module->segment_base) + total; - total += rbuf[i]; + total += rbuf[2 * i]; if (module->noncontig) { total += OPAL_ALIGN_PAD_AMOUNT(total, pagesize); } @@ -390,12 +515,19 @@ component_select(struct ompi_win_t *win, void **base, size_t size, ptrdiff_t dis } } + ompi_osc_sm_refresh_notify_bases(module); + + /* Zero only this process's own counters. */ + memset((void *) module->notify_bases[ompi_comm_rank(module->comm)], 0, + notify_reserved * sizeof(int64_t)); + free(rbuf); } /* initialize my state shared */ module->my_node_state = &module->node_states[ompi_comm_rank(module->comm)]; - memset (module->my_node_state, 0, sizeof(*module->my_node_state)); + module->my_node_state->complete_count = 0; + memset (&module->my_node_state->lock, 0, sizeof(module->my_node_state->lock)); *base = module->bases[ompi_comm_rank(module->comm)]; @@ -550,6 +682,10 @@ ompi_osc_sm_free(struct ompi_win_t *win) module->comm->c_coll->coll_barrier(module->comm, module->comm->c_coll->coll_barrier_module); + if (NULL != module->notify_segment_base) { + opal_shmem_segment_detach (&module->notify_seg_ds); + } + opal_shmem_segment_detach (&module->seg_ds); } else { free(module->node_states); @@ -558,7 +694,14 @@ ompi_osc_sm_free(struct ompi_win_t *win) mca_mpool_base_default_module->mpool_free(mca_mpool_base_default_module, module->bases[0]); } + /* free the counters for the single process case */ + /* cast away the atomic/volatile qualifier for free(), as in + * opal/runtime/opal_progress.c */ + if (NULL != module->notify_bases) { + free((void *) module->notify_bases[0]); + } } + free(module->notify_bases); free(module->disp_units); free(module->outstanding_locks); free(module->sizes); @@ -605,6 +748,13 @@ ompi_osc_sm_get_info(struct ompi_win_t *win, struct opal_info_t **info_used) (module->noncontig) ? "true" : "false"); } + /* Report the assertion back only when one was actually given. */ + if (0 != module->notify_max_assert) { + char value_str[16]; + snprintf(value_str, sizeof(value_str), "%u", module->notify_max_assert); + opal_info_set(info, "mpi_assert_max_num_notify", value_str); + } + *info_used = info; return OMPI_SUCCESS; diff --git a/ompi/mca/osc/ucx/osc_ucx.h b/ompi/mca/osc/ucx/osc_ucx.h index bc3dc8a91b3..f79db5d1070 100644 --- a/ompi/mca/osc/ucx/osc_ucx.h +++ b/ompi/mca/osc/ucx/osc_ucx.h @@ -27,6 +27,11 @@ #define OMPI_OSC_UCX_POST_PEER_MAX 32 #define OMPI_OSC_UCX_ATTACH_MAX 48 #define OMPI_OSC_UCX_MEM_ADDR_MAX_LEN 1024 +/* Default number of RMA notification counters reserved per MPI process in each + * window's registered memory region. Overridden per job by the + * osc_ucx_num_notify_counters MCA parameter and per window by the + * "mpi_assert_max_num_notify" info key. */ +#define OMPI_OSC_UCX_DEFAULT_NOTIFY_COUNTERS 16 typedef struct ompi_osc_ucx_component { @@ -43,6 +48,9 @@ typedef struct ompi_osc_ucx_component { bool no_locks; /* Default value of the no_locks info key for new windows */ bool acc_single_intrinsic; unsigned int priority; + /* Number of notification counters reserved per MPI process in each window, + * unless the window's info gives "mpi_assert_max_num_notify". */ + unsigned int num_notify_counters; /* directory where to place backing files */ char *backing_directory; } ompi_osc_ucx_component_t; @@ -122,6 +130,24 @@ typedef struct ompi_osc_ucx_module { struct ompi_communicator_t *comm; int flavor; size_t size; + int *notify_counts; /* per-rank number of notification counters *attached* at each + * rank (size comm_size), as set by MPI_WIN_SET_NUM_NOTIFY and + * kept consistent across the group by an allgather. Always + * <= notify_capacity. */ + unsigned int notify_capacity; /* notification counters currently reserved per rank. + * Agreed on across the group and uniform. Grown on + * demand by MPI_WIN_SET_NUM_NOTIFY unless + * notify_max_assert caps it. */ + unsigned int notify_max_assert; /* non-zero only if *every* rank passed + * "mpi_assert_max_num_notify" at window creation. + * Then the agreed reservation is a hard cap and the + * counters never grow (MPI-5.1 12.2: the assertion + * lets the implementation optimize the allocation). + * Zero means no rank asserted a bound, so the + * standard's "does not assume any limit" applies. */ + uint64_t *notify_addrs; /* per-rank base address of the notification counters + * (size comm_size) */ + void *notify_base; /* this rank's counters; notify_capacity uint64_t */ size_t *sizes; /* used if not every process has the same size */ uint64_t *addrs; uint64_t *state_addrs; @@ -149,6 +175,11 @@ typedef struct ompi_osc_ucx_module { opal_common_ucx_ctx_t *ctx; opal_common_ucx_wpmem_t *mem; opal_common_ucx_wpmem_t *state_mem; + /* Notification counters get their own registration rather than being + * appended to the window data: the data region for MPI_WIN_FLAVOR_CREATE + * belongs to the user and has no room for them, and a dynamic window has + * no data region at all. */ + opal_common_ucx_wpmem_t *notify_mem; ompi_osc_ucx_mem_ranges_t *epoc_outstanding_ops_mems; bool skip_sync_check; bool noncontig_shared_win; @@ -277,6 +308,75 @@ int ompi_osc_find_attached_region_position(ompi_osc_dynamic_win_info_t *dynamic_ int ompi_osc_ucx_dynamic_lock(ompi_osc_ucx_module_t *module, int target); int ompi_osc_ucx_dynamic_unlock(ompi_osc_ucx_module_t *module, int target); +int ompi_osc_ucx_put_notify(const void *origin_addr, size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, ptrdiff_t target_disp, size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, struct ompi_win_t *win); +int ompi_osc_ucx_get_notify(void *origin_addr, size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, ptrdiff_t target_disp, size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, struct ompi_win_t *win); +int ompi_osc_ucx_rput_notify(const void *origin_addr, size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, ptrdiff_t target_disp, size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, struct ompi_win_t *win, + struct ompi_request_t **request); +int ompi_osc_ucx_rget_notify(void *origin_addr, size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, ptrdiff_t target_disp, size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, struct ompi_win_t *win, + struct ompi_request_t **request); +int ompi_osc_ucx_accumulate_notify(const void *origin_addr, size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, ptrdiff_t target_disp, size_t target_count, + struct ompi_datatype_t *target_dt, + struct ompi_op_t *op, int notify, + struct ompi_win_t *win); +int ompi_osc_ucx_get_accumulate_notify(const void *origin_addr, size_t origin_count, + struct ompi_datatype_t *origin_dt, + void *result_addr, size_t result_count, + struct ompi_datatype_t *result_dt, + int target, ptrdiff_t target_disp, size_t target_count, + struct ompi_datatype_t *target_dt, + struct ompi_op_t *op, int notify, + struct ompi_win_t *win); +int ompi_osc_ucx_raccumulate_notify(const void *origin_addr, size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, ptrdiff_t target_disp, size_t target_count, + struct ompi_datatype_t *target_dt, + struct ompi_op_t *op, int notify, + struct ompi_win_t *win, + struct ompi_request_t **request); +int ompi_osc_ucx_rget_accumulate_notify(const void *origin_addr, size_t origin_count, + struct ompi_datatype_t *origin_dt, + void *result_addr, size_t result_count, + struct ompi_datatype_t *result_dt, + int target, ptrdiff_t target_disp, size_t target_count, + struct ompi_datatype_t *target_dt, + struct ompi_op_t *op, int notify, + struct ompi_win_t *win, + struct ompi_request_t **request); +int ompi_osc_ucx_win_get_notify_value(struct ompi_win_t *win, int notify, + OMPI_MPI_COUNT_TYPE *value); +int ompi_osc_ucx_win_get_notify_bounds(struct ompi_win_t *win, int *num_sb, int *num_ub, + OMPI_MPI_COUNT_TYPE *value_ub); +int ompi_osc_ucx_win_reset_notify_value(struct ompi_win_t *win, int notify, + OMPI_MPI_COUNT_TYPE *value); +int ompi_osc_ucx_win_set_num_notify(struct ompi_win_t *win, struct opal_info_t *info, + int num_notifications); +/* Collectively re-reserve new_capacity notification counters per rank, replacing + * the current registration. Defined in osc_ucx_component.c because it needs the + * component's address-exchange helper. */ +int ompi_osc_ucx_grow_notify_counters(ompi_osc_ucx_module_t *module, + unsigned int new_capacity); + +int ompi_osc_ucx_win_get_num_notify(struct ompi_win_t *win, int target_rank, + int *num_notifications); + /* returns the size at the peer */ static inline size_t ompi_osc_ucx_get_size(ompi_osc_ucx_module_t *module, int rank) { diff --git a/ompi/mca/osc/ucx/osc_ucx_comm.c b/ompi/mca/osc/ucx/osc_ucx_comm.c index 0354edb71c0..1f400f85fb9 100644 --- a/ompi/mca/osc/ucx/osc_ucx_comm.c +++ b/ompi/mca/osc/ucx/osc_ucx_comm.c @@ -17,9 +17,15 @@ #include "ompi/mca/osc/base/osc_base_obj_convert.h" #include "opal/mca/common/ucx/common_ucx.h" +#include + #include "osc_ucx.h" #include "osc_ucx_request.h" +#include + +#include "ompi/attribute/attribute.h" + #define CHECK_VALID_RKEY(_module, _target, _count) \ if (!((_module)->win_info_array[_target]).rkey_init && ((_count) > 0)) { \ @@ -603,6 +609,589 @@ int ompi_osc_ucx_get(void *origin_addr, size_t origin_count, } } +static int osc_ucx_request_over_flush(ompi_osc_ucx_module_t *module, + struct ompi_win_t *win, int target, + ucp_ep_h *ep, enum req_type req_type, + struct ompi_request_t **request); + +static int accumulate_req(const void *origin_addr, size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, ptrdiff_t target_disp, size_t target_count, + struct ompi_datatype_t *target_dt, + struct ompi_op_t *op, struct ompi_win_t *win, + ompi_osc_ucx_accumulate_request_t *ucx_req); + +static int get_accumulate_req(const void *origin_addr, size_t origin_count, + struct ompi_datatype_t *origin_dt, + void *result_addr, size_t result_count, + struct ompi_datatype_t *result_dt, + int target, ptrdiff_t target_disp, size_t target_count, + struct ompi_datatype_t *target_dt, + struct ompi_op_t *op, struct ompi_win_t *win, + ompi_osc_ucx_accumulate_request_t *ucx_req); + +/* Returns the remote address of notify counter[notify] for the given target. + * The counters have their own registered region (module->notify_mem), separate + * from the window data, so this is independent of the window's flavor and size. */ +static inline uint64_t +osc_ucx_notify_counter_addr(ompi_osc_ucx_module_t *module, int target, int notify) +{ + return module->notify_addrs[target] + (uint64_t)notify * sizeof(uint64_t); +} + +/* A region of module->notify_capacity notification counters is registered per + * rank at window creation (see osc_ucx_component.c), but only the first + * notify_counts[rank] of them are considered *attached* by + * MPI_WIN_SET_NUM_NOTIFY. Per the MPI Standard it is erroneous to reference a + * counter that is out of range at the target, so validate against the target + * rank's attached count. */ +#define CHECK_NOTIFY_IDX(module, notify, rank) \ + if ((notify) < 0 || (notify) >= (module)->notify_counts[rank]) { \ + return MPI_ERR_RMA_NOTIFICATION; \ + } + +/* Increments the target's notification counter once the preceding data + * operation has been ordered ahead of it. Shared by every notified + * operation; they differ only in which base operation they issue first and + * in whether a fence or a flush is needed to order it. + * + * Note that for the request-based variants the data operation has already been + * issued and *request already handed back by the time this can fail, so an + * error return leaves that request outstanding and the caller still has to + * complete it. A transport failure here is not recoverable in any case. */ +static inline int +osc_ucx_notify_target(ompi_osc_ucx_module_t *module, int target, int notify, + ucp_ep_h *ep) +{ + int ret = opal_common_ucx_wpmem_post(module->notify_mem, + UCP_ATOMIC_POST_OP_ADD, 1, + target, sizeof(uint64_t), + osc_ucx_notify_counter_addr(module, target, notify), + ep); + return (OPAL_SUCCESS == ret) ? OMPI_SUCCESS : OMPI_ERROR; +} + +int ompi_osc_ucx_win_get_notify_value(struct ompi_win_t *win, int notify, + OMPI_MPI_COUNT_TYPE *value) +{ + ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t *)win->w_osc_module; + int my_rank = ompi_comm_rank(module->comm); + + CHECK_NOTIFY_IDX(module, notify, my_rank); + + /* Origins increment this counter with a UCX atomic, which the transport may + * emulate in software on the local worker rather than offload to the NIC. + * In that case the counter only advances while the worker is progressed, so + * a consumer spinning on MPI_WIN_GET_NOTIFY_VALUE -- the natural way to wait + * for a notification -- would never observe the update. Progress the worker + * here so that such a loop makes forward progress on its own, as every other + * spin-wait in this component does. */ + opal_common_ucx_wpool_progress(mca_osc_ucx_component.wpool); + + volatile uint64_t *counter = + (volatile uint64_t *)osc_ucx_notify_counter_addr(module, my_rank, notify); + *value = (OMPI_MPI_COUNT_TYPE)*counter; + opal_atomic_rmb(); + return OMPI_SUCCESS; +} + +int ompi_osc_ucx_win_reset_notify_value(struct ompi_win_t *win, int notify, + OMPI_MPI_COUNT_TYPE *value) +{ + ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t *)win->w_osc_module; + int my_rank = ompi_comm_rank(module->comm); + uint64_t result_value = 0; + ucp_ep_h *ep; + int ret; + + CHECK_NOTIFY_IDX(module, notify, my_rank); + + OSC_UCX_GET_DEFAULT_EP(ep, module, my_rank); + + /* The counter is incremented by remote origins through UCX network atomic + * operations, so reset it with a UCX atomic swap (targeting our own rank) + * rather than a CPU atomic. That keeps the read-and-zero atomic with + * respect to those concurrent network atomics — a plain CPU swap is not + * ordered against them. The fetch returns the counter's previous value. */ + ret = opal_common_ucx_wpmem_fetch(module->notify_mem, + UCP_ATOMIC_FETCH_OP_SWAP, 0, + my_rank, &result_value, sizeof(result_value), + osc_ucx_notify_counter_addr(module, my_rank, notify), + ep); + if (OPAL_SUCCESS != ret) { + OSC_UCX_VERBOSE(1, "opal_common_ucx_wpmem_fetch failed: %d", ret); + return OMPI_ERROR; + } + + *value = (OMPI_MPI_COUNT_TYPE)result_value; + return OMPI_SUCCESS; +} + +int ompi_osc_ucx_win_get_num_notify(struct ompi_win_t *win, int target_rank, + int *num_notifications) +{ + ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t *)win->w_osc_module; + + if (target_rank < 0 || target_rank >= ompi_comm_size(module->comm)) { + return MPI_ERR_RANK; + } + + /* Local query (MPI_WIN_GET_NUM_NOTIFY, §12.6.1): return the number of + * notification counters currently attached at target_rank, as last + * published by MPI_WIN_SET_NUM_NOTIFY. */ + *num_notifications = module->notify_counts[target_rank]; + return OMPI_SUCCESS; +} + +int ompi_osc_ucx_win_set_num_notify(struct ompi_win_t *win, struct opal_info_t *info, + int num_notifications) +{ + ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t *)win->w_osc_module; + int comm_size = ompi_comm_size(module->comm); + int requested = num_notifications; + int *requested_counts; + unsigned int needed; + int ret, i; + + (void) info; /* "mpi_assert_same_num_notifications" is an optimization hint only */ + + /* When every rank asserted "mpi_assert_max_num_notify" at window creation, + * that value is a hard upper bound and asking for more is erroneous. + * Otherwise MPI-5.1 12.2 says no limit is assumed, so a request above the + * current reservation grows the counters below rather than failing. + * + * This is a synchronizing collective, so a rank with a bad argument must not + * return before the allgather below -- that would leave the rest of the + * group blocked in it. Mark the request instead and let every rank discover + * the error from the gathered values. */ + if (requested < 0 || + (0 != module->notify_max_assert && + (unsigned int) requested > module->notify_max_assert)) { + requested = -1; + } + + requested_counts = malloc(comm_size * sizeof(int)); + if (NULL == requested_counts) { + return OMPI_ERR_TEMP_OUT_OF_RESOURCE; + } + + /* All notification counters (existing and newly attached) are reset to zero + * by this call. Resetting before the allgather is what makes the standard's + * "will not return until ... all processes have adjusted the number of + * notification counters" hold: completing the collective implies every rank + * has already reset, so no rank can return and then have a peer wipe the + * notification it just delivered. It is erroneous to call this while an + * access epoch is open, so no concurrent network atomics touch the counters + * and a plain local reset is sufficient. */ + if (NULL != module->notify_base) { + memset(module->notify_base, 0, module->notify_capacity * sizeof(uint64_t)); + } + opal_atomic_wmb(); + + /* Publish every rank's requested count to the whole group so that origins + * can validate notification indices against the target's count. Gathering + * the requested value directly is what makes MPI_WIN_GET_NUM_NOTIFY return + * the value given here, including when it lowers the count. */ + ret = module->comm->c_coll->coll_allgather(&requested, 1, MPI_INT, + requested_counts, 1, MPI_INT, + module->comm, + module->comm->c_coll->coll_allgather_module); + if (OMPI_SUCCESS != ret) { + free(requested_counts); + return ret; + } + + for (i = 0; i < comm_size; i++) { + if (0 > requested_counts[i]) { + /* Some rank asked for a count outside [0, notify_capacity]. Every + * rank sees the same gathered array and bails identically, so the + * attached counts stay as they were rather than the group ending up + * half-reconfigured. The counters have been zeroed, which is + * harmless for a call that is erroneous anyway. */ + free(requested_counts); + return MPI_ERR_ARG; + } + } + + /* Every rank sees the same gathered array, so they all reach the same + * decision about whether to grow and to what size, without extra + * communication. The reservation is uniform across the window (window + * creation agrees it with an allreduce), so it grows to the largest request + * anyone made. Never shrink: a rank that lowered its count keeps the space + * it already has, so only genuine growth costs a re-registration and + * alternating high/low requests do not thrash the NIC. */ + needed = module->notify_capacity; + for (i = 0; i < comm_size; i++) { + if ((unsigned int) requested_counts[i] > needed) { + needed = (unsigned int) requested_counts[i]; + } + } + + if (needed > module->notify_capacity) { + ret = ompi_osc_ucx_grow_notify_counters(module, needed); + if (OMPI_SUCCESS != ret) { + free(requested_counts); + return ret; + } + + /* MPI_WIN_NOTIFICATION_NUM_SB is the count supported without paying for + * a re-registration, so it has to follow the reservation rather than + * stay at whatever was cached when the window was created. */ + ret = ompi_attr_set_int(WIN_ATTR, win, &win->w_keyhash, + MPI_WIN_NOTIFICATION_NUM_SB, + (int) module->notify_capacity, true); + if (OMPI_SUCCESS != ret) { + free(requested_counts); + return ret; + } + } + + memcpy(module->notify_counts, requested_counts, comm_size * sizeof(int)); + free(requested_counts); + + return OMPI_SUCCESS; +} + +int ompi_osc_ucx_win_get_notify_bounds(struct ompi_win_t *win, int *num_sb, int *num_ub, + OMPI_MPI_COUNT_TYPE *value_ub) +{ + ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t *)win->w_osc_module; + + /* The current reservation is what is supported without paying for a + * re-registration, so it is the suggested bound. The hard bound is only + * real when every rank asserted "mpi_assert_max_num_notify" at window + * creation; otherwise MPI_WIN_SET_NUM_NOTIFY grows the counters on demand + * and the only limit is what can be allocated. Neither depends on the + * window's flavor. */ + *num_sb = (int) module->notify_capacity; + *num_ub = (0 != module->notify_max_assert) ? (int) module->notify_max_assert + : INT_MAX; + + /* Counters are uint64_t and only ever incremented by one per notified + * operation, but they are returned to the user as a signed MPI_Count, so + * the representable maximum of that type is the real bound. */ + *value_ub = (OMPI_MPI_COUNT_TYPE) INT64_MAX; + + return OMPI_SUCCESS; +} + +int ompi_osc_ucx_put_notify(const void *origin_addr, size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, ptrdiff_t target_disp, size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, struct ompi_win_t *win) +{ + ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t *)win->w_osc_module; + ucp_ep_h *ep; + int ret; + + CHECK_NOTIFY_IDX(module, notify, target); + + OSC_UCX_GET_DEFAULT_EP(ep, module, target); + + ret = ompi_osc_ucx_put(origin_addr, origin_count, origin_dt, + target, target_disp, target_count, target_dt, win); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* Flush to ensure the PUT is visible at the target before the counter + * increment arrives. */ + ret = opal_common_ucx_wpmem_fence(module->mem); + if (OPAL_SUCCESS != ret) { + return OMPI_ERROR; + } + + /* Atomically increment the target's notify counter in-place using the + * same mem handle as the window data. */ + return osc_ucx_notify_target(module, target, notify, ep); +} + +int ompi_osc_ucx_get_notify(void *origin_addr, size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, ptrdiff_t target_disp, size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, struct ompi_win_t *win) +{ + ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t *)win->w_osc_module; + ucp_ep_h *ep; + int ret; + + CHECK_NOTIFY_IDX(module, notify, target); + + OSC_UCX_GET_DEFAULT_EP(ep, module, target); + + ret = ompi_osc_ucx_get(origin_addr, origin_count, origin_dt, + target, target_disp, target_count, target_dt, win); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* Flush to ensure the GET data is locally available before issuing the + * counter increment back to the target. */ + ret = opal_common_ucx_ctx_flush(module->ctx, OPAL_COMMON_UCX_SCOPE_EP, target); + if (OPAL_SUCCESS != ret) { + return OMPI_ERROR; + } + + return osc_ucx_notify_target(module, target, notify, ep); +} + +int ompi_osc_ucx_rput_notify(const void *origin_addr, size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, ptrdiff_t target_disp, size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, struct ompi_win_t *win, + struct ompi_request_t **request) +{ + ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t *)win->w_osc_module; + ucp_ep_h *ep; + int ret; + + CHECK_NOTIFY_IDX(module, notify, target); + + OSC_UCX_GET_DEFAULT_EP(ep, module, target); + + ret = check_sync_state(module, target, true); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* Issue the data movement and the notification first, and only then build + * the request over a flush -- that way the flush covers both, so completing + * the request means the counter update has been pushed out and not merely + * queued behind the origin's next MPI call. */ + ret = ompi_osc_ucx_put(origin_addr, origin_count, origin_dt, target, + target_disp, target_count, target_dt, win); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* Fence to order the PUT before the counter increment. */ + ret = opal_common_ucx_wpmem_fence(module->mem); + if (OPAL_SUCCESS != ret) { + return OMPI_ERROR; + } + + ret = osc_ucx_notify_target(module, target, notify, ep); + if (OMPI_SUCCESS != ret) { + return ret; + } + + return osc_ucx_request_over_flush(module, win, target, ep, RPUT_REQ, request); +} + +int ompi_osc_ucx_rget_notify(void *origin_addr, size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, ptrdiff_t target_disp, size_t target_count, + struct ompi_datatype_t *target_dt, + int notify, struct ompi_win_t *win, + struct ompi_request_t **request) +{ + ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t *)win->w_osc_module; + ucp_ep_h *ep; + int ret; + + CHECK_NOTIFY_IDX(module, notify, target); + + OSC_UCX_GET_DEFAULT_EP(ep, module, target); + + ret = check_sync_state(module, target, true); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* As in rput_notify, the data movement and the notification both precede + * the request-bearing flush so that the request covers both. */ + ret = ompi_osc_ucx_get(origin_addr, origin_count, origin_dt, target, + target_disp, target_count, target_dt, win); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* Blocking flush: the notification tells the target its window has been + * read, so the GET must have completed before the counter is incremented + * (MPI-5.1 12.6.4 requires that order). */ + ret = opal_common_ucx_ctx_flush(module->ctx, OPAL_COMMON_UCX_SCOPE_EP, target); + if (OPAL_SUCCESS != ret) { + return OMPI_ERROR; + } + + ret = osc_ucx_notify_target(module, target, notify, ep); + if (OMPI_SUCCESS != ret) { + return ret; + } + + return osc_ucx_request_over_flush(module, win, target, ep, RGET_REQ, request); +} + +int ompi_osc_ucx_accumulate_notify(const void *origin_addr, size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, ptrdiff_t target_disp, size_t target_count, + struct ompi_datatype_t *target_dt, + struct ompi_op_t *op, int notify, + struct ompi_win_t *win) +{ + ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t *)win->w_osc_module; + ucp_ep_h *ep; + int ret; + + CHECK_NOTIFY_IDX(module, notify, target); + + OSC_UCX_GET_DEFAULT_EP(ep, module, target); + + ret = ompi_osc_ucx_accumulate(origin_addr, origin_count, origin_dt, + target, target_disp, target_count, target_dt, + op, win); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* Fence so that the accumulate is applied at the target before the counter + * increment, as §12.6.4 requires. */ + ret = opal_common_ucx_wpmem_fence(module->mem); + if (OPAL_SUCCESS != ret) { + return OMPI_ERROR; + } + + return osc_ucx_notify_target(module, target, notify, ep); +} + +int ompi_osc_ucx_get_accumulate_notify(const void *origin_addr, size_t origin_count, + struct ompi_datatype_t *origin_dt, + void *result_addr, size_t result_count, + struct ompi_datatype_t *result_dt, + int target, ptrdiff_t target_disp, size_t target_count, + struct ompi_datatype_t *target_dt, + struct ompi_op_t *op, int notify, + struct ompi_win_t *win) +{ + ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t *)win->w_osc_module; + ucp_ep_h *ep; + int ret; + + CHECK_NOTIFY_IDX(module, notify, target); + + OSC_UCX_GET_DEFAULT_EP(ep, module, target); + + ret = ompi_osc_ucx_get_accumulate(origin_addr, origin_count, origin_dt, + result_addr, result_count, result_dt, + target, target_disp, target_count, target_dt, + op, win); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* Flush rather than fence: the result buffer must be locally valid, and the + * update must have been applied at the target, before the target may + * observe the notification. */ + ret = opal_common_ucx_ctx_flush(module->ctx, OPAL_COMMON_UCX_SCOPE_EP, target); + if (OPAL_SUCCESS != ret) { + return OMPI_ERROR; + } + + return osc_ucx_notify_target(module, target, notify, ep); +} + +int ompi_osc_ucx_raccumulate_notify(const void *origin_addr, size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, ptrdiff_t target_disp, size_t target_count, + struct ompi_datatype_t *target_dt, + struct ompi_op_t *op, int notify, + struct ompi_win_t *win, + struct ompi_request_t **request) +{ + ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t *)win->w_osc_module; + ucp_ep_h *ep; + int ret; + + CHECK_NOTIFY_IDX(module, notify, target); + + OSC_UCX_GET_DEFAULT_EP(ep, module, target); + + ret = check_sync_state(module, target, true); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* Passing a NULL request runs accumulate_req as the blocking accumulate: it + * ends with a blocking flush, so the read-modify-write has completed at the + * target before we return here. The notification issued below is therefore + * correctly ordered after the window access (MPI-5.1 12.6.4). Building our + * own request afterwards -- rather than taking the pre-completed one from + * ompi_osc_ucx_raccumulate -- is what lets that request also cover the + * notification. */ + ret = accumulate_req(origin_addr, origin_count, origin_dt, target, target_disp, + target_count, target_dt, op, win, NULL); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* Fence to order the accumulate ahead of the counter increment. */ + ret = opal_common_ucx_wpmem_fence(module->mem); + if (OPAL_SUCCESS != ret) { + return OMPI_ERROR; + } + + ret = osc_ucx_notify_target(module, target, notify, ep); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* The accumulate is already complete, so the request only has to represent + * the notification reaching the wire; RPUT_REQ selects the plain + * flush-completion behaviour rather than the accumulate state machine. */ + return osc_ucx_request_over_flush(module, win, target, ep, RPUT_REQ, request); +} + +int ompi_osc_ucx_rget_accumulate_notify(const void *origin_addr, size_t origin_count, + struct ompi_datatype_t *origin_dt, + void *result_addr, size_t result_count, + struct ompi_datatype_t *result_dt, + int target, ptrdiff_t target_disp, size_t target_count, + struct ompi_datatype_t *target_dt, + struct ompi_op_t *op, int notify, + struct ompi_win_t *win, + struct ompi_request_t **request) +{ + ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t *)win->w_osc_module; + ucp_ep_h *ep; + int ret; + + CHECK_NOTIFY_IDX(module, notify, target); + + OSC_UCX_GET_DEFAULT_EP(ep, module, target); + + ret = check_sync_state(module, target, true); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* As in raccumulate_notify: a NULL request makes this the blocking form, + * which ends with a blocking flush, so both the fetched result and the + * update at the target are complete before the notification is issued. */ + ret = get_accumulate_req(origin_addr, origin_count, origin_dt, + result_addr, result_count, result_dt, + target, target_disp, target_count, target_dt, + op, win, NULL); + if (OMPI_SUCCESS != ret) { + return ret; + } + + /* Fence to order the accumulate ahead of the counter increment. */ + ret = opal_common_ucx_wpmem_fence(module->mem); + if (OPAL_SUCCESS != ret) { + return OMPI_ERROR; + } + + ret = osc_ucx_notify_target(module, target, notify, ep); + if (OMPI_SUCCESS != ret) { + return ret; + } + + return osc_ucx_request_over_flush(module, win, target, ep, RGET_REQ, request); +} + static inline bool ompi_osc_need_acc_lock(ompi_osc_ucx_module_t *module, int target) { ompi_osc_ucx_lock_t *lock = NULL; @@ -1490,31 +2079,30 @@ int ompi_osc_ucx_get_accumulate_nb(const void *origin_addr, size_t origin_count, target_count, target_dt, op, win, GET_ACCUMULATE); } -int ompi_osc_ucx_rput(const void *origin_addr, size_t origin_count, - struct ompi_datatype_t *origin_dt, - int target, ptrdiff_t target_disp, size_t target_count, - struct ompi_datatype_t *target_dt, - struct ompi_win_t *win, struct ompi_request_t **request) { - ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module; - ucp_ep_h *ep; - OSC_UCX_GET_DEFAULT_EP(ep, module, target); +/* Attach an MPI request to a nonblocking worker flush. + * + * The request completes when ucp_worker_flush_nb completes, and that flush + * covers every operation already issued on this worker -- whichever memory + * registration they used. So the request waits for exactly what the caller + * issued *before* getting here. The notified variants rely on this: they issue + * their counter atomic first, so completing the request implies the + * notification has been pushed to the target rather than left queued locally + * (MPI-5.1 12.6.4, advice to implementors). + * + * Allocating the request last also keeps the failure contract clean: anything + * that can fail has already run, so callers return their errors with *request + * untouched, as MPI expects of a call that reports an error. */ +static int osc_ucx_request_over_flush(ompi_osc_ucx_module_t *module, + struct ompi_win_t *win, int target, + ucp_ep_h *ep, enum req_type req_type, + struct ompi_request_t **request) +{ opal_common_ucx_wpmem_t *mem = module->mem; uint64_t remote_addr = (module->state_addrs[target]) + OSC_UCX_STATE_REQ_FLAG_OFFSET; ompi_osc_ucx_generic_request_t *ucx_req = NULL; int ret = OMPI_SUCCESS; - ret = check_sync_state(module, target, true); - if (ret != OMPI_SUCCESS) { - return ret; - } - - ret = ompi_osc_ucx_put(origin_addr, origin_count, origin_dt, target, target_disp, - target_count, target_dt, win); - if (ret != OMPI_SUCCESS) { - return ret; - } - - OMPI_OSC_UCX_GENERIC_REQUEST_ALLOC(win, ucx_req, RPUT_REQ); + OMPI_OSC_UCX_GENERIC_REQUEST_ALLOC(win, ucx_req, req_type); ucx_req->super.module = module; OSC_UCX_INCREMENT_OUTSTANDING_NB_OPS(module); @@ -1544,17 +2132,14 @@ int ompi_osc_ucx_rput(const void *origin_addr, size_t origin_count, return ret; } -int ompi_osc_ucx_rget(void *origin_addr, size_t origin_count, +int ompi_osc_ucx_rput(const void *origin_addr, size_t origin_count, struct ompi_datatype_t *origin_dt, int target, ptrdiff_t target_disp, size_t target_count, - struct ompi_datatype_t *target_dt, struct ompi_win_t *win, - struct ompi_request_t **request) { + struct ompi_datatype_t *target_dt, + struct ompi_win_t *win, struct ompi_request_t **request) { ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module; ucp_ep_h *ep; OSC_UCX_GET_DEFAULT_EP(ep, module, target); - opal_common_ucx_wpmem_t *mem = module->mem; - uint64_t remote_addr = (module->state_addrs[target]) + OSC_UCX_STATE_REQ_FLAG_OFFSET; - ompi_osc_ucx_generic_request_t *ucx_req = NULL; int ret = OMPI_SUCCESS; ret = check_sync_state(module, target, true); @@ -1562,40 +2147,37 @@ int ompi_osc_ucx_rget(void *origin_addr, size_t origin_count, return ret; } - ret = ompi_osc_ucx_get(origin_addr, origin_count, origin_dt, target, target_disp, + ret = ompi_osc_ucx_put(origin_addr, origin_count, origin_dt, target, target_disp, target_count, target_dt, win); if (ret != OMPI_SUCCESS) { return ret; } - OMPI_OSC_UCX_GENERIC_REQUEST_ALLOC(win, ucx_req, RGET_REQ); - ucx_req->super.module = module; + return osc_ucx_request_over_flush(module, win, target, ep, RPUT_REQ, request); +} - OSC_UCX_INCREMENT_OUTSTANDING_NB_OPS(module); - ret = opal_common_ucx_wpmem_flush_ep_nb(mem, target, ompi_osc_ucx_req_completion, ucx_req, ep); +int ompi_osc_ucx_rget(void *origin_addr, size_t origin_count, + struct ompi_datatype_t *origin_dt, + int target, ptrdiff_t target_disp, size_t target_count, + struct ompi_datatype_t *target_dt, struct ompi_win_t *win, + struct ompi_request_t **request) { + ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module; + ucp_ep_h *ep; + OSC_UCX_GET_DEFAULT_EP(ep, module, target); + int ret = OMPI_SUCCESS; + ret = check_sync_state(module, target, true); if (ret != OMPI_SUCCESS) { - /* fallback to using an atomic op to acquire a request handle */ - ret = opal_common_ucx_wpmem_fence(mem); - if (ret != OMPI_SUCCESS) { - OSC_UCX_VERBOSE(1, "opal_common_ucx_mem_fence failed: %d", ret); - OMPI_OSC_UCX_REQUEST_RETURN(ucx_req); - return OMPI_ERROR; - } - - ret = opal_common_ucx_wpmem_fetch_nb(mem, UCP_ATOMIC_FETCH_OP_FADD, - 0, target, &(module->req_result), - sizeof(uint64_t), remote_addr & (~0x7), - ompi_osc_ucx_req_completion, ucx_req, ep); - if (ret != OMPI_SUCCESS) { - OMPI_OSC_UCX_REQUEST_RETURN(ucx_req); - return ret; - } + return ret; } - *request = &ucx_req->super.super; + ret = ompi_osc_ucx_get(origin_addr, origin_count, origin_dt, target, target_disp, + target_count, target_dt, win); + if (ret != OMPI_SUCCESS) { + return ret; + } - return ret; + return osc_ucx_request_over_flush(module, win, target, ep, RGET_REQ, request); } int ompi_osc_ucx_raccumulate(const void *origin_addr, size_t origin_count, diff --git a/ompi/mca/osc/ucx/osc_ucx_component.c b/ompi/mca/osc/ucx/osc_ucx_component.c index 635a53a3e0f..4c33b366543 100644 --- a/ompi/mca/osc/ucx/osc_ucx_component.c +++ b/ompi/mca/osc/ucx/osc_ucx_component.c @@ -25,6 +25,10 @@ #include "osc_ucx.h" #include "osc_ucx_request.h" #include "opal/util/sys_limits.h" +#include "opal/util/info.h" +#include "opal/class/opal_cstring.h" + +#include #define memcpy_off(_dst, _src, _len, _off) \ memcpy(((char*)(_dst)) + (_off), _src, _len); \ @@ -102,6 +106,20 @@ ompi_osc_ucx_module_t ompi_osc_ucx_module_template = { .osc_fetch_and_op = ompi_osc_ucx_fetch_and_op, .osc_get_accumulate = ompi_osc_ucx_get_accumulate, + .osc_put_notify = ompi_osc_ucx_put_notify, + .osc_get_notify = ompi_osc_ucx_get_notify, + .osc_rput_notify = ompi_osc_ucx_rput_notify, + .osc_rget_notify = ompi_osc_ucx_rget_notify, + .osc_accumulate_notify = ompi_osc_ucx_accumulate_notify, + .osc_get_accumulate_notify = ompi_osc_ucx_get_accumulate_notify, + .osc_raccumulate_notify = ompi_osc_ucx_raccumulate_notify, + .osc_rget_accumulate_notify = ompi_osc_ucx_rget_accumulate_notify, + .osc_win_get_notify_value = ompi_osc_ucx_win_get_notify_value, + .osc_win_reset_notify_value = ompi_osc_ucx_win_reset_notify_value, + .osc_win_set_num_notify = ompi_osc_ucx_win_set_num_notify, + .osc_win_get_num_notify = ompi_osc_ucx_win_get_num_notify, + .osc_win_get_notify_bounds = ompi_osc_ucx_win_get_notify_bounds, + .osc_rput = ompi_osc_ucx_rput, .osc_rget = ompi_osc_ucx_rget, .osc_raccumulate = ompi_osc_ucx_raccumulate, @@ -150,6 +168,53 @@ static bool check_config_value_bool (char *key, opal_info_t *info) return flag_value[0]; } +/* Read the mpi_assert_max_num_notify info key (MPI-5.1 section 12.2) to decide + * how many notification counters to reserve per MPI process initially, and + * report in *asserted whether the key was actually given. + * + * The key is an assertion by the caller that it will not ask + * MPI_WIN_SET_NUM_NOTIFY for more than this, which lets us size the + * registration once and treat it as a hard upper bound. Without it the + * standard is explicit that "the implementation does not assume any limit", so + * the reservation is only a starting size and the counters grow on demand. */ +static int osc_ucx_reserved_notify_counters(opal_info_t *info, unsigned int *reserved, + bool *asserted) +{ + opal_cstring_t *value_string; + int flag = 0, value = 0; + + *reserved = mca_osc_ucx_component.num_notify_counters; + *asserted = false; + + if (NULL == info) { + return OMPI_SUCCESS; + } + + if (OMPI_SUCCESS != opal_info_get(info, "mpi_assert_max_num_notify", + &value_string, &flag) || !flag) { + return OMPI_SUCCESS; + } + + if (OPAL_SUCCESS != opal_cstring_to_int(value_string, &value)) { + OBJ_RELEASE(value_string); + return MPI_ERR_INFO; + } + OBJ_RELEASE(value_string); + + /* A negative value is a malformed key rather than "no assertion"; only 0 + * carries the "assume nothing" meaning. */ + if (value < 0) { + return MPI_ERR_INFO; + } + + if (0 != value) { + *reserved = (unsigned int) value; + *asserted = true; + } + + return OMPI_SUCCESS; +} + static int component_open(void) { opal_common_ucx_mca_register(); @@ -220,6 +285,23 @@ static int component_register(void) { MCA_BASE_VAR_SCOPE_GROUP, &ompi_osc_ucx_outstanding_ops_flush_threshold); free(description_str); + mca_osc_ucx_component.num_notify_counters = OMPI_OSC_UCX_DEFAULT_NOTIFY_COUNTERS; + + opal_asprintf(&description_str, + "Number of RMA notification counters reserved per MPI process " + "in the registered memory region of each window. Windows whose " + "info gives an mpi_assert_max_num_notify value use that instead. " + "This is a hard upper bound: the counters share the window's " + "registration, so MPI_Win_set_num_notify cannot exceed it " + "(default: %u)", + mca_osc_ucx_component.num_notify_counters); + (void) mca_base_component_var_register(&mca_osc_ucx_component.super.osc_version, + "num_notify_counters", description_str, + MCA_BASE_VAR_TYPE_UNSIGNED_INT, NULL, 0, 0, + OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_GROUP, + &mca_osc_ucx_component.num_notify_counters); + free(description_str); + opal_common_ucx_mca_var_register(&mca_osc_ucx_component.super.osc_version); if (0 == access ("/dev/shm", W_OK)) { @@ -559,7 +641,8 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, pt opal_common_ucx_mem_type_t mem_type; char *my_mem_addr; int my_mem_addr_size; - uint64_t my_info[3] = {0}; + uint64_t my_info[4] = {0}; + void *notify_base = NULL; char *recv_buf = NULL; void *dynamic_base = NULL; unsigned long adjusted_size = size; @@ -672,6 +755,48 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, pt module->flavor = flavor; module->size = size; + + /* How many notification counters to reserve per MPI process. Read before + * the counter region is allocated below. A malformed info value must not + * make this rank skip the allreduce that follows -- that would leave the + * rest of the group blocked in window creation -- so the failure is carried + * through the collective as a negative reservation instead. */ + unsigned int notify_reserved = 0; + bool notify_asserted = false; + int notify_values[3]; + /* The reservation is exchanged as an int and is used to size an allocation, + * so a value that does not fit is a bad configuration rather than a request + * to honor. Both failures ride the flag, not the value: MPI_MAX would hide + * a sentinel value behind some other rank's larger reservation. */ + bool notify_bad = (OMPI_SUCCESS != osc_ucx_reserved_notify_counters(info, ¬ify_reserved, + ¬ify_asserted)) + || notify_reserved > (unsigned int) INT_MAX; + notify_values[0] = notify_bad ? 1 : 0; + notify_values[1] = notify_bad ? 0 : (int) notify_reserved; + /* Carried as "some rank did NOT assert" so that it combines under MPI_MAX + * along with the other two values. */ + notify_values[2] = notify_asserted ? 0 : 1; + + /* info is allowed to differ between MPI processes, so agree on one + * reservation for the whole window. Taking the maximum keeps every rank's + * own assertion satisfiable, and propagates any rank's failure flag. */ + ret = module->comm->c_coll->coll_allreduce(MPI_IN_PLACE, notify_values, 3, + MPI_INT, MPI_MAX, module->comm, + module->comm->c_coll->coll_allreduce_module); + if (OMPI_SUCCESS != ret) { + goto error; + } + if (0 != notify_values[0]) { + ret = MPI_ERR_INFO; + goto error; + } + module->notify_capacity = (unsigned int) notify_values[1]; + /* Only treat the reservation as a hard cap when *every* rank asserted a + * bound. A rank that gave no key made no promise, so the window has to + * stay growable for it -- MPI-5.1 12.2 says an absent (zero) key means the + * implementation assumes no limit. */ + module->notify_max_assert = notify_values[2] ? 0 : (unsigned int) notify_values[1]; + module->no_locks = check_config_value_bool ("no_locks", info); module->acc_single_intrinsic = check_config_value_bool ("acc_single_intrinsic", info); module->skip_sync_check = false; @@ -849,7 +974,6 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, pt goto error; } - for (i = 0, total = 0; i < comm_size ; ++i) { size_t peer_size = ompi_osc_ucx_get_size(module, i); if (peer_size || !module->noncontig_shared_win) { @@ -884,7 +1008,8 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, pt ret = OMPI_ERR_BAD_PARAM; goto error; } - ret = opal_common_ucx_wpmem_create(module->ctx, mem_base, module->size, + ret = opal_common_ucx_wpmem_create(module->ctx, mem_base, + module->size, mem_type, &exchange_len_info, OPAL_COMMON_UCX_WPMEM_ADDR_EXCHANGE_FULL, (void *)module->comm, @@ -899,6 +1024,37 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, pt ucp_rkey_buffer_release(my_mem_addr); } + /* Notification counters live in their own registered region, like the + * window state does. Appending them to the window data would mean writing + * past the end of the user's buffer for MPI_WIN_FLAVOR_CREATE, and would + * leave dynamic windows -- which have no data region -- with nowhere to put + * them. */ + if (0 != module->notify_capacity) { + module->notify_base = calloc(module->notify_capacity, sizeof(uint64_t)); + if (NULL == module->notify_base) { + ret = OMPI_ERR_TEMP_OUT_OF_RESOURCE; + goto error; + } + + notify_base = module->notify_base; + ret = opal_common_ucx_wpmem_create(module->ctx, ¬ify_base, + module->notify_capacity * sizeof(uint64_t), + OPAL_COMMON_UCX_MEM_MAP, + &exchange_len_info, + OPAL_COMMON_UCX_WPMEM_ADDR_EXCHANGE_FULL, + (void *)module->comm, + &my_mem_addr, &my_mem_addr_size, + &module->notify_mem); + if (ret != OMPI_SUCCESS) { + goto error; + } + + if (my_mem_addr_size != 0) { + /* rkey object is already distributed among comm processes */ + ucp_rkey_buffer_release(my_mem_addr); + } + } + state_base = (void *)&(module->state); ret = opal_common_ucx_wpmem_create(module->ctx, &state_base, sizeof(ompi_osc_ucx_state_t), @@ -926,6 +1082,7 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, pt } my_info[1] = (uint64_t)state_base; my_info[2] = ompi_comm_rank(&ompi_mpi_comm_world.comm); + my_info[3] = (uint64_t)module->notify_base; recv_buf = (char *)calloc(comm_size, sizeof(my_info)); ret = comm->c_coll->coll_allgather((void *)my_info, sizeof(my_info), @@ -941,10 +1098,17 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, pt module->addrs = calloc(comm_size, sizeof(uint64_t)); module->state_addrs = calloc(comm_size, sizeof(uint64_t)); module->comm_world_ranks = calloc(comm_size, sizeof(uint64_t)); + /* Number of notification counters attached at each rank; starts at zero + * everywhere (consistent without communication) and is updated by + * MPI_WIN_SET_NUM_NOTIFY. Counters must be attached before use. */ + module->notify_counts = calloc(comm_size, sizeof(int)); + module->notify_addrs = calloc(comm_size, sizeof(uint64_t)); for (i = 0; i < comm_size; i++) { - memcpy(&(module->addrs[i]), recv_buf + i * 3 * sizeof(uint64_t), sizeof(uint64_t)); - memcpy(&(module->state_addrs[i]), recv_buf + i * 3 * sizeof(uint64_t) + sizeof(uint64_t), sizeof(uint64_t)); - memcpy(&(module->comm_world_ranks[i]), recv_buf + i * 3 * sizeof(uint64_t) + 2 * sizeof(uint64_t), sizeof(uint64_t)); + const char *entry = recv_buf + i * sizeof(my_info); + memcpy(&(module->addrs[i]), entry, sizeof(uint64_t)); + memcpy(&(module->state_addrs[i]), entry + sizeof(uint64_t), sizeof(uint64_t)); + memcpy(&(module->comm_world_ranks[i]), entry + 2 * sizeof(uint64_t), sizeof(uint64_t)); + memcpy(&(module->notify_addrs[i]), entry + 3 * sizeof(uint64_t), sizeof(uint64_t)); } free(recv_buf); @@ -957,6 +1121,7 @@ static int component_select(struct ompi_win_t *win, void **base, size_t size, pt module->state.acc_lock = TARGET_LOCK_UNLOCKED; module->state.dynamic_lock = TARGET_LOCK_UNLOCKED; module->state.dynamic_win_count = 0; + for (i = 0; i < OMPI_OSC_UCX_ATTACH_MAX; i++) { module->local_dynamic_win_info[i].refcnt = 0; } @@ -1091,6 +1256,92 @@ int ompi_osc_ucx_dynamic_unlock(ompi_osc_ucx_module_t *module, int target) { return OMPI_SUCCESS; } +/* Collectively replace the notification-counter registration with a larger one. + * + * Called from MPI_WIN_SET_NUM_NOTIFY, which the standard defines as a blocking, + * synchronizing collective procedure -- that is what makes this safe. Every + * rank has to take part even if its own request fits, because registering the + * memory exchanges rkeys with the whole group. + * + * Two properties of MPI_WIN_SET_NUM_NOTIFY keep this simple. It resets every + * counter to zero, so a freshly calloc'd region is already the required + * contents and no value has to be carried across. And it is erroneous to call + * it while an access epoch is open or with an active notification-threshold + * request, so no remote atomic can be in flight against the old region while it + * is being replaced. + * + * The allgather of the new base addresses doubles as the barrier that lets the + * old region be released: once it completes, every rank has published its new + * address and no rank can issue a notified operation until it returns from the + * enclosing collective. */ +int ompi_osc_ucx_grow_notify_counters(ompi_osc_ucx_module_t *module, + unsigned int new_capacity) +{ + int comm_size = ompi_comm_size(module->comm); + opal_common_ucx_wpmem_t *new_mem = NULL; + void *new_base = NULL, *reg_base; + char *my_mem_addr = NULL; + uint64_t my_addr, *new_addrs = NULL; + int my_mem_addr_size = 0; + int ret; + + new_base = calloc(new_capacity, sizeof(uint64_t)); + if (NULL == new_base) { + return MPI_ERR_NO_MEM; + } + + new_addrs = calloc(comm_size, sizeof(uint64_t)); + if (NULL == new_addrs) { + free(new_base); + return MPI_ERR_NO_MEM; + } + + reg_base = new_base; + ret = opal_common_ucx_wpmem_create(module->ctx, ®_base, + new_capacity * sizeof(uint64_t), + OPAL_COMMON_UCX_MEM_MAP, + &exchange_len_info, + OPAL_COMMON_UCX_WPMEM_ADDR_EXCHANGE_FULL, + (void *)module->comm, + &my_mem_addr, &my_mem_addr_size, + &new_mem); + if (OMPI_SUCCESS != ret) { + free(new_addrs); + free(new_base); + return ret; + } + + if (0 != my_mem_addr_size) { + /* rkey object is already distributed among comm processes */ + ucp_rkey_buffer_release(my_mem_addr); + } + + my_addr = (uint64_t) new_base; + ret = module->comm->c_coll->coll_allgather(&my_addr, sizeof(uint64_t), MPI_BYTE, + new_addrs, sizeof(uint64_t), MPI_BYTE, + module->comm, + module->comm->c_coll->coll_allgather_module); + if (OMPI_SUCCESS != ret) { + opal_common_ucx_wpmem_free(new_mem); + free(new_addrs); + free(new_base); + return ret; + } + + if (NULL != module->notify_mem) { + opal_common_ucx_wpmem_free(module->notify_mem); + } + free(module->notify_base); + free(module->notify_addrs); + + module->notify_mem = new_mem; + module->notify_base = new_base; + module->notify_addrs = new_addrs; + module->notify_capacity = new_capacity; + + return OMPI_SUCCESS; +} + int ompi_osc_ucx_win_attach(struct ompi_win_t *win, void *base, size_t len) { ompi_osc_ucx_module_t *module = (ompi_osc_ucx_module_t*) win->w_osc_module; int insert_index = -1, contain_index; @@ -1245,11 +1496,17 @@ int ompi_osc_ucx_free(struct ompi_win_t *win) { free(module->addrs); free(module->state_addrs); free(module->comm_world_ranks); + free(module->notify_counts); + free(module->notify_addrs); opal_common_ucx_wpmem_free(module->state_mem); if (NULL != module->mem) { opal_common_ucx_wpmem_free(module->mem); } + if (NULL != module->notify_mem) { + opal_common_ucx_wpmem_free(module->notify_mem); + } + free(module->notify_base); opal_common_ucx_wpctx_release(module->ctx); diff --git a/ompi/mpi/bindings/ompi_bindings/consts.py b/ompi/mpi/bindings/ompi_bindings/consts.py index 7a523d7670d..4a249629079 100644 --- a/ompi/mpi/bindings/ompi_bindings/consts.py +++ b/ompi/mpi/bindings/ompi_bindings/consts.py @@ -100,6 +100,7 @@ 'MPI_T_ERR_PVAR_NO_ATOMIC', 'MPI_T_ERR_NOT_ACCESSIBLE', 'MPI_T_ERR_NOT_SUPPORTED', + 'MPI_ERR_RMA_NOTIFICATION', 'MPI_ERR_LASTCODE', ] diff --git a/ompi/mpi/c/Makefile.am b/ompi/mpi/c/Makefile.am index f532121bf28..e4b6367045f 100644 --- a/ompi/mpi/c/Makefile.am +++ b/ompi/mpi/c/Makefile.am @@ -56,6 +56,7 @@ headers = bindings.h prototype_sources = \ abort.c.in \ accumulate.c.in \ + accumulate_notify.c.in \ add_error_class.c.in \ add_error_code.c.in \ add_error_string.c.in \ @@ -221,8 +222,10 @@ prototype_sources = \ gatherv.c.in \ gatherv_init.c.in \ get_accumulate.c.in \ + get_accumulate_notify.c.in \ get_address.c.in \ get.c.in \ + get_notify.c.in \ get_count.c.in \ get_elements.c.in \ get_elements_x.c.in \ @@ -341,8 +344,10 @@ prototype_sources = \ psend_init.c.in \ publish_name.c.in \ put.c.in \ + put_notify.c.in \ query_thread.c.in \ raccumulate.c.in \ + raccumulate_notify.c.in \ recv.c.in \ recv_init.c.in \ reduce.c.in \ @@ -364,8 +369,11 @@ prototype_sources = \ request_get_status_any.c.in \ request_get_status_some.c.in \ rget_accumulate.c.in \ + rget_accumulate_notify.c.in \ rget.c.in \ + rget_notify.c.in \ rput.c.in \ + rput_notify.c.in \ rsend.c.in \ rsend_init.c.in \ scan.c.in \ @@ -484,6 +492,10 @@ prototype_sources = \ win_get_group.c.in \ win_get_info.c.in \ win_get_name.c.in \ + win_get_notify_value.c.in \ + win_reset_notify_value.c.in \ + win_get_num_notify.c.in \ + win_set_num_notify.c.in \ win_lock_all.c.in \ win_lock.c.in \ win_post.c.in \ @@ -526,6 +538,7 @@ endif interface_profile_sources = \ abort_generated.c \ accumulate_generated.c \ + accumulate_notify_generated.c \ add_error_class_generated.c \ add_error_code_generated.c \ add_error_string_generated.c \ @@ -691,6 +704,7 @@ interface_profile_sources = \ gatherv_generated.c \ gatherv_init_generated.c \ get_accumulate_generated.c \ + get_accumulate_notify_generated.c \ get_address_generated.c \ get_generated.c \ get_count_generated.c \ @@ -698,6 +712,7 @@ interface_profile_sources = \ get_elements_x_generated.c \ get_hw_resource_info_generated.c \ get_library_version_generated.c \ + get_notify_generated.c \ get_processor_name_generated.c \ get_version_generated.c \ graph_create_generated.c \ @@ -811,8 +826,10 @@ interface_profile_sources = \ psend_init_generated.c \ publish_name_generated.c \ put_generated.c \ + put_notify_generated.c \ query_thread_generated.c \ raccumulate_generated.c \ + raccumulate_notify_generated.c \ recv_generated.c \ recv_init_generated.c \ reduce_generated.c \ @@ -834,8 +851,11 @@ interface_profile_sources = \ request_get_status_any_generated.c \ request_get_status_some_generated.c \ rget_accumulate_generated.c \ + rget_accumulate_notify_generated.c \ rget_generated.c \ + rget_notify_generated.c \ rput_generated.c \ + rput_notify_generated.c \ rsend_generated.c \ rsend_init_generated.c \ scan_generated.c \ @@ -954,6 +974,10 @@ interface_profile_sources = \ win_get_group_generated.c \ win_get_info_generated.c \ win_get_name_generated.c \ + win_get_notify_value_generated.c \ + win_reset_notify_value_generated.c \ + win_get_num_notify_generated.c \ + win_set_num_notify_generated.c \ win_lock_all_generated.c \ win_lock_generated.c \ win_post_generated.c \ diff --git a/ompi/mpi/c/accumulate_notify.c.in b/ompi/mpi/c/accumulate_notify.c.in new file mode 100644 index 00000000000..395dd3cad24 --- /dev/null +++ b/ompi/mpi/c/accumulate_notify.c.in @@ -0,0 +1,141 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2020 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2009 Sun Microsystmes, Inc. All rights reserved. + * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2024 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" +#include +#include "ompi/mpi/c/bindings.h" +#include "ompi/runtime/params.h" +#include "ompi/communicator/communicator.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/win/win.h" +#include "ompi/mca/osc/osc.h" +#include "ompi/op/op.h" +#include "ompi/datatype/ompi_datatype.h" +#include "ompi/datatype/ompi_datatype_internal.h" +#include "ompi/memchecker.h" + +PROTOTYPE ERROR_CLASS accumulate_notify(BUFFER origin_addr, COUNT origin_count, DATATYPE origin_datatype, + INT target_rank, AINT target_disp, COUNT target_count, + DATATYPE target_datatype, OP op, INT notification_idx, + WIN win) +{ + int rc; + ompi_win_t *ompi_win = (ompi_win_t*) win; + + MEMCHECKER( + memchecker_datatype(origin_datatype); + memchecker_datatype(target_datatype); + memchecker_call(&opal_memchecker_base_isdefined, (void *) origin_addr, origin_count, origin_datatype); + ); + + if (MPI_PARAM_CHECK) { + rc = OMPI_SUCCESS; + + OMPI_ERR_INIT_FINALIZE(FUNC_NAME); + + if (ompi_win_invalid(win)) { + return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); + } else if (origin_count < 0 || target_count < 0) { + rc = MPI_ERR_COUNT; + } else if (ompi_win_peer_invalid(win, target_rank) && + (MPI_PROC_NULL != target_rank)) { + rc = MPI_ERR_RANK; + } else if (MPI_OP_NULL == op || MPI_NO_OP == op) { + rc = MPI_ERR_OP; + } else if (!ompi_op_is_intrinsic(op)) { + rc = MPI_ERR_OP; + } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) { + rc = MPI_ERR_DISP; + } else if (notification_idx < 0) { + /* The upper bound depends on how many counters the *target* has + attached, which only the osc module can see, so the range check + proper happens there. */ + rc = MPI_ERR_RMA_NOTIFICATION; + } else { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count); + if (OMPI_SUCCESS == rc) { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count); + } + if (OMPI_SUCCESS == rc) { + /* While technically the standard probably requires that the + datatypes used with MPI_REPLACE conform to all the rules + for other reduction operators, we don't require such + behavior, as checking for it is expensive here and we don't + care in implementation.. */ + if (op != &ompi_mpi_op_replace.op && op != &ompi_mpi_op_no_op.op) { + ompi_datatype_t *op_check_dt, *origin_check_dt; + char *msg; + + /* ACCUMULATE, unlike REDUCE, can use with derived + datatypes with predefinied operations, with some + restrictions outlined in MPI-3:11.3.4. The derived + datatype must be composed entirely from one predefined + datatype (so you can do all the construction you want, + but at the bottom, you can only use one datatype, say, + MPI_INT). If the datatype at the target isn't + predefined, then make sure it's composed of only one + datatype, and check that datatype against + ompi_op_is_valid(). */ + origin_check_dt = ompi_datatype_get_single_predefined_type_from_args(origin_datatype); + op_check_dt = ompi_datatype_get_single_predefined_type_from_args(target_datatype); + + if( !((origin_check_dt == op_check_dt) & (NULL != op_check_dt)) ) { + OMPI_ERRHANDLER_RETURN(MPI_ERR_ARG, win, MPI_ERR_ARG, FUNC_NAME); + } + + /* check to make sure primitive type is valid for + reduction. Should do this on the target, but + then can't get the errcode back for this + call */ + if (!ompi_op_is_valid(op, op_check_dt, &msg, FUNC_NAME)) { + int ret = OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_OP, msg); + free(msg); + return ret; + } + } + } + } + OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); + } + + if (NULL == win->w_osc_module->osc_accumulate_notify) { + return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_UNSUPPORTED_OPERATION, FUNC_NAME); + } + + if (MPI_PROC_NULL == target_rank) { + return MPI_SUCCESS; + } + + rc = ompi_win->w_osc_module->osc_accumulate_notify(origin_addr, + origin_count, + origin_datatype, + target_rank, + target_disp, + target_count, + target_datatype, + op, notification_idx, win); + OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); +} diff --git a/ompi/mpi/c/get_accumulate_notify.c.in b/ompi/mpi/c/get_accumulate_notify.c.in new file mode 100644 index 00000000000..2c16e79299c --- /dev/null +++ b/ompi/mpi/c/get_accumulate_notify.c.in @@ -0,0 +1,150 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2020 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2009 Sun Microsystmes, Inc. All rights reserved. + * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2024 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" +#include +#include "ompi/mpi/c/bindings.h" +#include "ompi/runtime/params.h" +#include "ompi/communicator/communicator.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/win/win.h" +#include "ompi/mca/osc/osc.h" +#include "ompi/op/op.h" +#include "ompi/datatype/ompi_datatype.h" +#include "ompi/datatype/ompi_datatype_internal.h" +#include "ompi/memchecker.h" + +PROTOTYPE ERROR_CLASS get_accumulate_notify(BUFFER origin_addr, COUNT origin_count, DATATYPE origin_datatype, + BUFFER_OUT result_addr, COUNT result_count, DATATYPE result_datatype, + INT target_rank, AINT target_disp, COUNT target_count, + DATATYPE target_datatype, OP op, INT notification_idx, + WIN win) +{ + int rc; + ompi_win_t *ompi_win = (ompi_win_t*) win; + + MEMCHECKER( + memchecker_datatype(origin_datatype); + memchecker_datatype(target_datatype); + memchecker_call(&opal_memchecker_base_isdefined, (void *) origin_addr, origin_count, origin_datatype); + ); + + if (MPI_PARAM_CHECK) { + rc = OMPI_SUCCESS; + + OMPI_ERR_INIT_FINALIZE(FUNC_NAME); + + if (ompi_win_invalid(win)) { + return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); + } else if (origin_count < 0 || target_count < 0) { + rc = MPI_ERR_COUNT; + } else if (ompi_win_peer_invalid(win, target_rank) && + (MPI_PROC_NULL != target_rank)) { + rc = MPI_ERR_RANK; + } else if (MPI_OP_NULL == op) { + rc = MPI_ERR_OP; + } else if (!ompi_op_is_intrinsic(op)) { + rc = MPI_ERR_OP; + } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) { + rc = MPI_ERR_DISP; + } else if (notification_idx < 0) { + /* The upper bound depends on how many counters the *target* has + attached, which only the osc module can see, so the range check + proper happens there. */ + rc = MPI_ERR_RMA_NOTIFICATION; + } else { + /* the origin datatype is meaningless when using MPI_OP_NO_OP */ + if (&ompi_mpi_op_no_op.op != op) { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count); + } else { + rc = OMPI_SUCCESS; + } + if (OMPI_SUCCESS == rc) { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count); + } + if (OMPI_SUCCESS == rc) { + /* While technically the standard probably requires that the + datatypes used with MPI_REPLACE conform to all the rules + for other reduction operators, we don't require such + behavior, as checking for it is expensive here and we don't + care in implementation.. */ + if (op != &ompi_mpi_op_replace.op && op != &ompi_mpi_op_no_op.op) { + ompi_datatype_t *op_check_dt, *origin_check_dt; + char *msg; + + /* GET_ACCUMULATE, unlike REDUCE, can use with derived + datatypes with predefinied operations, with some + restrictions outlined in MPI-3:11.3.4. The derived + datatype must be composed entirely from one predefined + datatype (so you can do all the construction you want, + but at the bottom, you can only use one datatype, say, + MPI_INT). If the datatype at the target isn't + predefined, then make sure it's composed of only one + datatype, and check that datatype against + ompi_op_is_valid(). */ + origin_check_dt = ompi_datatype_get_single_predefined_type_from_args(origin_datatype); + op_check_dt = ompi_datatype_get_single_predefined_type_from_args(target_datatype); + + if( !((origin_check_dt == op_check_dt) & (NULL != op_check_dt)) ) { + OMPI_ERRHANDLER_RETURN(MPI_ERR_ARG, win, MPI_ERR_ARG, FUNC_NAME); + } + + /* check to make sure primitive type is valid for + reduction. Should do this on the target, but + then can't get the errcode back for this + call */ + if (!ompi_op_is_valid(op, op_check_dt, &msg, FUNC_NAME)) { + int ret = OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_OP, msg); + free(msg); + return ret; + } + } + } + } + OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); + } + + if (NULL == win->w_osc_module->osc_get_accumulate_notify) { + return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_UNSUPPORTED_OPERATION, FUNC_NAME); + } + + if (MPI_PROC_NULL == target_rank) { + return MPI_SUCCESS; + } + + rc = ompi_win->w_osc_module->osc_get_accumulate_notify(origin_addr, + origin_count, + origin_datatype, + result_addr, + result_count, + result_datatype, + target_rank, + target_disp, + target_count, + target_datatype, + op, notification_idx, win); + OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); +} diff --git a/ompi/mpi/c/get_notify.c.in b/ompi/mpi/c/get_notify.c.in new file mode 100644 index 00000000000..6191eceaac2 --- /dev/null +++ b/ompi/mpi/c/get_notify.c.in @@ -0,0 +1,81 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2020 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2015 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2024 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "ompi_config.h" +#include + +#include "ompi/mpi/c/bindings.h" +#include "ompi/runtime/params.h" +#include "ompi/communicator/communicator.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/win/win.h" +#include "ompi/mca/osc/osc.h" +#include "ompi/datatype/ompi_datatype.h" +#include "ompi/runtime/ompi_spc.h" + +PROTOTYPE ERROR_CLASS get_notify(BUFFER_OUT origin_addr, COUNT origin_count, + DATATYPE origin_datatype, INT target_rank, + AINT target_disp, COUNT target_count, + DATATYPE target_datatype, INT notification_idx, WIN win) +{ + int rc; + + SPC_RECORD(OMPI_SPC_GET_NOTIFY, 1); + + if (MPI_PARAM_CHECK) { + rc = OMPI_SUCCESS; + + OMPI_ERR_INIT_FINALIZE(FUNC_NAME); + + if (ompi_win_invalid(win)) { + return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); + } else if (origin_count < 0 || target_count < 0) { + rc = MPI_ERR_COUNT; + } else if (ompi_win_peer_invalid(win, target_rank) && + (MPI_PROC_NULL != target_rank)) { + rc = MPI_ERR_RANK; + } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) { + rc = MPI_ERR_DISP; + } else if (notification_idx < 0) { + rc = MPI_ERR_RMA_NOTIFICATION; + } else { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count); + if (OMPI_SUCCESS == rc) { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count); + } + } + OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); + } + + if (NULL == win->w_osc_module->osc_get_notify) { + return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_UNSUPPORTED_OPERATION, FUNC_NAME); + } + + if (MPI_PROC_NULL == target_rank) return MPI_SUCCESS; + + rc = win->w_osc_module->osc_get_notify(origin_addr, origin_count, origin_datatype, + target_rank, target_disp, target_count, + target_datatype, notification_idx, win); + OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); +} diff --git a/ompi/mpi/c/put_notify.c.in b/ompi/mpi/c/put_notify.c.in new file mode 100644 index 00000000000..0141725c322 --- /dev/null +++ b/ompi/mpi/c/put_notify.c.in @@ -0,0 +1,84 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2020 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2024 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "ompi_config.h" +#include + +#include "ompi/mpi/c/bindings.h" +#include "ompi/runtime/params.h" +#include "ompi/communicator/communicator.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/win/win.h" +#include "ompi/mca/osc/osc.h" +#include "ompi/datatype/ompi_datatype.h" +#include "ompi/runtime/ompi_spc.h" + +PROTOTYPE ERROR_CLASS put_notify(BUFFER origin_addr, COUNT origin_count, DATATYPE origin_datatype, + INT target_rank, AINT target_disp, COUNT target_count, + DATATYPE target_datatype, INT notification_idx, WIN win) +{ + int rc; + + SPC_RECORD(OMPI_SPC_PUT_NOTIFY, 1); + + if (MPI_PARAM_CHECK) { + rc = OMPI_SUCCESS; + + OMPI_ERR_INIT_FINALIZE(FUNC_NAME); + + if (ompi_win_invalid(win)) { + return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); + } else if (origin_count < 0 || target_count < 0) { + rc = MPI_ERR_COUNT; + } else if (ompi_win_peer_invalid(win, target_rank) && + (MPI_PROC_NULL != target_rank)) { + rc = MPI_ERR_RANK; + } else if (NULL == target_datatype || + MPI_DATATYPE_NULL == target_datatype) { + rc = MPI_ERR_TYPE; + } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) { + rc = MPI_ERR_DISP; + } else if (notification_idx < 0) { + rc = MPI_ERR_RMA_NOTIFICATION; + } else { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count); + if (OMPI_SUCCESS == rc) { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count); + } + } + OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); + } + + if (NULL == win->w_osc_module->osc_put_notify) { + return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_UNSUPPORTED_OPERATION, FUNC_NAME); + } + + if (MPI_PROC_NULL == target_rank) return MPI_SUCCESS; + + rc = win->w_osc_module->osc_put_notify(origin_addr, origin_count, origin_datatype, + target_rank, target_disp, target_count, + target_datatype, notification_idx, win); + OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); +} diff --git a/ompi/mpi/c/raccumulate_notify.c.in b/ompi/mpi/c/raccumulate_notify.c.in new file mode 100644 index 00000000000..c9df87b74bc --- /dev/null +++ b/ompi/mpi/c/raccumulate_notify.c.in @@ -0,0 +1,143 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2020 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2009 Sun Microsystmes, Inc. All rights reserved. + * Copyright (c) 2011 Sandia National Laboratories. All rights reserved. + * Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2024 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "ompi_config.h" +#include +#include "ompi/mpi/c/bindings.h" +#include "ompi/runtime/params.h" +#include "ompi/communicator/communicator.h" +#include "ompi/request/request.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/win/win.h" +#include "ompi/mca/osc/osc.h" +#include "ompi/op/op.h" +#include "ompi/datatype/ompi_datatype.h" +#include "ompi/datatype/ompi_datatype_internal.h" +#include "ompi/memchecker.h" + +PROTOTYPE ERROR_CLASS raccumulate_notify(BUFFER origin_addr, COUNT origin_count, DATATYPE origin_datatype, + INT target_rank, AINT target_disp, COUNT target_count, + DATATYPE target_datatype, OP op, INT notification_idx, + WIN win, REQUEST_INOUT request) +{ + int rc; + ompi_win_t *ompi_win = (ompi_win_t*) win; + + MEMCHECKER( + memchecker_datatype(origin_datatype); + memchecker_datatype(target_datatype); + memchecker_call(&opal_memchecker_base_isdefined, (void *) origin_addr, origin_count, origin_datatype); + ); + + if (MPI_PARAM_CHECK) { + rc = OMPI_SUCCESS; + + OMPI_ERR_INIT_FINALIZE(FUNC_NAME); + + if (ompi_win_invalid(win)) { + return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); + } else if (origin_count < 0 || target_count < 0) { + rc = MPI_ERR_COUNT; + } else if (ompi_win_peer_invalid(win, target_rank) && + (MPI_PROC_NULL != target_rank)) { + rc = MPI_ERR_RANK; + } else if (MPI_OP_NULL == op || MPI_NO_OP == op) { + rc = MPI_ERR_OP; + } else if (!ompi_op_is_intrinsic(op)) { + rc = MPI_ERR_OP; + } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) { + rc = MPI_ERR_DISP; + } else if (notification_idx < 0) { + /* The upper bound depends on how many counters the *target* has + attached, which only the osc module can see, so the range check + proper happens there. */ + rc = MPI_ERR_RMA_NOTIFICATION; + } else { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count); + if (OMPI_SUCCESS == rc) { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count); + } + if (OMPI_SUCCESS == rc) { + /* While technically the standard probably requires that the + datatypes used with MPI_REPLACE conform to all the rules + for other reduction operators, we don't require such + behavior, as checking for it is expensive here and we don't + care in implementation.. */ + if (op != &ompi_mpi_op_replace.op && op != &ompi_mpi_op_no_op.op) { + ompi_datatype_t *op_check_dt, *origin_check_dt; + char *msg; + + /* ACCUMULATE, unlike REDUCE, can use with derived + datatypes with predefinied operations, with some + restrictions outlined in MPI-3:11.3.4. The derived + datatype must be composed entirely from one predefined + datatype (so you can do all the construction you want, + but at the bottom, you can only use one datatype, say, + MPI_INT). If the datatype at the target isn't + predefined, then make sure it's composed of only one + datatype, and check that datatype against + ompi_op_is_valid(). */ + origin_check_dt = ompi_datatype_get_single_predefined_type_from_args(origin_datatype); + op_check_dt = ompi_datatype_get_single_predefined_type_from_args(target_datatype); + + if( !((origin_check_dt == op_check_dt) & (NULL != op_check_dt)) ) { + OMPI_ERRHANDLER_RETURN(MPI_ERR_ARG, win, MPI_ERR_ARG, FUNC_NAME); + } + + /* check to make sure primitive type is valid for + reduction. Should do this on the target, but + then can't get the errcode back for this + call */ + if (!ompi_op_is_valid(op, op_check_dt, &msg, FUNC_NAME)) { + int ret = OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_OP, msg); + free(msg); + return ret; + } + } + } + } + OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); + } + + if (NULL == win->w_osc_module->osc_raccumulate_notify) { + return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_UNSUPPORTED_OPERATION, FUNC_NAME); + } + + if (MPI_PROC_NULL == target_rank) { + *request = &ompi_request_empty; + return MPI_SUCCESS; + } + + rc = ompi_win->w_osc_module->osc_raccumulate_notify(origin_addr, + origin_count, + origin_datatype, + target_rank, + target_disp, + target_count, + target_datatype, + op, notification_idx, win, request); + OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); +} diff --git a/ompi/mpi/c/rget_accumulate_notify.c.in b/ompi/mpi/c/rget_accumulate_notify.c.in new file mode 100644 index 00000000000..f3466f5fb0b --- /dev/null +++ b/ompi/mpi/c/rget_accumulate_notify.c.in @@ -0,0 +1,152 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2020 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2009 Sun Microsystmes, Inc. All rights reserved. + * Copyright (c) 2011 Sandia National Laboratories. All rights reserved. + * Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2024 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "ompi_config.h" +#include +#include "ompi/mpi/c/bindings.h" +#include "ompi/runtime/params.h" +#include "ompi/communicator/communicator.h" +#include "ompi/request/request.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/win/win.h" +#include "ompi/mca/osc/osc.h" +#include "ompi/op/op.h" +#include "ompi/datatype/ompi_datatype.h" +#include "ompi/datatype/ompi_datatype_internal.h" +#include "ompi/memchecker.h" + +PROTOTYPE ERROR_CLASS rget_accumulate_notify(BUFFER origin_addr, COUNT origin_count, DATATYPE origin_datatype, + BUFFER_OUT result_addr, COUNT result_count, DATATYPE result_datatype, + INT target_rank, AINT target_disp, COUNT target_count, + DATATYPE target_datatype, OP op, INT notification_idx, + WIN win, REQUEST_INOUT request) +{ + int rc; + ompi_win_t *ompi_win = (ompi_win_t*) win; + + MEMCHECKER( + memchecker_datatype(origin_datatype); + memchecker_datatype(target_datatype); + memchecker_call(&opal_memchecker_base_isdefined, (void *) origin_addr, origin_count, origin_datatype); + ); + + if (MPI_PARAM_CHECK) { + rc = OMPI_SUCCESS; + + OMPI_ERR_INIT_FINALIZE(FUNC_NAME); + + if (ompi_win_invalid(win)) { + return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); + } else if (origin_count < 0 || target_count < 0) { + rc = MPI_ERR_COUNT; + } else if (ompi_win_peer_invalid(win, target_rank) && + (MPI_PROC_NULL != target_rank)) { + rc = MPI_ERR_RANK; + } else if (MPI_OP_NULL == op) { + rc = MPI_ERR_OP; + } else if (!ompi_op_is_intrinsic(op)) { + rc = MPI_ERR_OP; + } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) { + rc = MPI_ERR_DISP; + } else if (notification_idx < 0) { + /* The upper bound depends on how many counters the *target* has + attached, which only the osc module can see, so the range check + proper happens there. */ + rc = MPI_ERR_RMA_NOTIFICATION; + } else { + /* the origin datatype is meaningless when using MPI_OP_NO_OP */ + if (&ompi_mpi_op_no_op.op != op) { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count); + } else { + rc = OMPI_SUCCESS; + } + if (OMPI_SUCCESS == rc) { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count); + } + if (OMPI_SUCCESS == rc) { + /* While technically the standard probably requires that the + datatypes used with MPI_REPLACE conform to all the rules + for other reduction operators, we don't require such + behavior, as checking for it is expensive here and we don't + care in implementation.. */ + if (op != &ompi_mpi_op_replace.op && op != &ompi_mpi_op_no_op.op) { + ompi_datatype_t *op_check_dt, *origin_check_dt; + char *msg; + + /* GET_ACCUMULATE, unlike REDUCE, can use with derived + datatypes with predefinied operations, with some + restrictions outlined in MPI-3:11.3.4. The derived + datatype must be composed entirely from one predefined + datatype (so you can do all the construction you want, + but at the bottom, you can only use one datatype, say, + MPI_INT). If the datatype at the target isn't + predefined, then make sure it's composed of only one + datatype, and check that datatype against + ompi_op_is_valid(). */ + origin_check_dt = ompi_datatype_get_single_predefined_type_from_args(origin_datatype); + op_check_dt = ompi_datatype_get_single_predefined_type_from_args(target_datatype); + + if( !((origin_check_dt == op_check_dt) & (NULL != op_check_dt)) ) { + OMPI_ERRHANDLER_RETURN(MPI_ERR_ARG, win, MPI_ERR_ARG, FUNC_NAME); + } + + /* check to make sure primitive type is valid for + reduction. Should do this on the target, but + then can't get the errcode back for this + call */ + if (!ompi_op_is_valid(op, op_check_dt, &msg, FUNC_NAME)) { + int ret = OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_OP, msg); + free(msg); + return ret; + } + } + } + } + OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); + } + + if (NULL == win->w_osc_module->osc_rget_accumulate_notify) { + return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_UNSUPPORTED_OPERATION, FUNC_NAME); + } + + if (MPI_PROC_NULL == target_rank) { + *request = &ompi_request_empty; + return MPI_SUCCESS; + } + + rc = ompi_win->w_osc_module->osc_rget_accumulate_notify(origin_addr, + origin_count, + origin_datatype, + result_addr, + result_count, + result_datatype, + target_rank, + target_disp, + target_count, + target_datatype, + op, notification_idx, win, request); + OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); +} diff --git a/ompi/mpi/c/rget_notify.c.in b/ompi/mpi/c/rget_notify.c.in new file mode 100644 index 00000000000..1241a84128a --- /dev/null +++ b/ompi/mpi/c/rget_notify.c.in @@ -0,0 +1,93 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2020 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2024 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "ompi_config.h" +#include + +#include "ompi/mpi/c/bindings.h" +#include "ompi/runtime/params.h" +#include "ompi/communicator/communicator.h" +#include "ompi/request/request.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/win/win.h" +#include "ompi/mca/osc/osc.h" +#include "ompi/datatype/ompi_datatype.h" +#include "ompi/runtime/ompi_spc.h" + +PROTOTYPE ERROR_CLASS rget_notify(BUFFER_OUT origin_addr, COUNT origin_count, DATATYPE origin_datatype, + INT target_rank, AINT target_disp, COUNT target_count, + DATATYPE target_datatype, INT notification_idx, WIN win, + REQUEST_INOUT request) +{ + int rc; + + SPC_RECORD(OMPI_SPC_RGET_NOTIFY, 1); + + if (MPI_PARAM_CHECK) { + rc = OMPI_SUCCESS; + + OMPI_ERR_INIT_FINALIZE(FUNC_NAME); + + if (ompi_win_invalid(win)) { + return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); + } else if (origin_count < 0 || target_count < 0) { + rc = MPI_ERR_COUNT; + } else if (ompi_win_peer_invalid(win, target_rank) && + (MPI_PROC_NULL != target_rank)) { + rc = MPI_ERR_RANK; + } else if (NULL == target_datatype || + MPI_DATATYPE_NULL == target_datatype) { + rc = MPI_ERR_TYPE; + } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) { + rc = MPI_ERR_DISP; + } else if (notification_idx < 0) { + /* The upper bound depends on how many counters the *target* has + attached, which only the osc module can see, so the range check + proper happens there. */ + rc = MPI_ERR_RMA_NOTIFICATION; + } else { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count); + if (OMPI_SUCCESS == rc) { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count); + } + } + OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); + } + + if (NULL == win->w_osc_module->osc_rget_notify) { + return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_UNSUPPORTED_OPERATION, FUNC_NAME); + } + + if (MPI_PROC_NULL == target_rank) { + *request = &ompi_request_empty; + return MPI_SUCCESS; + } + + rc = win->w_osc_module->osc_rget_notify(origin_addr, origin_count, origin_datatype, + target_rank, target_disp, target_count, + target_datatype, notification_idx, win, + request); + OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); +} diff --git a/ompi/mpi/c/rput_notify.c.in b/ompi/mpi/c/rput_notify.c.in new file mode 100644 index 00000000000..621bfadb10b --- /dev/null +++ b/ompi/mpi/c/rput_notify.c.in @@ -0,0 +1,93 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2020 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2024 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "ompi_config.h" +#include + +#include "ompi/mpi/c/bindings.h" +#include "ompi/runtime/params.h" +#include "ompi/communicator/communicator.h" +#include "ompi/request/request.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/win/win.h" +#include "ompi/mca/osc/osc.h" +#include "ompi/datatype/ompi_datatype.h" +#include "ompi/runtime/ompi_spc.h" + +PROTOTYPE ERROR_CLASS rput_notify(BUFFER origin_addr, COUNT origin_count, DATATYPE origin_datatype, + INT target_rank, AINT target_disp, COUNT target_count, + DATATYPE target_datatype, INT notification_idx, WIN win, + REQUEST_INOUT request) +{ + int rc; + + SPC_RECORD(OMPI_SPC_RPUT_NOTIFY, 1); + + if (MPI_PARAM_CHECK) { + rc = OMPI_SUCCESS; + + OMPI_ERR_INIT_FINALIZE(FUNC_NAME); + + if (ompi_win_invalid(win)) { + return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); + } else if (origin_count < 0 || target_count < 0) { + rc = MPI_ERR_COUNT; + } else if (ompi_win_peer_invalid(win, target_rank) && + (MPI_PROC_NULL != target_rank)) { + rc = MPI_ERR_RANK; + } else if (NULL == target_datatype || + MPI_DATATYPE_NULL == target_datatype) { + rc = MPI_ERR_TYPE; + } else if ( MPI_WIN_FLAVOR_DYNAMIC != win->w_flavor && target_disp < 0 ) { + rc = MPI_ERR_DISP; + } else if (notification_idx < 0) { + /* The upper bound depends on how many counters the *target* has + attached, which only the osc module can see, so the range check + proper happens there. */ + rc = MPI_ERR_RMA_NOTIFICATION; + } else { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, origin_datatype, origin_count); + if (OMPI_SUCCESS == rc) { + OMPI_CHECK_DATATYPE_FOR_ONE_SIDED(rc, target_datatype, target_count); + } + } + OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); + } + + if (NULL == win->w_osc_module->osc_rput_notify) { + return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_UNSUPPORTED_OPERATION, FUNC_NAME); + } + + if (MPI_PROC_NULL == target_rank) { + *request = &ompi_request_empty; + return MPI_SUCCESS; + } + + rc = win->w_osc_module->osc_rput_notify(origin_addr, origin_count, origin_datatype, + target_rank, target_disp, target_count, + target_datatype, notification_idx, win, + request); + OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); +} diff --git a/ompi/mpi/c/win_get_notify_value.c.in b/ompi/mpi/c/win_get_notify_value.c.in new file mode 100644 index 00000000000..4dfcafbab05 --- /dev/null +++ b/ompi/mpi/c/win_get_notify_value.c.in @@ -0,0 +1,45 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2026 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "ompi_config.h" + +#include "ompi/mpi/c/bindings.h" +#include "ompi/runtime/params.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/win/win.h" +#include "ompi/mca/osc/osc.h" + +PROTOTYPE ERROR_CLASS win_get_notify_value(WIN win, INT notification_idx, ELEMENT_COUNT value) +{ + int rc; + + if (MPI_PARAM_CHECK) { + rc = OMPI_SUCCESS; + + OMPI_ERR_INIT_FINALIZE(FUNC_NAME); + + if (ompi_win_invalid(win)) { + return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); + } else if (notification_idx < 0) { + rc = MPI_ERR_RMA_NOTIFICATION; + } else if (NULL == value) { + rc = MPI_ERR_ARG; + } + + OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); + } + + if (NULL == win->w_osc_module->osc_win_get_notify_value) { + return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_UNSUPPORTED_OPERATION, FUNC_NAME); + } + + rc = win->w_osc_module->osc_win_get_notify_value(win, notification_idx, value); + OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); +} diff --git a/ompi/mpi/c/win_get_num_notify.c.in b/ompi/mpi/c/win_get_num_notify.c.in new file mode 100644 index 00000000000..790a740c65a --- /dev/null +++ b/ompi/mpi/c/win_get_num_notify.c.in @@ -0,0 +1,48 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2026 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "ompi_config.h" + +#include "ompi/mpi/c/bindings.h" +#include "ompi/runtime/params.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/win/win.h" +#include "ompi/mca/osc/osc.h" + +PROTOTYPE ERROR_CLASS win_get_num_notify(WIN win, INT target_rank, INT_OUT num_notifications) +{ + int rc; + + if (MPI_PARAM_CHECK) { + rc = OMPI_SUCCESS; + + OMPI_ERR_INIT_FINALIZE(FUNC_NAME); + + if (ompi_win_invalid(win)) { + return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); + } else if (ompi_win_peer_invalid(win, target_rank)) { + /* target_rank is specified as a nonnegative integer in the group of + * the window; unlike the notified communication operations, + * MPI_PROC_NULL is not permitted here. */ + rc = MPI_ERR_RANK; + } else if (NULL == num_notifications) { + rc = MPI_ERR_ARG; + } + + OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); + } + + if (NULL == win->w_osc_module->osc_win_get_num_notify) { + return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_UNSUPPORTED_OPERATION, FUNC_NAME); + } + + rc = win->w_osc_module->osc_win_get_num_notify(win, target_rank, num_notifications); + OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); +} diff --git a/ompi/mpi/c/win_reset_notify_value.c.in b/ompi/mpi/c/win_reset_notify_value.c.in new file mode 100644 index 00000000000..09a132555f5 --- /dev/null +++ b/ompi/mpi/c/win_reset_notify_value.c.in @@ -0,0 +1,45 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2026 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "ompi_config.h" + +#include "ompi/mpi/c/bindings.h" +#include "ompi/runtime/params.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/win/win.h" +#include "ompi/mca/osc/osc.h" + +PROTOTYPE ERROR_CLASS win_reset_notify_value(WIN win, INT notification_idx, ELEMENT_COUNT value) +{ + int rc; + + if (MPI_PARAM_CHECK) { + rc = OMPI_SUCCESS; + + OMPI_ERR_INIT_FINALIZE(FUNC_NAME); + + if (ompi_win_invalid(win)) { + return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); + } else if (notification_idx < 0) { + rc = MPI_ERR_RMA_NOTIFICATION; + } else if (NULL == value) { + rc = MPI_ERR_ARG; + } + + OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); + } + + if (NULL == win->w_osc_module->osc_win_reset_notify_value) { + return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_UNSUPPORTED_OPERATION, FUNC_NAME); + } + + rc = win->w_osc_module->osc_win_reset_notify_value(win, notification_idx, value); + OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); +} diff --git a/ompi/mpi/c/win_set_num_notify.c.in b/ompi/mpi/c/win_set_num_notify.c.in new file mode 100644 index 00000000000..f85bf1c8fd8 --- /dev/null +++ b/ompi/mpi/c/win_set_num_notify.c.in @@ -0,0 +1,55 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2026 Triad National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "ompi_config.h" + +#include "ompi/mpi/c/bindings.h" +#include "ompi/runtime/params.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/win/win.h" +#include "ompi/info/info.h" +#include "ompi/mca/osc/osc.h" + +PROTOTYPE ERROR_CLASS win_set_num_notify(WIN win, INFO info, INT num_notifications) +{ + int rc; + + if (MPI_PARAM_CHECK) { + rc = OMPI_SUCCESS; + + OMPI_ERR_INIT_FINALIZE(FUNC_NAME); + + if (ompi_win_invalid(win)) { + return OMPI_ERRHANDLER_NOHANDLE_INVOKE(MPI_ERR_WIN, FUNC_NAME); + } else if (NULL != info && MPI_INFO_NULL != info && ompi_info_is_freed(info)) { + rc = MPI_ERR_INFO; + } + + /* num_notifications is deliberately *not* range-checked here. + MPI_WIN_SET_NUM_NOTIFY is a synchronizing collective whose count + argument is local -- MPI-5.1 12.6.1 allows it to differ between MPI + processes -- so a rank that rejected its own value here would return + while every other rank stayed blocked in the osc module's internal + collective, turning an erroneous argument into a hang. The module + carries each rank's validity through that collective instead, so all + ranks agree to fail together. */ + + OMPI_ERRHANDLER_CHECK(rc, win, rc, FUNC_NAME); + } + + if (NULL == win->w_osc_module->osc_win_set_num_notify) { + return OMPI_ERRHANDLER_INVOKE(win, MPI_ERR_UNSUPPORTED_OPERATION, FUNC_NAME); + } + + rc = win->w_osc_module->osc_win_set_num_notify(win, + (NULL != info && MPI_INFO_NULL != info) ? &(info->super) : NULL, + num_notifications); + OMPI_ERRHANDLER_RETURN(rc, win, rc, FUNC_NAME); +} diff --git a/ompi/runtime/ompi_spc.c b/ompi/runtime/ompi_spc.c index fb097ac6077..5e6a3dbb339 100644 --- a/ompi/runtime/ompi_spc.c +++ b/ompi/runtime/ompi_spc.c @@ -71,9 +71,13 @@ static const ompi_spc_event_t ompi_spc_events_desc[OMPI_SPC_NUM_COUNTERS] = { SET_COUNTER_ARRAY(OMPI_SPC_SENDRECV, "The number of times MPI_Sendrecv was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_SENDRECV_REPLACE, "The number of times MPI_Sendrecv_replace was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_PUT, "The number of times MPI_Put was called.", false, false), + SET_COUNTER_ARRAY(OMPI_SPC_PUT_NOTIFY, "The number of times MPI_Put_notify was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_RPUT, "The number of times MPI_Rput was called.", false, false), + SET_COUNTER_ARRAY(OMPI_SPC_RPUT_NOTIFY, "The number of times MPI_Rput_notify was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_GET, "The number of times MPI_Get was called.", false, false), + SET_COUNTER_ARRAY(OMPI_SPC_GET_NOTIFY, "The number of times MPI_Get_notify was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_RGET, "The number of times MPI_Rget was called.", false, false), + SET_COUNTER_ARRAY(OMPI_SPC_RGET_NOTIFY, "The number of times MPI_Rget_notify was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_PROBE, "The number of times MPI_Probe was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_IPROBE, "The number of times MPI_Iprobe was called.", false, false), SET_COUNTER_ARRAY(OMPI_SPC_BCAST, "The number of times MPI_Bcast was called.", false, false), diff --git a/ompi/runtime/ompi_spc.h b/ompi/runtime/ompi_spc.h index ca61aa8a409..5aebb282de8 100644 --- a/ompi/runtime/ompi_spc.h +++ b/ompi/runtime/ompi_spc.h @@ -58,9 +58,13 @@ typedef enum ompi_spc_counters { OMPI_SPC_SENDRECV, OMPI_SPC_SENDRECV_REPLACE, OMPI_SPC_PUT, + OMPI_SPC_PUT_NOTIFY, OMPI_SPC_RPUT, + OMPI_SPC_RPUT_NOTIFY, OMPI_SPC_GET, + OMPI_SPC_GET_NOTIFY, OMPI_SPC_RGET, + OMPI_SPC_RGET_NOTIFY, OMPI_SPC_PROBE, OMPI_SPC_IPROBE, OMPI_SPC_BCAST, diff --git a/ompi/test/general/Makefile.am b/ompi/test/general/Makefile.am index d3f06dab347..3ec643ca0a8 100644 --- a/ompi/test/general/Makefile.am +++ b/ompi/test/general/Makefile.am @@ -43,6 +43,8 @@ check_PROGRAMS = \ info_mpi \ request \ win \ + win_notify \ + win_notify_unsupported \ instance \ file \ message @@ -108,6 +110,14 @@ win_SOURCES = win.c win_LDADD = $(ompi_test_ldadd) win_DEPENDENCIES = $(ompi_test_ldadd) +win_notify_SOURCES = win_notify.c +win_notify_LDADD = $(ompi_test_ldadd) +win_notify_DEPENDENCIES = $(ompi_test_ldadd) + +win_notify_unsupported_SOURCES = win_notify_unsupported.c +win_notify_unsupported_LDADD = $(ompi_test_ldadd) +win_notify_unsupported_DEPENDENCIES = $(ompi_test_ldadd) + instance_SOURCES = instance.c instance_LDADD = $(ompi_test_ldadd) instance_DEPENDENCIES = $(ompi_test_ldadd) diff --git a/ompi/test/general/win_notify.c b/ompi/test/general/win_notify.c new file mode 100644 index 00000000000..05c2550024c --- /dev/null +++ b/ompi/test/general/win_notify.c @@ -0,0 +1,594 @@ +/* + * Copyright (c) 2026 Joseph Antony. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +/* + * Unit test for notified RMA communication (MPI-5.1 section 12.6): all + * eight notified communication operations, the notification counter + * management calls, and the counter accessors. + * + * Single process on MPI_COMM_SELF, so the target of every operation is + * this process itself. That is enough to pin down the semantics this test + * cares about -- that each notified operation moves the data its + * non-notified counterpart would, and then increments exactly one + * notification counter by exactly one -- without needing a launcher. + * + * osc/sm is forced because it is currently the only osc component that + * implements the notified operations; on any other component the module's + * notify function pointers are NULL. + * + * Note: the library is compiled with -DNDEBUG, so assert() is a no-op + * here -- all verification must go through test_verify(). + */ + +#include "ompi_config.h" + +#include +#include +#include +#include + +#include "support.h" + +#include "mpi.h" + +#define WIN_COUNT 8 +#define NUM_NOTIFY 4 + +/* Comfortably more than the osc_sm_num_notify_counters default of 16, so that + * asking for this many forces the counters to be reallocated. */ +#define NUM_NOTIFY_GROWN 100 + +static void test_counter_management(void); +static void test_blocking_ops(void); +static void test_request_ops(void); +static void test_notify_idx_errors(void); +static void test_counter_growth(void); +static void test_max_num_notify_assertion(void); +static void test_notify_attributes(void); + +/* Read notification counter "idx" and check it against "expect". */ +static void check_counter(MPI_Win win, int idx, MPI_Count expect, + const char *what) +{ + MPI_Count value = -1; + int rc = MPI_Win_get_notify_value(win, idx, &value); + test_verify(what, MPI_SUCCESS == rc && expect == value); +} + +int main(int argc, char *argv[]) +{ + /* Must be set before MPI_Init: component selection happens there. */ + setenv("OMPI_MCA_osc", "sm", 1); + + test_init("ompi win_notify"); + + int rc = MPI_Init(&argc, &argv); + test_verify("MPI_Init succeeds", MPI_SUCCESS == rc); + + test_counter_management(); + test_blocking_ops(); + test_request_ops(); + test_notify_idx_errors(); + test_counter_growth(); + test_max_num_notify_assertion(); + test_notify_attributes(); + + int r = test_finalize(); + MPI_Finalize(); + return r; +} + +/* ------------------------------------------------------------------ */ + +/* MPI-5.1 section 12.6.1: MPI_WIN_SET_NUM_NOTIFY / MPI_WIN_GET_NUM_NOTIFY, + * and the reset-to-zero behavior of the former. */ +static void test_counter_management(void) +{ + int *base = NULL; + MPI_Win win = MPI_WIN_NULL; + + int rc = MPI_Win_allocate(WIN_COUNT * sizeof(int), sizeof(int), + MPI_INFO_NULL, MPI_COMM_SELF, &base, &win); + test_verify("Win_allocate succeeds", MPI_SUCCESS == rc); + if (MPI_SUCCESS != rc) { + return; + } + + /* MPI-5.1 section 12.6.1 does not state how many counters are attached + * before the first MPI_WIN_SET_NUM_NOTIFY, and osc/sm and osc/ucx + * currently disagree (osc/sm pre-attaches its full reserved capacity, + * osc/ucx starts at zero -- see the note in osc_sm_component.c). So + * only require that the query works and reports something sane; a + * portable program must call MPI_WIN_SET_NUM_NOTIFY first regardless. */ + int num = -1; + rc = MPI_Win_get_num_notify(win, 0, &num); + test_verify("Win_get_num_notify succeeds before set", MPI_SUCCESS == rc); + test_verify("initial attached count is non-negative", num >= 0); + + rc = MPI_Win_set_num_notify(win, MPI_INFO_NULL, NUM_NOTIFY); + test_verify("Win_set_num_notify succeeds", MPI_SUCCESS == rc); + + /* "A subsequent call to MPI_WIN_GET_NUM_NOTIFY will return the value + * given to MPI_WIN_SET_NUM_NOTIFY." */ + num = -1; + rc = MPI_Win_get_num_notify(win, 0, &num); + test_verify("Win_get_num_notify returns what was set", + MPI_SUCCESS == rc && NUM_NOTIFY == num); + + /* All counters start at zero. */ + for (int i = 0; i < NUM_NOTIFY; ++i) { + check_counter(win, i, 0, "counter is zero after set_num_notify"); + } + + /* Bump a counter, then check that set_num_notify resets it: "All + * notification counters (both existing and newly attached) are reset to + * zero by this call." */ + MPI_Win_lock_all(0, win); + int src = 1; + rc = MPI_Put_notify(&src, 1, MPI_INT, 0, 0, 1, MPI_INT, 2, win); + test_verify("Put_notify succeeds", MPI_SUCCESS == rc); + MPI_Win_flush(0, win); + check_counter(win, 2, 1, "counter advanced before reset"); + MPI_Win_unlock_all(win); + + rc = MPI_Win_set_num_notify(win, MPI_INFO_NULL, NUM_NOTIFY); + test_verify("Win_set_num_notify succeeds again", MPI_SUCCESS == rc); + check_counter(win, 2, 0, "set_num_notify resets existing counters"); + + /* MPI_WIN_RESET_NOTIFY_VALUE is an atomic fetch-and-zero. */ + MPI_Win_lock_all(0, win); + rc = MPI_Put_notify(&src, 1, MPI_INT, 0, 0, 1, MPI_INT, 1, win); + test_verify("Put_notify succeeds for reset test", MPI_SUCCESS == rc); + rc = MPI_Put_notify(&src, 1, MPI_INT, 0, 0, 1, MPI_INT, 1, win); + test_verify("second Put_notify succeeds for reset test", MPI_SUCCESS == rc); + MPI_Win_flush(0, win); + MPI_Win_unlock_all(win); + + MPI_Count value = -1; + rc = MPI_Win_reset_notify_value(win, 1, &value); + test_verify("Win_reset_notify_value returns the prior value", + MPI_SUCCESS == rc && 2 == value); + check_counter(win, 1, 0, "Win_reset_notify_value zeroes the counter"); + + MPI_Win_free(&win); +} + +/* The four blocking notified operations. */ +static void test_blocking_ops(void) +{ + int *base = NULL; + MPI_Win win = MPI_WIN_NULL; + + int rc = MPI_Win_allocate(WIN_COUNT * sizeof(int), sizeof(int), + MPI_INFO_NULL, MPI_COMM_SELF, &base, &win); + test_verify("Win_allocate succeeds (blocking ops)", MPI_SUCCESS == rc); + if (MPI_SUCCESS != rc) { + return; + } + rc = MPI_Win_set_num_notify(win, MPI_INFO_NULL, NUM_NOTIFY); + test_verify("Win_set_num_notify succeeds (blocking ops)", MPI_SUCCESS == rc); + + memset(base, 0, WIN_COUNT * sizeof(int)); + + /* Notified operations are permitted only during a passive target + * epoch (MPI-5.1 section 12.3). */ + MPI_Win_lock_all(0, win); + + /* --- MPI_PUT_NOTIFY --- */ + int src = 42; + rc = MPI_Put_notify(&src, 1, MPI_INT, 0, 0, 1, MPI_INT, 0, win); + test_verify("Put_notify succeeds", MPI_SUCCESS == rc); + MPI_Win_flush(0, win); + test_verify("Put_notify moved the data", 42 == base[0]); + check_counter(win, 0, 1, "Put_notify incremented its counter by one"); + + /* --- MPI_GET_NOTIFY --- */ + int dst = 0; + rc = MPI_Get_notify(&dst, 1, MPI_INT, 0, 0, 1, MPI_INT, 0, win); + test_verify("Get_notify succeeds", MPI_SUCCESS == rc); + MPI_Win_flush(0, win); + test_verify("Get_notify moved the data", 42 == dst); + check_counter(win, 0, 2, "Get_notify incremented its counter by one"); + + /* --- MPI_ACCUMULATE_NOTIFY --- */ + src = 8; + rc = MPI_Accumulate_notify(&src, 1, MPI_INT, 0, 0, 1, MPI_INT, MPI_SUM, + 1, win); + test_verify("Accumulate_notify succeeds", MPI_SUCCESS == rc); + MPI_Win_flush(0, win); + test_verify("Accumulate_notify applied the op", 50 == base[0]); + check_counter(win, 1, 1, "Accumulate_notify incremented its counter by one"); + check_counter(win, 0, 2, "Accumulate_notify left other counters alone"); + + /* MPI_REPLACE takes the other branch in the osc/sm accumulate path. */ + src = 7; + rc = MPI_Accumulate_notify(&src, 1, MPI_INT, 0, 0, 1, MPI_INT, MPI_REPLACE, + 1, win); + test_verify("Accumulate_notify with MPI_REPLACE succeeds", MPI_SUCCESS == rc); + MPI_Win_flush(0, win); + test_verify("Accumulate_notify applied MPI_REPLACE", 7 == base[0]); + check_counter(win, 1, 2, "Accumulate_notify/REPLACE incremented its counter"); + + /* --- MPI_GET_ACCUMULATE_NOTIFY --- */ + src = 3; + int result = -1; + rc = MPI_Get_accumulate_notify(&src, 1, MPI_INT, &result, 1, MPI_INT, + 0, 0, 1, MPI_INT, MPI_SUM, 2, win); + test_verify("Get_accumulate_notify succeeds", MPI_SUCCESS == rc); + MPI_Win_flush(0, win); + test_verify("Get_accumulate_notify fetched the prior value", 7 == result); + test_verify("Get_accumulate_notify applied the op", 10 == base[0]); + check_counter(win, 2, 1, + "Get_accumulate_notify incremented its counter by one"); + + /* MPI_NO_OP fetches without modifying, and still notifies: the window + * was read, which is an access the notification covers. */ + result = -1; + rc = MPI_Get_accumulate_notify(&src, 1, MPI_INT, &result, 1, MPI_INT, + 0, 0, 1, MPI_INT, MPI_NO_OP, 2, win); + test_verify("Get_accumulate_notify with MPI_NO_OP succeeds", MPI_SUCCESS == rc); + MPI_Win_flush(0, win); + test_verify("Get_accumulate_notify/NO_OP fetched the value", 10 == result); + test_verify("Get_accumulate_notify/NO_OP left the window alone", 10 == base[0]); + check_counter(win, 2, 2, + "Get_accumulate_notify/NO_OP incremented its counter"); + + MPI_Win_unlock_all(win); + MPI_Win_free(&win); +} + +/* The four request-based notified operations. Completion of the request + * indicates completion at the origin (MPI-5.1 section 12.6.4). */ +static void test_request_ops(void) +{ + int *base = NULL; + MPI_Win win = MPI_WIN_NULL; + + int rc = MPI_Win_allocate(WIN_COUNT * sizeof(int), sizeof(int), + MPI_INFO_NULL, MPI_COMM_SELF, &base, &win); + test_verify("Win_allocate succeeds (request ops)", MPI_SUCCESS == rc); + if (MPI_SUCCESS != rc) { + return; + } + rc = MPI_Win_set_num_notify(win, MPI_INFO_NULL, NUM_NOTIFY); + test_verify("Win_set_num_notify succeeds (request ops)", MPI_SUCCESS == rc); + + memset(base, 0, WIN_COUNT * sizeof(int)); + + MPI_Win_lock_all(0, win); + + MPI_Request req = MPI_REQUEST_NULL; + + /* --- MPI_RPUT_NOTIFY --- */ + int src = 42; + rc = MPI_Rput_notify(&src, 1, MPI_INT, 0, 0, 1, MPI_INT, 0, win, &req); + test_verify("Rput_notify succeeds", MPI_SUCCESS == rc); + rc = MPI_Wait(&req, MPI_STATUS_IGNORE); + test_verify("Wait on Rput_notify request succeeds", MPI_SUCCESS == rc); + MPI_Win_flush(0, win); + test_verify("Rput_notify moved the data", 42 == base[0]); + check_counter(win, 0, 1, "Rput_notify incremented its counter by one"); + + /* --- MPI_RGET_NOTIFY --- */ + int dst = 0; + rc = MPI_Rget_notify(&dst, 1, MPI_INT, 0, 0, 1, MPI_INT, 0, win, &req); + test_verify("Rget_notify succeeds", MPI_SUCCESS == rc); + rc = MPI_Wait(&req, MPI_STATUS_IGNORE); + test_verify("Wait on Rget_notify request succeeds", MPI_SUCCESS == rc); + MPI_Win_flush(0, win); + test_verify("Rget_notify moved the data", 42 == dst); + check_counter(win, 0, 2, "Rget_notify incremented its counter by one"); + + /* --- MPI_RACCUMULATE_NOTIFY --- */ + src = 8; + rc = MPI_Raccumulate_notify(&src, 1, MPI_INT, 0, 0, 1, MPI_INT, MPI_SUM, + 1, win, &req); + test_verify("Raccumulate_notify succeeds", MPI_SUCCESS == rc); + rc = MPI_Wait(&req, MPI_STATUS_IGNORE); + test_verify("Wait on Raccumulate_notify request succeeds", MPI_SUCCESS == rc); + MPI_Win_flush(0, win); + test_verify("Raccumulate_notify applied the op", 50 == base[0]); + check_counter(win, 1, 1, "Raccumulate_notify incremented its counter by one"); + + /* --- MPI_RGET_ACCUMULATE_NOTIFY --- */ + src = 3; + int result = -1; + rc = MPI_Rget_accumulate_notify(&src, 1, MPI_INT, &result, 1, MPI_INT, + 0, 0, 1, MPI_INT, MPI_SUM, 2, win, &req); + test_verify("Rget_accumulate_notify succeeds", MPI_SUCCESS == rc); + rc = MPI_Wait(&req, MPI_STATUS_IGNORE); + test_verify("Wait on Rget_accumulate_notify request succeeds", + MPI_SUCCESS == rc); + MPI_Win_flush(0, win); + test_verify("Rget_accumulate_notify fetched the prior value", 50 == result); + test_verify("Rget_accumulate_notify applied the op", 53 == base[0]); + check_counter(win, 2, 1, + "Rget_accumulate_notify incremented its counter by one"); + + MPI_Win_unlock_all(win); + MPI_Win_free(&win); +} + +/* MPI-5.1 section 12.6: "Initiating a notified communication operation that + * references a notification counter that is out of range at the target is + * erroneous", reported as MPI_ERR_RMA_NOTIFICATION. */ +static void test_notify_idx_errors(void) +{ + int *base = NULL; + MPI_Win win = MPI_WIN_NULL; + + int rc = MPI_Win_allocate(WIN_COUNT * sizeof(int), sizeof(int), + MPI_INFO_NULL, MPI_COMM_SELF, &base, &win); + test_verify("Win_allocate succeeds (error cases)", MPI_SUCCESS == rc); + if (MPI_SUCCESS != rc) { + return; + } + MPI_Win_set_errhandler(win, MPI_ERRORS_RETURN); + + rc = MPI_Win_set_num_notify(win, MPI_INFO_NULL, NUM_NOTIFY); + test_verify("Win_set_num_notify succeeds (error cases)", MPI_SUCCESS == rc); + + memset(base, 0, WIN_COUNT * sizeof(int)); + + MPI_Win_lock_all(0, win); + + int src = 1; + int result = 0; + MPI_Request req = MPI_REQUEST_NULL; + + /* Negative index: rejected by the binding's parameter check. */ + rc = MPI_Put_notify(&src, 1, MPI_INT, 0, 0, 1, MPI_INT, -1, win); + test_verify("Put_notify rejects a negative index", + MPI_ERR_RMA_NOTIFICATION == rc); + rc = MPI_Accumulate_notify(&src, 1, MPI_INT, 0, 0, 1, MPI_INT, MPI_SUM, + -1, win); + test_verify("Accumulate_notify rejects a negative index", + MPI_ERR_RMA_NOTIFICATION == rc); + rc = MPI_Get_accumulate_notify(&src, 1, MPI_INT, &result, 1, MPI_INT, + 0, 0, 1, MPI_INT, MPI_SUM, -1, win); + test_verify("Get_accumulate_notify rejects a negative index", + MPI_ERR_RMA_NOTIFICATION == rc); + rc = MPI_Rput_notify(&src, 1, MPI_INT, 0, 0, 1, MPI_INT, -1, win, &req); + test_verify("Rput_notify rejects a negative index", + MPI_ERR_RMA_NOTIFICATION == rc); + rc = MPI_Rget_notify(&result, 1, MPI_INT, 0, 0, 1, MPI_INT, -1, win, &req); + test_verify("Rget_notify rejects a negative index", + MPI_ERR_RMA_NOTIFICATION == rc); + rc = MPI_Raccumulate_notify(&src, 1, MPI_INT, 0, 0, 1, MPI_INT, MPI_SUM, + -1, win, &req); + test_verify("Raccumulate_notify rejects a negative index", + MPI_ERR_RMA_NOTIFICATION == rc); + rc = MPI_Rget_accumulate_notify(&src, 1, MPI_INT, &result, 1, MPI_INT, + 0, 0, 1, MPI_INT, MPI_SUM, -1, win, &req); + test_verify("Rget_accumulate_notify rejects a negative index", + MPI_ERR_RMA_NOTIFICATION == rc); + + /* Index at or past the target's attached count: rejected by the osc + * module, which is the only layer that knows the target's count. */ + rc = MPI_Put_notify(&src, 1, MPI_INT, 0, 0, 1, MPI_INT, NUM_NOTIFY, win); + test_verify("Put_notify rejects an out-of-range index", + MPI_ERR_RMA_NOTIFICATION == rc); + rc = MPI_Accumulate_notify(&src, 1, MPI_INT, 0, 0, 1, MPI_INT, MPI_SUM, + NUM_NOTIFY, win); + test_verify("Accumulate_notify rejects an out-of-range index", + MPI_ERR_RMA_NOTIFICATION == rc); + rc = MPI_Get_accumulate_notify(&src, 1, MPI_INT, &result, 1, MPI_INT, + 0, 0, 1, MPI_INT, MPI_SUM, NUM_NOTIFY, win); + test_verify("Get_accumulate_notify rejects an out-of-range index", + MPI_ERR_RMA_NOTIFICATION == rc); + rc = MPI_Raccumulate_notify(&src, 1, MPI_INT, 0, 0, 1, MPI_INT, MPI_SUM, + NUM_NOTIFY, win, &req); + test_verify("Raccumulate_notify rejects an out-of-range index", + MPI_ERR_RMA_NOTIFICATION == rc); + rc = MPI_Rget_accumulate_notify(&src, 1, MPI_INT, &result, 1, MPI_INT, + 0, 0, 1, MPI_INT, MPI_SUM, NUM_NOTIFY, + win, &req); + test_verify("Rget_accumulate_notify rejects an out-of-range index", + MPI_ERR_RMA_NOTIFICATION == rc); + + /* An operation rejected for its index must not have touched the + * window, and must not have notified. */ + test_verify("a rejected notified operation does not touch the window", + 0 == base[0]); + for (int i = 0; i < NUM_NOTIFY; ++i) { + check_counter(win, i, 0, + "a rejected notified operation does not notify"); + } + + /* The accessors validate their index too. */ + MPI_Count value = -1; + rc = MPI_Win_get_notify_value(win, NUM_NOTIFY, &value); + test_verify("Win_get_notify_value rejects an out-of-range index", + MPI_ERR_RMA_NOTIFICATION == rc); + rc = MPI_Win_reset_notify_value(win, NUM_NOTIFY, &value); + test_verify("Win_reset_notify_value rejects an out-of-range index", + MPI_ERR_RMA_NOTIFICATION == rc); + + MPI_Win_unlock_all(win); + MPI_Win_free(&win); +} + +/* ------------------------------------------------------------------ */ + +/* MPI-5.1 section 12.2 defines the mpi_assert_max_num_notify info key with a + * default of 0, meaning "the implementation does not assume any limit on the + * number of notification counters". A window created without the key must + * therefore honour a request for more counters than osc/sm reserves up front, + * which it does by moving the counters to a larger allocation. */ +static void test_counter_growth(void) +{ + int *base = NULL; + MPI_Win win = MPI_WIN_NULL; + int rc; + + rc = MPI_Win_allocate(WIN_COUNT * sizeof(int), sizeof(int), MPI_INFO_NULL, + MPI_COMM_SELF, &base, &win); + test_verify("Win_allocate succeeds (growth)", MPI_SUCCESS == rc); + MPI_Win_set_errhandler(win, MPI_ERRORS_RETURN); + memset(base, 0, WIN_COUNT * sizeof(int)); + + rc = MPI_Win_set_num_notify(win, MPI_INFO_NULL, NUM_NOTIFY_GROWN); + test_verify("Win_set_num_notify grows past the reserved capacity", + MPI_SUCCESS == rc); + + int num = -1; + rc = MPI_Win_get_num_notify(win, 0, &num); + test_verify("Win_get_num_notify returns the grown count", + MPI_SUCCESS == rc && NUM_NOTIFY_GROWN == num); + + /* Every counter in the grown range must exist and read as zero. */ + for (int i = 0; i < NUM_NOTIFY_GROWN; ++i) { + check_counter(win, i, 0, "grown counter is zero"); + } + + MPI_Win_lock_all(0, win); + + /* An index only reachable after the growth must actually work end to end: + * the operation moves data and lands on the right counter. */ + int src = 99; + rc = MPI_Put_notify(&src, 1, MPI_INT, 0, 0, 1, MPI_INT, + NUM_NOTIFY_GROWN - 1, win); + test_verify("Put_notify succeeds on a counter that only growth provided", + MPI_SUCCESS == rc); + test_verify("Put_notify moved the data after growth", 99 == base[0]); + check_counter(win, NUM_NOTIFY_GROWN - 1, 1, + "the grown counter advanced by one"); + check_counter(win, 0, 0, "the grown counter did not disturb its neighbours"); + + /* One past the grown range is still out of range. */ + rc = MPI_Put_notify(&src, 1, MPI_INT, 0, 0, 1, MPI_INT, + NUM_NOTIFY_GROWN, win); + test_verify("Put_notify still rejects an index past the grown count", + MPI_ERR_RMA_NOTIFICATION == rc); + + MPI_Win_unlock_all(win); + MPI_Win_free(&win); +} + +/* A non-zero mpi_assert_max_num_notify is the user promising not to ask for + * more counters than that. osc/sm reserves exactly that many and holds the + * user to the promise rather than silently reallocating. */ +static void test_max_num_notify_assertion(void) +{ + int *base = NULL; + MPI_Win win = MPI_WIN_NULL; + MPI_Info info = MPI_INFO_NULL; + int rc; + + MPI_Info_create(&info); + MPI_Info_set(info, "mpi_assert_max_num_notify", "8"); + + rc = MPI_Win_allocate(WIN_COUNT * sizeof(int), sizeof(int), info, + MPI_COMM_SELF, &base, &win); + test_verify("Win_allocate succeeds with mpi_assert_max_num_notify", + MPI_SUCCESS == rc); + MPI_Info_free(&info); + MPI_Win_set_errhandler(win, MPI_ERRORS_RETURN); + memset(base, 0, WIN_COUNT * sizeof(int)); + + /* Up to the asserted bound is fine. */ + rc = MPI_Win_set_num_notify(win, MPI_INFO_NULL, 8); + test_verify("Win_set_num_notify accepts the asserted maximum", + MPI_SUCCESS == rc); + + int num = -1; + rc = MPI_Win_get_num_notify(win, 0, &num); + test_verify("Win_get_num_notify returns the asserted maximum", + MPI_SUCCESS == rc && 8 == num); + + /* Past it is an error rather than a reallocation: the window was sized on + * the strength of the assertion. */ + rc = MPI_Win_set_num_notify(win, MPI_INFO_NULL, 9); + test_verify("Win_set_num_notify refuses to exceed the asserted maximum", + MPI_ERR_ARG == rc); + + /* The refused call must not have disturbed the counters that do exist. */ + rc = MPI_Win_get_num_notify(win, 0, &num); + test_verify("a refused Win_set_num_notify leaves the count alone", + MPI_SUCCESS == rc && 8 == num); + + MPI_Win_lock_all(0, win); + int src = 7; + rc = MPI_Put_notify(&src, 1, MPI_INT, 0, 0, 1, MPI_INT, 7, win); + test_verify("Put_notify works on the last asserted counter", + MPI_SUCCESS == rc); + check_counter(win, 7, 1, "the last asserted counter advanced"); + MPI_Win_unlock_all(win); + + MPI_Win_free(&win); +} + +/* ------------------------------------------------------------------ */ + +/* MPI-5.1 section 12.2.6, Table 12.1: the three notification bounds are cached + * on every window at creation. NUM_SB and NUM_UB are int *, VALUE_UB is + * MPI_Count *. */ +static void test_notify_attributes(void) +{ + int *num_sb = NULL, *num_ub = NULL; + MPI_Count *value_ub = NULL; + int *base = NULL; + MPI_Win win = MPI_WIN_NULL; + MPI_Info info = MPI_INFO_NULL; + int flag = 0, rc; + + /* Without an assertion the reservation bounds what is served without + * reallocation, and nothing bounds what may be requested. */ + rc = MPI_Win_allocate(WIN_COUNT * sizeof(int), sizeof(int), MPI_INFO_NULL, + MPI_COMM_SELF, &base, &win); + test_verify("Win_allocate succeeds (attributes)", MPI_SUCCESS == rc); + MPI_Win_set_errhandler(win, MPI_ERRORS_RETURN); + + rc = MPI_Win_get_attr(win, MPI_WIN_NOTIFICATION_NUM_SB, &num_sb, &flag); + test_verify("MPI_WIN_NOTIFICATION_NUM_SB is present", + MPI_SUCCESS == rc && flag && NULL != num_sb); + test_verify("MPI_WIN_NOTIFICATION_NUM_SB is positive", *num_sb > 0); + + rc = MPI_Win_get_attr(win, MPI_WIN_NOTIFICATION_NUM_UB, &num_ub, &flag); + test_verify("MPI_WIN_NOTIFICATION_NUM_UB is present", + MPI_SUCCESS == rc && flag && NULL != num_ub); + test_verify("NUM_UB is unbounded when no assertion was given", + INT_MAX == *num_ub); + + /* A suggested maximum above the hard maximum would be nonsense. */ + test_verify("NUM_SB does not exceed NUM_UB", *num_sb <= *num_ub); + + rc = MPI_Win_get_attr(win, MPI_WIN_NOTIFICATION_VALUE_UB, &value_ub, &flag); + test_verify("MPI_WIN_NOTIFICATION_VALUE_UB is present", + MPI_SUCCESS == rc && flag && NULL != value_ub); + test_verify("VALUE_UB is the full range of the counter type", + INT64_MAX == *value_ub); + + /* Asking for exactly NUM_SB counters must not need a reallocation, and + * must be accepted. */ + int sb = *num_sb; + rc = MPI_Win_set_num_notify(win, MPI_INFO_NULL, sb); + test_verify("Win_set_num_notify accepts NUM_SB counters", MPI_SUCCESS == rc); + + MPI_Win_free(&win); + + /* With an assertion, both bounds collapse onto the asserted value. */ + MPI_Info_create(&info); + MPI_Info_set(info, "mpi_assert_max_num_notify", "8"); + rc = MPI_Win_allocate(WIN_COUNT * sizeof(int), sizeof(int), info, + MPI_COMM_SELF, &base, &win); + test_verify("Win_allocate succeeds (asserted attributes)", MPI_SUCCESS == rc); + MPI_Info_free(&info); + MPI_Win_set_errhandler(win, MPI_ERRORS_RETURN); + + rc = MPI_Win_get_attr(win, MPI_WIN_NOTIFICATION_NUM_UB, &num_ub, &flag); + test_verify("NUM_UB reports the asserted maximum", + MPI_SUCCESS == rc && flag && 8 == *num_ub); + + rc = MPI_Win_get_attr(win, MPI_WIN_NOTIFICATION_NUM_SB, &num_sb, &flag); + test_verify("NUM_SB reports the asserted maximum", + MPI_SUCCESS == rc && flag && 8 == *num_sb); + + MPI_Win_free(&win); +} diff --git a/ompi/test/general/win_notify_unsupported.c b/ompi/test/general/win_notify_unsupported.c new file mode 100644 index 00000000000..ac5754502f9 --- /dev/null +++ b/ompi/test/general/win_notify_unsupported.c @@ -0,0 +1,151 @@ +/* + * Copyright (c) 2026 Joseph Antony. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +/* + * Notified RMA (MPI-5.1 section 12.6) is optional: an osc component that + * does not implement it leaves the corresponding entries of the module + * struct NULL. Every notified entry point must report that as + * MPI_ERR_UNSUPPORTED_OPERATION rather than calling through a NULL + * function pointer. + * + * osc/rdma is forced because it is a general-purpose component that does + * not implement any of these, so it exercises the guard on all twelve + * entry points. If it cannot be selected in this build the test reports + * that and passes trivially. + * + * Note: the library is compiled with -DNDEBUG, so assert() is a no-op + * here -- all verification must go through test_verify(). + */ + +#include "ompi_config.h" + +#include +#include + +#include "support.h" + +#include "mpi.h" + +#define WIN_COUNT 8 + +int main(int argc, char *argv[]) +{ + /* Must be set before MPI_Init: component selection happens there. */ + setenv("OMPI_MCA_osc", "rdma", 1); + + test_init("ompi win_notify_unsupported"); + + int rc = MPI_Init(&argc, &argv); + test_verify("MPI_Init succeeds", MPI_SUCCESS == rc); + + int *base = NULL; + MPI_Win win = MPI_WIN_NULL; + rc = MPI_Win_allocate(WIN_COUNT * sizeof(int), sizeof(int), MPI_INFO_NULL, + MPI_COMM_SELF, &base, &win); + if (MPI_SUCCESS != rc) { + test_comment("osc/rdma not selectable in this build; skipping"); + int r = test_finalize(); + MPI_Finalize(); + return r; + } + MPI_Win_set_errhandler(win, MPI_ERRORS_RETURN); + memset(base, 0, WIN_COUNT * sizeof(int)); + + int src = 1; + int result = 0; + MPI_Count value = 0; + int num = 0; + MPI_Request req = MPI_REQUEST_NULL; + + /* Counter management and the accessors are usable outside an epoch. */ + rc = MPI_Win_set_num_notify(win, MPI_INFO_NULL, 4); + test_verify("Win_set_num_notify reports unsupported", + MPI_ERR_UNSUPPORTED_OPERATION == rc); + rc = MPI_Win_get_num_notify(win, 0, &num); + test_verify("Win_get_num_notify reports unsupported", + MPI_ERR_UNSUPPORTED_OPERATION == rc); + rc = MPI_Win_get_notify_value(win, 0, &value); + test_verify("Win_get_notify_value reports unsupported", + MPI_ERR_UNSUPPORTED_OPERATION == rc); + rc = MPI_Win_reset_notify_value(win, 0, &value); + test_verify("Win_reset_notify_value reports unsupported", + MPI_ERR_UNSUPPORTED_OPERATION == rc); + + MPI_Win_lock_all(0, win); + + /* The four blocking communication operations. */ + rc = MPI_Put_notify(&src, 1, MPI_INT, 0, 0, 1, MPI_INT, 0, win); + test_verify("Put_notify reports unsupported", + MPI_ERR_UNSUPPORTED_OPERATION == rc); + rc = MPI_Get_notify(&result, 1, MPI_INT, 0, 0, 1, MPI_INT, 0, win); + test_verify("Get_notify reports unsupported", + MPI_ERR_UNSUPPORTED_OPERATION == rc); + rc = MPI_Accumulate_notify(&src, 1, MPI_INT, 0, 0, 1, MPI_INT, MPI_SUM, + 0, win); + test_verify("Accumulate_notify reports unsupported", + MPI_ERR_UNSUPPORTED_OPERATION == rc); + rc = MPI_Get_accumulate_notify(&src, 1, MPI_INT, &result, 1, MPI_INT, + 0, 0, 1, MPI_INT, MPI_SUM, 0, win); + test_verify("Get_accumulate_notify reports unsupported", + MPI_ERR_UNSUPPORTED_OPERATION == rc); + + /* The four request-based communication operations. */ + rc = MPI_Rput_notify(&src, 1, MPI_INT, 0, 0, 1, MPI_INT, 0, win, &req); + test_verify("Rput_notify reports unsupported", + MPI_ERR_UNSUPPORTED_OPERATION == rc); + rc = MPI_Rget_notify(&result, 1, MPI_INT, 0, 0, 1, MPI_INT, 0, win, &req); + test_verify("Rget_notify reports unsupported", + MPI_ERR_UNSUPPORTED_OPERATION == rc); + rc = MPI_Raccumulate_notify(&src, 1, MPI_INT, 0, 0, 1, MPI_INT, MPI_SUM, + 0, win, &req); + test_verify("Raccumulate_notify reports unsupported", + MPI_ERR_UNSUPPORTED_OPERATION == rc); + rc = MPI_Rget_accumulate_notify(&src, 1, MPI_INT, &result, 1, MPI_INT, + 0, 0, 1, MPI_INT, MPI_SUM, 0, win, &req); + test_verify("Rget_accumulate_notify reports unsupported", + MPI_ERR_UNSUPPORTED_OPERATION == rc); + + /* The guard sits ahead of the MPI_PROC_NULL no-op, so an unsupported + * operation is reported identically no matter what the target is. */ + rc = MPI_Put_notify(&src, 1, MPI_INT, MPI_PROC_NULL, 0, 1, MPI_INT, 0, win); + test_verify("Put_notify to MPI_PROC_NULL reports unsupported", + MPI_ERR_UNSUPPORTED_OPERATION == rc); + rc = MPI_Rput_notify(&src, 1, MPI_INT, MPI_PROC_NULL, 0, 1, MPI_INT, 0, + win, &req); + test_verify("Rput_notify to MPI_PROC_NULL reports unsupported", + MPI_ERR_UNSUPPORTED_OPERATION == rc); + + /* Nothing above should have moved any data. */ + test_verify("no unsupported operation touched the window", 0 == base[0]); + + /* MPI-5.1 section 12.2.6: the notification bounds are cached on every + * window, including one whose component cannot do notified communication at + * all. Reporting zero there is the honest answer, and is consistent with + * every operation above having been refused. */ + int *num_sb = NULL, *num_ub = NULL; + MPI_Count *value_ub = NULL; + int flag = 0; + + rc = MPI_Win_get_attr(win, MPI_WIN_NOTIFICATION_NUM_SB, &num_sb, &flag); + test_verify("NUM_SB is present and zero without notification support", + MPI_SUCCESS == rc && flag && NULL != num_sb && 0 == *num_sb); + rc = MPI_Win_get_attr(win, MPI_WIN_NOTIFICATION_NUM_UB, &num_ub, &flag); + test_verify("NUM_UB is present and zero without notification support", + MPI_SUCCESS == rc && flag && NULL != num_ub && 0 == *num_ub); + rc = MPI_Win_get_attr(win, MPI_WIN_NOTIFICATION_VALUE_UB, &value_ub, &flag); + test_verify("VALUE_UB is present and zero without notification support", + MPI_SUCCESS == rc && flag && NULL != value_ub && 0 == *value_ub); + + MPI_Win_unlock_all(win); + MPI_Win_free(&win); + + int r = test_finalize(); + MPI_Finalize(); + return r; +} diff --git a/ompi/win/win.c b/ompi/win/win.c index aff2aa61ea1..69d096e53e9 100644 --- a/ompi/win/win.c +++ b/ompi/win/win.c @@ -249,6 +249,45 @@ config_window(void *base, size_t size, ptrdiff_t disp_unit, MPI_WIN_MODEL, model, true); if (OMPI_SUCCESS != ret) return ret; + /* MPI-5.1 section 12.2.6, Table 12.1: the notification bounds are cached on + * every window, whether or not its component implements notified + * communication. A component that does not leaves the query NULL and the + * attributes read as zero -- no counter may be attached to such a window, + * which is exactly what MPI_Win_set_num_notify would report by returning + * MPI_ERR_UNSUPPORTED_OPERATION. */ + int notify_num_sb = 0, notify_num_ub = 0; + MPI_Count notify_value_ub = 0; + + if (NULL != win->w_osc_module->osc_win_get_notify_bounds) { + ret = win->w_osc_module->osc_win_get_notify_bounds(win, ¬ify_num_sb, + ¬ify_num_ub, + ¬ify_value_ub); + if (OMPI_SUCCESS != ret) return ret; + } + + ret = ompi_attr_set_int(WIN_ATTR, win, + &win->w_keyhash, + MPI_WIN_NOTIFICATION_NUM_SB, notify_num_sb, true); + if (OMPI_SUCCESS != ret) return ret; + + ret = ompi_attr_set_int(WIN_ATTR, win, + &win->w_keyhash, + MPI_WIN_NOTIFICATION_NUM_UB, notify_num_ub, true); + if (OMPI_SUCCESS != ret) return ret; + + /* MPI-5.1 Table 12.1 types this attribute MPI_Count *, and the attribute + * machinery has no MPI_Count slot -- every other predefined attribute is + * integer- or address-valued. Storing it as an MPI_Aint is safe because + * MPI_Aint tracks the pointer width and Open MPI no longer supports 32-bit + * environments, so the two are the same width wherever this runs and the + * value round-trips to the user unchanged. Should 32-bit ever come back, + * this needs a real MPI_Count slot in attribute_value_t instead. */ + ret = ompi_attr_set_aint(WIN_ATTR, win, + &win->w_keyhash, + MPI_WIN_NOTIFICATION_VALUE_UB, + (MPI_Aint) notify_value_ub, true); + if (OMPI_SUCCESS != ret) return ret; + win->w_f_to_c_index = opal_pointer_array_add(&ompi_mpi_windows, win); if (-1 == win->w_f_to_c_index) return OMPI_ERR_OUT_OF_RESOURCE;