| 1 | #ifndef _CIVLMPI_
|
|---|
| 2 | #define _CIVLMPI_
|
|---|
| 3 |
|
|---|
| 4 | #include <civlc.cvh>
|
|---|
| 5 | #include <comm.cvh>
|
|---|
| 6 | #include <concurrency.cvh>
|
|---|
| 7 | #include <bundle.cvh>
|
|---|
| 8 | #include <collate.cvh>
|
|---|
| 9 | #include <stddef.h>
|
|---|
| 10 | #ifdef _MPI_NON_BLOCKING
|
|---|
| 11 | #include <civl-mpi-nonblocking.cvh>
|
|---|
| 12 | #elif defined(_MPI_BLOCKING)
|
|---|
| 13 | #include <civl-mpi-blocking.cvh>
|
|---|
| 14 | #endif
|
|---|
| 15 |
|
|---|
| 16 | #pragma CIVL ACSL
|
|---|
| 17 |
|
|---|
| 18 | #define BCAST_TAG 9999
|
|---|
| 19 | #define REDUCE_TAG 9998
|
|---|
| 20 | #define ALLREDUCE_TAG 9997
|
|---|
| 21 | #define GATHER_TAG 9996
|
|---|
| 22 | #define SCATTER_TAG 9995
|
|---|
| 23 | #define GATHERV_TAG 9994
|
|---|
| 24 | #define SCATTERV_TAG 9993
|
|---|
| 25 | #define ALLGATHER_TAG 9992
|
|---|
| 26 | #define RED_SCATTER_TAG 9991
|
|---|
| 27 | #define ALLTOALL_TAG 9990
|
|---|
| 28 | #define ALLTOALLV_TAG 9989
|
|---|
| 29 | #define ALLTOALLW_TAG 9988
|
|---|
| 30 | #define BARRIER_TAG 9987
|
|---|
| 31 | #define COMMDUP_TAG 9986
|
|---|
| 32 | #define COMMFREE_TAG 9985
|
|---|
| 33 | #define SCAN_TAG 9984
|
|---|
| 34 | #define EXSCAN_TAG 9983
|
|---|
| 35 |
|
|---|
| 36 | /* Definition of CIVL-MPI */
|
|---|
| 37 | typedef enum _mpi_state{
|
|---|
| 38 | _MPI_UNINIT,
|
|---|
| 39 | _MPI_INIT,
|
|---|
| 40 | _MPI_FINALIZED
|
|---|
| 41 | }$mpi_state;
|
|---|
| 42 |
|
|---|
| 43 | #ifdef _MPI_CONTRACT
|
|---|
| 44 |
|
|---|
| 45 | typedef int _MPI_INT_t;
|
|---|
| 46 | typedef int _MPI_2INT_t[2];
|
|---|
| 47 | typedef double _MPI_DOUBLE_t;
|
|---|
| 48 | typedef float _MPI_FLOAT_t;
|
|---|
| 49 |
|
|---|
| 50 | typedef enum MPI_COMM_MODE {
|
|---|
| 51 | P2P, COL
|
|---|
| 52 | } MPI_COMM_MODE;
|
|---|
| 53 |
|
|---|
| 54 | #endif
|
|---|
| 55 |
|
|---|
| 56 | /**************************** Redundant Definitions *******************************/
|
|---|
| 57 | /* The independence between civlmpi.cvh and mpi.h causes redundant definition */
|
|---|
| 58 | typedef struct CIVL_MPI_Comm * MPI_Comm;
|
|---|
| 59 | #ifdef __MPI__
|
|---|
| 60 | #else
|
|---|
| 61 | typedef struct MPI_Status MPI_Status;
|
|---|
| 62 | #endif
|
|---|
| 63 |
|
|---|
| 64 | #ifdef __MPI_DATATYPE__
|
|---|
| 65 | #else
|
|---|
| 66 | #define __MPI_DATATYPE__
|
|---|
| 67 | typedef enum {
|
|---|
| 68 | MPI_CHAR,
|
|---|
| 69 | MPI_CHARACTER,
|
|---|
| 70 | MPI_SIGNED_CHAR,
|
|---|
| 71 | MPI_UNSIGNED_CHAR,
|
|---|
| 72 | MPI_BYTE,
|
|---|
| 73 | MPI_WCHAR,
|
|---|
| 74 | MPI_SHORT,
|
|---|
| 75 | MPI_UNSIGNED_SHORT,
|
|---|
| 76 | MPI_INT,
|
|---|
| 77 | MPI_INT16_T,
|
|---|
| 78 | MPI_INT32_T,
|
|---|
| 79 | MPI_INT64_T,
|
|---|
| 80 | MPI_INT8_T,
|
|---|
| 81 | MPI_INTEGER,
|
|---|
| 82 | MPI_INTEGER1,
|
|---|
| 83 | MPI_INTEGER16,
|
|---|
| 84 | MPI_INTEGER2,
|
|---|
| 85 | MPI_INTEGER4,
|
|---|
| 86 | MPI_INTEGER8,
|
|---|
| 87 | MPI_UNSIGNED,
|
|---|
| 88 | MPI_LONG,
|
|---|
| 89 | MPI_UNSIGNED_LONG,
|
|---|
| 90 | MPI_FLOAT,
|
|---|
| 91 | MPI_DOUBLE,
|
|---|
| 92 | MPI_LONG_DOUBLE,
|
|---|
| 93 | MPI_LONG_LONG_INT,
|
|---|
| 94 | MPI_UNSIGNED_LONG_LONG,
|
|---|
| 95 | MPI_LONG_LONG,
|
|---|
| 96 | MPI_PACKED,
|
|---|
| 97 | MPI_LB,
|
|---|
| 98 | MPI_UB,
|
|---|
| 99 | MPI_UINT16_T,
|
|---|
| 100 | MPI_UINT32_T,
|
|---|
| 101 | MPI_UINT64_T,
|
|---|
| 102 | MPI_UINT8_T,
|
|---|
| 103 | MPI_FLOAT_INT,
|
|---|
| 104 | MPI_DOUBLE_INT,
|
|---|
| 105 | MPI_LONG_INT,
|
|---|
| 106 | MPI_SHORT_INT,
|
|---|
| 107 | MPI_2INT,
|
|---|
| 108 | MPI_LONG_DOUBLE_INT,
|
|---|
| 109 | MPI_AINT,
|
|---|
| 110 | MPI_OFFSET,
|
|---|
| 111 | MPI_2DOUBLE_PRECISION,
|
|---|
| 112 | MPI_2INTEGER,
|
|---|
| 113 | MPI_2REAL,
|
|---|
| 114 | MPI_C_BOOL,
|
|---|
| 115 | MPI_C_COMPLEX,
|
|---|
| 116 | MPI_C_DOUBLE_COMPLEX,
|
|---|
| 117 | MPI_C_FLOAT_COMPLEX,
|
|---|
| 118 | MPI_C_LONG_DOUBLE_COMPLEX,
|
|---|
| 119 | MPI_COMPLEX,
|
|---|
| 120 | MPI_COMPLEX16,
|
|---|
| 121 | MPI_COMPLEX32,
|
|---|
| 122 | MPI_COMPLEX4,
|
|---|
| 123 | MPI_COMPLEX8,
|
|---|
| 124 | MPI_REAL,
|
|---|
| 125 | MPI_REAL16,
|
|---|
| 126 | MPI_REAL2,
|
|---|
| 127 | MPI_REAL4,
|
|---|
| 128 | MPI_REAL8
|
|---|
| 129 | } MPI_Datatype;
|
|---|
| 130 | #endif
|
|---|
| 131 |
|
|---|
| 132 | /* Definition of CMPI_Gcomm and MPI_Comm */
|
|---|
| 133 | typedef struct $mpi_gcomm $mpi_gcomm;
|
|---|
| 134 |
|
|---|
| 135 | /****************************** Helper Functions **********************************/
|
|---|
| 136 | /*@ pure;
|
|---|
| 137 | @ depends_on \nothing;
|
|---|
| 138 | @ executes_when \true;
|
|---|
| 139 | @*/
|
|---|
| 140 | $state_f $atomic_f int sizeofDatatype(MPI_Datatype);
|
|---|
| 141 |
|
|---|
| 142 | /************************** MPI LIB Implementations *******************************/
|
|---|
| 143 | $abstract double $mpi_time(int i);
|
|---|
| 144 |
|
|---|
| 145 | $mpi_gcomm $mpi_gcomm_create($scope, int);
|
|---|
| 146 |
|
|---|
| 147 | void $mpi_gcomm_destroy($mpi_gcomm);
|
|---|
| 148 |
|
|---|
| 149 | MPI_Comm $mpi_comm_create($scope, $mpi_gcomm, int);
|
|---|
| 150 |
|
|---|
| 151 | void $mpi_comm_destroy(MPI_Comm, $mpi_state);
|
|---|
| 152 |
|
|---|
| 153 | int $mpi_send(const void *, int, MPI_Datatype, int, int, MPI_Comm);
|
|---|
| 154 |
|
|---|
| 155 | int $mpi_recv(void *, int, MPI_Datatype, int, int, MPI_Comm, MPI_Status *);
|
|---|
| 156 |
|
|---|
| 157 | int $mpi_isend(const void *buf, int count, MPI_Datatype datatype, int dest,
|
|---|
| 158 | int tag, MPI_Comm comm, MPI_Request * request);
|
|---|
| 159 |
|
|---|
| 160 | int $mpi_irecv(void *buf, int count, MPI_Datatype datatype, int source,
|
|---|
| 161 | int tag, MPI_Comm comm, MPI_Request * request);
|
|---|
| 162 |
|
|---|
| 163 | int $mpi_wait_send(MPI_Request * request, MPI_Status * status);
|
|---|
| 164 |
|
|---|
| 165 | int $mpi_wait_recv(MPI_Request * request, MPI_Status * status);
|
|---|
| 166 |
|
|---|
| 167 | int $mpi_wait(MPI_Request * request, MPI_Status * status);
|
|---|
| 168 |
|
|---|
| 169 | int $mpi_sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
|
|---|
| 170 | int dest, int sendtag, void *recvbuf, int recvcount,
|
|---|
| 171 | MPI_Datatype recvtype, int source, int recvtag,
|
|---|
| 172 | MPI_Comm comm, MPI_Status *status);
|
|---|
| 173 |
|
|---|
| 174 | int $mpi_collective_send(const void *, int, MPI_Datatype, int, int, MPI_Comm);
|
|---|
| 175 |
|
|---|
| 176 | int $mpi_collective_recv(void *, int, MPI_Datatype, int, int, MPI_Comm, MPI_Status *,
|
|---|
| 177 | char *);
|
|---|
| 178 |
|
|---|
| 179 | int $mpi_bcast(void *, int, MPI_Datatype, int, int, MPI_Comm, char *);
|
|---|
| 180 |
|
|---|
| 181 | int $mpi_reduce(const void*, void*, int,
|
|---|
| 182 | MPI_Datatype, MPI_Op, int, int,
|
|---|
| 183 | MPI_Comm, char *);
|
|---|
| 184 |
|
|---|
| 185 | int $mpi_gather(const void*, int, MPI_Datatype,
|
|---|
| 186 | void*, int, MPI_Datatype, int, int, MPI_Comm, char *);
|
|---|
| 187 |
|
|---|
| 188 | int $mpi_gatherv(const void*, int, MPI_Datatype,
|
|---|
| 189 | void*, const int[], const int[],
|
|---|
| 190 | MPI_Datatype, int, int,
|
|---|
| 191 | MPI_Comm, char *);
|
|---|
| 192 |
|
|---|
| 193 | int $mpi_scatter(const void*, int, MPI_Datatype,
|
|---|
| 194 | void*, int, MPI_Datatype, int,
|
|---|
| 195 | int, MPI_Comm, char *);
|
|---|
| 196 |
|
|---|
| 197 | int $mpi_scatterv(const void*, const int[], const
|
|---|
| 198 | int[], MPI_Datatype, void* ,
|
|---|
| 199 | int, MPI_Datatype, int, int,
|
|---|
| 200 | MPI_Comm, char *);
|
|---|
| 201 |
|
|---|
| 202 | int $mpi_scan(const void *, void *, int, MPI_Datatype, MPI_Op, MPI_Comm);
|
|---|
| 203 |
|
|---|
| 204 | int $mpi_exscan(const void *, void *, int, MPI_Datatype, MPI_Op, MPI_Comm);
|
|---|
| 205 |
|
|---|
| 206 | void* $mpi_pointer_add(const void*, int, MPI_Datatype);
|
|---|
| 207 |
|
|---|
| 208 | /*@ depends_on \nothing;
|
|---|
| 209 | @ executes_when \true;
|
|---|
| 210 | @*/
|
|---|
| 211 | $system[mpi] int $mpi_new_gcomm($scope, $mpi_gcomm);
|
|---|
| 212 |
|
|---|
| 213 | /*@ depends_on \nothing;
|
|---|
| 214 | @ executes_when \true;
|
|---|
| 215 | @*/
|
|---|
| 216 | $system[mpi] $mpi_gcomm $mpi_get_gcomm($scope, int);
|
|---|
| 217 |
|
|---|
| 218 | int $mpi_comm_dup($scope, MPI_Comm, MPI_Comm *, char *);
|
|---|
| 219 |
|
|---|
| 220 | int $mpi_comm_free(MPI_Comm *, $mpi_state);
|
|---|
| 221 |
|
|---|
| 222 | /* Assert that the basetype of the object pointed by 'buf' is sames as
|
|---|
| 223 | the basetype of the 'datatype'. Here a basetype of an MPI_Datatype
|
|---|
| 224 | means a indivisiable MPI_datatype, e.g. the basetype of MPI_2INT is
|
|---|
| 225 | MPI_INT.*/
|
|---|
| 226 | /*@ depends_on \access(buf);
|
|---|
| 227 | @ executes_when \true;
|
|---|
| 228 | @*/
|
|---|
| 229 | $system[mpi] void $mpi_check_buffer(const void * buf, int count, MPI_Datatype datatype);
|
|---|
| 230 |
|
|---|
| 231 | /* Create a $bundle type object which contains a sequence of entries
|
|---|
| 232 | that identify a group of consistent MPI collective routines. MPI
|
|---|
| 233 | collective routines should be reached by every process in the same
|
|---|
| 234 | communicator with the same order
|
|---|
| 235 | Arguments:
|
|---|
| 236 | comm: the MPI communicator all the processes are belong to.
|
|---|
| 237 | routineTag: the tag indicating a collective routine.
|
|---|
| 238 | root: rank of the root process
|
|---|
| 239 | op: a int type object representing the enumerator of the MPI_Op,
|
|---|
| 240 | if MPI_Op is not part of the arguments of the collective routine,
|
|---|
| 241 | this argument should be set 0.
|
|---|
| 242 | numDatatypes: the number of datatypes involved in the messages.
|
|---|
| 243 | datatypes: the datatype array stores datatypes enumerators involved in
|
|---|
| 244 | the message.
|
|---|
| 245 | */
|
|---|
| 246 | $bundle $mpi_create_coroutine_entry(int routineTag, int root,
|
|---|
| 247 | int op, int numDatatypes, int * datatypes);
|
|---|
| 248 |
|
|---|
| 249 | /* Compare two collective routine entries, report errors if there is
|
|---|
| 250 | * unmatched informations in these entries. */
|
|---|
| 251 | void $mpi_diff_coroutine_entries($bundle specEntry, $bundle mineEntry, int rank);
|
|---|
| 252 |
|
|---|
| 253 | /* Allocate memory space with MPI type signitures. Returns a pointer
|
|---|
| 254 | to the allocated memory space */
|
|---|
| 255 | void * $mpi_malloc(int count, MPI_Datatype datatype);
|
|---|
| 256 |
|
|---|
| 257 | /* Checking a boolean expression in a collective way. Participating
|
|---|
| 258 | processes are defined by the first argument. */
|
|---|
| 259 | void $mpi_coassert(MPI_Comm, _Bool);
|
|---|
| 260 |
|
|---|
| 261 | int $mpi_barrier(MPI_Comm);
|
|---|
| 262 |
|
|---|
| 263 | /********** Section: bridging MPI and the comm library ***********/
|
|---|
| 264 |
|
|---|
| 265 | /* Returns the `$comm` object, representing the point-to-point
|
|---|
| 266 | channel, referred by the given MPI communicator handle.
|
|---|
| 267 | */
|
|---|
| 268 | /*@ depends_on \nothing;
|
|---|
| 269 | @ executes_when \true;
|
|---|
| 270 | @*/
|
|---|
| 271 | $atomic_f $state_f $comm $mpi_get_p2p_channel(MPI_Comm comm);
|
|---|
| 272 |
|
|---|
| 273 | /* Returns the `$comm` object, representing the collective
|
|---|
| 274 | channel, referred by the given MPI communicator handle.
|
|---|
| 275 | */
|
|---|
| 276 | /*@ depends_on \nothing;
|
|---|
| 277 | @ executes_when \true;
|
|---|
| 278 | @*/
|
|---|
| 279 | $atomic_f $state_f $comm $mpi_get_col_channel(MPI_Comm comm);
|
|---|
| 280 |
|
|---|
| 281 | /* Returns the number of processes in the given MPI communicator.
|
|---|
| 282 | */
|
|---|
| 283 | /*@ depends_on \nothing;
|
|---|
| 284 | @ executes_when \true;
|
|---|
| 285 | @*/
|
|---|
| 286 | $atomic_f $state_f int $mpi_comm_size(MPI_Comm comm);
|
|---|
| 287 |
|
|---|
| 288 | /* Returns the place of the process owning the the given MPI
|
|---|
| 289 | communicator.
|
|---|
| 290 | */
|
|---|
| 291 | /*@ depends_on \nothing;
|
|---|
| 292 | @ executes_when \true;
|
|---|
| 293 | @*/
|
|---|
| 294 | $atomic_f $state_f int $mpi_comm_place(MPI_Comm comm);
|
|---|
| 295 |
|
|---|
| 296 | /* Returns the root scope of all processes in the given MPI communicator.
|
|---|
| 297 | */
|
|---|
| 298 | /*@ depends_on \nothing;
|
|---|
| 299 | @ executes_when \true;
|
|---|
| 300 | @*/
|
|---|
| 301 | $atomic_f $state_f $scope $mpi_root_scope(MPI_Comm);
|
|---|
| 302 |
|
|---|
| 303 | /* Returns the root scope of the process owning the given MPI
|
|---|
| 304 | communicator.
|
|---|
| 305 | */
|
|---|
| 306 | /*@ depends_on \nothing;
|
|---|
| 307 | @ executes_when \true;
|
|---|
| 308 | @*/
|
|---|
| 309 | $atomic_f $state_f $scope $mpi_proc_scope(MPI_Comm);
|
|---|
| 310 |
|
|---|
| 311 | /*@ depends_on \nothing;
|
|---|
| 312 | @ executes_when \true;
|
|---|
| 313 | @*/
|
|---|
| 314 | $system[mpi] $scope $mpi_root_scope_system($comm);
|
|---|
| 315 |
|
|---|
| 316 | /*@ depends_on \nothing;
|
|---|
| 317 | @ executes_when \true;
|
|---|
| 318 | @*/
|
|---|
| 319 | $system[mpi] $scope $mpi_proc_scope_system($comm);
|
|---|
| 320 |
|
|---|
| 321 | /*************** Bridging MPI and the collate library **************/
|
|---|
| 322 |
|
|---|
| 323 | /*
|
|---|
| 324 | Checks if all processes in a communicator are collectively executing
|
|---|
| 325 | a routine provided by the information in the $bundle.
|
|---|
| 326 | */
|
|---|
| 327 | $state_f $bundle $mpi_check_collective(MPI_Comm, $bundle);
|
|---|
| 328 |
|
|---|
| 329 | /*********** Section: contract helper functions ***********/
|
|---|
| 330 | #ifdef _MPI_CONTRACT
|
|---|
| 331 | /*@ executes_when \true;
|
|---|
| 332 | @ depends_on \nothing;
|
|---|
| 333 | */
|
|---|
| 334 | $atomic_f $collate_state $mpi_snapshot(MPI_Comm comm, $scope scope);
|
|---|
| 335 |
|
|---|
| 336 | /*@ executes_when \true;
|
|---|
| 337 | @ depends_on \nothing;
|
|---|
| 338 | */
|
|---|
| 339 | $atomic_f void $mpi_unsnapshot(MPI_Comm comm, $collate_state cs);
|
|---|
| 340 |
|
|---|
| 341 | /* Returns the number of chars of the given MPI_Datatype.
|
|---|
| 342 | * equivalent to returns (datatypeSizeof(datatype) / sizeof(char))
|
|---|
| 343 | */
|
|---|
| 344 | $state_f size_t $mpi_extentof(MPI_Datatype);
|
|---|
| 345 |
|
|---|
| 346 | /* Assigns fresh new symbolic constants to the given mpi region */
|
|---|
| 347 | void $mpi_assigns(void * buf, int count, MPI_Datatype datatype);
|
|---|
| 348 |
|
|---|
| 349 | $atomic_f void $mpi_comm_empty(MPI_Comm comm, MPI_COMM_MODE mode);
|
|---|
| 350 | #endif
|
|---|
| 351 | #endif
|
|---|
| 352 |
|
|---|
| 353 |
|
|---|
| 354 |
|
|---|