source: CIVL/examples/mpi-omp/AMG2013/struct_mv/struct_communication.c@ a389857

main test-branch
Last change on this file since a389857 was ea777aa, checked in by Alex Wilton <awilton@…>, 3 years ago

Moved examples, include, build_default.properties, common.xml, and README out from dev.civl.com into the root of the repo.

git-svn-id: ​svn://vsl.cis.udel.edu/civl/trunk@5704 fb995dde-84ed-4084-dfe6-e5aef3e2452c
  • Property mode set to 100644
File size: 38.9 KB
Line 
1/*BHEADER**********************************************************************
2 * Copyright (c) 2008, Lawrence Livermore National Security, LLC.
3 * Produced at the Lawrence Livermore National Laboratory.
4 * This file is part of HYPRE. See file COPYRIGHT for details.
5 *
6 * HYPRE is free software; you can redistribute it and/or modify it under the
7 * terms of the GNU Lesser General Public License (as published by the Free
8 * Software Foundation) version 2.1 dated February 1999.
9 *
10 * $Revision: 2.4 $
11 ***********************************************************************EHEADER*/
12
13
14
15#include "headers.h"
16
17#define DEBUG 0
18
19#if DEBUG
20char filename[255];
21FILE *file;
22#endif
23
24/* this computes a (large enough) size (in doubles) for the message prefix */
25#define hypre_CommPrefixSize(ne) \
26( ((ne*sizeof(int) + ne*sizeof(hypre_Box))/sizeof(double)) + 1 )
27
28/*--------------------------------------------------------------------------
29 * Create a communication package. A grid-based description of a
30 * communication exchange is passed in. This description is then
31 * compiled into an intermediate processor-based description of the
32 * communication. The intermediate processor-based description is
33 * used directly to pack and unpack buffers during the communications.
34 *
35 * Note: The input boxes and processes are destroyed.
36 *--------------------------------------------------------------------------*/
37
38int
39hypre_CommPkgCreate( hypre_CommInfo *comm_info,
40 hypre_BoxArray *send_data_space,
41 hypre_BoxArray *recv_data_space,
42 int num_values,
43 MPI_Comm comm,
44 hypre_CommPkg **comm_pkg_ptr )
45{
46 int ierr = 0;
47
48 hypre_BoxArrayArray *send_boxes;
49 hypre_BoxArrayArray *recv_boxes;
50 hypre_IndexRef send_stride;
51 hypre_IndexRef recv_stride;
52 int **send_processes;
53 int **recv_processes;
54 int **send_rboxnums;
55 hypre_BoxArrayArray *send_rboxes;
56
57 hypre_CommPkg *comm_pkg;
58 hypre_CommType *comm_types;
59 hypre_CommType *comm_type;
60 hypre_CommEntryType *ct_entries;
61 int *ct_loc_boxnums;
62 int *ct_rem_boxnums;
63 hypre_Box *ct_loc_boxes;
64 hypre_Box *ct_rem_boxes;
65 int *p_comm_types;
66 int num_comms, num_entries, comm_bufsize;
67
68 hypre_BoxArray *box_array;
69 hypre_Box *box;
70 hypre_BoxArray *rbox_array;
71 hypre_Box *rbox;
72 hypre_Box *data_box;
73 int *data_offsets;
74 int data_offset;
75
76 int i, j, k, p, m, size;
77 int num_procs, my_proc;
78
79 /*------------------------------------------------------
80 *------------------------------------------------------*/
81
82 send_boxes = hypre_CommInfoSendBoxes(comm_info);
83 recv_boxes = hypre_CommInfoRecvBoxes(comm_info);
84 send_stride = hypre_CommInfoSendStride(comm_info);
85 recv_stride = hypre_CommInfoRecvStride(comm_info);
86 send_processes = hypre_CommInfoSendProcesses(comm_info);
87 recv_processes = hypre_CommInfoRecvProcesses(comm_info);
88 send_rboxnums = hypre_CommInfoSendRBoxnums(comm_info);
89 send_rboxes = hypre_CommInfoSendRBoxes(comm_info);
90
91 MPI_Comm_size(comm, &num_procs );
92 MPI_Comm_rank(comm, &my_proc );
93
94 /*------------------------------------------------------
95 * Set up various entries in CommPkg
96 *------------------------------------------------------*/
97
98 comm_pkg = hypre_CTAlloc(hypre_CommPkg, 1);
99
100 hypre_CommPkgComm(comm_pkg) = comm;
101 hypre_CommPkgFirstSend(comm_pkg) = 1;
102 hypre_CommPkgFirstRecv(comm_pkg) = 1;
103 hypre_CommPkgNumValues(comm_pkg) = num_values;
104 hypre_CopyIndex(send_stride, hypre_CommPkgSendStride(comm_pkg));
105 hypre_CopyIndex(recv_stride, hypre_CommPkgRecvStride(comm_pkg));
106
107 /*------------------------------------------------------
108 * Set up send CommType information
109 *------------------------------------------------------*/
110
111 p_comm_types = hypre_CTAlloc(int, num_procs);
112
113 /* set send_data_offsets and send_data_space */
114 data_offsets = hypre_TAlloc(int, hypre_BoxArraySize(send_data_space));
115 data_offset = 0;
116 hypre_ForBoxI(i, send_data_space)
117 {
118 data_offsets[i] = data_offset;
119 data_box = hypre_BoxArrayBox(send_data_space, i);
120 data_offset += hypre_BoxVolume(data_box) * num_values;
121 }
122
123 /* compute num_comms and use -p_comm_types to compute num_entries */
124
125 num_comms = 0;
126 num_entries = 0;
127 hypre_ForBoxArrayI(i, send_boxes)
128 {
129 box_array = hypre_BoxArrayArrayBoxArray(send_boxes, i);
130
131 hypre_ForBoxI(j, box_array)
132 {
133 box = hypre_BoxArrayBox(box_array, j);
134 p = send_processes[i][j];
135
136 if (hypre_BoxVolume(box) != 0)
137 {
138 p_comm_types[p]--;
139 num_entries++;
140 if ((p_comm_types[p] == -1) && (p != my_proc))
141 {
142 num_comms++;
143 }
144 }
145 }
146 }
147
148 /* compute comm_types */
149
150 comm_types = hypre_CTAlloc(hypre_CommType, (num_comms + 1));
151 ct_entries = hypre_TAlloc(hypre_CommEntryType, num_entries);
152 ct_loc_boxnums = hypre_TAlloc(int, num_entries);
153 ct_rem_boxnums = hypre_TAlloc(int, num_entries);
154 ct_loc_boxes = hypre_TAlloc(hypre_Box, num_entries);
155 ct_rem_boxes = hypre_TAlloc(hypre_Box, num_entries);
156
157 /* initialize local copy type */
158 comm_type = &comm_types[0];
159 k = -p_comm_types[my_proc];
160 p_comm_types[my_proc] = 0;
161 hypre_CommTypeProc(comm_type) = my_proc;
162 hypre_CommTypeNumEntries(comm_type) = 0;
163 hypre_CommTypeEntries(comm_type) = ct_entries;
164 hypre_CommTypeLocBoxnums(comm_type) = ct_loc_boxnums;
165 hypre_CommTypeRemBoxnums(comm_type) = ct_rem_boxnums;
166 hypre_CommTypeLocBoxes(comm_type) = ct_loc_boxes;
167 hypre_CommTypeRemBoxes(comm_type) = ct_rem_boxes;
168 ct_entries += k;
169 ct_loc_boxnums += k;
170 ct_rem_boxnums += k;
171 ct_loc_boxes += k;
172 ct_rem_boxes += k;
173
174 m = 1;
175 comm_bufsize = 0;
176 hypre_ForBoxArrayI(i, send_boxes)
177 {
178 box_array = hypre_BoxArrayArrayBoxArray(send_boxes, i);
179 rbox_array = hypre_BoxArrayArrayBoxArray(send_rboxes, i);
180
181 hypre_ForBoxI(j, box_array)
182 {
183 box = hypre_BoxArrayBox(box_array, j);
184 rbox = hypre_BoxArrayBox(rbox_array, j);
185 p = send_processes[i][j];
186
187 if (hypre_BoxVolume(box) != 0)
188 {
189 /* initialize comm type for process p */
190 if (p_comm_types[p] < 0)
191 {
192 comm_type = &comm_types[m];
193 k = -p_comm_types[p];
194 p_comm_types[p] = m;
195 hypre_CommTypeProc(comm_type) = p;
196 size = hypre_CommPrefixSize(k);
197 hypre_CommTypeBufsize(comm_type) = size;
198 comm_bufsize += size;
199 hypre_CommTypeNumEntries(comm_type) = 0;
200 hypre_CommTypeEntries(comm_type) = ct_entries;
201 hypre_CommTypeLocBoxnums(comm_type) = ct_loc_boxnums;
202 hypre_CommTypeRemBoxnums(comm_type) = ct_rem_boxnums;
203 hypre_CommTypeLocBoxes(comm_type) = ct_loc_boxes;
204 hypre_CommTypeRemBoxes(comm_type) = ct_rem_boxes;
205 ct_entries += k;
206 ct_loc_boxnums += k;
207 ct_rem_boxnums += k;
208 ct_loc_boxes += k;
209 ct_rem_boxes += k;
210 m++;
211 }
212
213 comm_type = &comm_types[p_comm_types[p]];
214 k = hypre_CommTypeNumEntries(comm_type);
215 hypre_BoxGetStrideVolume(box, send_stride, &size);
216 hypre_CommTypeBufsize(comm_type) += (size*num_values);
217 comm_bufsize += (size*num_values);
218 hypre_CommTypeLocBoxnum(comm_type, k) = i;
219 hypre_CommTypeRemBoxnum(comm_type, k) = send_rboxnums[i][j];
220 hypre_CopyBox(box, hypre_CommTypeLocBox(comm_type, k));
221 hypre_CopyBox(rbox, hypre_CommTypeRemBox(comm_type, k));
222 hypre_CommTypeNumEntries(comm_type) ++;
223 }
224 }
225 }
226
227 /* Loop over comm_types and set entries */
228 for (m = 0; m < (num_comms + 1); m++)
229 {
230 comm_type = &comm_types[m];
231 hypre_CommTypeSetEntries(comm_type,
232 hypre_CommTypeLocBoxnums(comm_type),
233 hypre_CommTypeLocBoxes(comm_type),
234 hypre_CommPkgSendStride(comm_pkg),
235 hypre_CommPkgNumValues(comm_pkg),
236 send_data_space, data_offsets );
237 if (m > 0)
238 {
239 hypre_CommTypeLocBoxnums(comm_type) = NULL;
240 hypre_CommTypeLocBoxes(comm_type) = NULL;
241 }
242
243 /* reset p_comm_types to zero */
244 p_comm_types[hypre_CommTypeProc(comm_type)] = 0;
245 }
246
247 /* set send info in comm_pkg */
248 hypre_CommPkgSendBufsize(comm_pkg) = comm_bufsize;
249 hypre_CommPkgNumSends(comm_pkg) = num_comms;
250 hypre_CommPkgSendTypes(comm_pkg) = &comm_types[1];
251 hypre_CommPkgCopyFromType(comm_pkg) = &comm_types[0];
252
253 /* free up data_offsets */
254 hypre_TFree(data_offsets);
255
256 /* free up local box info */
257 comm_type = hypre_CommPkgCopyFromType(comm_pkg);
258 hypre_TFree(hypre_CommTypeLocBoxnums(comm_type));
259 hypre_TFree(hypre_CommTypeLocBoxes(comm_type));
260
261 /*------------------------------------------------------
262 * Set up recv CommType information
263 *------------------------------------------------------*/
264
265 /* set data_offsets and recv_data_space */
266 data_offsets = hypre_TAlloc(int, hypre_BoxArraySize(recv_data_space));
267 data_offset = 0;
268 hypre_ForBoxI(i, recv_data_space)
269 {
270 data_offsets[i] = data_offset;
271 data_box = hypre_BoxArrayBox(recv_data_space, i);
272 data_offset += hypre_BoxVolume(data_box) * num_values;
273 }
274 hypre_CommPkgRecvDataOffsets(comm_pkg) = data_offsets;
275 hypre_CommPkgRecvDataSpace(comm_pkg) =
276 hypre_BoxArrayDuplicate(recv_data_space);
277
278 /* compute num_comms and use -p_comm_types to compute num_entries */
279
280 num_comms = 0;
281 num_entries = 0;
282 hypre_ForBoxArrayI(i, recv_boxes)
283 {
284 box_array = hypre_BoxArrayArrayBoxArray(recv_boxes, i);
285
286 hypre_ForBoxI(j, box_array)
287 {
288 box = hypre_BoxArrayBox(box_array, j);
289 p = recv_processes[i][j];
290
291 if (hypre_BoxVolume(box) != 0)
292 {
293 p_comm_types[p]--;
294 num_entries++;
295 if ((p_comm_types[p] == -1) && (p != my_proc))
296 {
297 num_comms++;
298 }
299 }
300 }
301 }
302
303 /* compute comm_types */
304
305 comm_types = hypre_CTAlloc(hypre_CommType, (num_comms + 1));
306 ct_entries = hypre_TAlloc(hypre_CommEntryType, num_entries);
307
308 /* initialize local copy type */
309 comm_type = &comm_types[0];
310 k = -p_comm_types[my_proc];
311 p_comm_types[my_proc] = 0;
312 hypre_CommTypeProc(comm_type) = my_proc;
313 hypre_CommTypeNumEntries(comm_type) = 0;
314 hypre_CommTypeEntries(comm_type) = ct_entries;
315 ct_entries += k;
316
317 m = 1;
318 comm_bufsize = 0;
319 hypre_ForBoxArrayI(i, recv_boxes)
320 {
321 box_array = hypre_BoxArrayArrayBoxArray(recv_boxes, i);
322
323 hypre_ForBoxI(j, box_array)
324 {
325 box = hypre_BoxArrayBox(box_array, j);
326 p = recv_processes[i][j];
327
328 if (hypre_BoxVolume(box) != 0)
329 {
330 /* initialize comm type for process p */
331 if (p_comm_types[p] < 0)
332 {
333 comm_type = &comm_types[m];
334 k = -p_comm_types[p];
335 p_comm_types[p] = m;
336 hypre_CommTypeProc(comm_type) = p;
337 size = hypre_CommPrefixSize(k);
338 hypre_CommTypeBufsize(comm_type) = size;
339 comm_bufsize += size;
340 hypre_CommTypeNumEntries(comm_type) = 0;
341 hypre_CommTypeEntries(comm_type) = ct_entries;
342 ct_entries += k;
343 m++;
344 }
345
346 comm_type = &comm_types[p_comm_types[p]];
347 hypre_BoxGetStrideVolume(box, recv_stride, &size);
348 hypre_CommTypeBufsize(comm_type) += (size*num_values);
349 comm_bufsize += (size*num_values);
350 hypre_CommTypeNumEntries(comm_type) ++;
351 }
352 }
353 }
354
355 /* set copy type entries */
356 comm_type = hypre_CommPkgCopyFromType(comm_pkg);
357 hypre_CommTypeSetEntries(&comm_types[0],
358 hypre_CommTypeRemBoxnums(comm_type),
359 hypre_CommTypeRemBoxes(comm_type),
360 hypre_CommPkgRecvStride(comm_pkg),
361 hypre_CommPkgNumValues(comm_pkg),
362 hypre_CommPkgRecvDataSpace(comm_pkg),
363 hypre_CommPkgRecvDataOffsets(comm_pkg) );
364
365 /* set recv info in comm_pkg */
366 hypre_CommPkgRecvBufsize(comm_pkg) = comm_bufsize;
367 hypre_CommPkgNumRecvs(comm_pkg) = num_comms;
368 hypre_CommPkgRecvTypes(comm_pkg) = &comm_types[1];
369 hypre_CommPkgCopyToType(comm_pkg) = &comm_types[0];
370
371 /*------------------------------------------------------
372 * Debugging stuff
373 *------------------------------------------------------*/
374
375#if DEBUG
376{
377 MPI_Comm_rank(MPI_COMM_WORLD, &my_proc);
378
379 sprintf(filename, "zcommboxes.%05d", my_proc);
380
381 if ((file = fopen(filename, "a")) == NULL)
382 {
383 printf("Error: can't open output file %s\n", filename);
384 exit(1);
385 }
386
387 fprintf(file, "\n\n============================\n\n");
388 fprintf(file, "SEND boxes:\n\n");
389
390 fprintf(file, "Stride = (%d,%d,%d)\n",
391 hypre_IndexX(send_stride),
392 hypre_IndexY(send_stride),
393 hypre_IndexZ(send_stride));
394 fprintf(file, "BoxArrayArraySize = %d\n",
395 hypre_BoxArrayArraySize(send_boxes));
396 hypre_ForBoxArrayI(i, send_boxes)
397 {
398 box_array = hypre_BoxArrayArrayBoxArray(send_boxes, i);
399
400 fprintf(file, "BoxArraySize = %d\n", hypre_BoxArraySize(box_array));
401 hypre_ForBoxI(j, box_array)
402 {
403 box = hypre_BoxArrayBox(box_array, j);
404 fprintf(file, "(%d,%d): (%d,%d,%d) x (%d,%d,%d)\n",
405 i, j,
406 hypre_BoxIMinX(box),
407 hypre_BoxIMinY(box),
408 hypre_BoxIMinZ(box),
409 hypre_BoxIMaxX(box),
410 hypre_BoxIMaxY(box),
411 hypre_BoxIMaxZ(box));
412 fprintf(file, "(%d,%d): %d,%d\n",
413 i, j, send_processes[i][j], send_rboxnums[i][j]);
414 }
415 }
416
417 fprintf(file, "\n\n============================\n\n");
418 fprintf(file, "RECV boxes:\n\n");
419
420 fprintf(file, "Stride = (%d,%d,%d)\n",
421 hypre_IndexX(recv_stride),
422 hypre_IndexY(recv_stride),
423 hypre_IndexZ(recv_stride));
424 fprintf(file, "BoxArrayArraySize = %d\n",
425 hypre_BoxArrayArraySize(recv_boxes));
426 hypre_ForBoxArrayI(i, recv_boxes)
427 {
428 box_array = hypre_BoxArrayArrayBoxArray(recv_boxes, i);
429
430 fprintf(file, "BoxArraySize = %d\n", hypre_BoxArraySize(box_array));
431 hypre_ForBoxI(j, box_array)
432 {
433 box = hypre_BoxArrayBox(box_array, j);
434 fprintf(file, "(%d,%d): (%d,%d,%d) x (%d,%d,%d)\n",
435 i, j,
436 hypre_BoxIMinX(box),
437 hypre_BoxIMinY(box),
438 hypre_BoxIMinZ(box),
439 hypre_BoxIMaxX(box),
440 hypre_BoxIMaxY(box),
441 hypre_BoxIMaxZ(box));
442 fprintf(file, "(%d,%d): %d\n",
443 i, j, recv_processes[i][j]);
444 }
445 }
446
447 fflush(file);
448 fclose(file);
449}
450#endif
451
452#if DEBUG
453{
454 hypre_CommEntryType *comm_entry;
455 int offset, dim;
456 int *length;
457 int *stride;
458
459 MPI_Comm_rank(MPI_COMM_WORLD, &my_proc);
460
461 sprintf(filename, "zcommentries.%05d", my_proc);
462
463 if ((file = fopen(filename, "a")) == NULL)
464 {
465 printf("Error: can't open output file %s\n", filename);
466 exit(1);
467 }
468
469 fprintf(file, "\n\n============================\n\n");
470 fprintf(file, "SEND entries:\n\n");
471
472 fprintf(file, "num_sends = %d\n", hypre_CommPkgNumSends(comm_pkg));
473
474 comm_types = hypre_CommPkgCopyFromType(comm_pkg);
475 for (m = 0; m < (hypre_CommPkgNumSends(comm_pkg) + 1); m++)
476 {
477 comm_type = &comm_types[m];
478 fprintf(file, "process = %d\n", hypre_CommTypeProc(comm_type));
479 fprintf(file, "num_entries = %d\n", hypre_CommTypeNumEntries(comm_type));
480 for (i = 0; i < hypre_CommTypeNumEntries(comm_type); i++)
481 {
482 comm_entry = hypre_CommTypeEntry(comm_type, i);
483 offset = hypre_CommEntryTypeOffset(comm_entry);
484 dim = hypre_CommEntryTypeDim(comm_entry);
485 length = hypre_CommEntryTypeLengthArray(comm_entry);
486 stride = hypre_CommEntryTypeStrideArray(comm_entry);
487 fprintf(file, "%d: %d,%d,(%d,%d,%d,%d),(%d,%d,%d,%d)\n",
488 i, offset, dim,
489 length[0], length[1], length[2], length[3],
490 stride[0], stride[1], stride[2], stride[3]);
491 }
492 }
493
494 fprintf(file, "\n\n============================\n\n");
495 fprintf(file, "RECV entries:\n\n");
496
497 fprintf(file, "num_recvs = %d\n", hypre_CommPkgNumRecvs(comm_pkg));
498
499 comm_types = hypre_CommPkgCopyToType(comm_pkg);
500
501 comm_type = &comm_types[0];
502 fprintf(file, "process = %d\n", hypre_CommTypeProc(comm_type));
503 fprintf(file, "num_entries = %d\n", hypre_CommTypeNumEntries(comm_type));
504 for (i = 0; i < hypre_CommTypeNumEntries(comm_type); i++)
505 {
506 comm_entry = hypre_CommTypeEntry(comm_type, i);
507 offset = hypre_CommEntryTypeOffset(comm_entry);
508 dim = hypre_CommEntryTypeDim(comm_entry);
509 length = hypre_CommEntryTypeLengthArray(comm_entry);
510 stride = hypre_CommEntryTypeStrideArray(comm_entry);
511 fprintf(file, "%d: %d,%d,(%d,%d,%d,%d),(%d,%d,%d,%d)\n",
512 i, offset, dim,
513 length[0], length[1], length[2], length[3],
514 stride[0], stride[1], stride[2], stride[3]);
515 }
516
517 for (m = 1; m < (hypre_CommPkgNumRecvs(comm_pkg) + 1); m++)
518 {
519 comm_type = &comm_types[m];
520 fprintf(file, "process = %d\n", hypre_CommTypeProc(comm_type));
521 fprintf(file, "num_entries = %d\n", hypre_CommTypeNumEntries(comm_type));
522 }
523
524 fflush(file);
525 fclose(file);
526}
527#endif
528
529 /*------------------------------------------------------
530 * Clean up
531 *------------------------------------------------------*/
532
533 hypre_CommInfoDestroy(comm_info);
534 hypre_TFree(p_comm_types);
535
536 *comm_pkg_ptr = comm_pkg;
537
538 return ierr;
539}
540
541/*--------------------------------------------------------------------------
542 *--------------------------------------------------------------------------*/
543
544int
545hypre_CommTypeSetEntries( hypre_CommType *comm_type,
546 int *boxnums,
547 hypre_Box *boxes,
548 hypre_Index stride,
549 int num_values,
550 hypre_BoxArray *data_space,
551 int *data_offsets )
552{
553 int ierr = 0;
554
555 int num_entries = hypre_CommTypeNumEntries(comm_type);
556 hypre_CommEntryType *entries = hypre_CommTypeEntries(comm_type);
557 hypre_Box *box;
558 hypre_Box *data_box;
559 int i, j;
560
561 for (j = 0; j < num_entries; j++)
562 {
563 i = boxnums[j];
564 box = &boxes[j];
565 data_box = hypre_BoxArrayBox(data_space, i);
566
567 hypre_CommTypeSetEntry(box, stride, data_box, num_values, data_offsets[i],
568 &entries[j]);
569 }
570
571 return ierr;
572}
573
574/*--------------------------------------------------------------------------
575 *--------------------------------------------------------------------------*/
576
577int
578hypre_CommTypeSetEntry( hypre_Box *box,
579 hypre_Index stride,
580 hypre_Box *data_box,
581 int num_values,
582 int data_box_offset,
583 hypre_CommEntryType *comm_entry )
584{
585 int ierr = 0;
586 int *length_array;
587 int *stride_array;
588 hypre_Index size;
589 int i, j, dim;
590
591 /*------------------------------------------------------
592 * Set offset
593 *------------------------------------------------------*/
594
595 hypre_CommEntryTypeOffset(comm_entry) =
596 data_box_offset + hypre_BoxIndexRank(data_box, hypre_BoxIMin(box));
597
598 /*------------------------------------------------------
599 * Set length_array, stride_array, and dim
600 *------------------------------------------------------*/
601
602 length_array = hypre_CommEntryTypeLengthArray(comm_entry);
603 stride_array = hypre_CommEntryTypeStrideArray(comm_entry);
604
605 /* initialize length_array */
606 hypre_BoxGetStrideSize(box, stride, size);
607 for (i = 0; i < 3; i++)
608 {
609 length_array[i] = hypre_IndexD(size, i);
610 }
611 length_array[3] = num_values;
612
613 /* initialize stride_array */
614 for (i = 0; i < 3; i++)
615 {
616 stride_array[i] = hypre_IndexD(stride, i);
617 for (j = 0; j < i; j++)
618 {
619 stride_array[i] *= hypre_BoxSizeD(data_box, j);
620 }
621 }
622 stride_array[3] = hypre_BoxVolume(data_box);
623
624 /* eliminate dimensions with length_array = 1 */
625 dim = 4;
626 i = 0;
627 while (i < dim)
628 {
629 if(length_array[i] == 1)
630 {
631 for(j = i; j < (dim - 1); j++)
632 {
633 length_array[j] = length_array[j+1];
634 stride_array[j] = stride_array[j+1];
635 }
636 length_array[dim - 1] = 1;
637 stride_array[dim - 1] = 1;
638 dim--;
639 }
640 else
641 {
642 i++;
643 }
644 }
645
646#if 0
647 /* sort the array according to length_array (largest to smallest) */
648 for (i = (dim-1); i > 0; i--)
649 {
650 for (j = 0; j < i; j++)
651 {
652 if (length_array[j] < length_array[j+1])
653 {
654 i_tmp = length_array[j];
655 length_array[j] = length_array[j+1];
656 length_array[j+1] = i_tmp;
657
658 i_tmp = stride_array[j];
659 stride_array[j] = stride_array[j+1];
660 stride_array[j+1] = i_tmp;
661 }
662 }
663 }
664#endif
665
666 /* if every len was 1 we need to fix to communicate at least one */
667 if(!dim)
668 {
669 dim = 1;
670 }
671
672 hypre_CommEntryTypeDim(comm_entry) = dim;
673
674 return ierr;
675}
676
677/*--------------------------------------------------------------------------
678 * Initialize a non-blocking communication exchange.
679 *
680 * The communication buffers are created, the send buffer is manually
681 * packed, and the communication requests are posted.
682 *--------------------------------------------------------------------------*/
683
684int
685hypre_InitializeCommunication( hypre_CommPkg *comm_pkg,
686 double *send_data,
687 double *recv_data,
688 hypre_CommHandle **comm_handle_ptr )
689{
690 int ierr = 0;
691
692 hypre_CommHandle *comm_handle;
693
694 int num_sends = hypre_CommPkgNumSends(comm_pkg);
695 int num_recvs = hypre_CommPkgNumRecvs(comm_pkg);
696 MPI_Comm comm = hypre_CommPkgComm(comm_pkg);
697
698 int num_requests;
699 MPI_Request *requests;
700 MPI_Status *status;
701 double **send_buffers;
702 double **recv_buffers;
703
704 hypre_CommType *comm_type;
705 hypre_CommEntryType *comm_entry;
706 int num_entries;
707
708 int *length_array, length;
709 int *stride_array, stride;
710
711 double *dptr, *iptr, *jptr, *kptr, *lptr;
712 int *qptr;
713
714 int i, j, ii, jj, kk, ll;
715 int size;
716
717 /*--------------------------------------------------------------------
718 * allocate requests and status
719 *--------------------------------------------------------------------*/
720
721 num_requests = num_sends + num_recvs;
722 requests = hypre_CTAlloc(MPI_Request, num_requests);
723 status = hypre_CTAlloc(MPI_Status, num_requests);
724
725 /*--------------------------------------------------------------------
726 * allocate buffers
727 *--------------------------------------------------------------------*/
728
729 /* allocate send buffers */
730 send_buffers = hypre_TAlloc(double *, num_sends);
731 if (num_sends > 0)
732 {
733 size = hypre_CommPkgSendBufsize(comm_pkg);
734 send_buffers[0] = hypre_SharedTAlloc(double, size);
735 for (i = 1; i < num_sends; i++)
736 {
737 comm_type = hypre_CommPkgSendType(comm_pkg, i-1);
738 size = hypre_CommTypeBufsize(comm_type);
739 send_buffers[i] = send_buffers[i-1] + size;
740 }
741 }
742
743 /* allocate recv buffers */
744 recv_buffers = hypre_TAlloc(double *, num_recvs);
745 if (num_recvs > 0)
746 {
747 size = hypre_CommPkgRecvBufsize(comm_pkg);
748 recv_buffers[0] = hypre_SharedTAlloc(double, size);
749 for (i = 1; i < num_recvs; i++)
750 {
751 comm_type = hypre_CommPkgRecvType(comm_pkg, i-1);
752 size = hypre_CommTypeBufsize(comm_type);
753 recv_buffers[i] = recv_buffers[i-1] + size;
754 }
755 }
756
757 /*--------------------------------------------------------------------
758 * pack send buffers
759 *--------------------------------------------------------------------*/
760
761 for (i = 0; i < num_sends; i++)
762 {
763 comm_type = hypre_CommPkgSendType(comm_pkg, i);
764 num_entries = hypre_CommTypeNumEntries(comm_type);
765
766 dptr = (double *) send_buffers[i];
767
768 if ( hypre_CommPkgFirstSend(comm_pkg) )
769 {
770 qptr = (int *) send_buffers[i];
771
772 memcpy(qptr, hypre_CommTypeRemBoxnums(comm_type),
773 num_entries*sizeof(int));
774 qptr += num_entries;
775 memcpy(qptr, hypre_CommTypeRemBoxes(comm_type),
776 num_entries*sizeof(hypre_Box));
777
778 hypre_CommTypeRemBoxnums(comm_type) = NULL;
779 hypre_CommTypeRemBoxes(comm_type) = NULL;
780
781 dptr += hypre_CommPrefixSize(num_entries);
782 }
783
784 for (j = 0; j < num_entries; j++)
785 {
786 comm_entry = hypre_CommTypeEntry(comm_type, j);
787 length_array = hypre_CommEntryTypeLengthArray(comm_entry);
788 stride_array = hypre_CommEntryTypeStrideArray(comm_entry);
789
790 lptr = send_data + hypre_CommEntryTypeOffset(comm_entry);
791 for (ll = 0; ll < length_array[3]; ll++)
792 {
793 kptr = lptr;
794 for (kk = 0; kk < length_array[2]; kk++)
795 {
796 jptr = kptr;
797 for (jj = 0; jj < length_array[1]; jj++)
798 {
799 if (stride_array[0] == 1)
800 {
801 memcpy(dptr, jptr, length_array[0]*sizeof(double));
802 }
803 else
804 {
805 iptr = jptr;
806 length = length_array[0];
807 stride = stride_array[0];
808 for (ii = 0; ii < length; ii++)
809 {
810 dptr[ii] = *iptr;
811 iptr += stride;
812 }
813 }
814 dptr += length_array[0];
815 jptr += stride_array[1];
816 }
817 kptr += stride_array[2];
818 }
819 lptr += stride_array[3];
820 }
821 }
822 }
823
824 /*--------------------------------------------------------------------
825 * post receives and initiate sends
826 *--------------------------------------------------------------------*/
827
828 j = 0;
829 for(i = 0; i < num_recvs; i++)
830 {
831 comm_type = hypre_CommPkgRecvType(comm_pkg, i);
832 MPI_Irecv(recv_buffers[i],
833 hypre_CommTypeBufsize(comm_type)*sizeof(double), MPI_BYTE,
834 hypre_CommTypeProc(comm_type), 0, comm, &requests[j++]);
835 if ( hypre_CommPkgFirstRecv(comm_pkg) )
836 {
837 size = hypre_CommPrefixSize(hypre_CommTypeNumEntries(comm_type));
838 hypre_CommTypeBufsize(comm_type) -= size;
839 hypre_CommPkgRecvBufsize(comm_pkg) -= size;
840 }
841 }
842
843 for(i = 0; i < num_sends; i++)
844 {
845 comm_type = hypre_CommPkgSendType(comm_pkg, i);
846 MPI_Isend(send_buffers[i],
847 hypre_CommTypeBufsize(comm_type)*sizeof(double), MPI_BYTE,
848 hypre_CommTypeProc(comm_type), 0, comm, &requests[j++]);
849 if ( hypre_CommPkgFirstSend(comm_pkg) )
850 {
851 size = hypre_CommPrefixSize(hypre_CommTypeNumEntries(comm_type));
852 hypre_CommTypeBufsize(comm_type) -= size;
853 hypre_CommPkgSendBufsize(comm_pkg) -= size;
854 }
855 }
856
857 hypre_ExchangeLocalData(comm_pkg, send_data, recv_data);
858
859 /*--------------------------------------------------------------------
860 * free up remote boxnums and turn off first_send indicator
861 *--------------------------------------------------------------------*/
862
863 if ( hypre_CommPkgFirstSend(comm_pkg) )
864 {
865 comm_type = hypre_CommPkgCopyFromType(comm_pkg);
866 hypre_TFree(hypre_CommTypeRemBoxnums(comm_type));
867 hypre_TFree(hypre_CommTypeRemBoxes(comm_type));
868
869 hypre_CommPkgFirstSend(comm_pkg) = 0;
870 }
871
872 /*--------------------------------------------------------------------
873 * set up comm_handle and return
874 *--------------------------------------------------------------------*/
875
876 comm_handle = hypre_TAlloc(hypre_CommHandle, 1);
877
878 hypre_CommHandleCommPkg(comm_handle) = comm_pkg;
879 hypre_CommHandleSendData(comm_handle) = send_data;
880 hypre_CommHandleRecvData(comm_handle) = recv_data;
881 hypre_CommHandleNumRequests(comm_handle) = num_requests;
882 hypre_CommHandleRequests(comm_handle) = requests;
883 hypre_CommHandleStatus(comm_handle) = status;
884 hypre_CommHandleSendBuffers(comm_handle) = send_buffers;
885 hypre_CommHandleRecvBuffers(comm_handle) = recv_buffers;
886
887 *comm_handle_ptr = comm_handle;
888
889 return ierr;
890}
891
892/*--------------------------------------------------------------------------
893 * Finalize a communication exchange. This routine blocks until all
894 * of the communication requests are completed.
895 *
896 * The communication requests are completed, and the receive buffer is
897 * manually unpacked.
898 *--------------------------------------------------------------------------*/
899
900int
901hypre_FinalizeCommunication( hypre_CommHandle *comm_handle )
902{
903
904 int ierr = 0;
905
906 hypre_CommPkg *comm_pkg = hypre_CommHandleCommPkg(comm_handle);
907 double **send_buffers = hypre_CommHandleSendBuffers(comm_handle);
908 double **recv_buffers = hypre_CommHandleRecvBuffers(comm_handle);
909 int num_sends = hypre_CommPkgNumSends(comm_pkg);
910 int num_recvs = hypre_CommPkgNumRecvs(comm_pkg);
911
912 hypre_CommType *comm_type;
913 hypre_CommEntryType *comm_entry;
914 int num_entries;
915
916 int *length_array, length;
917 int *stride_array, stride;
918
919 double *iptr, *jptr, *kptr, *lptr;
920 double *dptr;
921 int *qptr;
922
923 int *boxnums;
924 hypre_Box *boxes;
925
926 int i, j, ii, jj, kk, ll;
927
928 /*--------------------------------------------------------------------
929 * finish communications
930 *--------------------------------------------------------------------*/
931
932 if (hypre_CommHandleNumRequests(comm_handle))
933 {
934 MPI_Waitall(hypre_CommHandleNumRequests(comm_handle),
935 hypre_CommHandleRequests(comm_handle),
936 hypre_CommHandleStatus(comm_handle));
937 }
938
939 /*--------------------------------------------------------------------
940 * unpack recv buffers
941 *--------------------------------------------------------------------*/
942
943 for (i = 0; i < num_recvs; i++)
944 {
945 comm_type = hypre_CommPkgRecvType(comm_pkg, i);
946 num_entries = hypre_CommTypeNumEntries(comm_type);
947
948 dptr = (double *) recv_buffers[i];
949
950 if ( hypre_CommPkgFirstRecv(comm_pkg) )
951 {
952 qptr = (int *) recv_buffers[i];
953 boxnums = (int *) qptr;
954 qptr += num_entries;
955 boxes = (hypre_Box *) qptr;
956
957 hypre_CommTypeSetEntries(comm_type, boxnums, boxes,
958 hypre_CommPkgRecvStride(comm_pkg),
959 hypre_CommPkgNumValues(comm_pkg),
960 hypre_CommPkgRecvDataSpace(comm_pkg),
961 hypre_CommPkgRecvDataOffsets(comm_pkg) );
962
963 dptr += hypre_CommPrefixSize(num_entries);
964 }
965
966 for (j = 0; j < num_entries; j++)
967 {
968 comm_entry = hypre_CommTypeEntry(comm_type, j);
969 length_array = hypre_CommEntryTypeLengthArray(comm_entry);
970 stride_array = hypre_CommEntryTypeStrideArray(comm_entry);
971
972 lptr = hypre_CommHandleRecvData(comm_handle) +
973 hypre_CommEntryTypeOffset(comm_entry);
974 for (ll = 0; ll < length_array[3]; ll++)
975 {
976 kptr = lptr;
977 for (kk = 0; kk < length_array[2]; kk++)
978 {
979 jptr = kptr;
980 for (jj = 0; jj < length_array[1]; jj++)
981 {
982 if (stride_array[0] == 1)
983 {
984 memcpy(jptr, dptr, length_array[0]*sizeof(double));
985 }
986 else
987 {
988 iptr = jptr;
989 length = length_array[0];
990 stride = stride_array[0];
991 for (ii = 0; ii < length; ii++)
992 {
993 *iptr = dptr[ii];
994 iptr += stride;
995 }
996 }
997 dptr += length_array[0];
998 jptr += stride_array[1];
999 }
1000 kptr += stride_array[2];
1001 }
1002 lptr += stride_array[3];
1003 }
1004 }
1005 }
1006
1007 /*--------------------------------------------------------------------
1008 * turn off first_recv indicator
1009 *--------------------------------------------------------------------*/
1010
1011 hypre_CommPkgFirstRecv(comm_pkg) = 0;
1012
1013 /*--------------------------------------------------------------------
1014 * Free up communication handle
1015 *--------------------------------------------------------------------*/
1016
1017 hypre_TFree(hypre_CommHandleRequests(comm_handle));
1018 hypre_TFree(hypre_CommHandleStatus(comm_handle));
1019 if (num_sends > 0)
1020 {
1021 hypre_SharedTFree(send_buffers[0]);
1022 }
1023 if (num_recvs > 0)
1024 {
1025 hypre_SharedTFree(recv_buffers[0]);
1026 }
1027 hypre_TFree(send_buffers);
1028 hypre_TFree(recv_buffers);
1029 hypre_TFree(comm_handle);
1030
1031 return ierr;
1032}
1033
1034/*--------------------------------------------------------------------------
1035 * Execute local data exchanges.
1036 *--------------------------------------------------------------------------*/
1037
1038int
1039hypre_ExchangeLocalData( hypre_CommPkg *comm_pkg,
1040 double *send_data,
1041 double *recv_data )
1042{
1043 hypre_CommType *copy_from_type;
1044 hypre_CommType *copy_to_type;
1045 hypre_CommEntryType *copy_from_entry;
1046 hypre_CommEntryType *copy_to_entry;
1047
1048 double *from_dp;
1049 int *from_stride_array;
1050 int from_i;
1051 double *to_dp;
1052 int *to_stride_array;
1053 int to_i;
1054
1055 int *length_array;
1056 int i0, i1, i2, i3;
1057
1058 int i;
1059 int ierr = 0;
1060
1061 /*--------------------------------------------------------------------
1062 * copy local data
1063 *--------------------------------------------------------------------*/
1064
1065 copy_from_type = hypre_CommPkgCopyFromType(comm_pkg);
1066 copy_to_type = hypre_CommPkgCopyToType(comm_pkg);
1067
1068 for (i = 0; i < hypre_CommTypeNumEntries(copy_from_type); i++)
1069 {
1070 copy_from_entry = hypre_CommTypeEntry(copy_from_type, i);
1071 copy_to_entry = hypre_CommTypeEntry(copy_to_type, i);
1072
1073 from_dp = send_data + hypre_CommEntryTypeOffset(copy_from_entry);
1074 to_dp = recv_data + hypre_CommEntryTypeOffset(copy_to_entry);
1075
1076 /* copy data only when necessary */
1077 if (to_dp != from_dp)
1078 {
1079 length_array = hypre_CommEntryTypeLengthArray(copy_from_entry);
1080
1081 from_stride_array = hypre_CommEntryTypeStrideArray(copy_from_entry);
1082 to_stride_array = hypre_CommEntryTypeStrideArray(copy_to_entry);
1083
1084 for (i3 = 0; i3 < length_array[3]; i3++)
1085 {
1086 for (i2 = 0; i2 < length_array[2]; i2++)
1087 {
1088 for (i1 = 0; i1 < length_array[1]; i1++)
1089 {
1090 from_i = (i3*from_stride_array[3] +
1091 i2*from_stride_array[2] +
1092 i1*from_stride_array[1] );
1093 to_i = (i3*to_stride_array[3] +
1094 i2*to_stride_array[2] +
1095 i1*to_stride_array[1] );
1096 for (i0 = 0; i0 < length_array[0]; i0++)
1097 {
1098 to_dp[to_i] = from_dp[from_i];
1099
1100 from_i += from_stride_array[0];
1101 to_i += to_stride_array[0];
1102 }
1103 }
1104 }
1105 }
1106 }
1107 }
1108
1109 return ( ierr );
1110}
1111
1112/*--------------------------------------------------------------------------
1113 *--------------------------------------------------------------------------*/
1114
1115int
1116hypre_CommPkgDestroy( hypre_CommPkg *comm_pkg )
1117{
1118 int ierr = 0;
1119 hypre_CommType *comm_types;
1120
1121 if (comm_pkg)
1122 {
1123 comm_types = hypre_CommPkgCopyToType(comm_pkg);
1124 hypre_TFree(hypre_CommTypeEntries(comm_types));
1125 hypre_TFree(hypre_CommTypeLocBoxnums(comm_types));
1126 hypre_TFree(hypre_CommTypeRemBoxnums(comm_types));
1127 hypre_TFree(hypre_CommTypeLocBoxes(comm_types));
1128 hypre_TFree(hypre_CommTypeRemBoxes(comm_types));
1129 hypre_TFree(comm_types);
1130
1131 comm_types = hypre_CommPkgCopyFromType(comm_pkg);
1132 hypre_TFree(hypre_CommTypeEntries(comm_types));
1133 hypre_TFree(hypre_CommTypeLocBoxnums(comm_types));
1134 hypre_TFree(hypre_CommTypeRemBoxnums(comm_types));
1135 hypre_TFree(hypre_CommTypeLocBoxes(comm_types));
1136 hypre_TFree(hypre_CommTypeRemBoxes(comm_types));
1137 hypre_TFree(comm_types);
1138
1139 hypre_TFree(hypre_CommPkgRecvDataOffsets(comm_pkg));
1140 hypre_BoxArrayDestroy(hypre_CommPkgRecvDataSpace(comm_pkg));
1141
1142 hypre_TFree(comm_pkg);
1143 }
1144
1145 return ierr;
1146}
Note: See TracBrowser for help on using the repository browser.