source: CIVL/examples/mpi-omp/AMG2013/struct_mv/struct_grid.c@ 397ae5f

main test-branch
Last change on this file since 397ae5f was ea777aa, checked in by Alex Wilton <awilton@…>, 3 years ago

Moved examples, include, build_default.properties, common.xml, and README out from dev.civl.com into the root of the repo.

git-svn-id: ​svn://vsl.cis.udel.edu/civl/trunk@5704 fb995dde-84ed-4084-dfe6-e5aef3e2452c
  • Property mode set to 100644
File size: 19.2 KB
Line 
1/*BHEADER**********************************************************************
2 * Copyright (c) 2008, Lawrence Livermore National Security, LLC.
3 * Produced at the Lawrence Livermore National Laboratory.
4 * This file is part of HYPRE. See file COPYRIGHT for details.
5 *
6 * HYPRE is free software; you can redistribute it and/or modify it under the
7 * terms of the GNU Lesser General Public License (as published by the Free
8 * Software Foundation) version 2.1 dated February 1999.
9 *
10 * $Revision: 2.4 $
11 ***********************************************************************EHEADER*/
12
13
14/******************************************************************************
15 *
16 * Member functions for hypre_StructGrid class.
17 *
18 *****************************************************************************/
19
20#include "headers.h"
21
22#define DEBUG 0
23
24#if DEBU
25char filename[255];
26FILE *file;
27int my_rank;
28#endif
29
30/*--------------------------------------------------------------------------
31 * hypre_StructGridCreate
32 *--------------------------------------------------------------------------*/
33
34int
35hypre_StructGridCreate( MPI_Comm comm,
36 int dim,
37 hypre_StructGrid **grid_ptr)
38{
39 hypre_StructGrid *grid;
40 int i;
41
42 grid = hypre_TAlloc(hypre_StructGrid, 1);
43
44 hypre_StructGridComm(grid) = comm;
45 hypre_StructGridDim(grid) = dim;
46 hypre_StructGridBoxes(grid) = hypre_BoxArrayCreate(0);
47 hypre_StructGridIDs(grid) = NULL;
48 hypre_StructGridNeighbors(grid) = NULL;
49 hypre_StructGridMaxDistance(grid) = 2;
50 hypre_StructGridBoundingBox(grid) = NULL;
51 hypre_StructGridLocalSize(grid) = 0;
52 hypre_StructGridGlobalSize(grid) = 0;
53 hypre_SetIndex(hypre_StructGridPeriodic(grid), 0, 0, 0);
54 hypre_StructGridRefCount(grid) = 1;
55
56 /* additional defaults for the grid ghosts GEC0902 */
57
58 hypre_StructGridGhlocalSize(grid) = 0;
59
60 for (i = 0; i < 6; i++)
61 {
62 hypre_StructGridNumGhost(grid)[i] = 1;
63 }
64
65 *grid_ptr = grid;
66
67 return 0;
68}
69
70/*--------------------------------------------------------------------------
71 * hypre_StructGridRef
72 *--------------------------------------------------------------------------*/
73
74int
75hypre_StructGridRef( hypre_StructGrid *grid,
76 hypre_StructGrid **grid_ref)
77{
78 hypre_StructGridRefCount(grid) ++;
79 *grid_ref = grid;
80
81 return 0;
82}
83
84/*--------------------------------------------------------------------------
85 * hypre_StructGridDestroy
86 *--------------------------------------------------------------------------*/
87
88int
89hypre_StructGridDestroy( hypre_StructGrid *grid )
90{
91 int ierr = 0;
92
93 if (grid)
94 {
95 hypre_StructGridRefCount(grid) --;
96 if (hypre_StructGridRefCount(grid) == 0)
97 {
98 hypre_BoxDestroy(hypre_StructGridBoundingBox(grid));
99 hypre_BoxNeighborsDestroy(hypre_StructGridNeighbors(grid));
100 hypre_TFree(hypre_StructGridIDs(grid));
101 hypre_BoxArrayDestroy(hypre_StructGridBoxes(grid));
102 hypre_TFree(grid);
103 }
104 }
105
106 return ierr;
107}
108
109/*--------------------------------------------------------------------------
110 * hypre_StructGridSetHoodInfo
111 *--------------------------------------------------------------------------*/
112
113int
114hypre_StructGridSetHoodInfo( hypre_StructGrid *grid,
115 int max_distance )
116{
117 int ierr = 0;
118
119 hypre_StructGridMaxDistance(grid) = max_distance;
120
121 return ierr;
122}
123
124/*--------------------------------------------------------------------------
125 * hypre_StructGridSetPeriodic
126 *--------------------------------------------------------------------------*/
127
128int
129hypre_StructGridSetPeriodic( hypre_StructGrid *grid,
130 hypre_Index periodic)
131{
132 int ierr = 0;
133
134 hypre_CopyIndex(periodic, hypre_StructGridPeriodic(grid));
135
136 return ierr;
137}
138
139/*--------------------------------------------------------------------------
140 * hypre_StructGridSetExtents
141 *--------------------------------------------------------------------------*/
142
143int
144hypre_StructGridSetExtents( hypre_StructGrid *grid,
145 hypre_Index ilower,
146 hypre_Index iupper )
147{
148 int ierr = 0;
149 hypre_Box *box;
150
151 box = hypre_BoxCreate();
152 hypre_BoxSetExtents(box, ilower, iupper);
153 hypre_AppendBox(box, hypre_StructGridBoxes(grid));
154 hypre_BoxDestroy(box);
155
156 return ierr;
157}
158
159/*--------------------------------------------------------------------------
160 * hypre_StructGridSetBoxes
161 *--------------------------------------------------------------------------*/
162
163int
164hypre_StructGridSetBoxes( hypre_StructGrid *grid,
165 hypre_BoxArray *boxes )
166{
167 int ierr = 0;
168
169 hypre_TFree(hypre_StructGridBoxes(grid));
170 hypre_StructGridBoxes(grid) = boxes;
171
172 return ierr;
173}
174
175/*--------------------------------------------------------------------------
176 * hypre_StructGridSetHood
177 *--------------------------------------------------------------------------*/
178
179int
180hypre_StructGridSetHood( hypre_StructGrid *grid,
181 hypre_BoxArray *hood_boxes,
182 int *hood_procs,
183 int *hood_ids,
184 int first_local,
185 int num_local,
186 hypre_Box *bounding_box )
187{
188 int ierr = 0;
189
190 hypre_BoxArray *boxes;
191 int *ids;
192 hypre_BoxNeighbors *neighbors;
193
194 int i, ilocal;
195
196 boxes = hypre_BoxArrayCreate(num_local);
197 ids = hypre_TAlloc(int, num_local);
198 for (i = 0; i < num_local; i++)
199 {
200 ilocal = first_local + i;
201 hypre_CopyBox(hypre_BoxArrayBox(hood_boxes, ilocal),
202 hypre_BoxArrayBox(boxes, i));
203 ids[i] = hood_ids[ilocal];
204 }
205 hypre_TFree(hypre_StructGridBoxes(grid));
206 hypre_TFree(hypre_StructGridIDs(grid));
207 hypre_StructGridBoxes(grid) = boxes;
208 hypre_StructGridIDs(grid) = ids;
209
210 hypre_BoxNeighborsCreate(hood_boxes, hood_procs, hood_ids,
211 first_local, num_local, &neighbors);
212 hypre_StructGridNeighbors(grid) = neighbors;
213
214 hypre_BoxDestroy(hypre_StructGridBoundingBox(grid));
215 hypre_StructGridBoundingBox(grid) = bounding_box;
216
217 return ierr;
218}
219
220/*--------------------------------------------------------------------------
221 * hypre_StructGridAssemble
222 *
223 * NOTE: Box ids are set here for the non-periodic boxes. They are
224 * globally unique, and appear in increasing order. The periodic
225 * boxes are definedin BoxNeighborsAssemble.
226 *
227 * NOTE: Box procs are set here. They appear in non-decreasing order
228 * for the non-periodic boxes.
229 *
230 *--------------------------------------------------------------------------*/
231
232int
233hypre_StructGridAssemble( hypre_StructGrid *grid )
234{
235 int ierr = 0;
236
237 MPI_Comm comm = hypre_StructGridComm(grid);
238 int dim = hypre_StructGridDim(grid);
239 hypre_BoxArray *boxes = hypre_StructGridBoxes(grid);
240 int *ids;
241 hypre_BoxNeighbors *neighbors = hypre_StructGridNeighbors(grid);
242 int max_distance = hypre_StructGridMaxDistance(grid);
243 hypre_Box *bounding_box = hypre_StructGridBoundingBox(grid);
244 hypre_IndexRef periodic = hypre_StructGridPeriodic(grid);
245
246 hypre_Box *box;
247 hypre_BoxArray *all_boxes;
248 int *all_procs;
249 int *all_ids;
250 int first_local;
251 int num_local;
252 int size;
253 int prune;
254 int i, d, idmin, idmax;
255 /* GEC new declarations for the ghost size local */
256 int *numghost;
257 int ghostsize;
258 hypre_Box *ghostbox;
259
260 prune = 1;
261
262 if (neighbors == NULL)
263 {
264 /* gather grid box info */
265 hypre_GatherAllBoxes(comm, boxes, &all_boxes, &all_procs, &first_local);
266
267 /* set bounding box */
268 bounding_box = hypre_BoxCreate();
269 for (d = 0; d < dim; d++)
270 {
271 idmin = hypre_BoxIMinD(hypre_BoxArrayBox(all_boxes, 0), d);
272 idmax = hypre_BoxIMaxD(hypre_BoxArrayBox(all_boxes, 0), d);
273 hypre_ForBoxI(i, all_boxes)
274 {
275 box = hypre_BoxArrayBox(all_boxes, i);
276 idmin = hypre_min(idmin, hypre_BoxIMinD(box, d));
277 idmax = hypre_max(idmax, hypre_BoxIMaxD(box, d));
278 }
279 hypre_BoxIMinD(bounding_box, d) = idmin;
280 hypre_BoxIMaxD(bounding_box, d) = idmax;
281 }
282 for (d = dim; d < 3; d++)
283 {
284 hypre_BoxIMinD(bounding_box, d) = 0;
285 hypre_BoxIMaxD(bounding_box, d) = 0;
286 }
287 hypre_StructGridBoundingBox(grid) = bounding_box;
288
289 /* set global size */
290 size = 0;
291 hypre_ForBoxI(i, all_boxes)
292 {
293 box = hypre_BoxArrayBox(all_boxes, i);
294 size += hypre_BoxVolume(box);
295 }
296 hypre_StructGridGlobalSize(grid) = size;
297 }
298
299 if (neighbors == NULL)
300 {
301 /* set all_ids */
302 all_ids = hypre_TAlloc(int, hypre_BoxArraySize(all_boxes));
303 hypre_ForBoxI(i, all_boxes)
304 {
305 all_ids[i] = i;
306 }
307
308 /* set neighbors */
309 num_local = hypre_BoxArraySize(boxes);
310 hypre_BoxNeighborsCreate(all_boxes, all_procs, all_ids,
311 first_local, num_local, &neighbors);
312 hypre_StructGridNeighbors(grid) = neighbors;
313
314 /* set ids */
315 ids = hypre_TAlloc(int, hypre_BoxArraySize(boxes));
316 hypre_ForBoxI(i, boxes)
317 {
318 ids[i] = all_ids[first_local + i];
319 }
320 hypre_StructGridIDs(grid) = ids;
321
322 prune = 1;
323 }
324
325 hypre_BoxNeighborsAssemble(neighbors, periodic, max_distance, prune);
326
327 /* compute local size */
328
329 size = 0;
330 ghostsize = 0;
331 hypre_ForBoxI(i, boxes)
332 {
333 box = hypre_BoxArrayBox(boxes, i);
334 size += hypre_BoxVolume(box);
335 }
336
337 hypre_StructGridLocalSize(grid) = size;
338
339 /* GEC0902 expand the box to include the ghosts. Create, copy and expand
340 * the ghostbox and finally inserting into the ghlocalsize. As a reminder
341 * the boxes variable is the localboxes of the grid (owned by the processor)
342 */
343
344 numghost = hypre_StructGridNumGhost(grid) ;
345 ghostsize = 0;
346 ghostbox = hypre_BoxCreate();
347 hypre_ForBoxI(i, boxes)
348 {
349 box = hypre_BoxArrayBox(boxes, i);
350
351 hypre_CopyBox(box, ghostbox);
352 hypre_BoxExpand(ghostbox, numghost);
353
354 /* for (d = 0; d < 3; d++)
355 * {
356 * hypre_BoxIminD(ghostbox, d) -= numghost[2*d];
357 * hypre_BoxImaxD(ghostbox, d) += numghost[2*d + 1];
358 * } */
359
360 ghostsize += hypre_BoxVolume(ghostbox);
361
362 }
363
364 hypre_StructGridGhlocalSize(grid) = ghostsize;
365 hypre_BoxDestroy(ghostbox);
366
367#if DEBUG
368{
369 hypre_BoxNeighbors *neighbors;
370 int *procs, *boxnums;
371 int num_neighbors, i;
372
373 MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
374
375 sprintf(filename, "zgrid.%05d", my_rank);
376
377 if ((file = fopen(filename, "a")) == NULL)
378 {
379 printf("Error: can't open output file %s\n", filename);
380 exit(1);
381 }
382
383 fprintf(file, "\n\n============================\n\n");
384
385 hypre_StructGridPrint(file, grid);
386
387 neighbors = hypre_StructGridNeighbors(grid);
388 num_neighbors = hypre_BoxArraySize(hypre_BoxNeighborsBoxes(neighbors));
389 procs = hypre_BoxNeighborsProcs(neighbors);
390 boxnums = hypre_BoxNeighborsBoxnums(neighbors);
391 fprintf(file, "num_neighbors = %d\n", num_neighbors);
392 for (i = 0; i < num_neighbors; i++)
393 {
394 fprintf(file, "%d: (%d, %d)\n", i, procs[i], boxnums[i]);
395 }
396
397 fflush(file);
398 fclose(file);
399}
400#endif
401
402 return ierr;
403}
404
405/*--------------------------------------------------------------------------
406 * hypre_GatherAllBoxes
407 *--------------------------------------------------------------------------*/
408
409int
410hypre_GatherAllBoxes(MPI_Comm comm,
411 hypre_BoxArray *boxes,
412 hypre_BoxArray **all_boxes_ptr,
413 int **all_procs_ptr,
414 int *first_local_ptr)
415{
416 hypre_BoxArray *all_boxes;
417 int *all_procs;
418 int first_local;
419 int all_boxes_size;
420
421 hypre_Box *box;
422 hypre_Index imin;
423 hypre_Index imax;
424
425 int num_all_procs, my_rank;
426
427 int *sendbuf;
428 int sendcount;
429 int *recvbuf;
430 int *recvcounts;
431 int *displs;
432 int recvbuf_size;
433
434 int i, p, b, d;
435 int ierr = 0;
436
437 /*-----------------------------------------------------
438 * Accumulate the box info
439 *-----------------------------------------------------*/
440
441 MPI_Comm_size(comm, &num_all_procs);
442 MPI_Comm_rank(comm, &my_rank);
443
444 /* compute recvcounts and displs */
445 sendcount = 7*hypre_BoxArraySize(boxes);
446 recvcounts = hypre_SharedTAlloc(int, num_all_procs);
447 displs = hypre_TAlloc(int, num_all_procs);
448 MPI_Allgather(&sendcount, 1, MPI_INT,
449 recvcounts, 1, MPI_INT, comm);
450 displs[0] = 0;
451 recvbuf_size = recvcounts[0];
452 for (p = 1; p < num_all_procs; p++)
453 {
454 displs[p] = displs[p-1] + recvcounts[p-1];
455 recvbuf_size += recvcounts[p];
456 }
457
458 /* allocate sendbuf and recvbuf */
459 sendbuf = hypre_TAlloc(int, sendcount);
460 recvbuf = hypre_SharedTAlloc(int, recvbuf_size);
461
462 /* put local box extents and process number into sendbuf */
463 i = 0;
464 for (b = 0; b < hypre_BoxArraySize(boxes); b++)
465 {
466 sendbuf[i++] = my_rank;
467
468 box = hypre_BoxArrayBox(boxes, b);
469 for (d = 0; d < 3; d++)
470 {
471 sendbuf[i++] = hypre_BoxIMinD(box, d);
472 sendbuf[i++] = hypre_BoxIMaxD(box, d);
473 }
474 }
475
476 /* get global grid info */
477 MPI_Allgatherv(sendbuf, sendcount, MPI_INT,
478 recvbuf, recvcounts, displs, MPI_INT, comm);
479
480 /* sort recvbuf by process rank? */
481
482 /*-----------------------------------------------------
483 * Create all_boxes, etc.
484 *-----------------------------------------------------*/
485
486 /* unpack recvbuf box info */
487 all_boxes_size = recvbuf_size / 7;
488 all_boxes = hypre_BoxArrayCreate(all_boxes_size);
489 all_procs = hypre_TAlloc(int, all_boxes_size);
490 first_local = -1;
491 i = 0;
492 b = 0;
493 box = hypre_BoxCreate();
494 while (i < recvbuf_size)
495 {
496 all_procs[b] = recvbuf[i++];
497 for (d = 0; d < 3; d++)
498 {
499 hypre_IndexD(imin, d) = recvbuf[i++];
500 hypre_IndexD(imax, d) = recvbuf[i++];
501 }
502 hypre_BoxSetExtents(box, imin, imax);
503 hypre_CopyBox(box, hypre_BoxArrayBox(all_boxes, b));
504
505 if ((first_local < 0) && (all_procs[b] == my_rank))
506 {
507 first_local = b;
508 }
509
510 b++;
511 }
512 hypre_BoxDestroy(box);
513
514 /*-----------------------------------------------------
515 * Return
516 *-----------------------------------------------------*/
517
518 hypre_TFree(sendbuf);
519 hypre_SharedTFree(recvbuf);
520 hypre_SharedTFree(recvcounts);
521 hypre_TFree(displs);
522
523 *all_boxes_ptr = all_boxes;
524 *all_procs_ptr = all_procs;
525 *first_local_ptr = first_local;
526
527 return ierr;
528}
529
530/*--------------------------------------------------------------------------
531 * hypre_ComputeBoxnums
532 *
533 * It is assumed that, for any process number in 'procs', all of that
534 * processes local boxes appear in the 'boxes' array.
535 *
536 * It is assumed that the boxes in 'boxes' are ordered by associated
537 * process number then by their local ordering on that process.
538 *
539 *--------------------------------------------------------------------------*/
540
541int
542hypre_ComputeBoxnums(hypre_BoxArray *boxes,
543 int *procs,
544 int **boxnums_ptr)
545{
546 int ierr = 0;
547
548 int *boxnums;
549 int num_boxes;
550 int p, b, boxnum;
551
552 /*-----------------------------------------------------
553 *-----------------------------------------------------*/
554
555 num_boxes = hypre_BoxArraySize(boxes);
556 boxnums = hypre_TAlloc(int, num_boxes);
557
558 p = -1;
559 for(b = 0; b < num_boxes; b++)
560 {
561 /* start boxnum count at zero for each new process */
562 if (procs[b] != p)
563 {
564 boxnum = 0;
565 p = procs[b];
566 }
567 boxnums[b] = boxnum;
568 boxnum++;
569 }
570
571 *boxnums_ptr = boxnums;
572
573 return ierr;
574}
575
576/*--------------------------------------------------------------------------
577 * hypre_StructGridPrint
578 *--------------------------------------------------------------------------*/
579
580int
581hypre_StructGridPrint( FILE *file,
582 hypre_StructGrid *grid )
583{
584 int ierr = 0;
585
586 hypre_BoxArray *boxes;
587 hypre_Box *box;
588
589 int i;
590
591 fprintf(file, "%d\n", hypre_StructGridDim(grid));
592
593 boxes = hypre_StructGridBoxes(grid);
594 fprintf(file, "%d\n", hypre_BoxArraySize(boxes));
595 hypre_ForBoxI(i, boxes)
596 {
597 box = hypre_BoxArrayBox(boxes, i);
598 fprintf(file, "%d: (%d, %d, %d) x (%d, %d, %d)\n",
599 i,
600 hypre_BoxIMinX(box),
601 hypre_BoxIMinY(box),
602 hypre_BoxIMinZ(box),
603 hypre_BoxIMaxX(box),
604 hypre_BoxIMaxY(box),
605 hypre_BoxIMaxZ(box));
606 }
607
608 return ierr;
609}
610
611/*--------------------------------------------------------------------------
612 * hypre_StructGridRead
613 *--------------------------------------------------------------------------*/
614
615int
616hypre_StructGridRead( MPI_Comm comm,
617 FILE *file,
618 hypre_StructGrid **grid_ptr )
619{
620 int ierr = 0;
621
622 hypre_StructGrid *grid;
623
624 hypre_Index ilower;
625 hypre_Index iupper;
626
627 int dim;
628 int num_boxes;
629
630 int i, idummy;
631
632 fscanf(file, "%d\n", &dim);
633 hypre_StructGridCreate(comm, dim, &grid);
634
635 fscanf(file, "%d\n", &num_boxes);
636 for (i = 0; i < num_boxes; i++)
637 {
638 fscanf(file, "%d: (%d, %d, %d) x (%d, %d, %d)\n",
639 &idummy,
640 &hypre_IndexX(ilower),
641 &hypre_IndexY(ilower),
642 &hypre_IndexZ(ilower),
643 &hypre_IndexX(iupper),
644 &hypre_IndexY(iupper),
645 &hypre_IndexZ(iupper));
646
647 hypre_StructGridSetExtents(grid, ilower, iupper);
648 }
649
650#ifdef HYPRE_NO_GLOBAL_PARTITION
651 hypre_StructGridAssembleWithAP(grid);
652#else
653 hypre_StructGridAssemble(grid);
654#endif
655 *grid_ptr = grid;
656
657 return ierr;
658}
659
660/*------------------------------------------------------------------------------
661 * GEC0902 hypre_StructGridSetNumGhost
662 *
663 * the purpose is to set num ghost in the structure grid. It is identical
664 * to the function that is used in the structure vector entity.
665 *-----------------------------------------------------------------------------*/
666
667int
668hypre_StructGridSetNumGhost( hypre_StructGrid *grid, int *num_ghost )
669{
670 int ierr = 0;
671 int i;
672
673 for (i = 0; i < 6; i++)
674 {
675 hypre_StructGridNumGhost(grid)[i] = num_ghost[i];
676 }
677
678 return ierr;
679}
Note: See TracBrowser for help on using the repository browser.