source: CIVL/examples/mpi-omp/AMG2013/parcsr_ls/par_rap_communication.c

main
Last change on this file was ea777aa, checked in by Alex Wilton <awilton@…>, 3 years ago

Moved examples, include, build_default.properties, common.xml, and README out from dev.civl.com into the root of the repo.

git-svn-id: ​svn://vsl.cis.udel.edu/civl/trunk@5704 fb995dde-84ed-4084-dfe6-e5aef3e2452c
  • Property mode set to 100644
File size: 11.0 KB
Line 
1/*BHEADER**********************************************************************
2 * Copyright (c) 2008, Lawrence Livermore National Security, LLC.
3 * Produced at the Lawrence Livermore National Laboratory.
4 * This file is part of HYPRE. See file COPYRIGHT for details.
5 *
6 * HYPRE is free software; you can redistribute it and/or modify it under the
7 * terms of the GNU Lesser General Public License (as published by the Free
8 * Software Foundation) version 2.1 dated February 1999.
9 *
10 * $Revision: 2.4 $
11 ***********************************************************************EHEADER*/
12
13
14
15
16#include "headers.h"
17
18int
19hypre_GetCommPkgRTFromCommPkgA( hypre_ParCSRMatrix *RT,
20 hypre_ParCSRMatrix *A,
21 int *fine_to_coarse,
22 int *tmp_map_offd)
23{
24 MPI_Comm comm = hypre_ParCSRMatrixComm(RT);
25 hypre_ParCSRCommPkg *comm_pkg_A = hypre_ParCSRMatrixCommPkg(A);
26 hypre_ParCSRCommHandle *comm_handle;
27 int num_recvs_A = hypre_ParCSRCommPkgNumRecvs(comm_pkg_A);
28 int *recv_procs_A = hypre_ParCSRCommPkgRecvProcs(comm_pkg_A);
29 int *recv_vec_starts_A = hypre_ParCSRCommPkgRecvVecStarts(comm_pkg_A);
30 int num_sends_A = hypre_ParCSRCommPkgNumSends(comm_pkg_A);
31 int *send_procs_A = hypre_ParCSRCommPkgSendProcs(comm_pkg_A);
32 int *send_map_starts_A = hypre_ParCSRCommPkgSendMapStarts(comm_pkg_A);
33
34 hypre_ParCSRCommPkg *comm_pkg;
35 int num_recvs_RT;
36 int *recv_procs_RT;
37 int *recv_vec_starts_RT;
38 int num_sends_RT;
39 int *send_procs_RT;
40 int *send_map_starts_RT;
41 int *send_map_elmts_RT;
42
43 HYPRE_BigInt *col_map_offd_RT = hypre_ParCSRMatrixColMapOffd(RT);
44 int num_cols_offd_RT = hypre_CSRMatrixNumCols( hypre_ParCSRMatrixOffd(RT));
45 int num_cols_A_offd = hypre_CSRMatrixNumCols( hypre_ParCSRMatrixOffd(A));
46 int n_fine = hypre_CSRMatrixNumRows(hypre_ParCSRMatrixDiag(A));
47
48 HYPRE_BigInt first_col_diag = hypre_ParCSRMatrixFirstColDiag(RT);
49 HYPRE_BigInt *fine_to_coarse_offd;
50 HYPRE_BigInt *big_buf_data;
51 HYPRE_BigInt *send_big_elmts;
52
53 int i, j;
54 int vec_len, vec_start;
55 int num_procs, my_id, num_threads;
56 int ierr = 0;
57 int num_requests;
58 int offd_col, proc_num;
59
60 int *proc_mark;
61 int *change_array;
62
63 int *coarse_counter;
64 int coarse_shift;
65 int ns, ne, index;
66 int size, rest, start;
67 HYPRE_BigInt my_first_cpt;
68
69 MPI_Request *requests;
70 MPI_Status *status;
71
72 MPI_Comm_size(comm,&num_procs);
73 MPI_Comm_rank(comm,&my_id);
74 num_threads = hypre_NumThreads();
75
76/*--------------------------------------------------------------------------
77 * determine num_recvs, recv_procs and recv_vec_starts for RT
78 *--------------------------------------------------------------------------*/
79
80 proc_mark = hypre_CTAlloc(int, num_recvs_A);
81
82 for (i=0; i < num_recvs_A; i++)
83 proc_mark[i] = 0;
84
85 proc_num = 0;
86 num_recvs_RT = 0;
87 if (num_cols_offd_RT)
88 {
89 for (i=0; i < num_recvs_A; i++)
90 {
91 for (j=recv_vec_starts_A[i]; j<recv_vec_starts_A[i+1]; j++)
92 {
93 offd_col = tmp_map_offd[proc_num];
94 if (offd_col == j)
95 {
96 proc_mark[i]++;
97 proc_num++;
98 if (proc_num == num_cols_offd_RT) break;
99 }
100 }
101 if (proc_mark[i]) num_recvs_RT++;
102 if (proc_num == num_cols_offd_RT) break;
103 }
104 }
105
106 fine_to_coarse_offd = hypre_CTAlloc(HYPRE_BigInt, num_cols_A_offd);
107 big_buf_data = hypre_CTAlloc(HYPRE_BigInt, send_map_starts_A[num_sends_A]);
108 coarse_counter = hypre_CTAlloc(int, num_threads);
109
110#ifdef HYPRE_NO_GLOBAL_PARTITION
111 my_first_cpt = hypre_ParCSRMatrixColStarts(RT)[0];
112#else
113 my_first_cpt = hypre_ParCSRMatrixColStarts(RT)[my_id];
114#endif
115
116#define HYPRE_SMP_PRIVATE i,j,ns,ne,size,rest,coarse_shift
117#include "../utilities/hypre_smp_forloop.h"
118 for (j = 0; j < num_threads; j++)
119 {
120 coarse_shift = 0;
121 if (j > 0) coarse_shift = coarse_counter[j-1];
122 size = n_fine/num_threads;
123 rest = n_fine - size*num_threads;
124 if (j < rest)
125 {
126 ns = j*size+j;
127 ne = (j+1)*size+j+1;
128 }
129 else
130 {
131 ns = j*size+rest;
132 ne = (j+1)*size+rest;
133 }
134 for (i = ns; i < ne; i++)
135 fine_to_coarse[i] += coarse_shift;
136 }
137 index = 0;
138 for (i = 0; i < num_sends_A; i++)
139 {
140 start = hypre_ParCSRCommPkgSendMapStart(comm_pkg_A, i);
141 for (j = start; j < hypre_ParCSRCommPkgSendMapStart(comm_pkg_A, i+1); j++)
142 big_buf_data[index++] = my_first_cpt+
143 (HYPRE_BigInt)fine_to_coarse[hypre_ParCSRCommPkgSendMapElmt(comm_pkg_A,j)];
144 }
145
146 comm_handle = hypre_ParCSRCommHandleCreate( 21, comm_pkg_A, big_buf_data,
147 fine_to_coarse_offd);
148
149 hypre_ParCSRCommHandleDestroy(comm_handle);
150
151 for (i=0; i < num_cols_offd_RT; i++)
152 col_map_offd_RT[i] = fine_to_coarse_offd[tmp_map_offd[i]];
153
154 hypre_TFree(fine_to_coarse_offd);
155 hypre_TFree(coarse_counter);
156
157 recv_procs_RT = hypre_CTAlloc(int,num_recvs_RT);
158 recv_vec_starts_RT = hypre_CTAlloc(int, num_recvs_RT+1);
159
160 j = 0;
161 recv_vec_starts_RT[0] = 0;
162 for (i=0; i < num_recvs_A; i++)
163 if (proc_mark[i])
164 {
165 recv_procs_RT[j] = recv_procs_A[i];
166 recv_vec_starts_RT[j+1] = recv_vec_starts_RT[j]+proc_mark[i];
167 j++;
168 }
169
170/*--------------------------------------------------------------------------
171 * send num_changes to recv_procs_A and receive change_array from send_procs_A
172 *--------------------------------------------------------------------------*/
173
174 num_requests = num_recvs_A+num_sends_A;
175 requests = hypre_CTAlloc(MPI_Request, num_requests);
176 status = hypre_CTAlloc(MPI_Status, num_requests);
177
178 change_array = hypre_CTAlloc(int, num_sends_A);
179
180 j = 0;
181 for (i=0; i < num_sends_A; i++)
182 MPI_Irecv(&change_array[i],1,MPI_INT,send_procs_A[i],0,comm,
183 &requests[j++]);
184
185 for (i=0; i < num_recvs_A; i++)
186 MPI_Isend(&proc_mark[i],1,MPI_INT,recv_procs_A[i],0,comm,
187 &requests[j++]);
188
189 MPI_Waitall(num_requests,requests,status);
190
191 hypre_TFree(proc_mark);
192
193/*--------------------------------------------------------------------------
194 * if change_array[i] is 0 , omit send_procs_A[i] in send_procs_RT
195 *--------------------------------------------------------------------------*/
196
197 num_sends_RT = 0;
198 for (i=0; i < num_sends_A; i++)
199 if (change_array[i])
200 {
201 num_sends_RT++;
202 }
203
204 send_procs_RT = hypre_CTAlloc(int, num_sends_RT);
205 send_map_starts_RT = hypre_CTAlloc(int, num_sends_RT+1);
206
207 j = 0;
208 send_map_starts_RT[0] = 0;
209 for (i=0; i < num_sends_A; i++)
210 if (change_array[i])
211 {
212 send_procs_RT[j] = send_procs_A[i];
213 send_map_starts_RT[j+1] = send_map_starts_RT[j]+change_array[i];
214 j++;
215 }
216
217/*--------------------------------------------------------------------------
218 * generate send_map_elmts
219 *--------------------------------------------------------------------------*/
220
221 send_map_elmts_RT = hypre_CTAlloc(int,send_map_starts_RT[num_sends_RT]);
222 send_big_elmts = hypre_CTAlloc(HYPRE_BigInt,send_map_starts_RT[num_sends_RT]);
223
224 j = 0;
225 for (i=0; i < num_sends_RT; i++)
226 {
227 vec_start = send_map_starts_RT[i];
228 vec_len = send_map_starts_RT[i+1]-vec_start;
229 MPI_Irecv(&send_big_elmts[vec_start],vec_len,MPI_HYPRE_BIG_INT,
230 send_procs_RT[i],0,comm,&requests[j++]);
231 }
232
233 for (i=0; i < num_recvs_RT; i++)
234 {
235 vec_start = recv_vec_starts_RT[i];
236 vec_len = recv_vec_starts_RT[i+1] - vec_start;
237 MPI_Isend(&col_map_offd_RT[vec_start],vec_len,MPI_HYPRE_BIG_INT,
238 recv_procs_RT[i],0,comm,&requests[j++]);
239 }
240
241 MPI_Waitall(j,requests,status);
242
243 for (i=0; i < send_map_starts_RT[num_sends_RT]; i++)
244 send_map_elmts_RT[i] = (int)(send_big_elmts[i] - first_col_diag);
245
246 comm_pkg = hypre_CTAlloc(hypre_ParCSRCommPkg,1);
247
248 hypre_ParCSRCommPkgComm(comm_pkg) = comm;
249 hypre_ParCSRCommPkgNumSends(comm_pkg) = num_sends_RT;
250 hypre_ParCSRCommPkgNumRecvs(comm_pkg) = num_recvs_RT;
251 hypre_ParCSRCommPkgSendProcs(comm_pkg) = send_procs_RT;
252 hypre_ParCSRCommPkgRecvProcs(comm_pkg) = recv_procs_RT;
253 hypre_ParCSRCommPkgRecvVecStarts(comm_pkg) = recv_vec_starts_RT;
254 hypre_ParCSRCommPkgSendMapStarts(comm_pkg) = send_map_starts_RT;
255 hypre_ParCSRCommPkgSendMapElmts(comm_pkg) = send_map_elmts_RT;
256
257 hypre_TFree(status);
258 hypre_TFree(requests);
259 hypre_TFree(send_big_elmts);
260
261 hypre_ParCSRMatrixCommPkg(RT) = comm_pkg;
262 hypre_TFree(change_array);
263
264 return ierr;
265}
266
267int
268hypre_GenerateSendMapAndCommPkg(MPI_Comm comm, int num_sends, int num_recvs,
269 int *recv_procs, int *send_procs,
270 int *recv_vec_starts, hypre_ParCSRMatrix *A)
271{
272 int *send_map_starts;
273 int *send_map_elmts;
274 int i, j;
275 int num_requests = num_sends+num_recvs;
276 MPI_Request *requests;
277 MPI_Status *status;
278 int vec_len, vec_start;
279 hypre_ParCSRCommPkg *comm_pkg;
280 HYPRE_BigInt *col_map_offd = hypre_ParCSRMatrixColMapOffd(A);
281 HYPRE_BigInt first_col_diag = hypre_ParCSRMatrixFirstColDiag(A);
282 HYPRE_BigInt *send_big_elmts;
283
284/*--------------------------------------------------------------------------
285 * generate send_map_starts and send_map_elmts
286 *--------------------------------------------------------------------------*/
287 requests = hypre_CTAlloc(MPI_Request,num_requests);
288 status = hypre_CTAlloc(MPI_Status,num_requests);
289 send_map_starts = hypre_CTAlloc(int, num_sends+1);
290 j = 0;
291 for (i=0; i < num_sends; i++)
292 MPI_Irecv(&send_map_starts[i+1],1,MPI_INT,send_procs[i],0,comm,
293 &requests[j++]);
294
295 for (i=0; i < num_recvs; i++)
296 {
297 vec_len = recv_vec_starts[i+1] - recv_vec_starts[i];
298 MPI_Isend(&vec_len,1,MPI_INT, recv_procs[i],0,comm,&requests[j++]);
299 }
300
301 MPI_Waitall(j,requests,status);
302
303 send_map_starts[0] = 0;
304 for (i=0; i < num_sends; i++)
305 send_map_starts[i+1] += send_map_starts[i];
306
307 send_map_elmts = hypre_CTAlloc(int,send_map_starts[num_sends]);
308 send_big_elmts = hypre_CTAlloc(HYPRE_BigInt,send_map_starts[num_sends]);
309
310 j = 0;
311 for (i=0; i < num_sends; i++)
312 {
313 vec_start = send_map_starts[i];
314 vec_len = send_map_starts[i+1]-vec_start;
315 MPI_Irecv(&send_big_elmts[vec_start],vec_len,MPI_HYPRE_BIG_INT,
316 send_procs[i],0,comm,&requests[j++]);
317 }
318
319 for (i=0; i < num_recvs; i++)
320 {
321 vec_start = recv_vec_starts[i];
322 vec_len = recv_vec_starts[i+1] - vec_start;
323 MPI_Isend(&col_map_offd[vec_start],vec_len,MPI_HYPRE_BIG_INT,
324 recv_procs[i],0,comm,&requests[j++]);
325 }
326
327 MPI_Waitall(j,requests,status);
328
329 for (i=0; i < send_map_starts[num_sends]; i++)
330 send_map_elmts[i] = (int)(send_big_elmts[i]-first_col_diag);
331
332 comm_pkg = hypre_CTAlloc(hypre_ParCSRCommPkg,1);
333
334 hypre_ParCSRCommPkgComm(comm_pkg) = comm;
335 hypre_ParCSRCommPkgNumSends(comm_pkg) = num_sends;
336 hypre_ParCSRCommPkgNumRecvs(comm_pkg) = num_recvs;
337 hypre_ParCSRCommPkgSendProcs(comm_pkg) = send_procs;
338 hypre_ParCSRCommPkgRecvProcs(comm_pkg) = recv_procs;
339 hypre_ParCSRCommPkgRecvVecStarts(comm_pkg) = recv_vec_starts;
340 hypre_ParCSRCommPkgSendMapStarts(comm_pkg) = send_map_starts;
341 hypre_ParCSRCommPkgSendMapElmts(comm_pkg) = send_map_elmts;
342
343 hypre_TFree(status);
344 hypre_TFree(requests);
345 hypre_TFree(send_big_elmts);
346
347 hypre_ParCSRMatrixCommPkg(A) = comm_pkg;
348 return 0;
349}
Note: See TracBrowser for help on using the repository browser.