source: CIVL/mods/dev.civl.com/examples/omp/nas-dc/DC/jobcntl.c@ cb4d4f4

main test-branch
Last change on this file since cb4d4f4 was aad342c, checked in by Stephen Siegel <siegel@…>, 3 years ago

Performing huge refactor to incorporate ABC, GMC, and SARL into CIVL repo and use Java modules.

git-svn-id: ​svn://vsl.cis.udel.edu/civl/trunk@5664 fb995dde-84ed-4084-dfe6-e5aef3e2452c
  • Property mode set to 100644
File size: 15.7 KB
Line 
1#include <stdio.h>
2#include <stdlib.h>
3#include <string.h>
4
5#include "adc.h"
6#include "macrodef.h"
7
8#ifdef UNIX
9/*
10#include <fcntl.h>
11#include <sys/file.h>
12*/
13#include <unistd.h>
14#endif
15
16uint32 NumberOfOnes(uint64 s);
17void swap8(void *a);
18void SetOneBit(uint64 *s, int32 pos){ uint64 ob = MLB; ob >>= pos; *s |= ob;}
19void SetOneBit32(uint32 *s, uint32 pos){
20 uint32 ob = 0x80000000;
21 ob >>= pos;
22 *s |= ob;
23}
24uint32 Mlo32(uint32 x){
25 uint32 om = 0x80000000;
26 uint32 i;
27 uint32 k;
28
29 for ( k = 0, i = 0; i < 32; i++ ) {
30 if (om&x) break;
31 om >>= 1;
32 k++;
33 }
34 return(k);
35}
36int32 mro32(uint32 x){
37 uint32 om = 0x00000001;
38 uint32 i;
39 uint32 k;
40
41 for ( k = 32, i = 0; i < 32; i++ ) {
42 if (om&x) break;
43 om <<= 1;
44 k--;
45 }
46 return(k);
47}
48uint32 setLeadingOnes32(uint32 n){
49 int32 om = 0x80000000;
50 uint32 x;
51 uint32 i;
52
53 for ( x = 0, i = 0; i < n; i++ ) {
54 x |= om;
55 om >>= 1;
56 }
57 return (x);
58}
59int32 DeleteOneFile(const char * file_name) {
60# ifdef WINNT
61 return(remove(file_name));
62# else
63 return(unlink(file_name));
64# endif
65}
66void WriteOne32Tuple(char * t, uint32 s, uint32 l, FILE * logf) {
67 uint64 ob = MLB32;
68 uint32 i;
69
70 fprintf(logf, "\n %s", t);
71 for ( i = 0; i < l; i++ ) {
72 if (s&ob) fprintf(logf, "1"); else fprintf(logf, "0");
73 ob >>= 1;
74 }
75}
76uint32 NumOfCombsFromNbyK( uint32 n, uint32 k ){
77 uint32 l, combsNbyK;
78 if ( k > n ) return 0;
79 for(combsNbyK=1, l=1;l<=k;l++)combsNbyK = combsNbyK*(n-l+1)/l;
80 return combsNbyK;
81}
82void JobPoolUpdate(ADC_VIEW_CNTL *avp){
83 uint32 l = avp->nv;
84 uint32 k;
85
86 k = avp->lpp[l].layerIndex + avp->lpp[l].layerCurrentPopulation;
87 avp->jpp[k].grpb = avp->groupby;
88 avp->jpp[k].nv = l;
89 avp->jpp[k].nRows = avp->nViewRows;
90 avp->jpp[k].viewOffset = avp->accViewFileOffset;
91 avp->lpp[l].layerCurrentPopulation++;
92}
93int32 GetParent(ADC_VIEW_CNTL *avp, uint32 binRepTuple){
94 uint32 level, levelPop, i;
95 uint32 ig;
96 uint32 igOfSmallestParent;
97 uint32 igOfPrefixedParent;
98 uint32 igOfSharedSortParent;
99 uint32 spMinNumOfRows;
100 uint32 pfMinNumOfRows;
101 uint32 ssMinNumOfRows;
102 uint32 tgrpb;
103 uint32 pg;
104 uint32 pfm;
105 uint32 mlo = 0;
106 uint32 lom;
107 uint32 l = NumberOfOnes(binRepTuple);
108 uint32 spFound;
109 uint32 pfFound;
110 uint32 ssFound;
111 uint32 found;
112 uint32 spFt;
113 uint32 pfFt;
114 uint32 ssFt;
115
116 found = noneParent;
117 pfm = setLeadingOnes32(mro32(avp->groupby));
118 SetOneBit32(&mlo, Mlo32(avp->groupby));
119 lom = setLeadingOnes32(Mlo32(avp->groupby));
120
121 for(spFound=pfFound=ssFound=0, level=l;level<=avp->nTopDims;level++){
122 levelPop = avp->lpp[level].layerCurrentPopulation;
123
124 if(levelPop != 0);
125 {
126 for ( spFt = pfFt = ssFt = 1, ig = avp->lpp[level].layerIndex,
127 i = 0; i < levelPop; i++ )
128 {
129 tgrpb = avp->jpp[ig].grpb;
130 if ( (avp->groupby & tgrpb) == avp->groupby ) {
131 spFound = 1;
132 if (spFt) { spMinNumOfRows = avp->jpp[ig].nRows;
133 igOfSmallestParent = ig; spFt = 0; }
134 else if ( spMinNumOfRows > avp->jpp[ig].nRows )
135 { spMinNumOfRows = avp->jpp[ig].nRows;
136 igOfSmallestParent = ig; }
137
138 pg = tgrpb & pfm;
139 if (pg == binRepTuple) {
140 pfFound = 1;
141 if (pfFt) { pfMinNumOfRows = avp->jpp[ig].nRows;
142 igOfPrefixedParent = ig; pfFt = 0; }
143 else if ( pfMinNumOfRows > avp->jpp[ig].nRows)
144 { pfMinNumOfRows = avp->jpp[ig].nRows;
145 igOfPrefixedParent = ig; }
146 }
147
148 if ( (tgrpb & mlo) && !(tgrpb & lom)) {
149 ssFound = 1;
150 if (ssFt) { ssMinNumOfRows = avp->jpp[ig].nRows;
151 igOfSharedSortParent = ig; ssFt = 0; }
152 else if ( ssMinNumOfRows > avp->jpp[ig].nRows)
153 { ssMinNumOfRows = avp->jpp[ig].nRows;
154 igOfSharedSortParent = ig; }
155 }
156 }
157 ig++;
158 }
159 }
160 if (pfFound) found = prefixedParent;
161 else if (ssFound) found = sharedSortParent;
162 else if (spFound) found = smallestParent;
163
164 switch(found){
165 case prefixedParent:
166 avp->smallestParentLevel = level;
167 avp->viewOffset = avp->jpp[igOfPrefixedParent].viewOffset;
168 avp->nParentViewRows = avp->jpp[igOfPrefixedParent].nRows;
169 avp->parBinRepTuple = avp->jpp[igOfPrefixedParent].grpb;
170 break;
171 case sharedSortParent:
172 avp->smallestParentLevel = level;
173 avp->viewOffset = avp->jpp[igOfSharedSortParent].viewOffset;
174 avp->nParentViewRows = avp->jpp[igOfSharedSortParent].nRows;
175 avp->parBinRepTuple = avp->jpp[igOfSharedSortParent].grpb;
176 break;
177 case smallestParent:
178 avp->smallestParentLevel = level;
179 avp->viewOffset = avp->jpp[igOfSmallestParent].viewOffset;
180 avp->nParentViewRows = avp->jpp[igOfSmallestParent].nRows;
181 avp->parBinRepTuple = avp->jpp[igOfSmallestParent].grpb;
182 break;
183 default: break;
184 }
185 if( found == prefixedParent
186 || found == sharedSortParent
187 || found == smallestParent) break;
188 }
189 return found;
190}
191uint32 GetSmallestParent(ADC_VIEW_CNTL *avp, uint32 binRepTuple){
192 uint32 found, level, levelPop, i, ig, igOfSmallestParent;
193 uint32 minNumOfRows;
194 uint32 tgrpb;
195 uint32 ft;
196 uint32 l = NumberOfOnes(binRepTuple);
197
198 for(found=0, level=l; level<=avp->nTopDims;level++){
199 levelPop = avp->lpp[level].layerCurrentPopulation;
200 if(levelPop){
201 for(ft=1, ig=avp->lpp[level].layerIndex, i=0;i<levelPop;i++){
202 tgrpb = avp->jpp[ig].grpb;
203 if ( (avp->groupby & tgrpb) == avp->groupby ) {
204 found = 1;
205 if(ft){
206 minNumOfRows=avp->jpp[ig].nRows;
207 igOfSmallestParent = ig;
208 ft = 0;
209 }else if(minNumOfRows > avp->jpp[ig].nRows){
210 minNumOfRows = avp->jpp[ig].nRows;
211 igOfSmallestParent = ig;
212 }
213 }
214 ig++;
215 }
216 }
217 if( found ){
218 avp->smallestParentLevel = level;
219 avp->viewOffset = avp->jpp[igOfSmallestParent].viewOffset;
220 avp->nParentViewRows = avp->jpp[igOfSmallestParent].nRows;
221 avp->parBinRepTuple = avp->jpp[igOfSmallestParent].grpb;
222 break;
223 }
224 }
225 return found;
226}
227int32 GetPrefixedParent(ADC_VIEW_CNTL *avp, uint32 binRepTuple){
228 uint32 found, level, levelPop, i, ig, igOfSmallestParent;
229 uint32 minNumOfRows;
230 uint32 tgrpb;
231 uint32 ft;
232 uint32 pg, tm;
233 uint32 l = NumberOfOnes(binRepTuple);
234
235 tm = setLeadingOnes32(mro32(avp->groupby));
236
237 for(found=0, level=l; level<=avp->nTopDims; level++){
238 levelPop = avp->lpp[level].layerCurrentPopulation;
239
240 if (levelPop != 0);
241 {
242 for(ft = 1, ig = avp->lpp[level].layerIndex,
243 i = 0; i < levelPop; i++ ) {
244 tgrpb = avp->jpp[ig].grpb;
245 if ( (avp->groupby & tgrpb) == avp->groupby ) {
246 pg = tgrpb & tm;
247 if (pg == binRepTuple) {
248 found = 1;
249 if (ft) { minNumOfRows = avp->jpp[ig].nRows;
250 igOfSmallestParent = ig; ft = 0; }
251 else if ( minNumOfRows > avp->jpp[ig].nRows)
252 { minNumOfRows = avp->jpp[ig].nRows;
253 igOfSmallestParent = ig; }
254 }
255 }
256 ig++;
257 }
258 }
259 if ( found ) {
260 avp->smallestParentLevel = level;
261 avp->viewOffset = avp->jpp[igOfSmallestParent].viewOffset;
262 avp->nParentViewRows = avp->jpp[igOfSmallestParent].nRows;
263 avp->parBinRepTuple = avp->jpp[igOfSmallestParent].grpb;
264 break;
265 }
266 }
267 return found;
268}
269void JobPoolInit(JOB_POOL *jpp, uint32 n, uint32 nd){
270 uint32 i;
271
272 for ( i = 0; i < n; i++ ) {
273 jpp[i].grpb = 0;
274 jpp[i].nv = 0;
275 jpp[i].nRows = 0;
276 jpp[i].viewOffset = 0;
277 }
278}
279void WriteOne64Tuple(char * t, uint64 s, uint32 l, FILE * logf){
280 uint64 ob = MLB;
281 uint32 i;
282
283 fprintf(logf, "\n %s", t);
284 for ( i = 0; i < l; i++ ) {
285 if (s&ob) fprintf(logf, "1"); else fprintf(logf, "0");
286 ob >>= 1;
287 }
288}
289uint32 NumberOfOnes(uint64 s){
290 uint64 ob = MLB;
291 uint32 i;
292 uint32 nOnes;
293
294 for ( nOnes = 0, i = 0; i < 64; i++ ) {
295 if (s&ob) nOnes++;
296 ob >>= 1;
297 }
298 return nOnes;
299}
300void GetRegTupleFromBin64(
301 uint64 binRepTuple,
302 uint32 *selTuple,
303 uint32 numDims,
304 uint32 *numOfUnits){
305 uint64 oc = MLB;
306 uint32 i;
307 uint32 j;
308
309 *numOfUnits = 0;
310 for( j = 0, i = 0; i < numDims; i++ ) {
311 if (binRepTuple & oc) { selTuple[j++] = i+1; (*numOfUnits)++;}
312 oc >>= 1;
313 }
314}
315void getRegTupleFromBin32(
316 uint32 binRepTuple,
317 uint32 *selTuple,
318 uint32 numDims,
319 uint32 *numOfUnits){
320 uint32 oc = MLB32;
321 uint32 i;
322 uint32 j;
323
324 *numOfUnits = 0;
325 for( j = 0, i = 0; i < numDims; i++ ) {
326 if (binRepTuple & oc) { selTuple[j++] = i+1; (*numOfUnits)++;}
327 oc >>= 1;
328 }
329}
330void GetRegTupleFromParent(
331 uint64 bin64RepTuple,
332 uint32 bin32RepTuple,
333 uint32 *selTuple,
334 uint32 nd){
335 uint32 oc = MLB32;
336 uint32 i, j, k;
337 uint32 ut32;
338
339 ut32 = (uint32)(bin64RepTuple>>(64-nd));
340 ut32 <<= (32-nd);
341
342 for ( j = 0, k = 0, i = 0; i < nd; i++ ) {
343 if (bin32RepTuple & oc) k++;
344 if (bin32RepTuple & oc && ut32 & oc) selTuple[j++] = k;
345 oc >>= 1;
346 }
347}
348void CreateBinTuple(uint64 *binRepTuple, uint32 *selTuple, uint32 numDims){
349 uint32 i;
350
351 *binRepTuple = 0;
352 for(i = 0; i < numDims; i++ ){
353 SetOneBit( binRepTuple, selTuple[i]-1 );
354 }
355}
356void d32v( char * t, uint32 *v, uint32 n){
357 uint32 i;
358
359 fprintf(stderr,"\n%s ", t);
360 for ( i = 0; i < n; i++ ) fprintf(stderr," %d", v[i]);
361}
362void WriteOne64Tuple(char * t, uint64 s, uint32 l, FILE * logf);
363int32 Comp8gbuf(const void *a, const void *b){
364 if ( a < b ) return -1;
365 else if (a > b) return 1;
366 else return 0;
367}
368void restore(TUPLE_VIEWSIZE x[], uint32 f, uint32 l ){
369 uint32 j, m, tj, mm1, jm1, hl;
370 uint64 iW;
371 uint64 iW64;
372
373 j = f;
374 hl = l>>1;
375 while( j <= hl ) {
376 tj = j*2;
377 if (tj < l && x[tj-1].viewsize < x[tj].viewsize) m = tj+1;
378 else m = tj;
379 mm1 = m - 1;
380 jm1 = j - 1;
381 if ( x[mm1].viewsize > x[jm1].viewsize ) {
382 iW = x[mm1].viewsize;
383 x[mm1].viewsize = x[jm1].viewsize;
384 x[jm1].viewsize = iW;
385 iW64 = x[mm1].tuple;
386 x[mm1].tuple = x[jm1].tuple;
387 x[jm1].tuple = iW64;
388 j = m;
389 }else j = l;
390 }
391}
392void vszsort( TUPLE_VIEWSIZE x[], uint32 n){
393 int32 i, im1;
394 uint64 iW;
395 uint64 iW64;
396
397 for ( i = n>>1; i >= 1; i-- ) restore( x, i, n );
398 for ( i = n; i >= 2; i-- ) {
399 im1 = i - 1;
400 iW = x[0].viewsize; x[0].viewsize = x[im1].viewsize; x[im1].viewsize = iW;
401 iW64 = x[0].tuple; x[0].tuple = x[im1].tuple; x[im1].tuple = iW64;
402 restore( x, 1, im1);
403 }
404}
405uint32 countTupleOnes(uint64 binRepTuple, uint32 numDims){
406 uint32 i, cnt = 0;
407 uint64 ob = 0x0000000000000001;
408
409 for(i = 0; i < numDims; i++ ){
410 if ( binRepTuple&ob) cnt++;
411 ob <<= 1;
412 }
413 return cnt;
414}
415void restoreo( TUPLE_ONES x[], uint32 f, uint32 l ){
416 uint32 j, m, tj, mm1, jm1, hl;
417 uint32 iW;
418 uint64 iW64;
419
420 j = f;
421 hl = l>>1;
422 while( j <= hl ) {
423 tj = j*2;
424 if (tj < l && x[tj-1].nOnes < x[tj].nOnes) m = tj+1;
425 else m = tj;
426 mm1 = m - 1; jm1 = j - 1;
427 if ( x[mm1].nOnes > x[jm1].nOnes ){
428 iW = x[mm1].nOnes;
429 x[mm1].nOnes = x[jm1].nOnes;
430 x[jm1].nOnes = iW;
431 iW64 = x[mm1].tuple;
432 x[mm1].tuple = x[jm1].tuple;
433 x[jm1].tuple = iW64;
434 j = m;
435 }else j = l;
436 }
437}
438void onessort( TUPLE_ONES x[], uint32 n){
439 int32 i, im1;
440 uint32 iW;
441 uint64 iW64;
442
443 for ( i = n>>1; i >= 1; i-- ) restoreo( x, i, n );
444 for ( i = n; i >= 2; i-- ) {
445 im1 = i - 1;
446 iW = x[0].nOnes;
447 x[0].nOnes = x[im1].nOnes;
448 x[im1].nOnes = iW;
449 iW64 = x[0].tuple;
450 x[0].tuple = x[im1].tuple;
451 x[im1].tuple = iW64;
452 restoreo( x, 1, im1);
453 }
454}
455uint32 MultiFileProcJobs( TUPLE_VIEWSIZE *tuplesAndSizes,
456 uint32 nViews,
457 ADC_VIEW_CNTL *avp ){
458 uint32 i;
459 int32 ii; /* it should be int */
460 uint32 j;
461 uint32 pn;
462 uint32 direction = 0;
463 uint32 dChange = 0;
464 uint32 gbi;
465 uint32 maxn;
466 uint64 *gbuf;
467 uint64 vszs[MAX_NUMBER_OF_TASKS];
468 uint32 nGroupbys[MAX_NUMBER_OF_TASKS];
469 TUPLE_ONES *toptr;
470
471 gbuf = (uint64*) &avp->memPool[0];
472
473 for(i = 0; i < avp->nTasks; i++ ){ nGroupbys[i] = 0; vszs[i] = 0; }
474
475 for(pn = 0, gbi = 0, ii = nViews-1; ii >= 0; ii-- ){
476 if(pn == avp->taskNumber) gbuf[gbi++]=tuplesAndSizes[ii].tuple;
477 nGroupbys[pn]++;
478 vszs[pn] += tuplesAndSizes[ii].viewsize;
479 if(direction == 0 && pn == avp->nTasks-1 ) {
480 direction = 1;
481 dChange = 1;
482 }
483 if(direction == 1 && pn == 0 ){
484 direction = 0;
485 dChange = 1;
486 }
487 if (!dChange){ if (direction) pn--; else pn++;}
488 dChange = 0;
489 }
490 for(maxn = 0, i = 0; i < avp->nTasks; i++)
491 if (nGroupbys[i] > maxn) maxn = nGroupbys[i];
492
493 toptr = (TUPLE_ONES*) malloc(sizeof(TUPLE_ONES)*maxn);
494 if(!toptr) return 1;
495
496 for(i = 0; i < avp->nTasks; i++ ){
497 if(i == avp->taskNumber){
498 for(j = 0; j < nGroupbys[i]; j++ ){
499 toptr[j].tuple = gbuf[j];
500 toptr[j].nOnes = countTupleOnes(gbuf[j], avp->nTopDims);
501 }
502 qsort((void*)gbuf, nGroupbys[i], 8, Comp8gbuf );
503 onessort(toptr, nGroupbys[i]);
504
505 for(j = 0; j < nGroupbys[i]; j++){
506 toptr[nGroupbys[i]-1-j].tuple <<= (64-avp->nTopDims);
507 swap8(&toptr[nGroupbys[i]-1-j].tuple);
508 fwrite(&toptr[nGroupbys[i]-1-j].tuple, 8, 1, avp->groupbyFile);
509 }
510 }
511 }
512 FSEEK(avp->groupbyFile, 0L, SEEK_SET);
513 if (toptr) free(toptr);
514 return 0;
515}
516int32 PartitionCube(ADC_VIEW_CNTL *avp){
517 TUPLE_VIEWSIZE *tuplesAndSizes;
518 uint32 it = 0;
519 uint64 sz;
520 uint32 sel[64];
521 uint32 k;
522 uint64 tx;
523 uint32 i;
524 char inps[256];
525
526 tuplesAndSizes =
527 (TUPLE_VIEWSIZE*) malloc(avp->nViewLimit*sizeof(TUPLE_VIEWSIZE));
528 if(tuplesAndSizes == NULL){
529 fprintf(stderr," PartitionCube(): memory allocation failure'\n");
530 return ADC_MEMORY_ALLOCATION_FAILURE;
531 }
532 k = 0;
533 while( fscanf(avp->adcViewSizesFile, "%s", inps) != EOF ){
534 if( strcmp(inps, "Selection:") == 0 ) {
535 while ( fscanf(avp->adcViewSizesFile, "%s", inps)) {
536 if ( strcmp(inps, "View") == 0 ) break;
537 sel[k++] = atoi(inps);
538 }
539 }
540 if( strcmp(inps, "Size:") == 0 ){
541 fscanf(avp->adcViewSizesFile, "%s", inps);
542 sz = atoi(inps);
543 CreateBinTuple(&tx, sel, k);
544 if (sz > avp->nInputRecs) sz = avp->nInputRecs;
545 tuplesAndSizes[it].viewsize = sz;
546 tuplesAndSizes[it].tuple = tx;
547 it++;
548 k = 0;
549 }
550 }
551 vszsort(tuplesAndSizes, it);
552 for( i = 0; i < it; i++){
553 tuplesAndSizes[i].tuple >>= (64-avp->nTopDims);
554 }
555 if(MultiFileProcJobs( tuplesAndSizes, it, avp )){
556 fprintf(stderr, "MultiFileProcJobs() is failed \n");
557 fprintf(avp->logf, "MultiFileProcJobs() is failed.\n");
558 fflush(avp->logf);
559 return 1;
560 }
561 FSEEK(avp->adcViewSizesFile, 0L, SEEK_SET);
562 free(tuplesAndSizes);
563 return 0;
564}
Note: See TracBrowser for help on using the repository browser.