source: CIVL/examples/omp/dataracebench-1.2.0/scripts/test-harness.sh@ 0c90cbf

1.23 2.0 main test-branch
Last change on this file since 0c90cbf was 86ee0b6, checked in by Ziqing Luo <ziqing@…>, 7 years ago

merged wenhao's branch for omp transformation

git-svn-id: ​svn://vsl.cis.udel.edu/civl/trunk@5337 fb995dde-84ed-4084-dfe6-e5aef3e2452c
  • Property mode set to 100755
File size: 16.6 KB
Line 
1#!/usr/bin/env bash
2
3# Copyright (c) 2017, Lawrence Livermore National Security, LLC.
4# Produced at the Lawrence Livermore National Laboratory
5# Written by Chunhua Liao, Pei-Hung Lin, Joshua Asplund,
6# Markus Schordan, and Ian Karlin
7# (email: liao6@llnl.gov, lin32@llnl.gov, asplund1@llnl.gov,
8# schordan1@llnl.gov, karlin1@llnl.gov)
9# LLNL-CODE-732144
10# All rights reserved.
11#
12# This file is part of DataRaceBench. For details, see
13# https://github.com/LLNL/dataracebench. Please also see the LICENSE file
14# for our additional BSD notice
15#
16# Redistribution of Backstroke and use in source and binary forms, with
17# or without modification, are permitted provided that the following
18# conditions are met:
19#
20# * Redistributions of source code must retain the above copyright
21# notice, this list of conditions and the disclaimer below.
22#
23# * Redistributions in binary form must reproduce the above copyright
24# notice, this list of conditions and the disclaimer (as noted below)
25# in the documentation and/or other materials provided with the
26# distribution.
27#
28# * Neither the name of the LLNS/LLNL nor the names of its contributors
29# may be used to endorse or promote products derived from this
30# software without specific prior written permission.
31#
32# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
33# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
34# INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
35# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
36# DISCLAIMED. IN NO EVENT SHALL LAWRENCE LIVERMORE NATIONAL
37# SECURITY, LLC, THE U.S. DEPARTMENT OF ENERGY OR CONTRIBUTORS BE
38# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
39# OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
40# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
42# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
43# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
44# IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
45# THE POSSIBILITY OF SUCH DAMAGE.
46
47CSV_HEADER="tool,id,filename,haverace,threads,dataset,races,elapsed-time(seconds),used-mem(KBs),compile-return,runtime-return"
48TESTS=($(grep -l main micro-benchmarks/*.cpp micro-benchmarks/*.c))
49OUTPUT_DIR="results"
50LOG_DIR="$OUTPUT_DIR/log"
51EXEC_DIR="$OUTPUT_DIR/exec"
52LOGFILE="$LOG_DIR/dataracecheck.log"
53
54
55MEMCHECK=${MEMCHECK:-"/usr/bin/time"}
56TIMEOUTCMD=${TIMEOUTCMD:-"timeout"}
57VALGRIND=${VALGRIND:-"valgrind"}
58VALGRIND_COMPILE_C_FLAGS="-g -std=c99 -fopenmp"
59VALGRIND_COMPILE_CPP_FLAGS="-g -fopenmp"
60
61CLANG=${CLANG:-"clang"}
62TSAN_COMPILE_FLAGS="-fopenmp -fsanitize=thread -g"
63
64ARCHER=${ARCHER:-"clang-archer"}
65ARCHER_COMPILE_FLAGS="-larcher"
66
67INSPECTOR=${INSPECTOR:-"inspxe-cl"}
68ICC_COMPILE_FLAGS="-O0 -fopenmp -std=c99 -qopenmp-offload=host"
69ICPC_COMPILE_FLAGS="-O0 -fopenmp -qopenmp-offload=host"
70
71ROMP_ROOT="/g/g92/lin32/opt/romp"
72ROMP_COMPILE_FLAGS="-g -O0 -L${ROMP_ROOT}/pkgs-src/romp-lib/lib -L${ROMP_ROOT}/pkgs-src/gperftools/gperftools-install/lib -L${ROMP_ROOT}/pkgs-src/llvm-openmp/openmp/llvm-openmp-install/lib -fopenmp -fpermissive -ltcmalloc"
73DYNINST_CLIENT="/g/g92/lin32/opt/romp/pkgs-src/dyninst-client/omp_race_client"
74
75POLYFLAG="micro-benchmarks/utilities/polybench.c -I micro-benchmarks -I micro-benchmarks/utilities -DPOLYBENCH_NO_FLUSH_CACHE -DPOLYBENCH_TIME -D_POSIX_C_SOURCE=200112L"
76
77VARLEN_PATTERN='[[:alnum:]]+-var-[[:alnum:]]+\.c'
78RACES_PATTERN='[[:alnum:]]+-[[:alnum:]]+-yes\.c'
79CPP_PATTERN='[[:alnum:]]+\.cpp'
80
81usage () {
82 echo
83 echo "Usage: $0 [--help] [OPTIONS]"
84 echo " --help : Show usage and options"
85 echo
86 echo "OPTIONS:"
87 echo " -x tool : Add the specified tool to test set."
88 echo " Value can be one of: gnu, clang, intel, helgrind, tsan-clang, tsan-gcc, archer, inspector, inspector-max-resources, romp."
89 echo " -n iterations : Run each setting the specified number of iterations."
90 echo " -t threads : Add the specified number of threads as a testcase."
91 echo " -d size : Add a specific dataset size to the varlen test suite."
92 echo " -s minutes : Add a specific timeout minutes."
93 echo
94}
95
96valid_tool_name () {
97 case "$1" in
98 gnu) return 0 ;;
99 clang) return 0 ;;
100 intel) return 0 ;;
101 helgrind) return 0 ;;
102 archer) return 0 ;;
103 tsan-clang) return 0 ;;
104 tsan-gcc) return 0 ;;
105 inspector) return 0 ;;
106 inspector-max-resources) return 0 ;;
107 romp) return 0;;
108 *) return 1 ;;
109 esac
110}
111
112check_return_code () {
113 case "$1" in
114 11) echo "Seg Fault"; testreturn=11 ;;
115 124) echo "Executime timeout";testreturn=124 ;;
116 139) echo "Seg Fault"; testreturn=11 ;;
117 *) testreturn=0 ;;
118 esac
119}
120
121if [[ "$1" == "--help" ]]; then
122 usage && exit 0;
123fi
124
125mkdir -p "$OUTPUT_DIR"
126mkdir -p "$LOG_DIR"
127mkdir -p "$EXEC_DIR"
128
129TOOLS=()
130DATASET_SIZES=()
131THREADLIST=()
132ITERATIONS=0
133TIMEOUTMIN="10"
134# Parse options
135while getopts "n:t:x:d:s:" opt; do
136 case $opt in
137 x) if valid_tool_name "${OPTARG}"; then TOOLS+=(${OPTARG});
138 else echo "Invalid tool name ${OPTARG}" && usage && exit 1
139 fi ;;
140 n) if [[ ${OPTARG} -gt 0 ]]; then ITERATIONS=${OPTARG};
141 else echo "Number of iterations must be greater than 0"
142 fi ;;
143 t) if [[ ${OPTARG} -gt 1 ]]; then THREADLIST+=(${OPTARG})
144 else echo "Number of threads must be greater than 1" && usage && exit 1;
145 fi ;;
146 d) if [[ ${OPTARG} -gt 1 ]]; then DATASET_SIZES+=(${OPTARG})
147 else echo "Dataset size must be greater than 1" && usage && exit 1;
148 fi ;;
149 s) if [[ ${OPTARG} -gt 0 ]]; then TIMEOUTMIN=(${OPTARG})
150 else echo "timeout must be greater than 0" && usage && exit 1;
151 fi ;;
152 esac
153done
154
155# Set default values
156if [[ ! ${#TOOLS[@]} -gt 0 ]]; then
157 echo "Default tool set will be used: gnu, clang, intel helgrind, tsan-clang, tsan-gcc, archer, inspector-max-resources."
158 TOOLS=( 'gnu' 'clang' 'intel' 'helgrind' 'tsan-clang' 'tsan-gcc' 'archer' 'inspector-max-resources' )
159else
160 echo "Tools: ${TOOLS[*]}";
161fi
162
163if [[ ! ${#DATASET_SIZES[@]} -gt 0 ]]; then
164 echo "Default dataset sizes will be used: 32, 64, 128, 256, 512, 1024."
165 DATASET_SIZES=('32' '64' '128' '256' '512' '1024')
166else
167 echo "Dataset sizes: ${DATASET_SIZES[*]}";
168fi
169
170if [[ ! ${#THREADLIST[@]} -gt 0 ]]; then
171 echo "Default thread counts will be used: 3, 36, 45, 72, 90, 180, 256."
172 THREADLIST=('3' '36' '45' '72' '90' '180' '256')
173else
174 echo "Thread counts: ${THREADLIST[*]}";
175fi
176
177if [[ ! $ITERATIONS -gt 0 ]]; then
178 echo "Default number of iterations will be used: 5"
179 ITERATIONS=5
180else
181 echo "Iterations: ${ITERATIONS}";
182fi
183
184if [[ ! $TIMEOUTMIN -gt 0 ]]; then
185 echo "Default timeout will be 10 minutes"
186 TIMEOUTMIN=10
187else
188 echo "Timeout minutes: ${TIMEOUTMIN}";
189fi
190
191if [[ -e "$LOGFILE" ]]; then rm "$LOGFILE"; fi
192
193# Increase stack size - Fixes crashes in some analyses
194ULIMITS=$(ulimit -s)
195ulimit -s unlimited
196
197TOOL_INDEX=0
198TEST_INDEX=0
199THREAD_INDEX=0
200SIZE_INDEX=0
201ITER=1
202
203cleanup () {
204 ulimit -s "$ULIMITS"
205 if [[ -e $exname ]]; then rm "$exname"; fi
206 echo "EXITING"
207 echo "--------------------------" >> "$LOGFILE"
208 echo "Unfinished tests:" >> "$LOGFILE"
209 for tool in "${TOOLS[@]:$TOOL_INDEX}"; do
210 if [[ $TEST_INDEX -eq 0 && $THREAD_INDEX -eq 0 && $SIZE_INDEX -eq 0 && $ITER -eq 1 ]]; then
211 echo "$tool,all,all,all,$ITERATIONS" >> "$LOGFILE"
212 else
213 for test in "${TESTS[@]:$TEST_INDEX}"; do
214 if [[ $THREAD_INDEX -eq 0 && $SIZE_INDEX -eq 0 && $ITER -eq 1 ]]; then
215 echo "$tool,\"$test\",all,all,$ITERATIONS" >> "$LOGFILE"
216 else
217 for thread in "${THREADLIST[@]:$THREAD_INDEX}"; do
218 if [[ $SIZE_INDEX -eq 0 && $ITER -eq 1 ]]; then
219 echo "$tool,\"$test\",$thread,all,$ITERATIONS" >> "$LOGFILE"
220 else
221 for size in "${SIZES[@]:$SIZE_INDEX}"; do
222 if [[ $ITER -eq 1 ]]; then echo "$tool,\"$test\",$thread,${size:-"N/A"},$ITERATIONS" >> "$LOGFILE"
223 else echo "$tool,\"$test\",$thread,all,$((ITERATIONS - ITER + 1))" >> "$LOGFILE"; ITER=1; fi
224 done
225 SIZE_INDEX=0
226 fi
227 done
228 THREAD_INDEX=0
229 fi
230 done
231 TEST_INDEX=0
232 fi
233 done
234 exit 1
235}
236
237trap cleanup SIGINT SIGTERM
238
239for tool in "${TOOLS[@]}"; do
240
241 MEMLOG="$LOG_DIR/$tool.memlog"
242 file="$OUTPUT_DIR/$tool.csv"
243 echo "Saving to: $file and $MEMLOG"
244 [ -e "$file" ] && rm "$file"
245 echo "$CSV_HEADER" >> "$file"
246
247 runtime_flags=''
248 case "$tool" in
249 'inspector-max-resources')
250 runtime_flags+=" -collect ti3 -knob scope=extreme -knob stack-depth=16 -knob use-maximum-resources=true"
251 tool='inspector'
252 ;;
253 'inspector')
254 runtime_flags+=" -collect ti2"
255 ;;
256 esac
257
258 TEST_INDEX=0
259 for test in "${TESTS[@]}"; do
260 additional_compile_flags=''
261 if [[ "$test" =~ $RACES_PATTERN ]]; then haverace=true; else haverace=false; fi
262 if [[ "$test" =~ $VARLEN_PATTERN ]]; then SIZES=("${DATASET_SIZES[@]}"); else SIZES=(''); fi
263 testname=$(basename $test)
264 id=${testname#DRB}
265 id=${id%%-*}
266 echo "$test has $testname and ID=$id"
267
268 # Compile
269 exname="$EXEC_DIR/$(basename "$test").$tool.out"
270 rompcompile="$(basename "$test").$tool.compile"
271 logname="$(basename "$test").$tool.log"
272 if [[ -e "$LOG_DIR/$logname" ]]; then rm "$LOG_DIR/$logname"; fi
273 if grep -q 'PolyBench' "$test"; then additional_compile_flags+=" $POLYFLAG"; fi
274
275 if [[ "$test" =~ $CPP_PATTERN ]]; then
276 echo "testing C++ code:$test"
277 case "$tool" in
278 gnu) g++ -g -fopenmp $additional_compile_flags $test -o $exname -lm ;;
279 clang) clang++ -fopenmp -g $additional_compile_flags $test -o $exname -lm ;;
280 intel) icpc $ICPC_COMPILE_FLAGS $additional_compile_flags $test -o $exname -lm ;;
281 helgrind) g++ $VALGRIND_COMPILE_CPP_FLAGS $additional_compile_flags $test -o $exname -lm ;;
282 archer) clang-archer++ $ARCHER_COMPILE_FLAGS $additional_compile_flags $test -o $exname -lm ;;
283 tsan-clang) clang++ $TSAN_COMPILE_FLAGS $additional_compile_flags $test -o $exname -lm ;;
284 tsan-gcc) g++ $TSAN_COMPILE_FLAGS $additional_compile_flags $test -o $exname -lm ;;
285 inspector) icpc $ICPC_COMPILE_FLAGS $additional_compile_flags $test -o $exname -lm ;;
286 romp) clang++ $ROMP_COMPILE_FLAGS $additional_compile_flags $test -o $rompcompile -lm;
287 $DYNINST_CLIENT $rompcompile;
288 mv instrumented_app $exname;;
289 esac
290 else
291 case "$tool" in
292 gnu) gcc -g -std=c99 -fopenmp $additional_compile_flags $test -o $exname -lm ;;
293 clang) clang -fopenmp -g $additional_compile_flags $test -o $exname -lm ;;
294 intel) icc $ICC_COMPILE_FLAGS $additional_compile_flags $test -o $exname -lm ;;
295 helgrind) gcc $VALGRIND_COMPILE_C_FLAGS $additional_compile_flags $test -o $exname -lm ;;
296 archer) clang-archer $ARCHER_COMPILE_FLAGS $additional_compile_flags $test -o $exname -lm ;;
297 tsan-clang) clang $TSAN_COMPILE_FLAGS $additional_compile_flags $test -o $exname -lm ;;
298 tsan-gcc) gcc $TSAN_COMPILE_FLAGS $additional_compile_flags $test -o $exname -lm ;;
299 inspector) icc $ICC_COMPILE_FLAGS $additional_compile_flags $test -o $exname -lm ;;
300 romp) clang $ROMP_COMPILE_FLAGS $additional_compile_flags $test -o $rompcompile -lm;
301 $DYNINST_CLIENT $rompcompile;
302 mv instrumented_app $exname;;
303 esac
304 fi
305 compilereturn=$?;
306 echo "compile return code: $compilereturn";
307
308 THREAD_INDEX=0
309 for thread in "${THREADLIST[@]}"; do
310 echo "Testing $test: with $thread threads"
311 export OMP_NUM_THREADS=$thread
312 SIZE_INDEX=0
313 for size in "${SIZES[@]}"; do
314 # Sanity check
315 if [[ ! -e "$exname" ]]; then
316 echo "$tool,$id,\"$testname\",$haverace,$thread,${size:-"N/A"},,,,$compilereturn," >> "$file";
317 echo "Executable for $testname with $thread threads and input size $size is not available" >> "$LOGFILE";
318 elif { "./$exname $size"; } 2>&1 | grep -Eq 'Segmentation fault'; then
319 echo "$tool,$id,\"$testname\",$haverace,$thread,${size:-"N/A"},,,,$compilereturn," >> "$file";
320 echo "Seg fault found in $testname with $thread threads and input size $size" >> "$LOGFILE";
321 else
322 ITER_INDEX=1
323 for ITER in $(seq 1 "$ITERATIONS"); do
324 echo -e "***** Log $ITER_INDEX for $testname with $thread threads and input size $size *****" >> "$LOG_DIR/$logname"
325 start=$(date +%s%6N)
326 case "$tool" in
327 gnu)
328 #races=$($TIMEOUTCMD $TIMEOUTMIN"m" $MEMCHECK -f "%M" -o "$MEMLOG" "./$exname" $size 2>&1 | tee -a "$LOG_DIR/$logname" | grep -ce 'Possible data race') ;;
329 ;&
330 clang)
331 #races=$($MEMCHECK -f "%M" -o "$MEMLOG" "./$exname" $size 2>&1 | tee -a "$LOG_DIR/$logname" | grep -ce 'Possible data race') ;;
332 ;&
333 intel)
334 #races=$($MEMCHECK -f "%M" -o "$MEMLOG" "./$exname" $size 2>&1 | tee -a "$LOG_DIR/$logname" | grep -ce 'Possible data race') ;;
335 $TIMEOUTCMD $TIMEOUTMIN"m" $MEMCHECK -f "%M" -o "$MEMLOG" "./$exname" $size &> tmp.log;
336 check_return_code $?;
337 echo "testname return $testreturn";
338 races="",
339 cat tmp.log >> "$LOG_DIR/$logname" || >tmp.log ;;
340 helgrind)
341# races=$($MEMCHECK -f "%M" -o "$MEMLOG" $VALGRIND --tool=helgrind "./$exname" $size 2>&1 | tee -a "$LOG_DIR/$logname" | grep -ce 'Possible data race') ;;
342 $TIMEOUTCMD $TIMEOUTMIN"m" $MEMCHECK -f "%M" -o "$MEMLOG" $VALGRIND --tool=helgrind "./$exname" $size &> tmp.log;
343 check_return_code $?;
344 echo "testname return $testreturn"
345 races=$(grep -ce 'Possible data race' tmp.log)
346 cat tmp.log >> "$LOG_DIR/$logname" || >tmp.log ;;
347 archer)
348 $TIMEOUTCMD $TIMEOUTMIN"m" $MEMCHECK -f "%M" -o "$MEMLOG" "./$exname" $size &> tmp.log;
349 check_return_code $?;
350 echo "testname return $testreturn"
351 races=$(grep -ce 'WARNING: ThreadSanitizer: data race' tmp.log)
352 cat tmp.log >> "$LOG_DIR/$logname" || >tmp.log ;;
353 tsan-clang)
354# races=$($MEMCHECK -f "%M" -o "$MEMLOG" "./$exname" $size 2>&1 | tee -a "$LOG_DIR/$logname" | grep -ce 'WARNING: ThreadSanitizer: data race') ;;
355 $TIMEOUTCMD $TIMEOUTMIN"m" $MEMCHECK -f "%M" -o "$MEMLOG" "./$exname" $size &> tmp.log;
356 check_return_code $?;
357 echo "testname return $testreturn"
358 races=$(grep -ce 'WARNING: ThreadSanitizer: data race' tmp.log)
359 cat tmp.log >> "$LOG_DIR/$logname" || >tmp.log ;;
360 tsan-gcc)
361# races=$($MEMCHECK -f "%M" -o "$MEMLOG" "./$exname" $size 2>&1 | tee -a "$LOG_DIR/$logname" | grep -ce 'WARNING: ThreadSanitizer: data race') ;;
362 $TIMEOUTCMD $TIMEOUTMIN"m" $MEMCHECK -f "%M" -o "$MEMLOG" "./$exname" $size &> tmp.log;
363 check_return_code $?;
364 echo "testname return $testreturn"
365 races=$(grep -ce 'WARNING: ThreadSanitizer: data race' tmp.log)
366 cat tmp.log >> "$LOG_DIR/$logname" || >tmp.log ;;
367 inspector)
368# races=$($MEMCHECK -f "%M" -o "$MEMLOG" $INSPECTOR $runtime_flags -- "./$exname" $size 2>&1 | tee -a "$LOG_DIR/$logname" | grep 'Data race' | sed -E 's/[[:space:]]*([[:digit:]]+).*/\1/') ;;
369 $TIMEOUTCMD $TIMEOUTMIN"m" $MEMCHECK -f "%M" -o "$MEMLOG" $INSPECTOR $runtime_flags -- "./$exname" $size &> tmp.log;
370 check_return_code $?;
371 echo "testname return $testreturn";
372 races=$(grep 'Data race' tmp.log | sed -E 's/[[:space:]]*([[:digit:]]+).*/\1/');
373 cat tmp.log >> "$LOG_DIR/$logname" || >tmp.log ;;
374 romp)
375 $TIMEOUTCMD $TIMEOUTMIN"m" $MEMCHECK -f "%M" -o "$MEMLOG" "./$exname" $size &> tmp.log;
376 check_return_code $?;
377 echo "testname return $testreturn"
378 races=$(grep -ce 'race found!' tmp.log)
379 cat tmp.log >> "$LOG_DIR/$logname" || >tmp.log ;;
380 #races=$("./$exname" $size 2>&1 | tee -a "$LOG_DIR/$logname" | grep -ce 'race found!') ;;
381 esac
382 end=$(date +%s%6N)
383 elapsedtime=$(echo "scale=3; ($end-$start)/1000000"|bc)
384 mem=$(cat $MEMLOG)
385 echo "$tool,$id,\"$testname\",$haverace,$thread,${size:-"N/A"},${races:-0},$elapsedtime,$mem,$compilereturn,$testreturn" >> "$file"
386 ITER_INDEX=$((ITER_INDEX+1))
387 done
388 fi
389 SIZE_INDEX=$((SIZE_INDEX+1))
390 done
391 THREAD_INDEX=$((THREAD_INDEX+1))
392 done
393 TEST_INDEX=$((TEST_INDEX+1))
394 #if [[ -e $exname ]]; then rm "$exname"; fi
395 done
396 TOOL_INDEX=$((TOOL_INDEX+1))
397done
398
399ulimit -s "$ULIMITS"
Note: See TracBrowser for help on using the repository browser.