Correlation_Compressibility/blocking_sensitivity.sh at master · FTHPC/Correlation_Compressibility · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
#!usr/bin/env bash
function echo_do(){
  echo $@
  "$@"
}

COMPRESS_HOME=/project/jonccal/fthpc/alpoulo/repositories/Correlation_Compressibility
DATA_HOME=/project/jonccal/fthpc/common/sdrbench
cd $COMPRESS_HOME/build

declare -A dset_size
dset_size[SDRBENCH-Miranda-256x384x384]="-d 256 -d 384 -d 384"
dset_size[qmcpack]="-d 69 -d 69 -d 115"
dset_size[hurricane]="-d 100 -d 500 -d 500"
dset_size[hurricane_V]="-d 100 -d 500 -d 500"
dset_size[CESM3D]="-d 26 -d 1800 -d 3600"
dset_size[NYX_z]="-d 512 -d 512 -d 512"
dset_size[SDRBENCH-SCALE-98x1200x1200]="-d 98 -d 1200 -d 1200"
dset_size[s3d]="-d 500 -d 500 -d 500"

declare -A dset_dtype
dset_dtype[SDRBENCH-Miranda-256x384x384]=float64
dset_dtype[qmcpack]=float32
dset_dtype[hurricane]=float32
dset_dtype[hurricane_V]=float32
dset_dtype[CESM3D]=float32
dset_dtype[NYX_z]=float32
dset_dtype[SDRBENCH-SCALE-98x1200x1200]=float32
dset_dtype[s3d]=float64

declare -A dset_loc
dset_loc[SDRBENCH-Miranda-256x384x384]=$COMPRESS_HOME/datasets
dset_loc[qmcpack]=$COMPRESS_HOME/datasets
dset_loc[hurricane]=$COMPRESS_HOME/datasets
dset_loc[hurricane_V]=$COMPRESS_HOME/datasets
dset_loc[CESM3D]=/zfs/fthpc/alpoulo/datasets
dset_loc[NYX_z]=$COMPRESS_HOME/datasets
dset_loc[SDRBENCH-SCALE-98x1200x1200]=$COMPRESS_HOME/datasets
dset_loc[s3d]=$DATA_HOME

walltime="12:00:00"

#app="SDRBENCH-SCALE-98x1200x1200"
#app="SDRBENCH-Miranda-256x384x384"
#app="hurricane"
#app="hurricane_V"
#app="NYX_z"
#app="CESM3D"
#app="qmcpack"
app="s3d"

ncores=28

max_block_count="1"

slurmdir="./slurm"
mkdir -p $slurmdir
#for compressor in "sz" "sz3" "zfp" "mgard" "tthresh" "digit_rounding" "fpzip" "bit_grooming"
for compressor in "sz" "sz3" "zfp" "sperr" "tthresh"
#for compressor in "bit_grooming"
do
  #for block_size in 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24
  #for block_size in 16 20 24 28 32
  for block_size in 4 6 8 10 12 14 16 20 24 28 32
  do
    blocks=${max_block_count}

    slurmout="${slurmdir}/${app}_${compressor}_${blocks}_${block_size}.sh"
    echo "#!/bin/bash" > $slurmout
    echo "#SBATCH --job-name compression_estimation_${app}_${compressor}_${block_size}" >> $slurmout
    echo "#SBATCH --nodes 1" >> $slurmout
    echo "#SBATCH --ntasks-per-node ${ncores}" >> $slurmout
    echo "#SBATCH --cpus-per-task 1" >> $slurmout
    echo "#SBATCH --mem 500gb" >> $slurmout
    echo "#SBATCH --time ${walltime}" >> $slurmout
    #echo "#SBATCH --partition fluxcapacitor" >> $slurmout

    printf "\nsource ~/.bashrc\n" >> $slurmout

    echo "function echo_do(){" >> $slurmout
    printf "\techo \$@\n" >> $slurmout
    printf "\t\"\$@\"\n" >> $slurmout
    printf "}\n\n" >> $slurmout

    #load spack
    echo "compress" >> $slurmout
    echo "cd $COMPRESS_HOME/build" >> $slurmout
    printf "\n" >> $slurmout

    echo "declare -A dset_size" >> $slurmout
    echo "dset_size[${app}]=\"${dset_size[${app}]}\"" >> $slurmout
    echo "declare -A dset_dtype" >> $slurmout >> $slurmout
    echo "dset_dtype[${app}]=\"${dset_dtype[${app}]}\"" >> $slurmout
    echo "declare -A dset_loc" >> $slurmout >> $slurmout
    echo "dset_loc[${app}]=\"${dset_loc[${app}]}\"" >> $slurmout
    printf "\n" >> $slurmout


    echo "for dataset in ${app}" >> $slurmout
    echo "do" >> $slurmout

    printf "\techo_do which mpiexec\n" >> $slurmout
    printf "\techo_do mpiexec -np ${ncores} ./compress_analysis -t \${dset_dtype[\${dataset}]} \\
      -i \${dataset} -r \${dset_loc[\${dataset}]} \${dset_size[\${dataset}]} \\
      -o \"\${dataset}_${compressor}_blocks128_block_size${block_size}.csv\" \\
      --blocks ${blocks} --block_size ${block_size} --method \"UNIFORM\" \\
      --compressor \"${compressor}\"" >> $slurmout

    printf "\n" >> $slurmout

    echo "done" >> $slurmout

    #echo "cp -r \$TMPDIR ${app}_${fname}_blocks_${block_size}" >> $slurmout

    sbatch $slurmout
  #done
  done
done