Gross cluster performance

From CCM
Jump to: navigation, search


Contents

December 2014 - Rocks+ 6.5

Bandwidth

[jmandel@gross ~]$ mpirun --host compute-0-0,compute-0-1 /usr/mpi/gcc/mvapich2-1.9/tests/osu-micro-benchmarks-4.0.1/osu_bw
# OSU MPI Bandwidth Test v4.0.1
# Size      Bandwidth (MB/s)
1                       1.83
2                       3.67
4                       7.29
8                      14.05
16                     28.22
32                     55.25
64                    107.70
128                   201.83
256                   372.28
512                   713.60
1024                 1248.54
2048                 2126.39
4096                 2945.46
8192                 3119.09
16384                3209.20
32768                3178.47
65536                3281.42
131072               3341.01
262144               3376.55
524288               3394.05
1048576              3403.35
2097152              3407.89
4194304              3409.30
[jmandel@gross ~]$ mpirun --host compute-0-0,compute-0-1 /usr/mpi/gcc/mvapich2-1.9/tests/osu-micro-benchmarks-4.0.1/osu_bibw 
# OSU MPI Bi-Directional Bandwidth Test v4.0.1
# Size    Bi-Bandwidth (MB/s)
1                       1.76
2                       3.56
4                       7.11
8                      13.90
16                     27.84
32                     54.89
64                    109.67
128                   193.62
256                   360.01
512                   659.78
1024                 1137.57
2048                 2021.76
4096                 3240.34
8192                 4591.53
16384                5497.73
32768                5943.11
65536                6188.33
131072               6374.44
262144               6483.57
524288               6536.34
1048576              6565.38
2097152              6580.22
4194304              6560.31 
[jmandel@gross ~]$ mpirun -np 12 -machinefile machines /usr/mpi/gcc/mvapich2-1.9/tests/osu-micro-benchmarks-4.0.1/osu_mbw_mr
# OSU MPI Multiple Bandwidth / Message Rate Test v4.0.1
# [ pairs: 6 ] [ window size: 64 ]
# Size                  MB/s        Messages/s
0                      10.75       10751164.44
1                      10.42       10417193.48
2                      20.92       10458916.02
4                      41.35       10337220.34
8                      80.31       10038711.33
16                    159.96        9997513.38
32                    315.62        9863069.51
64                    615.53        9617664.07
128                  1148.44        8972203.58
256                  2230.16        8711569.47
512                  4234.62        8270736.15
1024                 7390.83        7217610.31
2048                12660.36        6181815.31
4096                17647.47        4308464.28
8192                18698.58        2282541.64
16384               19289.68        1177348.49
32768               19080.40         582287.63
65536               19690.18         300448.26

Latency

[jmandel@gross ~]$ mpirun -np 2 -machinefile machines /usr/mpi/gcc/mvapich2-1.9/tests/osu-micro-benchmarks-4.0.1/osu_latency
# OSU MPI Latency Test v4.0.1
# Size          Latency (us)
0                       1.45
1                       1.61
2                       1.61
4                       1.60
8                       1.61
16                      1.64
32                      1.67
64                      1.76
128                     1.94
256                     2.54
512                     2.78
1024                    3.35
2048                    4.40
4096                    5.44
8192                    7.68
16384                  12.40
32768                  17.18
65536                  26.72
131072                 45.98
262144                 84.44
524288                161.29
1048576               315.18
2097152               622.72
4194304              1241.21
[jmandel@gross ~]$ mpirun -np 144 -machinefile machines /usr/mpi/gcc/mvapich2-1.9/tests/osu-micro-benchmarks-4.0.1/osu_multi_lat 
# OSU MPI Multi Latency Test v4.0.1
# Size          Latency (us)
0                       0.61
1                       0.65
2                       0.64
4                       0.65
8                       0.77
16                      0.70
32                      0.73
64                      0.80
128                     0.89
256                     1.00
512                     1.20
1024                    1.64
2048                    2.02
4096                    3.42
8192                    6.59
16384                  14.88
32768                  34.42
65536                  57.03
131072                110.01
262144                193.99
524288                416.63
1048576              1026.67
2097152              2159.10
4194304              4326.27

RAID

Note: front end (gross) has 128GB memory

[jmandel@gross ~]$ dd if=/dev/zero of=file bs=1G count=100
100+0 records in
100+0 records out
107374182400 bytes (107 GB) copied, 89.7158 s, 1.2 GB/s
[jmandel@gross ~]$ dd if=file of=/dev/null  bs=1G count=100
100+0 records in
100+0 records out
107374182400 bytes (107 GB) copied, 28.8458 s, 3.7 GB/s
[jmandel@gross ~]$ 

Note: nodes have 24GB memory

[jmandel@compute-0-9 ~]$ dd if=/dev/zero of=file bs=1G count=100
^C
85+0 records in
85+0 records out
91268055040 bytes (91 GB) copied, 773.855 s, 118 MB/s

June 2010 - Aeon-installed OFED

Bandwidth

[root@frontend ~]# mpirun -np 2 -machinefile machines /usr/mpi/gcc/mvapich-1.2.0/tests/osu_benchmarks-3.1.1/osu_bw 
# OSU MPI Bandwidth Test v3.1.1
# Size        Bandwidth (MB/s)
1                         2.56
2                         5.10
4                        10.25
8                        20.31
16                       39.41
32                       76.42
64                      149.93
128                     249.22
256                     487.33
512                     858.70
1024                   1461.55
2048                   2076.22
4096                   2637.79
8192                   2874.39
16384                  2815.35
32768                  3081.33
65536                  3234.97
131072                 3321.83
262144                 3367.06
524288                 3389.63
1048576                3401.02
2097152                3406.80
4194304                3408.92
[root@frontend ~]# mpirun -np 2 -machinefile machines /usr/mpi/gcc/mvapich-1.2.0/tests/osu_benchmarks-3.1.1/osu_bibw 
# OSU MPI Bi-Directional Bandwidth Test v3.1.1
# Size     Bi-Bandwidth (MB/s)
1                         2.69
2                         5.38
4                        10.89
8                        21.49
16                       41.85
32                       82.27
64                      162.70
128                     283.36
256                     546.32
512                     986.10
1024                   1676.33
2048                   2827.57
4096                   4410.23
8192                   4866.69
16384                  4855.08
32768                  5573.09
65536                  6034.75
131072                 6303.20
262144                 6443.61
524288                 6516.34
1048576                6553.15
2097152                6572.78
4194304                6546.40
[root@frontend ~]# mpirun -np 12 -machinefile machines-1-12 /usr/mpi/gcc/mvapich-1.2.0/tests/osu_benchmarks-3.1.1/osu_
osu_alltoall   osu_bcast      osu_bibw       osu_bw         osu_latency    osu_mbw_mr     osu_multi_lat  
[root@frontend ~]# mpirun -np 12 -machinefile machines-1-12 /usr/mpi/gcc/mvapich-1.2.0/tests/osu_benchmarks-3.1.1/osu_mbw_mr 
# OSU MPI Multiple Bandwidth / Message Rate Test v3.1.1
# [ pairs: 6 ] [ window size: 64 ]
# Size                    MB/s          Messages/s
1                        13.88         13877846.04
2                        27.73         13862815.88
4                        55.63         13908004.35
8                       111.10         13887884.27
16                      215.05         13440672.03
32                      417.25         13039049.24
64                      821.94         12842809.36
128                    1415.67         11059907.83
256                    2770.69         10822998.87
512                    4946.11          9660377.36
1024                   8544.46          8344198.17
2048                  12459.32          6083650.19
4096                  15787.05          3854260.76
8192                  17148.54          2093327.52
16384                 16652.87          1016410.80
32768                 18334.42           559522.07
65536                 19319.69           294795.03
131072                19875.47           151637.81
262144                20176.04            76965.48
524288                20319.91            38757.15
1048576               20393.91            19449.14
2097152               20430.67             9742.10
4194304               20444.58             4874.37
[root@frontend ~]# vi machines-1-12 
[root@frontend ~]# mpirun -np 144 -machinefile machines-1-12 /usr/mpi/gcc/mvapich-1.2.0/tests/osu_benchmarks-3.1.1/osu_mbw_mr 
# OSU MPI Multiple Bandwidth / Message Rate Test v3.1.1
# [ pairs: 72 ] [ window size: 64 ]
# Size                    MB/s          Messages/s
1                        18.64         18640776.70
2                        37.43         18714970.35
4                        75.21         18803558.31
8                       150.74         18842772.44
16                      300.93         18808163.27
32                      549.35         17167126.15
64                     1078.05         16844567.92
128                    2066.73         16146326.08
256                    3960.94         15472433.01
512                    7785.17         15205411.65
1024                  11752.41         11476961.39
2048                  13624.36          6652518.52
4096                  15533.18          3792280.47
8192                  17031.55          2079047.10
16384                 18849.39          1150475.62
32768                 19815.48           604720.44
65536                 20094.95           306624.57
131072                20347.63           155240.11
262144                20478.87            78120.70
524288                20517.88            39134.76
1048576               19562.97            18656.70
2097152               19549.92             9322.13
4194304               19552.55             4661.69

Latency

[root@frontend ~]# mpirun -np 2 -machinefile machines /usr/mpi/gcc/mvapich-1.2.0/tests/osu_benchmarks-3.1.1/osu_latency 
# OSU MPI Latency Test v3.1.1
# Size            Latency (us)
0                         1.53
1                         1.53
2                         1.54
4                         1.54
8                         1.54
16                        1.55
32                        1.60
64                        1.70
128                       2.85
256                       3.02
512                       3.25
1024                      3.84
2048                      4.91
4096                      5.91
8192                      7.97
16384                    12.96
32768                    17.53
65536                    27.25
131072                   46.65
262144                   85.49
524288                  163.15
1048576                 318.56
2097152                 629.39
4194304                1256.06
[root@frontend ~]# mpirun -np 144 -machinefile machines-1-12 /usr/mpi/gcc/mvapich-1.2.0/tests/osu_benchmarks-3.1.1/osu_
osu_alltoall   osu_bcast      osu_bibw       osu_bw         osu_latency    osu_mbw_mr     osu_multi_lat  
[root@frontend ~]# mpirun -np 144 -machinefile machines-1-12 /usr/mpi/gcc/mvapich-1.2.0/tests/osu_benchmarks-3.1.1/osu_multi_lat 
# OSU MPI Multi Latency Test v3.1.1
# Size            Latency (us)
1                         1.85
2                         1.71
4                         1.70
8                         1.70
16                        1.71
32                        1.74
64                        1.81
128                       3.28
256                       3.61
512                       4.04
1024                      4.80
2048                      6.69
4096                     10.94
8192                     18.69
16384                    35.96
32768                    67.24
65536                   149.90
131072                  258.78
262144                  483.78
524288                 1027.92
1048576                2066.93
2097152                4162.77
4194304                8407.82

RAID

# Raid performance over nfs from node05

[root@node05 ~]# dd if=/dev/zero of=/storage/file bs=1G count=100
100+0 records in
100+0 records out
107374182400 bytes (107 GB) copied, 192.852 seconds, 557 MB/s
[root@node05 ~]# dd if=/storage/file of=/dev/null bs=1G count=100
100+0 records in
100+0 records out
107374182400 bytes (107 GB) copied, 106.808 seconds, 1.0 GB/s
 
Personal tools
Namespaces

Variants
Actions
Navigation
Toolbox