I did tests with BeeGFS 7.4.5 on Rocky Linux 8.10. Results shows that increasing number of metadata services is decreasing metadata performance.
Benchmark
How fast client can unarchive openmpi archive:
curl -O https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.6.tar.bz2
time tar xaf openmpi-4.1.6.tar.bz2
NOTE: Pre-warming effect. I was repeating unarchive benchmarks 10 times to demonstrate pre-warming effect. After some time performance is decreasing again.
Benchmark script benchmark_unrchive_openmpi.sh
#!/bin/bash
TIMEFORMAT=%R
for i in {1..10}
do
rm -rf openmpi-4.1.6
time tar xaf openmpi-4.1.6.tar.bz2
done
Results in seconds. The first line is clean first start (without pre-warming):
| Repeat # | 32 metas | 16 metas | 8 metas | 4 metas | 2 metas | 1 meta |
|---|---|---|---|---|---|---|
| 1 | 251 | 123 | 43 | 18 | 13 | 10 |
| 2 | 138 | 27 | 11 | 12 | 11 | 11 |
| 3 | 79 | 13 | 11 | 11 | 11 | 11 |
| 4 | 49 | 11 | 12 | 11 | 11 | 11 |
| 5 | 31 | 11 | 11 | 11 | 11 | 11 |
| 6 | 22 | 11 | 11 | 11 | 11 | 10 |
| 7 | 17 | 12 | 11 | 11 | 11 | 10 |
| 8 | 16 | 11 | 11 | 11 | 11 | 10 |
| 9 | 16 | 11 | 12 | 11 | 10 | 10 |
| 10 | 14 | 11 | 12 | 11 | 10 | 10 |
How to recreate my setups and results
5 physycal servers:
- mds-1 - one disk for os, two disks for 8 metas, 1 disk for mgmtd
- mds-2 - one disk for os, two disks for 8 metas
- mds-3 - one disk for os, two disks for 8 metas
- mds-4 - one disk for os, two disks for 8 metas
- oss-1 - one disk for os, one disks for one storage target
mgmtd on mds-1
systemctl stop beegfs-mgmtd.service
umount /beegfs/mgmtd
mkfs.ext4 -m 0 /dev/sda
mount /dev/sda /beegfs/mgmtd
/opt/beegfs/sbin/beegfs-setup-mgmtd -p /beegfs/mgmtd
systemctl start beegfs-mgmtd.service
metadata targets on mds-1
systemctl stop beegfs-me\*
rm -rf /beegfs/disk1/meta1
rm -rf /beegfs/disk1/meta2
rm -rf /beegfs/disk1/meta3
rm -rf /beegfs/disk1/meta4
rm -rf /beegfs/disk2/meta1
rm -rf /beegfs/disk2/meta2
rm -rf /beegfs/disk2/meta3
rm -rf /beegfs/disk2/meta4
mkdir /beegfs/disk1/meta1
mkdir /beegfs/disk1/meta2
mkdir /beegfs/disk1/meta3
mkdir /beegfs/disk1/meta4
mkdir /beegfs/disk2/meta1
mkdir /beegfs/disk2/meta2
mkdir /beegfs/disk2/meta3
mkdir /beegfs/disk2/meta4
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk1-meta1.conf -p /beegfs/disk1/meta1 -s 111 -S mds-1-d1m1 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk1-meta2.conf -p /beegfs/disk1/meta2 -s 112 -S mds-1-d1m2 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk1-meta3.conf -p /beegfs/disk1/meta3 -s 113 -S mds-1-d1m3 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk1-meta4.conf -p /beegfs/disk1/meta4 -s 114 -S mds-1-d1m4 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk2-meta1.conf -p /beegfs/disk2/meta1 -s 121 -S mds-1-d2m1 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk2-meta2.conf -p /beegfs/disk2/meta2 -s 122 -S mds-1-d2m2 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk2-meta3.conf -p /beegfs/disk2/meta3 -s 123 -S mds-1-d2m3 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk2-meta4.conf -p /beegfs/disk2/meta4 -s 124 -S mds-1-d2m4 -m 10.3.10.18 -C
systemctl start beegfs-meta-disk1-meta1
systemctl start beegfs-meta-disk1-meta2
systemctl start beegfs-meta-disk1-meta3
systemctl start beegfs-meta-disk1-meta4
systemctl start beegfs-meta-disk2-meta1
systemctl start beegfs-meta-disk2-meta2
systemctl start beegfs-meta-disk2-meta3
systemctl start beegfs-meta-disk2-meta4
metadata targets on mds-2
systemctl stop beegfs-me\*
rm -rf /beegfs/disk1/meta1
rm -rf /beegfs/disk1/meta2
rm -rf /beegfs/disk1/meta3
rm -rf /beegfs/disk1/meta4
rm -rf /beegfs/disk2/meta1
rm -rf /beegfs/disk2/meta2
rm -rf /beegfs/disk2/meta3
rm -rf /beegfs/disk2/meta4
mkdir /beegfs/disk1/meta1
mkdir /beegfs/disk1/meta2
mkdir /beegfs/disk1/meta3
mkdir /beegfs/disk1/meta4
mkdir /beegfs/disk2/meta1
mkdir /beegfs/disk2/meta2
mkdir /beegfs/disk2/meta3
mkdir /beegfs/disk2/meta4
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk1-meta1.conf -p /beegfs/disk1/meta1 -s 211 -S mds-2-d1m1 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk1-meta2.conf -p /beegfs/disk1/meta2 -s 212 -S mds-2-d1m2 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk1-meta3.conf -p /beegfs/disk1/meta3 -s 213 -S mds-2-d1m3 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk1-meta4.conf -p /beegfs/disk1/meta4 -s 214 -S mds-2-d1m4 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk2-meta1.conf -p /beegfs/disk2/meta1 -s 221 -S mds-2-d2m1 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk2-meta2.conf -p /beegfs/disk2/meta2 -s 222 -S mds-2-d2m2 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk2-meta3.conf -p /beegfs/disk2/meta3 -s 223 -S mds-2-d2m3 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk2-meta4.conf -p /beegfs/disk2/meta4 -s 224 -S mds-2-d2m4 -m 10.3.10.18 -C
systemctl start beegfs-meta-disk1-meta1
systemctl start beegfs-meta-disk1-meta2
systemctl start beegfs-meta-disk1-meta3
systemctl start beegfs-meta-disk1-meta4
systemctl start beegfs-meta-disk2-meta1
systemctl start beegfs-meta-disk2-meta2
systemctl start beegfs-meta-disk2-meta3
systemctl start beegfs-meta-disk2-meta4
metadata targets on mds-3
systemctl stop beegfs-me\*
rm -rf /beegfs/disk1/meta1
rm -rf /beegfs/disk1/meta2
rm -rf /beegfs/disk1/meta3
rm -rf /beegfs/disk1/meta4
rm -rf /beegfs/disk2/meta1
rm -rf /beegfs/disk2/meta2
rm -rf /beegfs/disk2/meta3
rm -rf /beegfs/disk2/meta4
mkdir /beegfs/disk1/meta1
mkdir /beegfs/disk1/meta2
mkdir /beegfs/disk1/meta3
mkdir /beegfs/disk1/meta4
mkdir /beegfs/disk2/meta1
mkdir /beegfs/disk2/meta2
mkdir /beegfs/disk2/meta3
mkdir /beegfs/disk2/meta4
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk1-meta1.conf -p /beegfs/disk1/meta1 -s 311 -S mds-3-d1m1 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk1-meta2.conf -p /beegfs/disk1/meta2 -s 312 -S mds-3-d1m2 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk1-meta3.conf -p /beegfs/disk1/meta3 -s 313 -S mds-3-d1m3 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk1-meta4.conf -p /beegfs/disk1/meta4 -s 314 -S mds-3-d1m4 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk2-meta1.conf -p /beegfs/disk2/meta1 -s 321 -S mds-3-d2m1 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk2-meta2.conf -p /beegfs/disk2/meta2 -s 322 -S mds-3-d2m2 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk2-meta3.conf -p /beegfs/disk2/meta3 -s 323 -S mds-3-d2m3 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk2-meta4.conf -p /beegfs/disk2/meta4 -s 324 -S mds-3-d2m4 -m 10.3.10.18 -C
systemctl start beegfs-meta-disk1-meta1
systemctl start beegfs-meta-disk1-meta2
systemctl start beegfs-meta-disk1-meta3
systemctl start beegfs-meta-disk1-meta4
systemctl start beegfs-meta-disk2-meta1
systemctl start beegfs-meta-disk2-meta2
systemctl start beegfs-meta-disk2-meta3
systemctl start beegfs-meta-disk2-meta4
metadata targets on mds-4
systemctl stop beegfs-me\*
rm -rf /beegfs/disk1/meta1
rm -rf /beegfs/disk1/meta2
rm -rf /beegfs/disk1/meta3
rm -rf /beegfs/disk1/meta4
rm -rf /beegfs/disk2/meta1
rm -rf /beegfs/disk2/meta2
rm -rf /beegfs/disk2/meta3
rm -rf /beegfs/disk2/meta4
mkdir /beegfs/disk1/meta1
mkdir /beegfs/disk1/meta2
mkdir /beegfs/disk1/meta3
mkdir /beegfs/disk1/meta4
mkdir /beegfs/disk2/meta1
mkdir /beegfs/disk2/meta2
mkdir /beegfs/disk2/meta3
mkdir /beegfs/disk2/meta4
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk1-meta1.conf -p /beegfs/disk1/meta1 -s 411 -S mds-4-d1m1 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk1-meta2.conf -p /beegfs/disk1/meta2 -s 412 -S mds-4-d1m2 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk1-meta3.conf -p /beegfs/disk1/meta3 -s 413 -S mds-4-d1m3 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk1-meta4.conf -p /beegfs/disk1/meta4 -s 414 -S mds-4-d1m4 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk2-meta1.conf -p /beegfs/disk2/meta1 -s 421 -S mds-4-d2m1 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk2-meta2.conf -p /beegfs/disk2/meta2 -s 422 -S mds-4-d2m2 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk2-meta3.conf -p /beegfs/disk2/meta3 -s 423 -S mds-4-d2m3 -m 10.3.10.18 -C
/opt/beegfs/sbin/beegfs-setup-meta -c /etc/beegfs/beegfs-meta-disk2-meta4.conf -p /beegfs/disk2/meta4 -s 424 -S mds-4-d2m4 -m 10.3.10.18 -C
systemctl start beegfs-meta-disk1-meta1
systemctl start beegfs-meta-disk1-meta2
systemctl start beegfs-meta-disk1-meta3
systemctl start beegfs-meta-disk1-meta4
systemctl start beegfs-meta-disk2-meta1
systemctl start beegfs-meta-disk2-meta2
systemctl start beegfs-meta-disk2-meta3
systemctl start beegfs-meta-disk2-meta4
storage targets on oss-1
systemctl stop beegfs-storage2.service
umount /dev/sda
rm -rf /beegfs/storage1
mkfs.xfs -f /dev/sda
mkdir /beegfs/storage1
mount /dev/sda /beegfs/storage1/
/opt/beegfs/sbin/beegfs-setup-storage -c /etc/beegfs/beegfs-storage1.conf -p /beegfs/storage1 -s 1111 -S oss-1-d1 -i 1111 -m 10.3.10.18 -C
systemctl restart beegfs-storage1.service
Client stop start
systemctl stop beegfs-client.service
systemctl stop beegfs-helperd.service
systemctl start beegfs-helperd.service
systemctl start beegfs-client.service