ethblk/scripts/ethblk-setup at master · vtl/ethblk · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
#!/bin/bash

ini=1  # should be 1 on initiator and 0 on target
tgt=0  # should be 1 on target and 0 on initiator

tgt_nic=enp33s0
tgt_ip=192.168.50.3
tgt_disk=/dev/nullb0

ini_nic=enp193s0
ini_ip=192.168.50.2
ini_mac=50:6b:4b:cc:f3:e6

#ini_nic=enp1s0f4
#ini_ip=192.168.50.4
#ini_mac=00:07:43:3e:db:60

if [[ $ini == 1 ]]; then
    nic=$ini_nic
else
    nic=$tgt_nic
fi

set_irq_affinity.sh ${nic} # from mlnx-en-utils
max_rings=$(ethtool -g ${nic} | grep [RT]X: | head -2 | cut -c 1-3,6- | tr ':' ' ' | tr '[:upper:]' '[:lower:]')
ethtool -G ${nic} $max_rings
ethtool --set-priv-flags ${nic} rx_cqe_compress on
ethtool -A ${nic} rx on tx on
#ethtool -A ${nic} rx off tx off
ifconfig ${nic} txqueuelen 100000

sysctl -w net.core.netdev_budget=5000
sysctl -w net.core.netdev_budget_usecs=10000

systemctl stop irqbalance
systemctl stop tuned

#mlnx_tune -p HIGH_THROUGHPUT

rmmod ethblk
insmod ethblk.ko initiator=$ini target=$tgt disk_major=154 queue_depth=256 rps=0 ip_ports=256 lat_stat=0
#insmod ethblk.ko initiator=$ini target=$tgt disk_major=154 rps=1
# uncomment for debug
#echo module ethblk +p > /sys/kernel/debug/dynamic_debug/control

if [[ $ini -eq 1 ]]; then
    ifconfig $ini_nic $ini_ip mtu 9216
    echo 1 > /sys/kernel/ethblk/initiator/discover
fi

if [[ $tgt -eq 1 ]]; then
    ifconfig $tgt_nic $tgt_ip mtu 9216
    modprobe null_blk bs=512 nr_devices=1 hw_queue_depth=256 queue_mode=0 submit_queues=32
    for i in {0..3}; do
        echo $i /dev/nvme${i}n1 > /sys/kernel/ethblk/target/create_disk
        echo $ini_mac $tgt_nic > /sys/kernel/ethblk/target/eda${i}/initiators/add
    done
    echo 4 $tgt_disk > /sys/kernel/ethblk/target/create_disk
    echo $ini_mac $tgt_nic > /sys/kernel/ethblk/target/eda4/initiators/add
fi