How to Set Network Adapter Multi-Queue Properties

1[root@instance-0wazpxeh ~]# ethtool -l eth0  #Query network interface card eth0 queue count
2Channel parameters for eth0:
3Pre-set maximums:
4RX:               0
5TX:               0
6Other:            0
7 Combined:         4 # indicates that the network interface card supports up to 4 queues enabled
8Current hardware settings:
9RX:               0
10TX:               0
11Other:            0
12 Combined:         4 # indicates that 4 queues are currently enabled

1usr/lib/systemd/system/virt-net-init.service
2[Unit]
3Description=Virt IO Network Init
4After=local-fs.target
5Before=network-pre.target
6
7[Service]
8Type=oneshot
9ExecStart=/usr/libexec/virt-net-init.sh
10RemainAfterExit=True
11
12[Install]
13WantedBy=multi-user.target

1#!/bin/bash
2
3rps_flow_cnt=4096
4#get specified mask
5function get_specified_cpumask(){
6    local cpu_num=$1
7    quotient=$((${cpu_num}/32))
8    remainder=$((${cpu_num}-32*$quotient))
9    if [ ${quotient} -gt 0 ]; then
10        res_tail=""
11        res_head="80000000"
12        while [ $quotient -gt 1 ]
13        do
14            res_tail="${res_tail},00000000"
15            ((quotient--))
16        done
17        if [ $remainder -ne 0 ];then
18            res_tail="${res_tail},00000000"
19            res_head=$((1<<($remainder-1)))
20            res_head=`printf "%x" ${res_head}`
21        fi
22        result="${res_head}${res_tail}"
23    else
24        result=$((1<<(${cpu_num}-1)))
25        result=`printf "%x" $result`
26    fi
27    echo $result
28}
29
30#get all mask for rps/xps
31function get_all_cpus() {
32    local cpu_nums=$(grep -c processor /proc/cpuinfo)
33    quotient=$((${cpu_nums}/32))
34    remainder=$((${cpu_nums}-32*$quotient))
35    if [ ${quotient} -gt 0 ];then
36        res_head="ffffffff"
37        res_tail=""
38        while [ $quotient -gt 1 ]
39        do
40            res_tail="${res_tail},ffffffff"
41            ((quotient--))
42        done
43        if [ $remainder -ne 0 ];then
44            res_tail="${res_tail},ffffffff"
45            res_head=$(((1<<$remainder)-1))
46            res_head=`printf "%x" ${res_head}`
47        fi
48        result="${res_head}${res_tail}"
49    else
50        result=$(((1<<${cpu_nums})-1))
51        result=`printf "%x" $result`
52    fi
53    echo $result
54}
55
56#set xps
57function set_xps() {
58    dev=$1
59    for xps_file in `ls /sys/class/net/$dev/queues/tx-*/xps_cpus`
60    do
61        xps_cpus=$(get_all_cpus)
62        echo "[INFO] set ${xps_cpus} into ${xps_file}." | logger -i -t 'virt-net-init'
63        echo ${xps_cpus} > ${xps_file}
64    done
65}
66
67#set rps/rfs for multicore when multiqueue is disbale
68function set_rps_and_rfs(){
69    rps_cpus=$(get_all_cpus)
70    dev=$1
71    queues=`ls -ld /sys/class/net/$dev/queues/rx-* | wc -l`
72    num=0
73    while [ $num -lt $queues ]
74    do
75        echo ${rps_cpus} > /sys/class/net/$dev/queues/rx-$num/rps_cpus
76        echo "[INFO] set ${rps_cpus} into /sys/class/net/$dev/queues/rx-$num/rps_cpus." | logger -i -t 'virt-net-init'
77        echo ${rps_flow_cnt} > /sys/class/net/$dev/queues/rx-$num/rps_flow_cnt
78        echo "[INFO] set ${rps_flow_cnt} into /sys/class/net/$dev/queues/rx-$num/rps_flow_cnt." | logger -i -t 'virt-net-init'
79        ((num++))
80    done
81
82}
83
84function set_smp_affinity(){
85    local dev=$1
86    pci_dbdf=$(ethtool -i $dev | grep bus-info | cut -d' ' -f2)
87    if [ -z $pci_dbdf ]; then
88        echo "[ERR] No LiquidIO NIC detected" | logger -i -t 'virt-net-init'
89    else
90        dir=$(find /sys/devices/ -type d | grep $pci_dbdf | grep "/net$")
91        result=${dir%/*}
92        # get virtioX
93        virtio_num=${result##*/}
94
95        cpunums=`cat /proc/cpuinfo | grep processor | wc -l`
96        if [ ${cpunums} -lt 2 ];then
97            echo "[INFO] don't need set affinity for net interrupt!!" | logger -i -t 'virt-net-init'
98        else
99            irq_input=$(grep ${virtio_num}-input /proc/interrupts | sed "s/: .*//g" | sed "s/^ *//g")
100            cur_cpunum=0
101            for irqnum in ${irq_input}; do
102                num=$((${cur_cpunum}%$cpunums+1))
103                mask=`get_specified_cpumask $num`
104                #echo "[INFO] irq:${irqnum} bind to cpu:$mask." | logger -i -t 'virt-net-init'
105                echo "[INFO] echo $mask > /proc/irq/$irqnum/smp_affinity" | logger -i -t 'virt-net-init'
106                echo $mask > /proc/irq/$irqnum/smp_affinity
107                ((cur_cpunum++))
108            done
109             
110            irq_output=$(grep ${virtio_num}-output /proc/interrupts | sed "s/: .*//g" | sed "s/^ *//g")
111            cur_cpunum=0
112            for irqnum in ${irq_output}; do
113                num=$((${cur_cpunum}%$cpunums+1))
114                mask=`get_specified_cpumask $num`
115                #echo "[INFO] irq:${irqnum} bind to cpu:$mask." | logger -i -t 'virt-net-init'
116                echo "[INFO] echo $mask > /proc/irq/$irqnum/smp_affinity" | logger -i -t 'virt-net-init'
117                echo $mask > /proc/irq/$irqnum/smp_affinity
118                ((cur_cpunum++))
119            done
120        fi
121    fi
122}
123
124#set multiqueue
125function set_multiqueue() {
126    local dev=$1
127    local pre_set=`ethtool -l ${dev} | grep Combined | head -n 1 | cut -f 2`
128    local cur_set=`ethtool -l ${dev} | grep Combined | tail -n 1 | cut -f 2`
129    #virtio-net will enable multiqueue by default
130    if [ ${pre_set} -gt ${cur_set} ];then
131        ethtool -L ${dev} combined ${pre_set} 2>&1 | logger -i -t 'virt-net-init'
132        if [ "${PIPESTATUS[0]}" -ne 0 ]; then
133            echo "[WARN] set multiqueue[chnum = ${cur_set}] for $dev failed. Ignore this if multiqueue is already set or chnum = 1." | logger -i -t 'virt-net-init'
134        fi
135    else
136        echo "[INFO] multiqueue[chnum= ${cur_set}] for $dev is enabled." | logger -i -t 'virt-net-init'
137    fi
138}
139
140function main(){
141    for (( i = 0; i < 5; i++ )); do
142        dir_list=$(find /sys/devices/ -type d | grep virtio | grep "/net$")
143        if [[ ! -z "${dir_list}" ]]; then
144            break
145        fi
146        sleep 0.5
147    done
148
149    total_queues=0
150    for dir in ${dir_list[*]}; do
151        dev=$(ls $dir)
152        if [[ -n "$dev" ]]; then
153            #check if multiqueue is available
154            queue_nums=`ethtool -l $dev 2>&- | grep "Combined" | head -n 1 | awk '{print $2}'`
155            if [ "$?" == "0" ] && [ ${queue_nums} -gt 1 ];then
156                #set multiqueue and affinity
157                #echo "set multiqueue for ${dev},and rps/rfs don't need to set."
158                set_multiqueue ${dev}
159                set_smp_affinity ${dev}
160                #continue
161            else
162                #set rps and rfs
163                set_rps_and_rfs ${dev}
164                queues=`ls -ld /sys/class/net/$dev/queues/rx-* | wc -l`
165                total_queues=$((${total_queues}+$queues))
166                set_xps ${dev}
167            fi
168        else
169            echo "[WARN] net device isn't exist." | logger -i -t 'virt-net-init'
170        fi
171    done
172
173    if [ ${total_queues} -ne 0 ];then
174        rps_sock_flow_entries=$((total_queues*${rps_flow_cnt}))
175        echo ${rps_sock_flow_entries} > /proc/sys/net/core/rps_sock_flow_entries
176        echo "[INFO] set ${rps_sock_flow_entries} for rfs." | logger -i -t 'virt-net-init'
177    fi
178}
179main