🐞 fix: Support modified RSS hash based on hardware supportv4.5.8-20230417rel-4.5dev-4.5

Support modified RSS hash based on hardware support

3 files changed, 158 insertions, 77 deletions

diff --git a/service/src/phydev.c b/service/src/phydev.c
index d556121..aeb3159 100644
--- a/service/src/phydev.c
+++ b/service/src/phydev.c
@@ -29,7 +29,7 @@
 static struct rte_eth_conf eth_conf_default = {
     .rxmode =
         {
-            .mq_mode = ETH_MQ_RX_RSS,
+            .mq_mode = ETH_MQ_RX_NONE,
             .split_hdr_size = 0,
         },
     .txmode =
@@ -154,11 +154,11 @@ static int gen_phydev_qconf(struct phydev * dev, struct rte_eth_rxconf * rxconf,
 }
 
 /* 用户参数解析：网卡参数设置 */
-static int gen_phydev_ethconf(struct phydev * dev, struct rte_eth_conf * out_eth_conf)
+static int gen_phydev_ethconf(struct phydev * dev, unsigned nr_rxq_use, struct rte_eth_conf * out_eth_conf)
 {
     struct rte_eth_conf eth_conf = eth_conf_default;
 
-    if (dev->info.type == __PHYDEV_INFO_TYPE_PCI)
+    if ((dev->info.type == __PHYDEV_INFO_TYPE_PCI) && (nr_rxq_use > 1))
     {
         /* only PCI devices can run at RSS mode. */
         eth_conf.rxmode.mq_mode = ETH_MQ_RX_RSS;
@@ -183,6 +183,30 @@ static int gen_phydev_ethconf(struct phydev * dev, struct rte_eth_conf * out_eth
                                                    ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_NONFRAG_IPV6_UDP;
             eth_conf.rx_adv_conf.rss_conf.rss_key = NULL;
         }
+
+        /* According to dev info reset rss conf */
+        struct rte_eth_conf request_eth_conf = eth_conf;
+        struct rte_eth_dev_info dev_info = {};
+
+        /* Get dev info */
+        rte_eth_dev_info_get(dev->port_id, &dev_info);
+        if (dev_info.flow_type_rss_offloads == 0)
+        {
+            memcpy(&eth_conf, &eth_conf_default, sizeof(eth_conf));
+            MR_WARNING("The port '%s' no support rss.", dev->symbol);
+        }
+        else
+        {
+            /* Check request rss_hf the dev supported or not */
+            eth_conf.rx_adv_conf.rss_conf.rss_hf &= dev_info.flow_type_rss_offloads;
+            if (eth_conf.rx_adv_conf.rss_conf.rss_hf != request_eth_conf.rx_adv_conf.rss_conf.rss_hf)
+            {
+                MR_WARNING("The port %s modified RSS hash function based on hardware support,"
+                           "requested:%#" PRIx64 " configured:%#" PRIx64 "\n",
+                           dev->symbol, request_eth_conf.rx_adv_conf.rss_conf.rss_hf,
+                           eth_conf.rx_adv_conf.rss_conf.rss_hf);
+            }
+        }
     }
     else if (dev->info.type == __PHYDEV_INFO_TYPE_VDEV)
     {
@@ -470,10 +494,6 @@ static int phydev_setup(struct sc_main * sc, struct phydev_main * ctx, struct ph
         return RT_ERR;
     }
 
-    // 配置端口信息
-    struct rte_eth_conf local_eth_conf;
-    gen_phydev_ethconf(dev, &local_eth_conf);
-
     unsigned nr_rxq_use = 0;
     unsigned nr_txq_use = 0;
 
@@ -508,6 +528,10 @@ static int phydev_setup(struct sc_main * sc, struct phydev_main * ctx, struct ph
     dev->nr_hairpin_q = dev->en_smartoffload ? 1 : 0;
     calc_phydev_queue(dev, &nr_rxq_use, &nr_txq_use);
 
+    // 配置端口信息
+    struct rte_eth_conf local_eth_conf;
+    gen_phydev_ethconf(dev, nr_rxq_use, &local_eth_conf);
+
     retval = rte_eth_dev_configure(dev->port_id, nr_rxq_use, nr_txq_use, &local_eth_conf);
     if (retval != 0)
     {
diff --git a/tools/devbind/devbind.py b/tools/devbind/devbind.py
index 1243f0e..3fc3cc0 100644
--- a/tools/devbind/devbind.py
+++ b/tools/devbind/devbind.py
@@ -39,7 +39,10 @@ import getopt
 import subprocess
 import json
 import argparse
+import platform
 from os.path import exists, abspath, dirname, basename
+from glob import glob
+from os.path import join as path_join
 
 # The PCI base class for NETWORK devices
 NETWORK_BASE_CLASS = "02"
@@ -57,6 +60,8 @@ default_gcfg_location = '/usr/local/etc/mrglobal.conf'
 
 # This is roughly compatible with check_output function in subprocess module
 # which is only available in python 2.7.
+
+
 def check_output(args, stderr=None):
     '''Run a command and capture its output'''
     return subprocess.Popen(args, stdout=subprocess.PIPE,
@@ -101,6 +106,7 @@ def check_modules():
 
     # change DPDK driver list to only contain drivers that are loaded
     dpdk_drivers = [mod["Name"] for mod in mods if mod["Found"]]
+    print dpdk_drivers
 
 
 def has_driver(devices, dev_id):
@@ -222,7 +228,7 @@ def dev_id_from_dev_name(devices, dev_name, quite=False):
 
     if not quite:
         print("Unknown device: %s. "
-          "Please specify device in \"bus:slot.func\" format" % dev_name)
+              "Please specify device in \"bus:slot.func\" format" % dev_name)
         sys.exit(1)
 
     return None
@@ -232,14 +238,14 @@ def unbind_one(devices, dev_id, force):
     '''Unbind the device identified by "dev_id" from its current driver'''
     dev = devices[dev_id]
     if not has_driver(devices, dev_id):
-        print("%s %s %s is not currently managed by any driver\n" %
+        print("Notice: %s %s %s is not currently managed by any driver" %
               (dev["Slot"], dev["Device_str"], dev["Interface"]))
         return
 
     # prevent us disconnecting ourselves
     if dev["Ssh_if"] and not force:
-        print("Routing table indicates that interface %s is active. "
-              "Skipping unbind" % (dev_id))
+        print("Warning: routing table indicates that interface %s is active. "
+              "Skipping unbind" % dev_id)
         return
 
     # write to /sys to unbind
@@ -247,13 +253,13 @@ def unbind_one(devices, dev_id, force):
     try:
         f = open(filename, "a")
     except:
-        print("Error: unbind failed for %s - Cannot open %s"
-              % (dev_id, filename))
-        sys.exit(1)
+        sys.exit("Error: unbind failed for %s - Cannot open %s" %
+                 (dev_id, filename))
     f.write(dev_id)
     f.close()
 
-def is_mlx_dev(dev,devices):
+
+def is_mlx_dev(dev, devices):
     dev_item = devices.get(dev)
     dev_driver = dev_item.get('Driver_str')
     if 'mlx5_core' == dev_driver:
@@ -261,57 +267,77 @@ def is_mlx_dev(dev,devices):
     else:
         return False
 
+
 def bind_one(devices, dev_id, driver, force):
     '''Bind the device given by "dev_id" to the driver "driver". If the device
     is already bound to a different driver, it will be unbound first'''
-
     dev = devices[dev_id]
     saved_driver = None  # used to rollback any unbind in case of failure
 
-    if driver == '' or driver == None:
-        return
-
     # prevent disconnection of our ssh session
     if dev["Ssh_if"] and not force:
-        print("Routing table indicates that interface %s is active. "
-              "Not modifying" % (dev_id))
+        print("Warning: routing table indicates that interface %s is active. "
+              "Not modifying" % dev_id)
         return
 
     # unbind any existing drivers we don't want
     if has_driver(devices, dev_id):
         if dev["Driver_str"] == driver:
-            print("%s already bound to driver %s, skipping\n"
-                  % (dev_id, driver))
+            print("Notice: %s already bound to driver %s, skipping" %
+                  (dev_id, driver))
             return
-        else:
-            saved_driver = dev["Driver_str"]
-            unbind_one(devices, dev_id, force)
-            dev["Driver_str"] = ""  # clear driver string
-
-    # if we are binding to one of DPDK drivers, add PCI id's to that driver
+        saved_driver = dev["Driver_str"]
+        unbind_one(devices, dev_id, force)
+        dev["Driver_str"] = ""  # clear driver string
+
+    # For kernels >= 3.15 driver_override can be used to specify the driver
+    # for a device rather than relying on the driver to provide a positive
+    # match of the device.  The existing process of looking up
+    # the vendor and device ID, adding them to the driver new_id,
+    # will erroneously bind other devices too which has the additional burden
+    # of unbinding those devices
     if driver in dpdk_drivers:
-        filename = "/sys/bus/pci/drivers/%s/new_id" % driver
-        try:
-            f = open(filename, "w")
-        except:
-            print("Error: bind failed for %s - Cannot open %s"
-                  % (dev_id, filename))
-            return
-        try:
-            f.write("%04x %04x" % (dev["Vendor"], dev["Device"]))
-            f.close()
-        except:
-            print("Error: bind failed for %s - Cannot write new PCI ID to "
-                  "driver %s" % (dev_id, driver))
-            return
+        filename = "/sys/bus/pci/devices/%s/driver_override" % dev_id
+        if exists(filename):
+            try:
+                f = open(filename, "w")
+            except:
+                print("Error: bind failed for %s - Cannot open %s" %
+                      (dev_id, filename))
+                return
+            try:
+                f.write("%s" % driver)
+                f.close()
+            except:
+                print("Error: bind failed for %s - Cannot write driver %s " %
+                      (dev_id, driver))
+                return
+        # For kernels < 3.15 use new_id to add PCI id's to the driver
+        else:
+            filename = "/sys/bus/pci/drivers/%s/new_id" % driver
+            try:
+                f = open(filename, "w")
+            except:
+                print("Error: bind failed for %s - Cannot open %s" %
+                      (dev_id, filename))
+                return
+            try:
+                # Convert Device and Vendor Id to int to write to new_id
+                f.write("%04x %04x" % (int(dev["Vendor"], 16),
+                                       int(dev["Device"], 16)))
+                f.close()
+            except:
+                print(
+                    "Error: bind failed for %s - Cannot write new PCI ID to " "driver %s" % (dev_id, driver))
+                return
 
     # do the bind by writing to /sys
     filename = "/sys/bus/pci/drivers/%s/bind" % driver
     try:
         f = open(filename, "a")
     except:
-        print("Error: bind failed for %s - Cannot open %s"
-              % (dev_id, filename))
+        print("Error: bind failed for %s - Cannot open %s" %
+              (dev_id, filename))
         if saved_driver is not None:  # restore any previous driver
             bind_one(devices, dev_id, saved_driver, force)
         return
@@ -325,12 +351,29 @@ def bind_one(devices, dev_id, driver, force):
         tmp = get_pci_device_details(dev_id)
         if "Driver_str" in tmp and tmp["Driver_str"] == driver:
             return
-        print("Error: bind failed for %s - Cannot bind to driver %s"
-              % (dev_id, driver))
+        print("Error: bind failed for %s - Cannot bind to driver %s" %
+              (dev_id, driver))
         if saved_driver is not None:  # restore any previous driver
             bind_one(devices, dev_id, saved_driver, force)
         return
 
+    # For kernels > 3.15 driver_override is used to bind a device to a driver.
+    # Before unbinding it, overwrite driver_override with empty string so that
+    # the device can be bound to any other driver
+    filename = "/sys/bus/pci/devices/%s/driver_override" % dev_id
+    if exists(filename):
+        try:
+            f = open(filename, "w")
+        except:
+            sys.exit("Error: unbind failed for %s - Cannot open %s" %
+                     (dev_id, filename))
+        try:
+            f.write("\00")
+            f.close()
+        except:
+            sys.exit("Error: unbind failed for %s - Cannot write %s" %
+                     (dev_id, filename))
+
 
 def unbind_all(devices, dev_list, force=False):
     """Unbind method, takes a list of device locations"""
@@ -344,7 +387,7 @@ def bind_all(devices, dev_list, driver, force=False):
 
     dev_list = map(lambda x: dev_id_from_dev_name(devices, x), dev_list)
     for d in dev_list:
-        if is_mlx_dev(d,devices) == False:
+        if is_mlx_dev(d, devices) == False:
             bind_one(devices, d, driver, force)
 
     # when binding devices to a generic driver (i.e. one that doesn't have a
@@ -365,6 +408,7 @@ def bind_all(devices, dev_list, driver, force=False):
         if "Driver_str" in devices[d]:
             unbind_one(devices, d, force)
 
+
 def hwfile_encode(location):
     devices = get_nic_details()
     _dirname = os.path.dirname(location)
@@ -375,39 +419,48 @@ def hwfile_encode(location):
         json.dump(devices, json_fp)
     return
 
+
 def hwfile_decode(location):
     with open(location, 'r') as json_fp:
         devices = json.load(json_fp)
     return devices
 
+
 def setup_argv_parser():
 
     parser = argparse.ArgumentParser(description='Marsio ZeroCopy Tools -- Network Interface Card Tools',
-                                     version = 'Marsio ZeroCopy Tools Suite 4.1')
-
-    parser.add_argument('--hwfile', help = 'Hardware file location', nargs=1, metavar = 'FILE')
-    parser.add_argument('--action', help = 'Action, bind modules or unbind modules, or dump hwfiles',
-                                    choices=['bind','unbind','dump'])
-    parser.add_argument('--gcfg', help = 'Global configure file location', nargs=1, metavar = 'FILE')
-    parser.add_argument('interfaces', metavar='INTERFACES', help = 'symbol of interfaces', nargs ='*')
+                                     version='Marsio ZeroCopy Tools Suite 4.1')
+
+    parser.add_argument(
+        '--hwfile', help='Hardware file location', nargs=1, metavar='FILE')
+    parser.add_argument('--action', help='Action, bind modules or unbind modules, or dump hwfiles',
+                        choices=['bind', 'unbind', 'dump'])
+    parser.add_argument(
+        '--gcfg', help='Global configure file location', nargs=1, metavar='FILE')
+    parser.add_argument('interfaces', metavar='INTERFACES',
+                        help='symbol of interfaces', nargs='*')
     return parser.parse_args()
 
+
 def global_configure_parser(g_file):
     import ConfigParser
     config = ConfigParser.ConfigParser()
     config.read(g_file)
-    return config.get('device','device', 0).split(',')
+    return config.get('device', 'device', 0).split(',')
+
 
 def nics_bind(hwinfo, niclist):
     devices_info = get_nic_details()
+    print dpdk_drivers
     bind_all(devices_info, niclist, dpdk_drivers[0], True)
     return
 
+
 def nics_unbind(hwinfo, niclist):
     devices_info = get_nic_details()
     dev_addr_list = map(lambda x: dev_id_from_dev_name(hwinfo, x), niclist)
     kmod_list = map(lambda x: hwinfo[x]["Driver_str"], dev_addr_list)
-    map(lambda x,y: bind_one(devices_info, x, y, True), dev_addr_list, kmod_list)
+    map(lambda x, y: bind_one(devices_info, x, y, True), dev_addr_list, kmod_list)
     return
 
 
@@ -415,7 +468,8 @@ def main():
     r_options = setup_argv_parser()
     hwfile_location = r_options.hwfile[0] if r_options.hwfile else default_hwfile_location
     gcfg_location = r_options.gcfg[0] if r_options.gcfg else default_gcfg_location
-    niclist = r_options.interfaces if r_options.interfaces else global_configure_parser(gcfg_location)
+    niclist = r_options.interfaces if r_options.interfaces else global_configure_parser(
+        gcfg_location)
 
     if r_options.action == 'dump':
         return hwfile_encode(hwfile_location)
@@ -430,7 +484,8 @@ def main():
     hwinfo = hwfile_decode(hwfile_location)
     check_modules()
 
-    niclist = [ d for d in niclist if dev_id_from_dev_name(hwinfo, d, quite=True)]
+    niclist = [d for d in niclist if dev_id_from_dev_name(
+        hwinfo, d, quite=True)]
     print niclist
 
     if r_options.action == 'bind':
@@ -439,5 +494,6 @@ def main():
         return nics_unbind(hwinfo, niclist)
     return
 
+
 if __name__ == "__main__":
-    main()
\ No newline at end of file
+    main()
diff --git a/tools/systemd/mrenv-script b/tools/systemd/mrenv-script
index d769ab2..074c569 100644
--- a/tools/systemd/mrenv-script
+++ b/tools/systemd/mrenv-script
@@ -60,7 +60,7 @@ function module_load_kni()
 		echo -n "$KNI_MODULE_FILENAME does not existed. "
 		return 1
 	fi
-	
+
 	module_unload_kni
 	insmod $MODULE_PATH/$KNI_MODULE_FILENAME > /dev/null 2>&1
 	[ ! $? -eq 0 ] && echo -n " $MODULE_PATH/$KNI_MODULE_FILENAME load failure." && return 1
@@ -93,9 +93,10 @@ function module_unload_vfio_pci()
 
 function module_load_vfio_pci()
 {
-	modprobe vfio enable_unsafe_noiommu_mode=1
+	modprobe vfio
 	[ ! $? -eq 0 ] && echo "failed at modprobe vfio" && return 1
 
+    echo 1 > /sys/module/vfio/parameters/enable_unsafe_noiommu_mode
 	modprobe vfio_pci
 	return $?
 }
@@ -124,7 +125,7 @@ function huge_release_1G()
 		umount $HUGEPAGE_PATH_1G > /dev/null
 		[ ! $? -eq 0 ] && echo "umount 1G hugepage failure, maybe in use" && return 1
 	fi
-	
+
 	if [ -d $HUGEPAGE_PATH_1G ] ; then
 		rm -R $HUGEPAGE_PATH_1G
 		[ ! $? -eq 0 ] && echo "Cannot delete 2M hugepage" && return 1
@@ -133,7 +134,7 @@ function huge_release_1G()
 	if [ $HUGEPAGE_NUM_1G -ne 0 ]; then
 		echo -n 0 > $NR_HUGEPAGE_FILE_1G
 	fi
-	
+
 	return 0
 }
 
@@ -146,16 +147,16 @@ function huge_release_2M()
 		umount $HUGEPAGE_PATH_2M > /dev/null
 		[ ! $? -eq 0 ] && echo "umount 2M hugepage failure, maybe in use" && return 1
 	fi
-	
+
 	if [ -d $HUGEPAGE_PATH_2M ] ; then
 		rm -R $HUGEPAGE_PATH_2M
 		[ ! $? -eq 0 ] && echo "Cannot delete 2M hugepage" && return 1
 	fi
-	
+
 	if [ $HUGEPAGE_NUM_2M -ne 0 ]; then
 		echo -n 0 > $NR_HUGEPAGE_FILE_2M
 	fi
-	
+
 	return 0
 }
 
@@ -164,7 +165,7 @@ function huge_alloc_1G()
 	if [ ! -e $NR_HUGEPAGE_FILE_1G ]; then
 		return 1
 	fi
-	
+
 	echo -n $HUGEPAGE_NUM_1G > $NR_HUGEPAGE_FILE_1G
 	[ $? -eq 0 ] && return 0
 	return 1
@@ -175,7 +176,7 @@ function huge_alloc_2M()
 	if [ ! -e $NR_HUGEPAGE_FILE_2M ]; then
 		return 1
 	fi
-	
+
 	echo -n $HUGEPAGE_NUM_2M > $NR_HUGEPAGE_FILE_2M
 	[ $? -eq 0 ] && return 0
 	return 1
@@ -186,13 +187,13 @@ function huge_mount_1G()
 	if [ ! -e $NR_HUGEPAGE_FILE_1G ]; then
 		return 0
 	fi
-	
+
 	SIZE_1G=$(cat $NR_HUGEPAGE_FILE_1G)
 	if [ $SIZE_1G -eq 0 ]; then
 		return 0
 	fi
 
-	mkdir -p $HUGEPAGE_PATH_1G	
+	mkdir -p $HUGEPAGE_PATH_1G
 	mount -t hugetlbfs nodev -o pagesize=1G $HUGEPAGE_PATH_1G
 	[ $? -eq 0 ] && return 0
 	return 1
@@ -203,13 +204,13 @@ function huge_mount_2M()
 	if [ ! -e $NR_HUGEPAGE_FILE_2M ]; then
 		return 0
 	fi
-	
+
 	SIZE_2M=$(cat $NR_HUGEPAGE_FILE_2M)
 	if [ $SIZE_2M -eq 0 ]; then
 		return 0
 	fi
 
-	mkdir -p $HUGEPAGE_PATH_2M	
+	mkdir -p $HUGEPAGE_PATH_2M
 	mount -t hugetlbfs nodev -o pagesize=2M $HUGEPAGE_PATH_2M
 	[ $? -eq 0 ] && return 0
 	return 1
@@ -256,7 +257,7 @@ function nic_unbind_ixgbe()
 	OLDIFS=$IFS; IFS=,
 	for ITER_PCI_ADDR in $PCI_ADDR_IXGBE; do
 		$NICTOOL --force -b ixgbe $ITER_PCI_ADDR
-		[ ! $? -eq 0 ] && return 1	
+		[ ! $? -eq 0 ] && return 1
 	done
 	IFS=$OLDIFS
 	return 0
@@ -373,7 +374,7 @@ function module_load()
 	if [ "$DEFAULT_UIO_MODULE" == "vfio_pci" ]; then
 		module_load_vfio_pci
 		return $?
-	fi	
+	fi
 
 	return 0
 }
@@ -412,8 +413,8 @@ function stop()
 	[ ! $? -eq 0 ] && return 1
 	return 0
 }
-                                                   
-function restart() 
+
+function restart()
 {
         stop
         start $*