diff options
| -rw-r--r-- | service/src/phydev.c | 38 | ||||
| -rw-r--r-- | tools/devbind/devbind.py | 162 | ||||
| -rw-r--r-- | tools/systemd/mrenv-script | 35 |
3 files changed, 158 insertions, 77 deletions
diff --git a/service/src/phydev.c b/service/src/phydev.c index d556121..aeb3159 100644 --- a/service/src/phydev.c +++ b/service/src/phydev.c @@ -29,7 +29,7 @@ static struct rte_eth_conf eth_conf_default = { .rxmode = { - .mq_mode = ETH_MQ_RX_RSS, + .mq_mode = ETH_MQ_RX_NONE, .split_hdr_size = 0, }, .txmode = @@ -154,11 +154,11 @@ static int gen_phydev_qconf(struct phydev * dev, struct rte_eth_rxconf * rxconf, } /* 用户参数解析：网卡参数设置 */ -static int gen_phydev_ethconf(struct phydev * dev, struct rte_eth_conf * out_eth_conf) +static int gen_phydev_ethconf(struct phydev * dev, unsigned nr_rxq_use, struct rte_eth_conf * out_eth_conf) { struct rte_eth_conf eth_conf = eth_conf_default; - if (dev->info.type == __PHYDEV_INFO_TYPE_PCI) + if ((dev->info.type == __PHYDEV_INFO_TYPE_PCI) && (nr_rxq_use > 1)) { /* only PCI devices can run at RSS mode. */ eth_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; @@ -183,6 +183,30 @@ static int gen_phydev_ethconf(struct phydev * dev, struct rte_eth_conf * out_eth ETH_RSS_NONFRAG_IPV4_UDP | ETH_RSS_NONFRAG_IPV6_UDP; eth_conf.rx_adv_conf.rss_conf.rss_key = NULL; } + + /* According to dev info reset rss conf */ + struct rte_eth_conf request_eth_conf = eth_conf; + struct rte_eth_dev_info dev_info = {}; + + /* Get dev info */ + rte_eth_dev_info_get(dev->port_id, &dev_info); + if (dev_info.flow_type_rss_offloads == 0) + { + memcpy(ð_conf, ð_conf_default, sizeof(eth_conf)); + MR_WARNING("The port '%s' no support rss.", dev->symbol); + } + else + { + /* Check request rss_hf the dev supported or not */ + eth_conf.rx_adv_conf.rss_conf.rss_hf &= dev_info.flow_type_rss_offloads; + if (eth_conf.rx_adv_conf.rss_conf.rss_hf != request_eth_conf.rx_adv_conf.rss_conf.rss_hf) + { + MR_WARNING("The port %s modified RSS hash function based on hardware support," + "requested:%#" PRIx64 " configured:%#" PRIx64 "\n", + dev->symbol, request_eth_conf.rx_adv_conf.rss_conf.rss_hf, + eth_conf.rx_adv_conf.rss_conf.rss_hf); + } + } } else if (dev->info.type == __PHYDEV_INFO_TYPE_VDEV) { @@ -470,10 +494,6 @@ static int phydev_setup(struct sc_main * sc, struct phydev_main * ctx, struct ph return RT_ERR; } - // 配置端口信息 - struct rte_eth_conf local_eth_conf; - gen_phydev_ethconf(dev, &local_eth_conf); - unsigned nr_rxq_use = 0; unsigned nr_txq_use = 0; @@ -508,6 +528,10 @@ static int phydev_setup(struct sc_main * sc, struct phydev_main * ctx, struct ph dev->nr_hairpin_q = dev->en_smartoffload ? 1 : 0; calc_phydev_queue(dev, &nr_rxq_use, &nr_txq_use); + // 配置端口信息 + struct rte_eth_conf local_eth_conf; + gen_phydev_ethconf(dev, nr_rxq_use, &local_eth_conf); + retval = rte_eth_dev_configure(dev->port_id, nr_rxq_use, nr_txq_use, &local_eth_conf); if (retval != 0) { diff --git a/tools/devbind/devbind.py b/tools/devbind/devbind.py index 1243f0e..3fc3cc0 100644 --- a/tools/devbind/devbind.py +++ b/tools/devbind/devbind.py @@ -39,7 +39,10 @@ import getopt import subprocess import json import argparse +import platform from os.path import exists, abspath, dirname, basename +from glob import glob +from os.path import join as path_join # The PCI base class for NETWORK devices NETWORK_BASE_CLASS = "02" @@ -57,6 +60,8 @@ default_gcfg_location = '/usr/local/etc/mrglobal.conf' # This is roughly compatible with check_output function in subprocess module # which is only available in python 2.7. + + def check_output(args, stderr=None): '''Run a command and capture its output''' return subprocess.Popen(args, stdout=subprocess.PIPE, @@ -101,6 +106,7 @@ def check_modules(): # change DPDK driver list to only contain drivers that are loaded dpdk_drivers = [mod["Name"] for mod in mods if mod["Found"]] + print dpdk_drivers def has_driver(devices, dev_id): @@ -222,7 +228,7 @@ def dev_id_from_dev_name(devices, dev_name, quite=False): if not quite: print("Unknown device: %s. " - "Please specify device in \"bus:slot.func\" format" % dev_name) + "Please specify device in \"bus:slot.func\" format" % dev_name) sys.exit(1) return None @@ -232,14 +238,14 @@ def unbind_one(devices, dev_id, force): '''Unbind the device identified by "dev_id" from its current driver''' dev = devices[dev_id] if not has_driver(devices, dev_id): - print("%s %s %s is not currently managed by any driver\n" % + print("Notice: %s %s %s is not currently managed by any driver" % (dev["Slot"], dev["Device_str"], dev["Interface"])) return # prevent us disconnecting ourselves if dev["Ssh_if"] and not force: - print("Routing table indicates that interface %s is active. " - "Skipping unbind" % (dev_id)) + print("Warning: routing table indicates that interface %s is active. " + "Skipping unbind" % dev_id) return # write to /sys to unbind @@ -247,13 +253,13 @@ def unbind_one(devices, dev_id, force): try: f = open(filename, "a") except: - print("Error: unbind failed for %s - Cannot open %s" - % (dev_id, filename)) - sys.exit(1) + sys.exit("Error: unbind failed for %s - Cannot open %s" % + (dev_id, filename)) f.write(dev_id) f.close() -def is_mlx_dev(dev,devices): + +def is_mlx_dev(dev, devices): dev_item = devices.get(dev) dev_driver = dev_item.get('Driver_str') if 'mlx5_core' == dev_driver: @@ -261,57 +267,77 @@ def is_mlx_dev(dev,devices): else: return False + def bind_one(devices, dev_id, driver, force): '''Bind the device given by "dev_id" to the driver "driver". If the device is already bound to a different driver, it will be unbound first''' - dev = devices[dev_id] saved_driver = None # used to rollback any unbind in case of failure - if driver == '' or driver == None: - return - # prevent disconnection of our ssh session if dev["Ssh_if"] and not force: - print("Routing table indicates that interface %s is active. " - "Not modifying" % (dev_id)) + print("Warning: routing table indicates that interface %s is active. " + "Not modifying" % dev_id) return # unbind any existing drivers we don't want if has_driver(devices, dev_id): if dev["Driver_str"] == driver: - print("%s already bound to driver %s, skipping\n" - % (dev_id, driver)) + print("Notice: %s already bound to driver %s, skipping" % + (dev_id, driver)) return - else: - saved_driver = dev["Driver_str"] - unbind_one(devices, dev_id, force) - dev["Driver_str"] = "" # clear driver string - - # if we are binding to one of DPDK drivers, add PCI id's to that driver + saved_driver = dev["Driver_str"] + unbind_one(devices, dev_id, force) + dev["Driver_str"] = "" # clear driver string + + # For kernels >= 3.15 driver_override can be used to specify the driver + # for a device rather than relying on the driver to provide a positive + # match of the device. The existing process of looking up + # the vendor and device ID, adding them to the driver new_id, + # will erroneously bind other devices too which has the additional burden + # of unbinding those devices if driver in dpdk_drivers: - filename = "/sys/bus/pci/drivers/%s/new_id" % driver - try: - f = open(filename, "w") - except: - print("Error: bind failed for %s - Cannot open %s" - % (dev_id, filename)) - return - try: - f.write("%04x %04x" % (dev["Vendor"], dev["Device"])) - f.close() - except: - print("Error: bind failed for %s - Cannot write new PCI ID to " - "driver %s" % (dev_id, driver)) - return + filename = "/sys/bus/pci/devices/%s/driver_override" % dev_id + if exists(filename): + try: + f = open(filename, "w") + except: + print("Error: bind failed for %s - Cannot open %s" % + (dev_id, filename)) + return + try: + f.write("%s" % driver) + f.close() + except: + print("Error: bind failed for %s - Cannot write driver %s " % + (dev_id, driver)) + return + # For kernels < 3.15 use new_id to add PCI id's to the driver + else: + filename = "/sys/bus/pci/drivers/%s/new_id" % driver + try: + f = open(filename, "w") + except: + print("Error: bind failed for %s - Cannot open %s" % + (dev_id, filename)) + return + try: + # Convert Device and Vendor Id to int to write to new_id + f.write("%04x %04x" % (int(dev["Vendor"], 16), + int(dev["Device"], 16))) + f.close() + except: + print( + "Error: bind failed for %s - Cannot write new PCI ID to " "driver %s" % (dev_id, driver)) + return # do the bind by writing to /sys filename = "/sys/bus/pci/drivers/%s/bind" % driver try: f = open(filename, "a") except: - print("Error: bind failed for %s - Cannot open %s" - % (dev_id, filename)) + print("Error: bind failed for %s - Cannot open %s" % + (dev_id, filename)) if saved_driver is not None: # restore any previous driver bind_one(devices, dev_id, saved_driver, force) return @@ -325,12 +351,29 @@ def bind_one(devices, dev_id, driver, force): tmp = get_pci_device_details(dev_id) if "Driver_str" in tmp and tmp["Driver_str"] == driver: return - print("Error: bind failed for %s - Cannot bind to driver %s" - % (dev_id, driver)) + print("Error: bind failed for %s - Cannot bind to driver %s" % + (dev_id, driver)) if saved_driver is not None: # restore any previous driver bind_one(devices, dev_id, saved_driver, force) return + # For kernels > 3.15 driver_override is used to bind a device to a driver. + # Before unbinding it, overwrite driver_override with empty string so that + # the device can be bound to any other driver + filename = "/sys/bus/pci/devices/%s/driver_override" % dev_id + if exists(filename): + try: + f = open(filename, "w") + except: + sys.exit("Error: unbind failed for %s - Cannot open %s" % + (dev_id, filename)) + try: + f.write("\00") + f.close() + except: + sys.exit("Error: unbind failed for %s - Cannot write %s" % + (dev_id, filename)) + def unbind_all(devices, dev_list, force=False): """Unbind method, takes a list of device locations""" @@ -344,7 +387,7 @@ def bind_all(devices, dev_list, driver, force=False): dev_list = map(lambda x: dev_id_from_dev_name(devices, x), dev_list) for d in dev_list: - if is_mlx_dev(d,devices) == False: + if is_mlx_dev(d, devices) == False: bind_one(devices, d, driver, force) # when binding devices to a generic driver (i.e. one that doesn't have a @@ -365,6 +408,7 @@ def bind_all(devices, dev_list, driver, force=False): if "Driver_str" in devices[d]: unbind_one(devices, d, force) + def hwfile_encode(location): devices = get_nic_details() _dirname = os.path.dirname(location) @@ -375,39 +419,48 @@ def hwfile_encode(location): json.dump(devices, json_fp) return + def hwfile_decode(location): with open(location, 'r') as json_fp: devices = json.load(json_fp) return devices + def setup_argv_parser(): parser = argparse.ArgumentParser(description='Marsio ZeroCopy Tools -- Network Interface Card Tools', - version = 'Marsio ZeroCopy Tools Suite 4.1') - - parser.add_argument('--hwfile', help = 'Hardware file location', nargs=1, metavar = 'FILE') - parser.add_argument('--action', help = 'Action, bind modules or unbind modules, or dump hwfiles', - choices=['bind','unbind','dump']) - parser.add_argument('--gcfg', help = 'Global configure file location', nargs=1, metavar = 'FILE') - parser.add_argument('interfaces', metavar='INTERFACES', help = 'symbol of interfaces', nargs ='*') + version='Marsio ZeroCopy Tools Suite 4.1') + + parser.add_argument( + '--hwfile', help='Hardware file location', nargs=1, metavar='FILE') + parser.add_argument('--action', help='Action, bind modules or unbind modules, or dump hwfiles', + choices=['bind', 'unbind', 'dump']) + parser.add_argument( + '--gcfg', help='Global configure file location', nargs=1, metavar='FILE') + parser.add_argument('interfaces', metavar='INTERFACES', + help='symbol of interfaces', nargs='*') return parser.parse_args() + def global_configure_parser(g_file): import ConfigParser config = ConfigParser.ConfigParser() config.read(g_file) - return config.get('device','device', 0).split(',') + return config.get('device', 'device', 0).split(',') + def nics_bind(hwinfo, niclist): devices_info = get_nic_details() + print dpdk_drivers bind_all(devices_info, niclist, dpdk_drivers[0], True) return + def nics_unbind(hwinfo, niclist): devices_info = get_nic_details() dev_addr_list = map(lambda x: dev_id_from_dev_name(hwinfo, x), niclist) kmod_list = map(lambda x: hwinfo[x]["Driver_str"], dev_addr_list) - map(lambda x,y: bind_one(devices_info, x, y, True), dev_addr_list, kmod_list) + map(lambda x, y: bind_one(devices_info, x, y, True), dev_addr_list, kmod_list) return @@ -415,7 +468,8 @@ def main(): r_options = setup_argv_parser() hwfile_location = r_options.hwfile[0] if r_options.hwfile else default_hwfile_location gcfg_location = r_options.gcfg[0] if r_options.gcfg else default_gcfg_location - niclist = r_options.interfaces if r_options.interfaces else global_configure_parser(gcfg_location) + niclist = r_options.interfaces if r_options.interfaces else global_configure_parser( + gcfg_location) if r_options.action == 'dump': return hwfile_encode(hwfile_location) @@ -430,7 +484,8 @@ def main(): hwinfo = hwfile_decode(hwfile_location) check_modules() - niclist = [ d for d in niclist if dev_id_from_dev_name(hwinfo, d, quite=True)] + niclist = [d for d in niclist if dev_id_from_dev_name( + hwinfo, d, quite=True)] print niclist if r_options.action == 'bind': @@ -439,5 +494,6 @@ def main(): return nics_unbind(hwinfo, niclist) return + if __name__ == "__main__": - main()
\ No newline at end of file + main() diff --git a/tools/systemd/mrenv-script b/tools/systemd/mrenv-script index d769ab2..074c569 100644 --- a/tools/systemd/mrenv-script +++ b/tools/systemd/mrenv-script @@ -60,7 +60,7 @@ function module_load_kni() echo -n "$KNI_MODULE_FILENAME does not existed. " return 1 fi - + module_unload_kni insmod $MODULE_PATH/$KNI_MODULE_FILENAME > /dev/null 2>&1 [ ! $? -eq 0 ] && echo -n " $MODULE_PATH/$KNI_MODULE_FILENAME load failure." && return 1 @@ -93,9 +93,10 @@ function module_unload_vfio_pci() function module_load_vfio_pci() { - modprobe vfio enable_unsafe_noiommu_mode=1 + modprobe vfio [ ! $? -eq 0 ] && echo "failed at modprobe vfio" && return 1 + echo 1 > /sys/module/vfio/parameters/enable_unsafe_noiommu_mode modprobe vfio_pci return $? } @@ -124,7 +125,7 @@ function huge_release_1G() umount $HUGEPAGE_PATH_1G > /dev/null [ ! $? -eq 0 ] && echo "umount 1G hugepage failure, maybe in use" && return 1 fi - + if [ -d $HUGEPAGE_PATH_1G ] ; then rm -R $HUGEPAGE_PATH_1G [ ! $? -eq 0 ] && echo "Cannot delete 2M hugepage" && return 1 @@ -133,7 +134,7 @@ function huge_release_1G() if [ $HUGEPAGE_NUM_1G -ne 0 ]; then echo -n 0 > $NR_HUGEPAGE_FILE_1G fi - + return 0 } @@ -146,16 +147,16 @@ function huge_release_2M() umount $HUGEPAGE_PATH_2M > /dev/null [ ! $? -eq 0 ] && echo "umount 2M hugepage failure, maybe in use" && return 1 fi - + if [ -d $HUGEPAGE_PATH_2M ] ; then rm -R $HUGEPAGE_PATH_2M [ ! $? -eq 0 ] && echo "Cannot delete 2M hugepage" && return 1 fi - + if [ $HUGEPAGE_NUM_2M -ne 0 ]; then echo -n 0 > $NR_HUGEPAGE_FILE_2M fi - + return 0 } @@ -164,7 +165,7 @@ function huge_alloc_1G() if [ ! -e $NR_HUGEPAGE_FILE_1G ]; then return 1 fi - + echo -n $HUGEPAGE_NUM_1G > $NR_HUGEPAGE_FILE_1G [ $? -eq 0 ] && return 0 return 1 @@ -175,7 +176,7 @@ function huge_alloc_2M() if [ ! -e $NR_HUGEPAGE_FILE_2M ]; then return 1 fi - + echo -n $HUGEPAGE_NUM_2M > $NR_HUGEPAGE_FILE_2M [ $? -eq 0 ] && return 0 return 1 @@ -186,13 +187,13 @@ function huge_mount_1G() if [ ! -e $NR_HUGEPAGE_FILE_1G ]; then return 0 fi - + SIZE_1G=$(cat $NR_HUGEPAGE_FILE_1G) if [ $SIZE_1G -eq 0 ]; then return 0 fi - mkdir -p $HUGEPAGE_PATH_1G + mkdir -p $HUGEPAGE_PATH_1G mount -t hugetlbfs nodev -o pagesize=1G $HUGEPAGE_PATH_1G [ $? -eq 0 ] && return 0 return 1 @@ -203,13 +204,13 @@ function huge_mount_2M() if [ ! -e $NR_HUGEPAGE_FILE_2M ]; then return 0 fi - + SIZE_2M=$(cat $NR_HUGEPAGE_FILE_2M) if [ $SIZE_2M -eq 0 ]; then return 0 fi - mkdir -p $HUGEPAGE_PATH_2M + mkdir -p $HUGEPAGE_PATH_2M mount -t hugetlbfs nodev -o pagesize=2M $HUGEPAGE_PATH_2M [ $? -eq 0 ] && return 0 return 1 @@ -256,7 +257,7 @@ function nic_unbind_ixgbe() OLDIFS=$IFS; IFS=, for ITER_PCI_ADDR in $PCI_ADDR_IXGBE; do $NICTOOL --force -b ixgbe $ITER_PCI_ADDR - [ ! $? -eq 0 ] && return 1 + [ ! $? -eq 0 ] && return 1 done IFS=$OLDIFS return 0 @@ -373,7 +374,7 @@ function module_load() if [ "$DEFAULT_UIO_MODULE" == "vfio_pci" ]; then module_load_vfio_pci return $? - fi + fi return 0 } @@ -412,8 +413,8 @@ function stop() [ ! $? -eq 0 ] && return 1 return 0 } - -function restart() + +function restart() { stop start $* |