diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/devbind/devbind.py | 162 | ||||
| -rw-r--r-- | tools/systemd/mrenv-script | 35 |
2 files changed, 127 insertions, 70 deletions
diff --git a/tools/devbind/devbind.py b/tools/devbind/devbind.py index 1243f0e..3fc3cc0 100644 --- a/tools/devbind/devbind.py +++ b/tools/devbind/devbind.py @@ -39,7 +39,10 @@ import getopt import subprocess import json import argparse +import platform from os.path import exists, abspath, dirname, basename +from glob import glob +from os.path import join as path_join # The PCI base class for NETWORK devices NETWORK_BASE_CLASS = "02" @@ -57,6 +60,8 @@ default_gcfg_location = '/usr/local/etc/mrglobal.conf' # This is roughly compatible with check_output function in subprocess module # which is only available in python 2.7. + + def check_output(args, stderr=None): '''Run a command and capture its output''' return subprocess.Popen(args, stdout=subprocess.PIPE, @@ -101,6 +106,7 @@ def check_modules(): # change DPDK driver list to only contain drivers that are loaded dpdk_drivers = [mod["Name"] for mod in mods if mod["Found"]] + print dpdk_drivers def has_driver(devices, dev_id): @@ -222,7 +228,7 @@ def dev_id_from_dev_name(devices, dev_name, quite=False): if not quite: print("Unknown device: %s. " - "Please specify device in \"bus:slot.func\" format" % dev_name) + "Please specify device in \"bus:slot.func\" format" % dev_name) sys.exit(1) return None @@ -232,14 +238,14 @@ def unbind_one(devices, dev_id, force): '''Unbind the device identified by "dev_id" from its current driver''' dev = devices[dev_id] if not has_driver(devices, dev_id): - print("%s %s %s is not currently managed by any driver\n" % + print("Notice: %s %s %s is not currently managed by any driver" % (dev["Slot"], dev["Device_str"], dev["Interface"])) return # prevent us disconnecting ourselves if dev["Ssh_if"] and not force: - print("Routing table indicates that interface %s is active. " - "Skipping unbind" % (dev_id)) + print("Warning: routing table indicates that interface %s is active. " + "Skipping unbind" % dev_id) return # write to /sys to unbind @@ -247,13 +253,13 @@ def unbind_one(devices, dev_id, force): try: f = open(filename, "a") except: - print("Error: unbind failed for %s - Cannot open %s" - % (dev_id, filename)) - sys.exit(1) + sys.exit("Error: unbind failed for %s - Cannot open %s" % + (dev_id, filename)) f.write(dev_id) f.close() -def is_mlx_dev(dev,devices): + +def is_mlx_dev(dev, devices): dev_item = devices.get(dev) dev_driver = dev_item.get('Driver_str') if 'mlx5_core' == dev_driver: @@ -261,57 +267,77 @@ def is_mlx_dev(dev,devices): else: return False + def bind_one(devices, dev_id, driver, force): '''Bind the device given by "dev_id" to the driver "driver". If the device is already bound to a different driver, it will be unbound first''' - dev = devices[dev_id] saved_driver = None # used to rollback any unbind in case of failure - if driver == '' or driver == None: - return - # prevent disconnection of our ssh session if dev["Ssh_if"] and not force: - print("Routing table indicates that interface %s is active. " - "Not modifying" % (dev_id)) + print("Warning: routing table indicates that interface %s is active. " + "Not modifying" % dev_id) return # unbind any existing drivers we don't want if has_driver(devices, dev_id): if dev["Driver_str"] == driver: - print("%s already bound to driver %s, skipping\n" - % (dev_id, driver)) + print("Notice: %s already bound to driver %s, skipping" % + (dev_id, driver)) return - else: - saved_driver = dev["Driver_str"] - unbind_one(devices, dev_id, force) - dev["Driver_str"] = "" # clear driver string - - # if we are binding to one of DPDK drivers, add PCI id's to that driver + saved_driver = dev["Driver_str"] + unbind_one(devices, dev_id, force) + dev["Driver_str"] = "" # clear driver string + + # For kernels >= 3.15 driver_override can be used to specify the driver + # for a device rather than relying on the driver to provide a positive + # match of the device. The existing process of looking up + # the vendor and device ID, adding them to the driver new_id, + # will erroneously bind other devices too which has the additional burden + # of unbinding those devices if driver in dpdk_drivers: - filename = "/sys/bus/pci/drivers/%s/new_id" % driver - try: - f = open(filename, "w") - except: - print("Error: bind failed for %s - Cannot open %s" - % (dev_id, filename)) - return - try: - f.write("%04x %04x" % (dev["Vendor"], dev["Device"])) - f.close() - except: - print("Error: bind failed for %s - Cannot write new PCI ID to " - "driver %s" % (dev_id, driver)) - return + filename = "/sys/bus/pci/devices/%s/driver_override" % dev_id + if exists(filename): + try: + f = open(filename, "w") + except: + print("Error: bind failed for %s - Cannot open %s" % + (dev_id, filename)) + return + try: + f.write("%s" % driver) + f.close() + except: + print("Error: bind failed for %s - Cannot write driver %s " % + (dev_id, driver)) + return + # For kernels < 3.15 use new_id to add PCI id's to the driver + else: + filename = "/sys/bus/pci/drivers/%s/new_id" % driver + try: + f = open(filename, "w") + except: + print("Error: bind failed for %s - Cannot open %s" % + (dev_id, filename)) + return + try: + # Convert Device and Vendor Id to int to write to new_id + f.write("%04x %04x" % (int(dev["Vendor"], 16), + int(dev["Device"], 16))) + f.close() + except: + print( + "Error: bind failed for %s - Cannot write new PCI ID to " "driver %s" % (dev_id, driver)) + return # do the bind by writing to /sys filename = "/sys/bus/pci/drivers/%s/bind" % driver try: f = open(filename, "a") except: - print("Error: bind failed for %s - Cannot open %s" - % (dev_id, filename)) + print("Error: bind failed for %s - Cannot open %s" % + (dev_id, filename)) if saved_driver is not None: # restore any previous driver bind_one(devices, dev_id, saved_driver, force) return @@ -325,12 +351,29 @@ def bind_one(devices, dev_id, driver, force): tmp = get_pci_device_details(dev_id) if "Driver_str" in tmp and tmp["Driver_str"] == driver: return - print("Error: bind failed for %s - Cannot bind to driver %s" - % (dev_id, driver)) + print("Error: bind failed for %s - Cannot bind to driver %s" % + (dev_id, driver)) if saved_driver is not None: # restore any previous driver bind_one(devices, dev_id, saved_driver, force) return + # For kernels > 3.15 driver_override is used to bind a device to a driver. + # Before unbinding it, overwrite driver_override with empty string so that + # the device can be bound to any other driver + filename = "/sys/bus/pci/devices/%s/driver_override" % dev_id + if exists(filename): + try: + f = open(filename, "w") + except: + sys.exit("Error: unbind failed for %s - Cannot open %s" % + (dev_id, filename)) + try: + f.write("\00") + f.close() + except: + sys.exit("Error: unbind failed for %s - Cannot write %s" % + (dev_id, filename)) + def unbind_all(devices, dev_list, force=False): """Unbind method, takes a list of device locations""" @@ -344,7 +387,7 @@ def bind_all(devices, dev_list, driver, force=False): dev_list = map(lambda x: dev_id_from_dev_name(devices, x), dev_list) for d in dev_list: - if is_mlx_dev(d,devices) == False: + if is_mlx_dev(d, devices) == False: bind_one(devices, d, driver, force) # when binding devices to a generic driver (i.e. one that doesn't have a @@ -365,6 +408,7 @@ def bind_all(devices, dev_list, driver, force=False): if "Driver_str" in devices[d]: unbind_one(devices, d, force) + def hwfile_encode(location): devices = get_nic_details() _dirname = os.path.dirname(location) @@ -375,39 +419,48 @@ def hwfile_encode(location): json.dump(devices, json_fp) return + def hwfile_decode(location): with open(location, 'r') as json_fp: devices = json.load(json_fp) return devices + def setup_argv_parser(): parser = argparse.ArgumentParser(description='Marsio ZeroCopy Tools -- Network Interface Card Tools', - version = 'Marsio ZeroCopy Tools Suite 4.1') - - parser.add_argument('--hwfile', help = 'Hardware file location', nargs=1, metavar = 'FILE') - parser.add_argument('--action', help = 'Action, bind modules or unbind modules, or dump hwfiles', - choices=['bind','unbind','dump']) - parser.add_argument('--gcfg', help = 'Global configure file location', nargs=1, metavar = 'FILE') - parser.add_argument('interfaces', metavar='INTERFACES', help = 'symbol of interfaces', nargs ='*') + version='Marsio ZeroCopy Tools Suite 4.1') + + parser.add_argument( + '--hwfile', help='Hardware file location', nargs=1, metavar='FILE') + parser.add_argument('--action', help='Action, bind modules or unbind modules, or dump hwfiles', + choices=['bind', 'unbind', 'dump']) + parser.add_argument( + '--gcfg', help='Global configure file location', nargs=1, metavar='FILE') + parser.add_argument('interfaces', metavar='INTERFACES', + help='symbol of interfaces', nargs='*') return parser.parse_args() + def global_configure_parser(g_file): import ConfigParser config = ConfigParser.ConfigParser() config.read(g_file) - return config.get('device','device', 0).split(',') + return config.get('device', 'device', 0).split(',') + def nics_bind(hwinfo, niclist): devices_info = get_nic_details() + print dpdk_drivers bind_all(devices_info, niclist, dpdk_drivers[0], True) return + def nics_unbind(hwinfo, niclist): devices_info = get_nic_details() dev_addr_list = map(lambda x: dev_id_from_dev_name(hwinfo, x), niclist) kmod_list = map(lambda x: hwinfo[x]["Driver_str"], dev_addr_list) - map(lambda x,y: bind_one(devices_info, x, y, True), dev_addr_list, kmod_list) + map(lambda x, y: bind_one(devices_info, x, y, True), dev_addr_list, kmod_list) return @@ -415,7 +468,8 @@ def main(): r_options = setup_argv_parser() hwfile_location = r_options.hwfile[0] if r_options.hwfile else default_hwfile_location gcfg_location = r_options.gcfg[0] if r_options.gcfg else default_gcfg_location - niclist = r_options.interfaces if r_options.interfaces else global_configure_parser(gcfg_location) + niclist = r_options.interfaces if r_options.interfaces else global_configure_parser( + gcfg_location) if r_options.action == 'dump': return hwfile_encode(hwfile_location) @@ -430,7 +484,8 @@ def main(): hwinfo = hwfile_decode(hwfile_location) check_modules() - niclist = [ d for d in niclist if dev_id_from_dev_name(hwinfo, d, quite=True)] + niclist = [d for d in niclist if dev_id_from_dev_name( + hwinfo, d, quite=True)] print niclist if r_options.action == 'bind': @@ -439,5 +494,6 @@ def main(): return nics_unbind(hwinfo, niclist) return + if __name__ == "__main__": - main()
\ No newline at end of file + main() diff --git a/tools/systemd/mrenv-script b/tools/systemd/mrenv-script index d769ab2..074c569 100644 --- a/tools/systemd/mrenv-script +++ b/tools/systemd/mrenv-script @@ -60,7 +60,7 @@ function module_load_kni() echo -n "$KNI_MODULE_FILENAME does not existed. " return 1 fi - + module_unload_kni insmod $MODULE_PATH/$KNI_MODULE_FILENAME > /dev/null 2>&1 [ ! $? -eq 0 ] && echo -n " $MODULE_PATH/$KNI_MODULE_FILENAME load failure." && return 1 @@ -93,9 +93,10 @@ function module_unload_vfio_pci() function module_load_vfio_pci() { - modprobe vfio enable_unsafe_noiommu_mode=1 + modprobe vfio [ ! $? -eq 0 ] && echo "failed at modprobe vfio" && return 1 + echo 1 > /sys/module/vfio/parameters/enable_unsafe_noiommu_mode modprobe vfio_pci return $? } @@ -124,7 +125,7 @@ function huge_release_1G() umount $HUGEPAGE_PATH_1G > /dev/null [ ! $? -eq 0 ] && echo "umount 1G hugepage failure, maybe in use" && return 1 fi - + if [ -d $HUGEPAGE_PATH_1G ] ; then rm -R $HUGEPAGE_PATH_1G [ ! $? -eq 0 ] && echo "Cannot delete 2M hugepage" && return 1 @@ -133,7 +134,7 @@ function huge_release_1G() if [ $HUGEPAGE_NUM_1G -ne 0 ]; then echo -n 0 > $NR_HUGEPAGE_FILE_1G fi - + return 0 } @@ -146,16 +147,16 @@ function huge_release_2M() umount $HUGEPAGE_PATH_2M > /dev/null [ ! $? -eq 0 ] && echo "umount 2M hugepage failure, maybe in use" && return 1 fi - + if [ -d $HUGEPAGE_PATH_2M ] ; then rm -R $HUGEPAGE_PATH_2M [ ! $? -eq 0 ] && echo "Cannot delete 2M hugepage" && return 1 fi - + if [ $HUGEPAGE_NUM_2M -ne 0 ]; then echo -n 0 > $NR_HUGEPAGE_FILE_2M fi - + return 0 } @@ -164,7 +165,7 @@ function huge_alloc_1G() if [ ! -e $NR_HUGEPAGE_FILE_1G ]; then return 1 fi - + echo -n $HUGEPAGE_NUM_1G > $NR_HUGEPAGE_FILE_1G [ $? -eq 0 ] && return 0 return 1 @@ -175,7 +176,7 @@ function huge_alloc_2M() if [ ! -e $NR_HUGEPAGE_FILE_2M ]; then return 1 fi - + echo -n $HUGEPAGE_NUM_2M > $NR_HUGEPAGE_FILE_2M [ $? -eq 0 ] && return 0 return 1 @@ -186,13 +187,13 @@ function huge_mount_1G() if [ ! -e $NR_HUGEPAGE_FILE_1G ]; then return 0 fi - + SIZE_1G=$(cat $NR_HUGEPAGE_FILE_1G) if [ $SIZE_1G -eq 0 ]; then return 0 fi - mkdir -p $HUGEPAGE_PATH_1G + mkdir -p $HUGEPAGE_PATH_1G mount -t hugetlbfs nodev -o pagesize=1G $HUGEPAGE_PATH_1G [ $? -eq 0 ] && return 0 return 1 @@ -203,13 +204,13 @@ function huge_mount_2M() if [ ! -e $NR_HUGEPAGE_FILE_2M ]; then return 0 fi - + SIZE_2M=$(cat $NR_HUGEPAGE_FILE_2M) if [ $SIZE_2M -eq 0 ]; then return 0 fi - mkdir -p $HUGEPAGE_PATH_2M + mkdir -p $HUGEPAGE_PATH_2M mount -t hugetlbfs nodev -o pagesize=2M $HUGEPAGE_PATH_2M [ $? -eq 0 ] && return 0 return 1 @@ -256,7 +257,7 @@ function nic_unbind_ixgbe() OLDIFS=$IFS; IFS=, for ITER_PCI_ADDR in $PCI_ADDR_IXGBE; do $NICTOOL --force -b ixgbe $ITER_PCI_ADDR - [ ! $? -eq 0 ] && return 1 + [ ! $? -eq 0 ] && return 1 done IFS=$OLDIFS return 0 @@ -373,7 +374,7 @@ function module_load() if [ "$DEFAULT_UIO_MODULE" == "vfio_pci" ]; then module_load_vfio_pci return $? - fi + fi return 0 } @@ -412,8 +413,8 @@ function stop() [ ! $? -eq 0 ] && return 1 return 0 } - -function restart() + +function restart() { stop start $* |