#! /usr/bin/env python # # BSD LICENSE # # Copyright(c) 2010-2014 Intel Corporation. All rights reserved. # Copyright(c) 2017-2020 Institute of Information Engineering, CAS # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # * Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in # the documentation and/or other materials provided with the # distribution. # * Neither the name of Intel Corporation nor the names of its # contributors may be used to endorse or promote products derived # from this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # import sys import os import getopt import subprocess import json import argparse from os.path import exists, abspath, dirname, basename # The PCI base class for NETWORK devices NETWORK_BASE_CLASS = "02" CRYPTO_BASE_CLASS = "0b" # global dict ethernet devices present. Dictionary indexed by PCI address. # Each device within this is itself a dictionary of device properties devices = {} # list of supported DPDK drivers dpdk_drivers = ["uio_pci_generic", "igb_uio", "vfio-pci"] # hwfile location default_hwfile_location = '/var/run/mrzcpd/hwfile.json' default_gcfg_location = '/usr/local/etc/mrglobal.conf' # This is roughly compatible with check_output function in subprocess module # which is only available in python 2.7. def check_output(args, stderr=None): '''Run a command and capture its output''' return subprocess.Popen(args, stdout=subprocess.PIPE, stderr=stderr).communicate()[0] def check_modules(): '''Checks that igb_uio is loaded''' global dpdk_drivers # list of supported modules mods = [{"Name": driver, "Found": False} for driver in dpdk_drivers] # first check if module is loaded try: # Get list of sysfs modules (both built-in and dynamically loaded) sysfs_path = '/sys/module/' # Get the list of directories in sysfs_path sysfs_mods = [os.path.join(sysfs_path, o) for o in os.listdir(sysfs_path) if os.path.isdir(os.path.join(sysfs_path, o))] # Extract the last element of '/sys/module/abc' in the array sysfs_mods = [a.split('/')[-1] for a in sysfs_mods] # special case for vfio_pci (module is named vfio-pci, # but its .ko is named vfio_pci) sysfs_mods = map(lambda a: a if a != 'vfio_pci' else 'vfio-pci', sysfs_mods) for mod in mods: if mod["Name"] in sysfs_mods: mod["Found"] = True except: pass # check if we have at least one loaded module if True not in [mod["Found"] for mod in mods]: print("Error - no supported modules(DPDK driver) are loaded") sys.exit(1) # change DPDK driver list to only contain drivers that are loaded dpdk_drivers = [mod["Name"] for mod in mods if mod["Found"]] def has_driver(devices, dev_id): '''return true if a device is assigned to a driver. False otherwise''' return "Driver_str" in devices[dev_id] def get_pci_device_details(dev_id): '''This function gets additional details for a PCI device''' device = {} extra_info = check_output(["lspci", "-vmmks", dev_id]).splitlines() # parse lspci details for line in extra_info: if len(line) == 0: continue name, value = line.decode().split("\t", 1) name = name.strip(":") + "_str" device[name] = value # check for a unix interface name device["Interface"] = "" for base, dirs, _ in os.walk("/sys/bus/pci/devices/%s/" % dev_id): if "net" in dirs: device["Interface"] = \ ",".join(os.listdir(os.path.join(base, "net"))) break # check if a port is used for ssh connection device["Ssh_if"] = False device["Active"] = "" return device def get_nic_details(): '''This function populates the "devices" dictionary. The keys used are the pci addresses (domain:bus:slot.func). The values are themselves dictionaries - one for each NIC.''' global dpdk_drivers # clear any old data devices = {} # first loop through and read details for all devices # request machine readable format, with numeric IDs dev = {} dev_lines = check_output(["lspci", "-Dvmmn"]).splitlines() for dev_line in dev_lines: if len(dev_line) == 0: if dev["Class"][0:2] == NETWORK_BASE_CLASS: # convert device and vendor ids to numbers, then add to global dev["Vendor"] = int(dev["Vendor"], 16) dev["Device"] = int(dev["Device"], 16) # use dict to make copy of dev devices[dev["Slot"]] = dict(dev) else: name, value = dev_line.decode().split("\t", 1) dev[name.rstrip(":")] = value # check what is the interface if any for an ssh connection if # any to this host, so we can mark it later. ssh_if = [] route = check_output(["ip", "-o", "route"]) # filter out all lines for 169.254 routes route = "\n".join(filter(lambda ln: not ln.startswith("169.254"), route.decode().splitlines())) rt_info = route.split() for i in range(len(rt_info) - 1): if rt_info[i] == "dev": ssh_if.append(rt_info[i+1]) # based on the basic info, get extended text details for d in devices.keys(): # get additional info and add it to existing data devices[d] = devices[d].copy() devices[d].update(get_pci_device_details(d).items()) for _if in ssh_if: if _if in devices[d]["Interface"].split(","): devices[d]["Ssh_if"] = True devices[d]["Active"] = "*Active*" break # add igb_uio to list of supporting modules if needed if "Module_str" in devices[d]: for driver in dpdk_drivers: if driver not in devices[d]["Module_str"]: devices[d]["Module_str"] = \ devices[d]["Module_str"] + ",%s" % driver else: devices[d]["Module_str"] = ",".join(dpdk_drivers) # make sure the driver and module strings do not have any duplicates if has_driver(devices, d): modules = devices[d]["Module_str"].split(",") if devices[d]["Driver_str"] in modules: modules.remove(devices[d]["Driver_str"]) devices[d]["Module_str"] = ",".join(modules) return devices def dev_id_from_dev_name(devices, dev_name, quite=False): '''Take a device "name" - a string passed in by user to identify a NIC device, and determine the device id - i.e. the domain:bus:slot.func - for it, which can then be used to index into the devices array''' # check if it's already a suitable index if dev_name in devices: return dev_name # check if it's an index just missing the domain part elif "0000:" + dev_name in devices: return "0000:" + dev_name else: # check if it's an interface name, e.g. eth1 for d in devices.keys(): if dev_name in devices[d]["Interface"].split(","): return devices[d]["Slot"] # if nothing else matches - error if not quite: print("Unknown device: %s. " "Please specify device in \"bus:slot.func\" format" % dev_name) sys.exit(1) return None def unbind_one(devices, dev_id, force): '''Unbind the device identified by "dev_id" from its current driver''' dev = devices[dev_id] if not has_driver(devices, dev_id): print("%s %s %s is not currently managed by any driver\n" % (dev["Slot"], dev["Device_str"], dev["Interface"])) return # prevent us disconnecting ourselves if dev["Ssh_if"] and not force: print("Routing table indicates that interface %s is active. " "Skipping unbind" % (dev_id)) return # write to /sys to unbind filename = "/sys/bus/pci/drivers/%s/unbind" % dev["Driver_str"] try: f = open(filename, "a") except: print("Error: unbind failed for %s - Cannot open %s" % (dev_id, filename)) sys.exit(1) f.write(dev_id) f.close() def bind_one(devices, dev_id, driver, force): '''Bind the device given by "dev_id" to the driver "driver". If the device is already bound to a different driver, it will be unbound first''' dev = devices[dev_id] saved_driver = None # used to rollback any unbind in case of failure if driver == '' or driver == None: return # prevent disconnection of our ssh session if dev["Ssh_if"] and not force: print("Routing table indicates that interface %s is active. " "Not modifying" % (dev_id)) return # unbind any existing drivers we don't want if has_driver(devices, dev_id): if dev["Driver_str"] == driver: print("%s already bound to driver %s, skipping\n" % (dev_id, driver)) return else: saved_driver = dev["Driver_str"] unbind_one(devices, dev_id, force) dev["Driver_str"] = "" # clear driver string # if we are binding to one of DPDK drivers, add PCI id's to that driver if driver in dpdk_drivers: filename = "/sys/bus/pci/drivers/%s/new_id" % driver try: f = open(filename, "w") except: print("Error: bind failed for %s - Cannot open %s" % (dev_id, filename)) return try: f.write("%04x %04x" % (dev["Vendor"], dev["Device"])) f.close() except: print("Error: bind failed for %s - Cannot write new PCI ID to " "driver %s" % (dev_id, driver)) return # do the bind by writing to /sys filename = "/sys/bus/pci/drivers/%s/bind" % driver try: f = open(filename, "a") except: print("Error: bind failed for %s - Cannot open %s" % (dev_id, filename)) if saved_driver is not None: # restore any previous driver bind_one(devices, dev_id, saved_driver, force) return try: f.write(dev_id) f.close() except: # for some reason, closing dev_id after adding a new PCI ID to new_id # results in IOError. however, if the device was successfully bound, # we don't care for any errors and can safely ignore IOError tmp = get_pci_device_details(dev_id) if "Driver_str" in tmp and tmp["Driver_str"] == driver: return print("Error: bind failed for %s - Cannot bind to driver %s" % (dev_id, driver)) if saved_driver is not None: # restore any previous driver bind_one(devices, dev_id, saved_driver, force) return def unbind_all(devices, dev_list, force=False): """Unbind method, takes a list of device locations""" dev_list = map(lambda x: dev_id_from_dev_name(devices, x), dev_list) for d in dev_list: unbind_one(devices, d, force) def bind_all(devices, dev_list, driver, force=False): """Bind method, takes a list of device locations""" dev_list = map(lambda x: dev_id_from_dev_name(devices, x), dev_list) for d in dev_list: bind_one(devices, d, driver, force) # when binding devices to a generic driver (i.e. one that doesn't have a # PCI ID table), some devices that are not bound to any other driver could # be bound even if no one has asked them to. hence, we check the list of # drivers again, and see if some of the previously-unbound devices were # erroneously bound. for d in devices.keys(): # skip devices that were already bound or that we know should be bound if "Driver_str" in devices[d] or d in dev_list: continue # update information about this device devices[d] = dict(devices[d].items() + get_pci_device_details(d).items()) # check if updated information indicates that the device was bound if "Driver_str" in devices[d]: unbind_one(devices, d, force) def hwfile_encode(location): devices = get_nic_details() _dirname = os.path.dirname(location) if not os.path.exists(_dirname): os.makedirs(_dirname) with open(location, 'w') as json_fp: json.dump(devices, json_fp) return def hwfile_decode(location): with open(location, 'r') as json_fp: devices = json.load(json_fp) return devices def setup_argv_parser(): parser = argparse.ArgumentParser(description='Marsio ZeroCopy Tools -- Network Interface Card Tools', version = 'Marsio ZeroCopy Tools Suite 4.1') parser.add_argument('--hwfile', help = 'Hardware file location', nargs=1, metavar = 'FILE') parser.add_argument('--action', help = 'Action, bind modules or unbind modules, or dump hwfiles', choices=['bind','unbind','dump']) parser.add_argument('--gcfg', help = 'Global configure file location', nargs=1, metavar = 'FILE') parser.add_argument('interfaces', metavar='INTERFACES', help = 'symbol of interfaces', nargs ='*') return parser.parse_args() def global_configure_parser(g_file): import ConfigParser config = ConfigParser.ConfigParser() config.read(g_file) return config.get('device','device', 0).split(',') def nics_bind(hwinfo, niclist): devices_info = get_nic_details() bind_all(devices_info, niclist, dpdk_drivers[0], True) return def nics_unbind(hwinfo, niclist): devices_info = get_nic_details() dev_addr_list = map(lambda x: dev_id_from_dev_name(hwinfo, x), niclist) kmod_list = map(lambda x: hwinfo[x]["Driver_str"], dev_addr_list) map(lambda x,y: bind_one(devices_info, x, y, True), dev_addr_list, kmod_list) return def main(): r_options = setup_argv_parser() hwfile_location = r_options.hwfile[0] if r_options.hwfile else default_hwfile_location gcfg_location = r_options.gcfg[0] if r_options.gcfg else default_gcfg_location niclist = r_options.interfaces if r_options.interfaces else global_configure_parser(gcfg_location) if r_options.action == 'dump': return hwfile_encode(hwfile_location) if not os.path.exists(hwfile_location): print('Hardware information file %s is not exists.' % hwfile_location) sys.exit(1) if not os.path.exists(gcfg_location): print('Global configure file %s is not exists.' % gcfg_location) sys.exit(1) hwinfo = hwfile_decode(hwfile_location) check_modules() niclist = [ d for d in niclist if dev_id_from_dev_name(hwinfo, d, quite=True)] print niclist if r_options.action == 'bind': return nics_bind(hwinfo, niclist) if r_options.action == 'unbind': return nics_unbind(hwinfo, niclist) return if __name__ == "__main__": main()