✨ feat(DPISDN-2): 将 mrenv 拆分为 hugepages setup 和 hwdb setup 两个servicev4.6.18-20230317

将 mrenv 拆分为 hugepages setup 和 hwdb setup 两个service

1 files changed, 86 insertions, 461 deletions

diff --git a/tools/devbind/devbind.py b/tools/devbind/devbind.py
index cac493c..62d4a5b 100644
--- a/tools/devbind/devbind.py
+++ b/tools/devbind/devbind.py
@@ -1,473 +1,98 @@
-#! /usr/bin/env python
-#
-#   BSD LICENSE
-#
-#   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
-#   Copyright(c) 2017-2020 Institute of Information Engineering, CAS
-#   All rights reserved.
-#
-#   Redistribution and use in source and binary forms, with or without
-#   modification, are permitted provided that the following conditions
-#   are met:
-#
-#     * Redistributions of source code must retain the above copyright
-#       notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above copyright
-#       notice, this list of conditions and the following disclaimer in
-#       the documentation and/or other materials provided with the
-#       distribution.
-#     * Neither the name of Intel Corporation nor the names of its
-#       contributors may be used to endorse or promote products derived
-#       from this software without specific prior written permission.
-#
-#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-#   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-#   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-#   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-#   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-#   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-#   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-#   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-#   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-#   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-#   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-#
-
-import sys
 import os
-import getopt
 import subprocess
+import configparser
+import re
 import json
-import argparse
-import platform
-from os.path import exists, abspath, dirname, basename
-from glob import glob
-from os.path import join as path_join
-
-# The PCI base class for NETWORK devices
-NETWORK_BASE_CLASS = "02"
-CRYPTO_BASE_CLASS = "0b"
-
-# global dict ethernet devices present. Dictionary indexed by PCI address.
-# Each device within this is itself a dictionary of device properties
-devices = {}
-# list of supported DPDK drivers
-dpdk_drivers = ["uio_pci_generic", "igb_uio", "vfio-pci"]
-
-# hwfile location
-default_hwfile_location = '/var/run/mrzcpd/hwfile.json'
-default_gcfg_location = '/usr/local/etc/mrglobal.conf'
-
-# This is roughly compatible with check_output function in subprocess module
-# which is only available in python 2.7.
-def check_output(args, stderr=None):
-    '''Run a command and capture its output'''
-    return subprocess.Popen(args, stdout=subprocess.PIPE,
-                            stderr=stderr).communicate()[0]
-
-
-def check_modules():
-    '''Checks that igb_uio is loaded'''
-    global dpdk_drivers
-
-    # list of supported modules
-    mods = [{"Name": driver, "Found": False} for driver in dpdk_drivers]
-
-    # first check if module is loaded
-    try:
-        # Get list of sysfs modules (both built-in and dynamically loaded)
-        sysfs_path = '/sys/module/'
-
-        # Get the list of directories in sysfs_path
-        sysfs_mods = [os.path.join(sysfs_path, o) for o
-                      in os.listdir(sysfs_path)
-                      if os.path.isdir(os.path.join(sysfs_path, o))]
-
-        # Extract the last element of '/sys/module/abc' in the array
-        sysfs_mods = [a.split('/')[-1] for a in sysfs_mods]
-
-        # special case for vfio_pci (module is named vfio-pci,
-        # but its .ko is named vfio_pci)
-        sysfs_mods = map(lambda a:
-                         a if a != 'vfio_pci' else 'vfio-pci', sysfs_mods)
-
-        for mod in mods:
-            if mod["Name"] in sysfs_mods:
-                mod["Found"] = True
-    except:
-        pass
-
-    # check if we have at least one loaded module
-    if True not in [mod["Found"] for mod in mods]:
-        print("Error - no supported modules(DPDK driver) are loaded")
-        sys.exit(1)
-
-    # change DPDK driver list to only contain drivers that are loaded
-    dpdk_drivers = [mod["Name"] for mod in mods if mod["Found"]]
-    print dpdk_drivers
-
-
-def has_driver(devices, dev_id):
-    '''return true if a device is assigned to a driver. False otherwise'''
-    return "Driver_str" in devices[dev_id]
-
-
-def get_pci_device_details(dev_id):
-    '''This function gets additional details for a PCI device'''
-    device = {}
-
-    extra_info = check_output(["lspci", "-vmmks", dev_id]).splitlines()
-
-    # parse lspci details
-    for line in extra_info:
-        if len(line) == 0:
-            continue
-        name, value = line.decode().split("\t", 1)
-        name = name.strip(":") + "_str"
-        device[name] = value
-    # check for a unix interface name
-    device["Interface"] = ""
-    for base, dirs, _ in os.walk("/sys/bus/pci/devices/%s/" % dev_id):
-        if "net" in dirs:
-            device["Interface"] = \
-                ",".join(os.listdir(os.path.join(base, "net")))
-            break
-    # check if a port is used for ssh connection
-    device["Ssh_if"] = False
-    device["Active"] = ""
-
-    return device
-
-
-def get_nic_details():
-    '''This function populates the "devices" dictionary. The keys used are
-    the pci addresses (domain:bus:slot.func). The values are themselves
-    dictionaries - one for each NIC.'''
-    global dpdk_drivers
-
-    # clear any old data
-    devices = {}
-    # first loop through and read details for all devices
-    # request machine readable format, with numeric IDs
-    dev = {}
-    dev_lines = check_output(["lspci", "-Dvmmn"]).splitlines()
-    for dev_line in dev_lines:
-        if len(dev_line) == 0:
-            if dev["Class"][0:2] == NETWORK_BASE_CLASS:
-                # convert device and vendor ids to numbers, then add to global
-                dev["Vendor"] = int(dev["Vendor"], 16)
-                dev["Device"] = int(dev["Device"], 16)
-                # use dict to make copy of dev
-                devices[dev["Slot"]] = dict(dev)
-        else:
-            name, value = dev_line.decode().split("\t", 1)
-            dev[name.rstrip(":")] = value
-
-    # check what is the interface if any for an ssh connection if
-    # any to this host, so we can mark it later.
-    ssh_if = []
-    route = check_output(["ip", "-o", "route"])
-    # filter out all lines for 169.254 routes
-    route = "\n".join(filter(lambda ln: not ln.startswith("169.254"),
-                             route.decode().splitlines()))
-    rt_info = route.split()
-    for i in range(len(rt_info) - 1):
-        if rt_info[i] == "dev":
-            ssh_if.append(rt_info[i+1])
-
-    # based on the basic info, get extended text details
-    for d in devices.keys():
-        # get additional info and add it to existing data
-        devices[d] = devices[d].copy()
-        devices[d].update(get_pci_device_details(d).items())
-
-        for _if in ssh_if:
-            if _if in devices[d]["Interface"].split(","):
-                devices[d]["Ssh_if"] = True
-                devices[d]["Active"] = "*Active*"
-                break
-
-        # add igb_uio to list of supporting modules if needed
-        if "Module_str" in devices[d]:
-            for driver in dpdk_drivers:
-                if driver not in devices[d]["Module_str"]:
-                    devices[d]["Module_str"] = \
-                        devices[d]["Module_str"] + ",%s" % driver
-        else:
-            devices[d]["Module_str"] = ",".join(dpdk_drivers)
-
-        # make sure the driver and module strings do not have any duplicates
-        if has_driver(devices, d):
-            modules = devices[d]["Module_str"].split(",")
-            if devices[d]["Driver_str"] in modules:
-                modules.remove(devices[d]["Driver_str"])
-                devices[d]["Module_str"] = ",".join(modules)
-
-    return devices
-
-
-def dev_id_from_dev_name(devices, dev_name, quite=False):
-    '''Take a device "name" - a string passed in by user to identify a NIC
-    device, and determine the device id - i.e. the domain:bus:slot.func - for
-    it, which can then be used to index into the devices array'''
-
-    # check if it's already a suitable index
-    if dev_name in devices:
-        return dev_name
-    # check if it's an index just missing the domain part
-    elif "0000:" + dev_name in devices:
-        return "0000:" + dev_name
-    else:
-        # check if it's an interface name, e.g. eth1
-        for d in devices.keys():
-            if dev_name in devices[d]["Interface"].split(","):
-                return devices[d]["Slot"]
-    # if nothing else matches - error
+import sys
 
-    if not quite:
-        print("Unknown device: %s. "
-          "Please specify device in \"bus:slot.func\" format" % dev_name)
+mrzcpd_root = "/opt/tsg/mrzcpd"
+mrzcpd_cfg = ""
+dpdk_bind = ""
+dpdk_driver = "vfio-pci"
+interfaces = []
+hwfile_location = '/var/run/mrzcpd/hwfile.json'
+hwinfo = {}
+
+
+def init():
+    global mrzcpd_root
+    global mrzcpd_cfg
+    global dpdk_bind
+    global dpdk_driver
+    global interfaces
+    global hwinfo
+
+    # Get driver mode
+    with open("/etc/sysconfig/mrzcpd", 'r') as drive_fp:
+        for line in drive_fp.readlines():
+            if re.search("DEFAULT_UIO_MODULE=*", line):
+                dpdk_driver = line.split('DEFAULT_UIO_MODULE=')[1]
+            if re.search("MRZCPD_ROOT=*", line):
+                mrzcpd_root = line.split('MRZCPD_ROOT=')[1]
+
+    # Get cfg path
+    mrzcpd_cfg = mrzcpd_root.replace("\n", "") + "/etc/mrglobal.conf"
+    dpdk_bind = mrzcpd_root.replace("\n", "") + "/bin/dpdk-devbind.py"
+
+    # Check driver mode
+    if dpdk_driver.strip() not in {"vfio-pci", "uio_pci_generic", "igb_uio"}:
+        print("driver mode invalid.", dpdk_driver)
         sys.exit(1)
 
-    return None
-
-def unbind_one(devices, dev_id, force):
-    '''Unbind the device identified by "dev_id" from its current driver'''
-    dev = devices[dev_id]
-    if not has_driver(devices,dev_id):
-        print("Notice: %s %s %s is not currently managed by any driver" %
-              (dev["Slot"], dev["Device_str"], dev["Interface"]))
-        return
-
-    # prevent us disconnecting ourselves
-    if dev["Ssh_if"] and not force:
-        print("Warning: routing table indicates that interface %s is active. "
-              "Skipping unbind" % dev_id)
-        return
-
-    # write to /sys to unbind
-    filename = "/sys/bus/pci/drivers/%s/unbind" % dev["Driver_str"]
-    try:
-        f = open(filename, "a")
-    except :
-        sys.exit("Error: unbind failed for %s - Cannot open %s" %(dev_id, filename))
-    f.write(dev_id)
-    f.close()
-
-def is_mlx_dev(dev,devices):
-    dev_item = devices.get(dev)
-    dev_driver = dev_item.get('Driver_str')
-    if 'mlx5_core' == dev_driver:
-        return True
-    else:
-        return False
-
-
-def bind_one(devices,dev_id, driver, force):
-    '''Bind the device given by "dev_id" to the driver "driver". If the device
-    is already bound to a different driver, it will be unbound first'''
-    dev = devices[dev_id]
-    saved_driver = None  # used to rollback any unbind in case of failure
-
-    # prevent disconnection of our ssh session
-    if dev["Ssh_if"] and not force:
-        print("Warning: routing table indicates that interface %s is active. "
-              "Not modifying" % dev_id)
-        return
-
-    # unbind any existing drivers we don't want
-    if has_driver(devices,dev_id):
-        if dev["Driver_str"] == driver:
-            print("Notice: %s already bound to driver %s, skipping" %
-                  (dev_id, driver))
-            return
-        saved_driver = dev["Driver_str"]
-        unbind_one(devices,dev_id, force)
-        dev["Driver_str"] = ""  # clear driver string
-
-    # For kernels >= 3.15 driver_override can be used to specify the driver
-    # for a device rather than relying on the driver to provide a positive
-    # match of the device.  The existing process of looking up
-    # the vendor and device ID, adding them to the driver new_id,
-    # will erroneously bind other devices too which has the additional burden
-    # of unbinding those devices
-    if driver in dpdk_drivers:
-        filename = "/sys/bus/pci/devices/%s/driver_override" % dev_id
-        if exists(filename):
-            try:
-                f = open(filename, "w")
-            except :
-                print("Error: bind failed for %s - Cannot open %s" % (dev_id, filename))
-                return
-            try:
-                f.write("%s" % driver)
-                f.close()
-            except :
-                print("Error: bind failed for %s - Cannot write driver %s " % (dev_id, driver))
-                return
-        # For kernels < 3.15 use new_id to add PCI id's to the driver
-        else:
-            filename = "/sys/bus/pci/drivers/%s/new_id" % driver
-            try:
-                f = open(filename, "w")
-            except :
-                print("Error: bind failed for %s - Cannot open %s" % (dev_id, filename))
-                return
-            try:
-                # Convert Device and Vendor Id to int to write to new_id
-                f.write("%04x %04x" % (int(dev["Vendor"], 16),
-                                       int(dev["Device"], 16)))
-                f.close()
-            except :
-                print("Error: bind failed for %s - Cannot write new PCI ID to " "driver %s" % (dev_id, driver))
-                return
-
-    # do the bind by writing to /sys
-    filename = "/sys/bus/pci/drivers/%s/bind" % driver
-    try:
-        f = open(filename, "a")
-    except :
-        print("Error: bind failed for %s - Cannot open %s"% (dev_id, filename))
-        if saved_driver is not None:  # restore any previous driver
-            bind_one(devices,dev_id, saved_driver, force)
-        return
-    try:
-        f.write(dev_id)
-        f.close ()
-    except :
-        # for some reason, closing dev_id after adding a new PCI ID to new_id
-        # results in IOError. however, if the device was successfully bound,
-        # we don't care for any errors and can safely ignore IOError
-        tmp = get_pci_device_details(dev_id)
-        if "Driver_str" in tmp and tmp["Driver_str"] == driver:
-            return
-        print("Error: bind failed for %s - Cannot bind to driver %s"% (dev_id, driver))
-        if saved_driver is not None:  # restore any previous driver
-            bind_one(devices,dev_id, saved_driver, force)
-        return
-
-    # For kernels > 3.15 driver_override is used to bind a device to a driver.
-    # Before unbinding it, overwrite driver_override with empty string so that
-    # the device can be bound to any other driver
-    filename = "/sys/bus/pci/devices/%s/driver_override" % dev_id
-    if exists(filename):
-        try:
-            f = open(filename, "w")
-        except :
-            sys.exit("Error: unbind failed for %s - Cannot open %s"% (dev_id, filename))
-        try:
-            f.write("\00")
-            f.close()
-        except :
-            sys.exit("Error: unbind failed for %s - Cannot write %s"% (dev_id, filename))
-
-def unbind_all(devices, dev_list, force=False):
-    """Unbind method, takes a list of device locations"""
-    dev_list = map(lambda x: dev_id_from_dev_name(devices, x), dev_list)
-    for d in dev_list:
-        unbind_one(devices, d, force)
-
-
-def bind_all(devices, dev_list, driver, force=False):
-    """Bind method, takes a list of device locations"""
-
-    dev_list = map(lambda x: dev_id_from_dev_name(devices, x), dev_list)
-    for d in dev_list:
-        if is_mlx_dev(d,devices) == False:
-            bind_one(devices, d, driver, force)
-
-    # when binding devices to a generic driver (i.e. one that doesn't have a
-    # PCI ID table), some devices that are not bound to any other driver could
-    # be bound even if no one has asked them to. hence, we check the list of
-    # drivers again, and see if some of the previously-unbound devices were
-    # erroneously bound.
-    for d in devices.keys():
-        # skip devices that were already bound or that we know should be bound
-        if "Driver_str" in devices[d] or d in dev_list:
-            continue
-
-        # update information about this device
-        devices[d] = dict(devices[d].items() +
-                          get_pci_device_details(d).items())
-
-        # check if updated information indicates that the device was bound
-        if "Driver_str" in devices[d]:
-            unbind_one(devices, d, force)
-
-def hwfile_encode(location):
-    devices = get_nic_details()
-    _dirname = os.path.dirname(location)
-    if not os.path.exists(_dirname):
-        os.makedirs(_dirname)
-
-    with open(location, 'w') as json_fp:
-        json.dump(devices, json_fp)
-    return
-
-def hwfile_decode(location):
-    with open(location, 'r') as json_fp:
-        devices = json.load(json_fp)
-    return devices
-
-def setup_argv_parser():
-
-    parser = argparse.ArgumentParser(description='Marsio ZeroCopy Tools -- Network Interface Card Tools',
-                                     version = 'Marsio ZeroCopy Tools Suite 4.1')
-
-    parser.add_argument('--hwfile', help = 'Hardware file location', nargs=1, metavar = 'FILE')
-    parser.add_argument('--action', help = 'Action, bind modules or unbind modules, or dump hwfiles',
-                                    choices=['bind','unbind','dump'])
-    parser.add_argument('--gcfg', help = 'Global configure file location', nargs=1, metavar = 'FILE')
-    parser.add_argument('interfaces', metavar='INTERFACES', help = 'symbol of interfaces', nargs ='*')
-    return parser.parse_args()
-
-def global_configure_parser(g_file):
-    import ConfigParser
-    config = ConfigParser.ConfigParser()
-    config.read(g_file)
-    return config.get('device','device', 0).split(',')
-
-def nics_bind(hwinfo, niclist):
-    devices_info = get_nic_details()
-    print dpdk_drivers
-    bind_all(devices_info, niclist, dpdk_drivers[0], True)
-    return
-
-def nics_unbind(hwinfo, niclist):
-    devices_info = get_nic_details()
-    dev_addr_list = map(lambda x: dev_id_from_dev_name(hwinfo, x), niclist)
-    kmod_list = map(lambda x: hwinfo[x]["Driver_str"], dev_addr_list)
-    map(lambda x,y: bind_one(devices_info, x, y, True), dev_addr_list, kmod_list)
-    return
+    # Get dev list from mrglobal
+    config = configparser.ConfigParser()
+    config.read(mrzcpd_cfg)
+    interfaces = config.get('device', 'device').split(',')
+
+    # hwfile decode
+    with open(hwfile_location, 'r') as json_fp:
+        json_info = json.load(json_fp)
+        for item in json_info:
+            dev_info = {}
+            dev_name = item.get('Interface', None)
+            dev_info['pci'] = item.get('Slot_str', None)
+            dev_info['driver'] = item.get('Driver_str', None)
+            hwinfo[dev_name] = dev_info
+
+
+def module_load():
+    # vfio_pci
+    if dpdk_driver.strip() == "vfio-pci".strip():
+        if os.system("modprobe vfio") or os.system("echo 1 > /sys/module/vfio/parameters/enable_unsafe_noiommu_mode") or os.system("modprobe vfio_pci"):
+            print("modprode vfio err")
+            sys.exit(1)
+
+    # uio_pci_generic
+    if dpdk_driver.strip() == "uio_pci_generic".strip():
+        if os.system("modprobe uio_pci_generic"):
+            print("modprode uio_pci_generic err")
+            sys.exit(1)
+
+    # igb_uio
+    if dpdk_driver.strip() == "igb_uio".strip():
+        if os.system("modprobe igb_uio"):
+            print("modprode igb_uio err")
+            sys.exit(1)
+
+
+def nic_bind():
+    for d in interfaces:
+        if d in hwinfo and hwinfo.get(d).get('driver') != 'mlx5_core':
+            bind_cmd = dpdk_bind + ' --bind=' + \
+                dpdk_driver.replace("\n", " ") + hwinfo.get(d).get('pci')
+            print("bind cmd:", bind_cmd)
+            if os.system(bind_cmd):
+                print("bind dev err")
+                sys.exit(1)
 
 
 def main():
-    r_options = setup_argv_parser()
-    hwfile_location = r_options.hwfile[0] if r_options.hwfile else default_hwfile_location
-    gcfg_location = r_options.gcfg[0] if r_options.gcfg else default_gcfg_location
-    niclist = r_options.interfaces if r_options.interfaces else global_configure_parser(gcfg_location)
-
-    if r_options.action == 'dump':
-        return hwfile_encode(hwfile_location)
-
-    if not os.path.exists(hwfile_location):
-        print('Hardware information file %s is not exists.' % hwfile_location)
-        sys.exit(1)
-    if not os.path.exists(gcfg_location):
-        print('Global configure file %s is not exists.' % gcfg_location)
-        sys.exit(1)
-
-    hwinfo = hwfile_decode(hwfile_location)
-    check_modules()
-
-    niclist = [ d for d in niclist if dev_id_from_dev_name(hwinfo, d, quite=True)]
-    print niclist
+    # module load
+    module_load()
+    # bind dev
+    nic_bind()
 
-    if r_options.action == 'bind':
-        return nics_bind(hwinfo, niclist)
-    if r_options.action == 'unbind':
-        return nics_unbind(hwinfo, niclist)
-    return
 
 if __name__ == "__main__":
-    main()
\ No newline at end of file
+    init()
+    main()