HAL渲染Clixon DefaultDB

6 files changed, 181 insertions, 259 deletions

diff --git a/ansible/roles/tsg-os-HAL/files/conf/tsg-os-HAL.conf b/ansible/roles/tsg-os-HAL/files/conf/tsg-os-HAL.conf
index aab24ca2..ceade6c8 100644
--- a/ansible/roles/tsg-os-HAL/files/conf/tsg-os-HAL.conf
+++ b/ansible/roles/tsg-os-HAL/files/conf/tsg-os-HAL.conf
@@ -1,7 +1,5 @@
 
 # HUGEPAGES=
-# KUBE_RESERVED=
-# SYSTEM_RESERVED=
 
 # NUMA_NODE_CNT=
 # IOCORE=
diff --git a/ansible/roles/tsg-os-HAL/files/script/build_clixon_default_db.sh b/ansible/roles/tsg-os-HAL/files/script/build_clixon_default_db.sh
new file mode 100644
index 00000000..070760dd
--- /dev/null
+++ b/ansible/roles/tsg-os-HAL/files/script/build_clixon_default_db.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+build_default_db()
+{
+    local resources="<config>
+   <system xmlns=\"urn:yang:tsg-os-mgnt-srv\">
+      <resources>
+         <packet_io_engine_hugepages>${HUGEPAGES}</packet_io_engine_hugepages>
+         <packet_io_engine_core>${IOCORE}</packet_io_engine_core>
+         <workload_core>${WORKLOAD_CORE}</workload_core>
+      </resources>
+   </system>
+</config>"
+    echo ${resources} | xmlstarlet fo > ${tsg_clixon_default_db_file}
+    sed -i '/<?xml version="1.0"?>/d' ${tsg_clixon_default_db_file}
+}
\ No newline at end of file
diff --git a/ansible/roles/tsg-os-HAL/files/script/cpu_amd.sh b/ansible/roles/tsg-os-HAL/files/script/cpu_amd.sh
index 9a404be3..2c477c49 100644
--- a/ansible/roles/tsg-os-HAL/files/script/cpu_amd.sh
+++ b/ansible/roles/tsg-os-HAL/files/script/cpu_amd.sh
@@ -1,114 +1,17 @@
 #!/bin/sh
 
-write_devices_disable()
-{
-(cat <<EOF1
-[device]
-rxcore_enable=0
-
-EOF1
-) >> $1
-}
-
-write_devices_enable()
-{
-(cat <<EOF1
-[device]
-rxcore_enable=1
-device=$1
-
-EOF1
-) >> $2
-}
-
-write_device_rxcore_conf()
-{
-(cat <<EOF1
-[device:$1]
-rx_cores=$2
-
-EOF1
-) >> $3
-}
-
-del_device_rxcore_conf()
-{
-    sed -i '/^\[device.*\]/,/^$/d' ${tsg_clixon_cfg_file}
-}
-
-set_mrzcpd_rx_core()
-{
-    local devices
-    local str_device
-
-    del_device_rxcore_conf
-
-    devices=$(lshw -c network -businfo | grep network | awk '{print $2}')
-    for device in $devices
-    do
-        if [ $device == "network" ]; then
-            continue
-        fi
-        businfo=$(ethtool -i $device | grep "bus-info:"| awk '{print $2}')
-        numa_node=$(lspci -s $businfo -vv | grep "NUMA node:" | awk '{print $3}')
-
-        if [ ! -n "$numa_node" ] || [ $numa_node -ge $NUMA_NODE_CNT ]; then
-            continue
-        fi
-        write_device_rxcore_conf $device ${mrzcpd_rx_core[$numa_node]} ${tsg_clixon_cfg_file}
-        str_device=$str_device","$device
-    done
-    str_device=${str_device#*,}
-
-    if [ ! -n "$str_device" ]; then
-        write_devices_disable ${tsg_clixon_cfg_file}
-    else
-        write_devices_enable $str_device ${tsg_clixon_cfg_file}
-    fi
-}
-
-output_common_str()
-{
-    local common_list
-
-    list1=$(echo $1 | tr ',' ' ')
-    list2=$(echo $2 | tr ',' ' ')
-
-    for numa_core in $list1;
-    do
-        for iocore in $list2;
-        do
-            if [ $numa_core -eq $iocore ]; then
-                common_list=$common_list","$numa_core
-                break
-            fi
-        done
-    done
-    common_list=${common_list#*,}
-    echo $common_list
-}
-
 allocate_cpu()
 {
-    local numa_node_cpu_cnt=0
-    local numa_cpu_cnt_min=0
-    local numa_list
+    local cpus
+    local numa_cpus
+    local clixon_cpus
     local mrzcpd_start
     local mrzcpd_end
     local clixon_start
+    local numa_cpu_cnt_min=0
+    local numa_node_cpu_cnt=0
 
-    if [ ! -n "$NUMA_NODE_CNT" ]; then
-        NUMA_NODE_CNT=`lscpu | grep "NUMA node(s):" | head -n 1 | sed  -r 's/NUMA node\(s\):\s{1,}//g'`
-    fi
-
-    if [ -n "$IOCORE" ] && [ -n "$WORKLOAD_CORE" ]; then
-        for((i=0;i<$NUMA_NODE_CNT;i++));
-        do
-            numa_list=$(numactl --hardware | grep "node $i cpus: " | sed -r "s/node $i cpus: //g" | tr ' ' ',')
-            mrzcpd_core=$(output_common_str $numa_list $IOCORE)
-            mrzcpd_rx_core+=($mrzcpd_core)
-        done
-    else
+    if [ ! -n "$IOCORE" ] || [ ! -n "$WORKLOAD_CORE" ]; then
         unset IOCORE
         unset WORKLOAD_CORE
 
@@ -140,16 +43,13 @@ allocate_cpu()
 
         for((i=0;i<$NUMA_NODE_CNT;i++));
         do
-            numa_list=$(numactl --hardware | grep "node $i cpus: " | sed -r "s/node $i cpus: //g")
-            mrzcpd_core=$(echo $numa_list | cut -d ' ' -f $mrzcpd_start-$mrzcpd_end | tr ' ' ',')
-            IOCORE=$IOCORE","$mrzcpd_core
-            mrzcpd_rx_core+=($mrzcpd_core)
-            clixon_core=$(echo $numa_list | cut -d ' ' -f $clixon_start- | tr ' ' ',')
-            WORKLOAD_CORE=$WORKLOAD_CORE","$clixon_core
+            numa_cpus=$(numactl --hardware | grep "node $i cpus: " | sed -r "s/node $i cpus: //g")
+            cpus=$(echo $numa_cpus | cut -d ' ' -f $mrzcpd_start-$mrzcpd_end | tr ' ' ',')
+            IOCORE=$IOCORE","$cpus
+            clixon_cpus=$(echo $numa_cpus | cut -d ' ' -f $clixon_start- | tr ' ' ',')
+            WORKLOAD_CORE=$WORKLOAD_CORE","$clixon_cpus
         done
         IOCORE=${IOCORE#*,}
         WORKLOAD_CORE=${WORKLOAD_CORE#*,}
     fi
-
-    set_mrzcpd_rx_core
 }
diff --git a/ansible/roles/tsg-os-HAL/files/script/cpu_default.sh b/ansible/roles/tsg-os-HAL/files/script/cpu_default.sh
index 349af0ad..8e406ad8 100644
--- a/ansible/roles/tsg-os-HAL/files/script/cpu_default.sh
+++ b/ansible/roles/tsg-os-HAL/files/script/cpu_default.sh
@@ -1,31 +1,14 @@
 #!/bin/sh
 
-write_devices_conf()
-{
-(cat <<EOF1
-[device]
-rxcore_enable=0
-
-EOF1
-) >> ${tsg_clixon_cfg_file}
-}
-
-del_device_rxcore_conf()
-{
-    sed -i '/^\[device.*\]/,/^$/d' ${tsg_clixon_cfg_file}
-}
-
 allocate_cpu()
 {
-    local numa_node_cpu_cnt=0
+    local cpus
+    local numa_cpus
+    local clixon_cpus
+    local mrzcpd_end
+    local clixon_start
     local numa_cpu_cnt_min=0
-    local numa_list
-    local mrzcpd_cnt
-    local clixon_min_index
-
-    if [ ! -n "$NUMA_NODE_CNT" ]; then
-        NUMA_NODE_CNT=`lscpu | grep "NUMA node(s):" | head -n 1 | sed  -r 's/NUMA node\(s\):\s{1,}//g'`
-    fi
+    local numa_node_cpu_cnt=0
 
     if [ ! -n "$IOCORE" ] || [ ! -n "$WORKLOAD_CORE" ]; then
         unset IOCORE
@@ -40,27 +23,25 @@ allocate_cpu()
         done
 
         if [ $numa_cpu_cnt_min -le 4 ]; then
-            mrzcpd_cnt=1
-            clixon_min_index=2
+            mrzcpd_end=1
+            clixon_start=2
         elif [ $numa_cpu_cnt_min -le 16 ]; then
-            mrzcpd_cnt=2
-            clixon_min_index=3
+            mrzcpd_end=2
+            clixon_start=3
         else
-            mrzcpd_cnt=4
-            clixon_min_index=5
+            mrzcpd_end=4
+            clixon_start=5
         fi
 
         for((i=0;i<$NUMA_NODE_CNT;i++));
         do
-            numa_list=$(numactl --hardware | grep "node $i cpus: " | sed -r "s/node $i cpus: //g" | sed -r "s/^0 //g")
-            mrzcpd_core=$(echo $numa_list | cut -d ' ' -f 1-$mrzcpd_cnt | tr ' ' ',')
-            IOCORE=$IOCORE","$mrzcpd_core
-            clixon_core=$(echo $numa_list | cut -d ' ' -f $clixon_min_index- | tr ' ' ',')
-            WORKLOAD_CORE=$WORKLOAD_CORE","$clixon_core
+            numa_cpus=$(numactl --hardware | grep "node $i cpus: " | sed -r "s/node $i cpus: //g" | sed -r "s/^0 //g")
+            cpus=$(echo $numa_cpus | cut -d ' ' -f 1-$mrzcpd_end | tr ' ' ',')
+            IOCORE=$IOCORE","$cpus
+            clixon_cpus=$(echo $numa_cpus | cut -d ' ' -f $clixon_start- | tr ' ' ',')
+            WORKLOAD_CORE=$WORKLOAD_CORE","$clixon_cpus
         done
         IOCORE=${IOCORE#*,}
         WORKLOAD_CORE=${WORKLOAD_CORE#*,}
     fi
-    del_device_rxcore_conf
-    write_devices_conf
 }
diff --git a/ansible/roles/tsg-os-HAL/files/script/set_nic_cpu_affinity.sh b/ansible/roles/tsg-os-HAL/files/script/set_nic_cpu_affinity.sh
new file mode 100644
index 00000000..072a8f2c
--- /dev/null
+++ b/ansible/roles/tsg-os-HAL/files/script/set_nic_cpu_affinity.sh
@@ -0,0 +1,101 @@
+#!/bin/sh
+
+nic_cpu_affinity_disable()
+{
+(cat <<EOF1
+[device]
+rxcore_enable=0
+
+EOF1
+) >> ${tsg_clixon_cfg_file}
+}
+
+nic_cpu_affinity_enable()
+{
+(cat <<EOF1
+[device]
+rxcore_enable=1
+device=$1
+
+EOF1
+) >> ${tsg_clixon_cfg_file}
+}
+
+set_nic_cpu_affinity()
+{
+(cat <<EOF1
+[device:$1]
+rx_cores=$2
+
+EOF1
+) >> ${tsg_clixon_cfg_file}
+}
+
+delete_nic_cpu_affinity_config()
+{
+    sed -i '/^\[device.*\]/,/^$/d' ${tsg_clixon_cfg_file}
+}
+
+output_common_cpu()
+{
+    local common_list
+
+    list1=$(echo $1 | tr ',' ' ')
+    list2=$(echo $2 | tr ',' ' ')
+
+    for numa_core in $list1;
+    do
+        for iocore in $list2;
+        do
+            if [ $numa_core -eq $iocore ]; then
+                common_list=$common_list","$numa_core
+                break
+            fi
+        done
+    done
+    common_list=${common_list#*,}
+    echo $common_list
+}
+
+build_nic_cpu_affinity_configuration()
+{
+    local cpus
+    local cpu_list
+    local numa_cpus
+    local devices
+    local device_list
+    local NIC_CPU_Affinity_Switch=$1
+
+    delete_nic_cpu_affinity_config
+    if [ -z "$NIC_CPU_Affinity_Switch" ] || [ "$NIC_CPU_Affinity_Switch" == "0" ]; then
+        nic_cpu_affinity_disable
+        return
+    fi
+
+    for((i=0;i<$NUMA_NODE_CNT;i++));
+    do
+        numa_cpus=$(numactl --hardware | grep "node $i cpus: " | sed -r "s/node $i cpus: //g" | tr ' ' ',')
+        cpus=$(output_common_cpu $numa_cpus $IOCORE)
+        cpu_list+=("$cpus")
+    done
+    
+    devices=$(lshw -c network -businfo | grep network | awk '{print $2}')
+    for device in $devices
+    do
+        if [ $device == "network" ]; then
+            continue
+        fi
+        businfo=$(ethtool -i $device | grep "bus-info:"| awk '{print $2}')
+        numa_node=$(lspci -s $businfo -vv | grep "NUMA node:" | awk '{print $3}')
+
+        if [ -z "$numa_node" ] || [ $numa_node -ge $NUMA_NODE_CNT ] || [ -z ${cpu_list[$numa_node]} ]; then
+            continue
+        fi
+        set_nic_cpu_affinity $device ${cpu_list[$numa_node]} ${tsg_clixon_cfg_file}
+        device_list=$device_list","$device
+    done
+
+    device_list=${device_list#*,}
+    nic_cpu_affinity_enable $device_list
+}
+
diff --git a/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh b/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh
index 3a5d6009..148f9a05 100644
--- a/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh
+++ b/ansible/roles/tsg-os-HAL/files/script/tsg-os-HAL.sh
@@ -4,11 +4,10 @@ set -x
 
 tsg_os_HAL_cfg_file="/etc/sysconfig/tsg-os-HAL.conf"
 
-grub_cfg_file="/boot/grub/grub.cfg"
 tsg_clixon_cfg_file="/opt/tsg/clixon/etc/mgnt-srv.conf"
-k3s_cfg_file="/etc/rancher/k3s/config.yaml"
 mrzcpd_conf_file="/var/run/share/container_mrzcpd.conf"
 cpu_partitioning_conf_file='/etc/tuned/cpu-partitioning-variables.conf'
+tsg_clixon_default_db_file='/etc/clixon/default_db'
 
 cpu_core_num=$(lscpu | grep "CPU(s):" | head -n 1 | sed  -r 's/CPU\(s\):\s{1,}//g')
 mem_num=$(free -m | grep Mem | awk '{print $2}')
@@ -77,50 +76,15 @@ load_tsg_os_HAL_config()
     fi
 }
 
-set_k3s_config()
+set_clixon_configuration()
 {
-    local mem_num="$1"
-    local system_mem
-
-    if [ ! -n "$KUBE_RESERVED" ]; then
-        KUBE_RESERVED=2
-    fi
-
-    if [ ! -n "$SYSTEM_RESERVED" ]; then
-        if [ $mem_num -le "32768" ]; then
-            SYSTEM_RESERVED=8
-        elif [ $mem_num -le "65536" ]; then
-            SYSTEM_RESERVED=8
-        elif [ $mem_num -le "131072" ]; then
-            SYSTEM_RESERVED=16
-        else
-            SYSTEM_RESERVED=32
-        fi
-    fi
-
-    system_mem=$(($HUGEPAGES + $SYSTEM_RESERVED))
-cat <<EOF > ${k3s_cfg_file}
-kubelet-arg:
-  - "kube-reserved=memory=${KUBE_RESERVED}Gi"
-  - "system-reserved=memory=${system_mem}Gi"
-  - "sync-frequency=1s"
-kube-apiserver-arg:
-  - "event-ttl=48h0m0s"
-EOF
-}
-
-set_tsg_clixon_conf()
-{
-    [ -f ${tsg_clixon_cfg_file} ] && sed -i "s/^cpu_range=.*$/cpu_range=${WORKLOAD_CORE}/g" ${tsg_clixon_cfg_file}
-    [ -f ${tsg_clixon_cfg_file} ] && sed -i "s/^hugepages=.*$/hugepages=${HUGEPAGES}/g" ${tsg_clixon_cfg_file}
     [ -f ${tsg_clixon_cfg_file} ] && sed -i "s/^nf_count=.*$/nf_count=${NF_COUNT}/g" ${tsg_clixon_cfg_file}
 }
 
-set_mrzcpd_conf()
+set_mrzcpd_configuration()
 {
     echo "sz_indirect_pktmbuf=${MRZCPD_INDIRECT_PKTMBUF}" > ${mrzcpd_conf_file}
     echo "sz_direct_pktmbuf=${MRZCPD_DIRECT_PKTMBUF}" >> ${mrzcpd_conf_file}
-    echo "iocore=${IOCORE}" >> ${mrzcpd_conf_file}
     echo "poll_wait_throttle=${MRZCPD_POLL_WAIT_THROTTLE}" >> ${mrzcpd_conf_file}
     echo "sz_data=${MRZCPD_SZ_DATA}" >> ${mrzcpd_conf_file}
     echo "sz_tunnel=${MRZCPD_SZ_TUNNEL}" >> ${mrzcpd_conf_file}
@@ -128,7 +92,7 @@ set_mrzcpd_conf()
     echo "create_mode=${MRZCPD_CREATE_MODE}" >> ${mrzcpd_conf_file}
 }
 
-set_hugepages()
+calculate_hugepages()
 {
     if [ ! -n "$HUGEPAGES" ]; then
         if [ $mem_num -le "32768" ]; then
@@ -143,14 +107,14 @@ set_hugepages()
     fi
 }
 
-set_mrzcpd_indirect_pktmbuf()
+calculate_mrzcpd_indirect_pktmbuf()
 {
     if [ ! -n "$MRZCPD_INDIRECT_PKTMBUF" ]; then
         MRZCPD_INDIRECT_PKTMBUF=`expr 8192 / $NUMA_NODE_CNT`
     fi
 }
 
-set_mrzcpd_sz_tunnel()
+calculate_mrzcpd_sz_tunnel()
 {
     if [ ! -n "$MRZCPD_SZ_TUNNEL" ]; then
         if [ $mem_num -le "32768" ]; then
@@ -165,7 +129,7 @@ set_mrzcpd_sz_tunnel()
     fi
 }
 
-set_mrzcpd_direct_pktmbuf()
+calculate_mrzcpd_direct_pktmbuf()
 {
     if [ ! -n "$MRZCPD_DIRECT_PKTMBUF" ]; then
         if [ $mem_num -le "32768" ]; then
@@ -180,17 +144,16 @@ set_mrzcpd_direct_pktmbuf()
     fi
 }
 
-set_cpu_list()
+calculate_cpu_list()
 {
     local vendor
 
     vendor=$(lscpu | grep "^Vendor ID" | awk '{print $3}')
-    if [ -n "$NIC_CPU_Affinity_Switch" ] && [ "$NIC_CPU_Affinity_Switch" == "1" ]; then
-        . /opt/tsg/tsg-os-HAL/scripts/cpu_amd.sh
-    elif [ -n "$NIC_CPU_Affinity_Switch" ] && [ "$NIC_CPU_Affinity_Switch" == "0" ]; then
-        . /opt/tsg/tsg-os-HAL/scripts/cpu_default.sh
-    elif [ "$vendor" == 'AuthenticAMD' ]; then
+    if [ "$vendor" == 'AuthenticAMD' ]; then
         . /opt/tsg/tsg-os-HAL/scripts/cpu_amd.sh
+        if [ ! -n "$NIC_CPU_Affinity_Switch" ]; then
+            NIC_CPU_Affinity_Switch=1
+        fi
     else
         . /opt/tsg/tsg-os-HAL/scripts/cpu_default.sh
     fi
@@ -198,42 +161,6 @@ set_cpu_list()
     allocate_cpu
 }
 
-set_grub_cmdline_file()
-{
-    local result=0
-    local grub_cmdline_key="$1"
-    local grub_cmdline_value="$2"
-
-    if ! grep -q " ${grub_cmdline_key}=" "$grub_cfg_file"; then
-        sed -i "/ linux / s/$/ ${grub_cmdline_key}=${grub_cmdline_value} /" $grub_cfg_file
-        result=1
-    elif ! grep -qE " ${grub_cmdline_key}=${grub_cmdline_value}\s| ${grub_cmdline_key}=${grub_cmdline_value}$" "$grub_cfg_file"; then
-        sed -i "s/ ${grub_cmdline_key}=\(\w\+,\)*\(\w\+-\w\+,*\)*\w*,*\s*/ ${grub_cmdline_key}=${grub_cmdline_value} /" $grub_cfg_file
-        result=1
-    fi
-    return $result
-}
-
-set_grub_cmdline()
-{
-    local set_grub_flag=0
-
-    grub_cmdline_key=("hugepagesz" "hugepages" "isolcpus")
-    grub_cmdline_value=("1G" ${HUGEPAGES} ${IOCORE})
-
-    for index in "${!grub_cmdline_key[@]}"
-    do
-        set_grub_cmdline_file ${grub_cmdline_key[$index]} ${grub_cmdline_value[$index]}
-        if [ "$?" == "1" ]; then
-            set_grub_flag=1
-        fi
-    done
-
-    if [ "$set_grub_flag" == "1" ]; then
-        reboot
-    fi
-}
-
 build_and_mount_traffic_engine_logs_vdisk()
 {
     local vdisk_path=${TRAFFIC_ENGINE_LOGS_VDISK_PATH}
@@ -283,14 +210,6 @@ build_and_mount_traffic_engine_logs_vdisk()
     fi
 }
 
-
-set_cpu_partitioning()
-{
-cat <<EOF > ${cpu_partitioning_conf_file}
-isolated_cores=${IOCORE}
-EOF
-}
-
 read_device_type()
 {
     product_name=`ipmitool fru list | grep 'Product Name' | awk '{print $4}' | head -n 1`
@@ -346,33 +265,40 @@ metadata:
 EOF
 }
 
-load_tsg_os_HAL_config
-mkdir -p /run/mrzcpd/hugepages
 mkdir -p /var/run/share
-
-build_and_mount_traffic_engine_logs_vdisk
-set_hugepages
-
-set_cpu_list
-set_grub_cmdline
-
-set_cpu_partitioning
-
+mkdir -p /run/mrzcpd/hugepages
 mount -t hugetlbfs -o pagesize=1073741824 nodev /var/run/mrzcpd/hugepages
 modprobe vfio
 echo 1 > /sys/module/vfio/parameters/enable_unsafe_noiommu_mode
 modprobe vfio_pci
 
-set_mrzcpd_indirect_pktmbuf
-set_mrzcpd_sz_tunnel
-set_mrzcpd_direct_pktmbuf
+load_tsg_os_HAL_config
 
-set_k3s_config ${mem_num}
-set_tsg_clixon_conf
-set_mrzcpd_conf
+if [ ! -n "$NUMA_NODE_CNT" ]; then
+    NUMA_NODE_CNT=`lscpu | grep "NUMA node(s):" | head -n 1 | sed  -r 's/NUMA node\(s\):\s{1,}//g'`
+fi
+
+build_and_mount_traffic_engine_logs_vdisk
+
+calculate_hugepages
+calculate_cpu_list
+
+. /opt/tsg/tsg-os-HAL/scripts/set_nic_cpu_affinity.sh
+build_nic_cpu_affinity_configuration ${NIC_CPU_Affinity_Switch}
+
+calculate_mrzcpd_indirect_pktmbuf
+calculate_mrzcpd_sz_tunnel
+calculate_mrzcpd_direct_pktmbuf
+
+set_k3s_configuration ${mem_num}
+set_clixon_configuration
+set_mrzcpd_configuration
 
 # add sn to k8s node annotations.
 read_device_type
 read_sn
 retry_read_sn_when_sn_unavailable
 add_node_metadata
+
+. /opt/tsg/tsg-os-HAL/scripts/build_clixon_default_db.sh
+build_default_db
\ No newline at end of file