#!/bin/bash

PATH=/bin:/sbin:/usr/bin:/usr/sbin:usr/local/bin:/usr/local/sbin:/usr/lib/ksvd/bin
export PATH

DVS_CONF="/var/lib/ksvd/dvs.conf"
BOND_CONF="/var/lib/ksvd/bond.conf"
MANAGE_BOND_CONF="/var/lib/ksvd/manage_bond.conf"
STORAGE_NET_CONF="/var/lib/ksvd/storage_net.conf"
STORAGE_BOND="storage_bond"
LOG_DIR="/var/log/ksvd-network/"
LOGFILE="/var/log/ksvd-network/ksvd_net_vsctl.log"

SYS_NETWORK_DIR="/etc/sysconfig/network-scripts"

DEFAULT_VALUE='DEFAULT'
OVS="/usr/lib/ksvd/bin/ovs-vsctl --db=unix:/run/openvswitch/db.sock"

usage(){
        cat <<EOT

Desc:  used for network configuration .
Log file: default /var/log/ksvd-network/ksvd_net_vsctl.log
Use Format: 
	ksvd_net_vsctl --action  action_value [--option option_value] ...

[Options]
[key]            [description]                                          [value]
-h,--help:       print help info                                        none
-o,--action:     actions                                                list-phy-iface,add-dvs,del-dvs,list-subnet,add-subnet,del-subnet
-t,--type:       type                                                   storage network type: share/single/bond
-d,--dvsname:    distribute virtual switch name                         dvs0
-I,--iface:      physical nic                                           nic
-u,--uplink:     up link                                                p4p1 or eth1
-s,--subnetname: subnet name                                            subnet10 
-v,--vlan:       vlan,default 0                                         1-4096
-n,--network:    ip network                                             192.168.10.2,192.168.10.254
-i,--ip:         ipaddress,default is last but one of network           192.168.10.253
-m,--mask:       netmask                                                255.255.255.0
-p,--prefix:     network prefix                                         24
-g,--gateway:    gateway,,default is the last one of network            192.168.10.254
--mirrorname:    mirrorname                                             m0
--srctype:       src mirror type                                        vlan or piface or viface
--srcparam:      src mirror paramter,port name or vlan id               tap0 or eth1 or 110
--dsttype:       dst mirror type                                        vlan or piface or viface
--dstparam:      dst mirror paramter,port name or vlan id               tap0 or eth1 or 110
--bondname:      bond port name,default:dvsname + .b                    dvs0.b or br0_bond
--bondmode:      mode0:balance-rr;
                 mode1:active-backup;
                 mode2:balance-xor;
                 mode3:broadcast;
                 mode4:802.3ad;
                 mode5:balance-tlb;
                 mode6:balance-alb

examples:
	ksvd_net_vsctl --help
	ksvd_net_vsctl --action=add-macvlan0-to-manage-port
	ksvd_net_vsctl --action=delete-macvlan0-from-manage-port
	[iface]
	ksvd_net_vsctl --action=list-phy-iface
	ksvd_net_vsctl --action=list-available-uplink
	ksvd_net_vsctl --action=list-phy-iface-brief
	ksvd_net_vsctl --action=active-phy-iface
	ksvd_net_vsctl --action=list-all-iface-ip
	ksvd_net_vsctl --action=get-manage-network-prefix
	[bond]
	ksvd_net_vsctl --action=list-bond-brief
	ksvd_net_vsctl --action=add-bond --bondname=bond0 --uplink=p1p1,p1p2 --bondmode=active-backup
	ksvd_net_vsctl --action=set-bond --bondname=bond0 --uplink=p1p1,p1p2 --bondmode=balance-rr
	ksvd_net_vsctl --action=del-bond --bondname=bond0
	ksvd_net_vsctl --action=clear-bond
	[manage bond]
	ksvd_net_vsctl --action=add-manage-bond --bondname=bond0 --uplink=p1p1,p1p2 --bondmode=active-backup
	ksvd_net_vsctl --action=set-manage-bond --bondname=bond0 --uplink=p1p1,p1p2 --bondmode=broadcast
	ksvd_net_vsctl --action=del-manage-bond --bondname=bond0
	[storage network]
	ksvd_net_vsctl --action=get-storage-net
	ksvd_net_vsctl --action=add-storage-net --type=single --iface=eth2 --ip=192.168.222.33 --prefix=24
	ksvd_net_vsctl --action=add-storage-net --type=bond --iface=ethx1,ethx2 --ip=192.168.222.33 --prefix=24 --bondmode=active-backup
	ksvd_net_vsctl --action=del-storage-net
	[docker network]
	ksvd_net_vsctl --action=reback-docker-net

	# below command need openvswitch #
	[all conf]
	ksvd_net_vsctl --action=load-conf
	ksvd_net_vsctl --action=clear-conf
	[dvs]
	ksvd_net_vsctl --action=list-dvs
	ksvd_net_vsctl --action=list-uplink --dvsname=dvs0
	ksvd_net_vsctl --action=add-dvs --dvsname=dvs0 --uplink=p4p1
	ksvd_net_vsctl --action=del-dvs --dvsname=dvs0
	ksvd_net_vsctl --action=update-dvs --dvsname=dvs0 --uplink=p1p1
	[subnet]
	ksvd_net_vsctl --action=list-subnet --dvsname=dvs0
	ksvd_net_vsctl --action=add-subnet --dvsname=dvs0 --subnetname=sn10 --vlan=10 --network=192.168.10.0 --mask=255.255.255.0 --ip=192.168.10.254 --dhcprange=192.168.10.2,192.168.10.253 --gateway=192.168.10.1 --dnsserver=114.114.114.114,8.8.8.8
	ksvd_net_vsctl --action=del-subnet --dvsname=dvs0 --subnetname=sn10
	ksvd_net_vsctl --action=clear-all-subnet
	[mirror]
	ksvd_net_vsctl --action=add-mirror --dvsname=dvs0 --mirrorname=m0 --srctype=viface --srcparam=tap0 --dsttype=viface --dstparam=tap1
	ksvd_net_vsctl --action=del-mirror --dvsname=dvs0

NOTE: subnet interface name equals dvsname+subnetname.

EOT
}

log(){
	local prefix=`date "+%Y-%m-%d %H:%M:%S"`" : "
	if [[ "$@" == *"set-portsec"* && "$@" != *"reload"* ]]; then
		return
	fi
	echo "${prefix}" $@ >> ${LOGFILE}
}

warn(){
	local prefix=`date "+%Y-%m-%d %H:%M:%S"`" : WARN : "
	echo "${prefix}" $@ >> ${LOGFILE}
}

fatal(){
	local prefix=`date "+%Y-%m-%d %H:%M:%S"`" : ERROR : "
	echo "${prefix}" $@ >> ${LOGFILE}
	exit 1
}

get_key_value(){
	local key=$1
	local file=$2
	[ ! -e ${file} ] && return 1
	local tmp=`grep "$1=" ${file}`
	if [ "${tmp}"x == x ];then
		return 1
	else
		echo "${tmp}"|sed s/\"//g |sed s/" "//g |awk -F "=" '{print $2}'
	fi
}

set_key_value(){
	#used in sed, value excludes escape characters, avoid conflict when writting at the same time .
	local key="$1"
	local value="$2"
	local file="$3"

	[ -e "${file}" ] || touch ${file}
	grep "${key}=" ${file} > /dev/null 2>&1
	if [ $? -eq 0 ];then
		sed -i 's/'"${key}""=.*"'/'"${key}=${value}"'/g' ${file}
	else
		echo "${key}=${value}" >> ${file}
	fi
}

del_key_value(){
	local key=$1
	local file=$2
	[ ! -e "${file}" ] && return 0
	sed -i '/'"^${key}"'/d' "${file}"
}

get_bond_mode(){
    log "Enter: get_bond_mode $@"
    local bondname=$1
    local t_mode=6
    if [ "$bondname" = "balance-rr" ]; then
        t_mode=0
    elif [ "$bondname" = "active-backup" ]; then
        t_mode=1
    elif [ "$bondname" = "balance-xor" ]; then
        t_mode=2
    elif [ "$bondname" = "broadcast" ]; then
        t_mode=3
    elif [ "$bondname" = "802.3ad" ]; then
        t_mode=4
    elif [ "$bondname" = "balance-tlb" ]; then
        t_mode=5
    elif [ "$bondname" = "balance-alb" ]; then
        t_mode=6
    fi
    log "End: get_bond_mode $t_mode"
    echo "$t_mode"
}

check_bond_mode(){
    log "Enter: check_bond_mode $@"
    local bondname=$1
    local t_mode=0
    if [ "$bondname" = "balance-rr" ]; then
        t_mode=0
    elif [ "$bondname" = "active-backup" ]; then
        t_mode=0
    elif [ "$bondname" = "balance-xor" ]; then
        t_mode=0
    elif [ "$bondname" = "broadcast" ]; then
        t_mode=0
    elif [ "$bondname" = "802.3ad" ]; then
        t_mode=0
    elif [ "$bondname" = "balance-tlb" ]; then
        t_mode=0
    elif [ "$bondname" = "balance-alb" ]; then
        t_mode=0
    else
        t_mode=1
    fi
    log "End: check_bond_mode $t_mode"
    return $t_mode
}

get_default_ip(){
	local broadcast=(`ipcalc --broadcast "$1" "$2"|awk -F '=' '{print $2}'|sed s/'\.'/" "/g`)
	let tmp_num=${broadcast[3]}-1
	echo "${broadcast[0]}.${broadcast[1]}.${broadcast[2]}.${tmp_num}"
}

get_default_gateway(){
	local broadcast=(`ipcalc --broadcast "$1" "$2"|awk -F '=' '{print $2}'|sed s/'\.'/" "/g`)
	let tmp_num=${broadcast[3]}-1
	echo "${broadcast[0]}.${broadcast[1]}.${broadcast[2]}.${tmp_num}"
}

get_default_dhcprange(){
        local network=(`ipcalc --network "$1" "$2"|awk -F '=' '{print $2}'|sed s/'\.'/" "/g`)
        let tmp_net_num=${network[3]}+1
        local broadcast=(`ipcalc --broadcast "$1" "$2"|awk -F '=' '{print $2}'|sed s/'\.'/" "/g`)
        let tmp_bcast_num=${broadcast[3]}-1
        [ $tmp_net_num -gt $tmp_bcast_num ] && fatal "num of valid ip address is not enough，please change mask"

        local gateway=(`echo ${gateway_value} | sed s/'\.'/" "/g`)

        if [ $tmp_net_num -eq ${gateway[3]} ];then
                tmp_net_num=$((tmp_net_num+1))
        elif [ $tmp_bcast_num -eq ${gateway[3]} ];then
                tmp_bcast_num=$((tmp_bcast_num-1))
        else
                local a=$((gateway[3]-tmp_net_num ))
                local b=$((tmp_bcast_num-gateway[3]))
                if [ $a -gt $b ];then
                        tmp_bcast_num=$((gateway[3]-1))
                else
                        tmp_net_num=$((gateway[3]+1))
                fi
        fi

        echo "${network[0]}.${network[1]}.${network[2]}.${tmp_net_num},${broadcast[0]}.${broadcast[1]}.${broadcast[2]}.${tmp_bcast_num}"
}

# 
# get all hardware nic name
# input: null
# output: hardware nic name list ,such as p4p1 p1p1 p1p2
#
list_all_phy_iface(){
	log "Enter: list_all_phy_iface $@"
	[ $# -ne 0 ] && fatal "list_all_phy_iface $@  : the number of args is wrong, so exit"
	local i=0
	local network_iface=""
	local virtual_network_iface=""
	local hardware_iface=()
	local is_virtual_iface=0
	local output=""
	for network_iface in `ls -F /sys/class/net/|grep "@$"|sed s/'@$'//g`;do
		is_virtual_network_iface=0
		for virtual_network_iface in `ls -F /sys/devices/virtual/net/|grep "/$"|sed s/'\/$'//g`;do
			[ "${network_iface}"x == "${virtual_network_iface}"x ] && is_virtual_network_iface=1 && break
		done
		if [ -e "/sys/class/net/${network_iface}/device/physfn/net" ] ; then
			is_virtual_network_iface=1
		fi
		[ ${is_virtual_network_iface} -eq 0 ] && hardware_iface[$i]="${network_iface}" && let i=i+1
	done
	output=${hardware_iface[@]}
	echo "${output}"
	log "END  : list_all_phy_iface output: ${output}"
}

# 
# fix manage net mac 
# intput: null 
# output: null  
#
fix_manage_net_mac()
{
	local manage_net_conf="/etc/sysconfig/network-scripts/manage_net.conf"
	if [ -e "${manage_net_conf}" ];then
		source "${manage_net_conf}"
		if [ -n "$macaddr" ];then
			ifconfig macvlan0 hw ether "${macaddr}" >> ${LOGFILE} 2>&1
			log "ifconfig macvlan0 hw ether ${macaddr} >> ${LOGFILE} 2>&1"
		fi
	else
		macaddr=$(ip a s macvlan0|grep "link/ether "|awk '{print $2}')
		echo "macaddr=\"$macaddr\"" > "${manage_net_conf}"
		log "cmd: echo macaddr=\"$macaddr\" > ${manage_net_conf}"
	fi
}

# 
# add macvlan0 to manage port
# intput: null 
# output: null  
#
add_macvlan0_to_manage_port()
{
	log "Enter: add_macvlan0_to_manage_port"
	local ifcfg=""
	local port=()
	local isVde=0
	if [ -e "/etc/.kyinfo" ];then
		egrep -q "^installtype = ((ksvd vde)|(KSVD-VDE))" /etc/.kyinfo
		[ $? -eq 0 ] && fatal "Vde does not need macvlan0"
	else
		fatal "/etc/.kyinfo does not exist"
	fi

	if [ -e "/sys/class/net/macvlan0" ] ; then
		warn "END  : add_macvlan0_to_manage_port: macvlan0 exists"
		return 0
	fi

	local line=`grep GATEWAY ${SYS_NETWORK_DIR}/ifcfg-*|wc -l`	
	[ `grep GATEWAY ${SYS_NETWORK_DIR}/ifcfg-*|awk -F: '$2 !~ /0.0.0.0/'|wc -l` -ne 1 ] && fatal "Find manage port via gateway, but gateway is not unique"

	for ifcfg in `find ${SYS_NETWORK_DIR}/ifcfg-*|grep -v ifcfg-lo`;do
		unset DEVICE IPADDR PREFIX GATEWAY ONBOOT 
		. ${ifcfg}
		if [ -n "${GATEWAY}" -a -n "${DEVICE}" ];then
			if [ ${ONBOOT} != "no" ];then
				if [ ! -n "${PREFIX}" ];then
					PREFIX=${NETMASK}
				fi
				ifconfig "${DEVICE}" 0 >> ${LOGFILE} 2>&1
				ip link add macvlan0 link ${DEVICE} type macvlan mode bridge >> ${LOGFILE} 2>&1
				fix_manage_net_mac
				ip link set macvlan0 up >> ${LOGFILE} 2>&1
				ip addr add ${IPADDR}/${PREFIX} dev macvlan0 >> ${LOGFILE} 2>&1
				ip route add default via ${GATEWAY} dev macvlan0 >> ${LOGFILE} 2>&1
				ifconfig macvlan0 promisc
				/sbin/arping -q -A -c 1 -I macvlan0 ${IPADDR} >> ${LOGFILE} 2>&1
				sleep 1
				/sbin/arping -q -U -c 1 -I macvlan0 ${IPADDR} >> ${LOGFILE} 2>&1
				log "END  : add_macvlan0_to_manage_port: ${DEVICE} ${IPADDR} ${PREFIX} ${GATEWAY} onboot:${ONBOOT}" >> ${LOGFILE} 2>&1
				return 0
			fi
		fi
	done
	fatal "END  : add_macvlan0_to_manage_port: No manage port"
}

# 
# delete macvlan0 to manage port
# intput: null 
# output: null  
#
delete_macvlan0_from_manage_port()
{
	log "Enter: delete_macvlan0_to_manage_port"
	# get manage port
	local parent_port=`ip a s macvlan0 2>>${LOGFILE} |grep "macvlan0@"|awk -F ':' '{print $2}'|awk -F '@' '{print $2}'`
	[ ! -e "/sys/class/net/${parent_port}" -o -z "$parent_port" ] && return 0 

	# del macvlan0
	ip link delete macvlan0 >> ${LOGFILE} 2>&1

	# reback manage port
	ifdown "${parent_port}" >> ${LOGFILE} 2>&1
	ifup "${parent_port}" >> ${LOGFILE} 2>&1
	log "END  : delete_macvlan0_to_manage_port"
}

# 
# get manage nic name, hareware nic or bond as manage port
# intput: array: all interface array
# output: array: management hardware nic name array, such as p1p1,or bond0 p1p1 p1p2
#
get_manage_iface(){
	log "Enter: get_manage_iface $@"
	local ifaces=$@
	local tmp_port=""

	if [ -e "/sys/class/net/macvlan0" ] ; then
		tmp_port=`ip a s|grep "macvlan0@"|awk -F '@' '{print $2}'|awk -F ':' '{print $1}'`
	else
		[ ! -e "/var/lib/ksvd/settings.node" ] && return 0 
		source /var/lib/ksvd/settings.node
		tmp_port=`ip a s|grep "$UNIQB_KSVD_PUBADDR"|awk '{print $NF}'`
	fi

	# manage port is bond, print salves
	if [ -e "/sys/class/net/${tmp_port}/bonding/slaves" ];then
		iface=`cat "/sys/class/net/${tmp_port}/bonding/slaves"`
		echo "$tmp_port $iface"
		log "END   : get_manage_iface bond: output: $tmp_port $iface"
        	return 0
	fi
    
	# manage port is physical iface, print it
	if [ "${tmp_port}" != "" ];then
		echo "${tmp_port}"
		log "END   : get_manage_iface phy: output: ${tmp_port}"
		return 0
	fi
}

# 
# get available hardware nic name, which is not used
# intput: null
# output: block, available hardware nic name list, such as p1p1 p1p2
#
get_storage_iface(){
	log "Enter: get_storage_iface $@"
	local output=""

	# rc1 not configured, output block.
	[ ! -e "${STORAGE_NET_CONF}" ] && return 0

	source "${STORAGE_NET_CONF}"
	# rc2 configured, output blcok
	if [ "${type}" == "share" ];then
		return 0
	# rc3 configured, output: single nic, such as: p1p1 
	elif [ "${type}" == "single" ];then
		output="${iface}"
	# rc4 configured, output: bond, such as: storage-bond p1p1 p1p2
	elif [ "${type}" == "bond" ];then
		output="${STORAGE_BOND} ${iface//,/ }"
	else
	# configured, unkown type, so fatal 
		fatal "error type: ${type}"
	fi
	echo "${output}"
	log "END   : get_storage_iface: output: ${output}"
	return 0 
}

# 
# revert iface system configuration 
# intput: iface name
# output: null
#
revert_iface(){
	log "Enter: revert_iface $@"
	local iface=$1
	local iface_conf="${SYS_NETWORK_DIR}/ifcfg-${iface}"
	if [ -e "${iface_conf}" ];then
        	ifdown "${iface_conf}" >> ${LOGFILE} 2>&1
	fi

	cat > ${iface_conf} <<EOF
NAME=${iface}
DEVICE=${iface}
TYPE=Ethernet
ONBOOT=yes
PROXY_METHOD=none
BROWSER_ONLY=no
DEFROUTE=no
IPV4_FAILURE_FATAL=yes
IPV6INIT=no
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
EOF
	log "Enter: record sync time"
sync
	log "End: record sync time"
	ifup "${iface_conf}" >> ${LOGFILE} 2>&1

	log "END   : revert_iface"
	return 0
}

reback_iface(){
	log "Enter: reback_iface $@"
	local iface=$1
	local iface_conf="${SYS_NETWORK_DIR}/ifcfg-${iface}"
	#if [ -e "${iface_conf}" ];then
		#ifdown "${iface_conf}" >> ${LOGFILE} 2>&1
	#fi

	cat > ${iface_conf} <<EOF
NAME=${iface}
DEVICE=${iface}
TYPE=Ethernet
ONBOOT=yes
PROXY_METHOD=none
BROWSER_ONLY=no
DEFROUTE=no
IPV4_FAILURE_FATAL=yes
IPV6INIT=no
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
EOF
	#ifup "${iface_conf}" >> ${LOGFILE} 2>&1

	log "END   : reback_iface"
	return 0
}

reback_manage_iface(){
	log "Enter: reback_manage_iface $@"
	local iface=$1
	local ipaddr=$2
	local prefix=$3
	local gateway=$4
	local iface_conf="${SYS_NETWORK_DIR}/ifcfg-${iface}"
	#if [ -e "${iface_conf}" ];then
		#ifdown "${iface_conf}" >> ${LOGFILE} 2>&1
	#fi

	cat > ${iface_conf} <<EOF
NAME=${iface}
DEVICE=${iface}
TYPE=Ethernet
ONBOOT=yes
PROXY_METHOD=none
BROWSER_ONLY=no
DEFROUTE=no
IPV4_FAILURE_FATAL=yes
IPADDR=${ipaddr}
PREFIX=${prefix}
GATEWAY=${gateway}
IPV6INIT=no
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
EOF
	#ifup "${iface_conf}" >> ${LOGFILE} 2>&1

	log "END   : reback_manage_iface"
	return 0
}
# 
# add and active system nic
# intput: iface ip prefix 
# output: null, but update system network file
#
add_system_iface(){
	log "Enter: add_system_iface $@"
	local iface=$1
	local ip=$2
	local prefix=$3
	local iface_conf="${SYS_NETWORK_DIR}/ifcfg-${iface}"
	if [ -e "${iface_conf}" ];then
        	ifdown "${iface_conf}" >> ${LOGFILE} 2>&1
	fi
	cat > ${iface_conf} <<EOF
NAME=${iface}
DEVICE=${iface}
TYPE=Ethernet
IPADDR=${ip}
PREFIX=${prefix}
ONBOOT=yes
PROXY_METHOD=none
BROWSER_ONLY=no
DEFROUTE=no
IPV4_FAILURE_FATAL=yes
IPV6INIT=no
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
EOF
	log "Enter: record sync time"
sync
	log "Enter: record sync time"
	ifup "${iface_conf}" >> ${LOGFILE} 2>&1
	log "END   : add_system_iface"
	return 0
}

# 
# make sure physical nic is up 
# intput: null
# output: null
#
active_phy_iface(){
        log "Enter: active_phy_iface "
        ifaces=`list_all_phy_iface`
        [ $? -ne 0 ] && exit 1
        local iface=""
        for iface in ${ifaces};do
                ifconfig "${iface}" up
        done
        log "END   : active_phy_iface ${iface[@]}"
        return 0
}

# 
# add and active system network of bond 
# intput: STORAGE_BOND iface ip prefix
# output: null, but create system network file
#
add_system_bond(){
	log "Enter: add_system_bond $@"
	# create bond
	local type=$1
	local iface=$2
	local ip=$3
	local prefix=$4
	local bondmode=$5
	local t_mode=$(get_bond_mode $bondmode)
	local iface_conf="${SYS_NETWORK_DIR}/ifcfg-$1"   
	log "Enter: add_system_bond get bond_mode : $t_mode"
	[ ! -e "/sys/class/net/bonding_masters" ] && modprobe bonding max_bonds=0 && echo "options bonding max_bonds=0" > /etc/modprobe.d/bonding.conf
	# create bond
	#if [ -e "${iface_conf}" ];then
		#ifdown "${iface_conf}"
		#ip link delete ${STORAGE_BOND}
	#fi
	cat > ${iface_conf} <<EOF
DEVICE=${STORAGE_BOND}
NAME=${STORAGE_BOND}
TYPE=Bond
ONBOOT=yes
IPADDR=${ip}
PREFIX=${prefix}
BONDING_MASTER=yes
BOOTPROTO=static
DEFROUTE=no
IPV4_FAILURE_FATAL=no
IPV6INIT=no
EOF
	set_key_value "BONDING_OPTS" "\"mode=${t_mode} miimon=100\"" "${iface_conf}"
	log "Enter: record sync time"
sync
	log "End: record sync time"
	ifdown ${iface_conf} >> ${LOGFILE} 2>&1
	ifup ${iface_conf} >> ${LOGFILE} 2>&1
	# add slaves
	local slave=""
	for slave in ${iface//\,/ };do
		local slave_conf="${SYS_NETWORK_DIR}/ifcfg-${slave}"
		ifdown ${slave_conf} >> ${LOGFILE} 2>&1
		cat > ${slave_conf} <<EOF
TYPE=Ethernet
NAME=${slave}
DEVICE=${slave}
ONBOOT=yes
MASTER=${STORAGE_BOND}
SLAVE=yes
EOF

		ifup ${slave_conf} >> ${LOGFILE} 2>&1
	done
	
	#set bond3+
	if [ "$t_mode" = "3" ]; then
		log "Get current mode is bond3+"
		local uplink_value=""
		local t_gatway=$(grep GATEWAY /etc/sysconfig/network-scripts/ifcfg-*|awk -F: '$2 !~ /0.0.0.0/' |awk -F "=" 'NR==1 {print $2}'|sed 's/"//g')
		if [ -e "/sys/devices/virtual/net/${STORAGE_BOND}/bonding/gateway_ip" -a -n "$t_gatway" ];then
			echo "$t_gatway" > "/sys/devices/virtual/net/${STORAGE_BOND}/bonding/gateway_ip"
		fi
	fi
	ifconfig ${STORAGE_BOND} up >> ${LOGFILE} 2>&1
	log "END   : add_system_bond"
	return 0
}


# 
# update storage network file
# intput: type iface ip prefix
# output: null, but update file
#
add_storage_conf(){
	log "Enter: add_storage_conf $@"
	local type="$1"
	local iface="$2"
	local ip="$3"
	local prefix="$4"
	local bondmode="$5"
	if [ "${type}" == "share" ];then
		cat > ${STORAGE_NET_CONF} <<EOF
type="share"
EOF
	elif [ "${type}" == "single" ];then
		cat > ${STORAGE_NET_CONF} <<EOF
type="${type}"
iface="${iface}"
ip="${ip}"
prefix="${prefix}"
EOF
	elif [ "${type}" == "bond" ];then
		cat > ${STORAGE_NET_CONF} <<EOF
type="${type}"
iface="${iface}"
ip="${ip}"
prefix="${prefix}"
bondmode="${bondmode}"
EOF
	else
        # configured, unkown type, so fatal 
		fatal "add_storage_conf: error type: ${type}"
	fi

	log "END   : add_storage_conf"
	return 0
}

# 
# get storage network info
# intput: null
# output: block, available hardware nic name list, such as p1p1 p1p2
#
get_storage_net(){
	log "Enter: get_storage_net $@"
	local output=""
	
	# 1 not configured, output: |||
	if [ ! -e "${STORAGE_NET_CONF}" ];then
		output="|||"
		echo "${output}"
		log "END   : get_storage_net: output: ${output}"
		return 0
	fi
	
	source "${STORAGE_NET_CONF}"
	# 2 configured share
	if [ "${type}" == "share" ];then
		output="${type}|||"
	# 3 configured single 
	elif [ "${type}" == "single" ];then
		output="${type}|${iface}|${ip}|${prefix}"
	# 4 configured bond
	elif [ "${type}" == "bond" ];then
		output="${type}|${iface}|${ip}|${prefix}"
	else
	# configured, unkown type, so fatal 
	        fatal "get_storage_net: error type: ${type}"
	fi
	echo "${output}"
	
	log "END   : get_storage_net: output: ${output}"
	return 0
}

# 
# delete storage network, revert to org 
# intput: null
# output: null
#
del_storage_net(){
	log "Enter: del_storage_net"

	if [ ! -e "${STORAGE_NET_CONF}" ];then
		log "END   : del_storage_iface: file: ${STORAGE_NET_CONF} does not exist"
		return 0
	fi

	# revert nic configuration(ifcfg) to default nic
	source "${STORAGE_NET_CONF}"
	if [ "${type}" == "single" ];then
		revert_iface "${iface}"
	elif [ "${type}" == "bond" ];then
		local ifaces=(${iface/\,/ })
		local iface_conf=""
		local slave=""

		for slave in ${ifaces[@]};do
			revert_iface "${slave}"
		done

		ifdown ${STORAGE_BOND} >> ${LOGFILE} 2>&1
		ip link delete ${STORAGE_BOND} >> ${LOGFILE} 2>&1
		rm -rf "${SYS_NETWORK_DIR}/ifcfg-${STORAGE_BOND}"
	fi
	rm -rf "${STORAGE_NET_CONF}"

	log "END   : del_storage_net"
	return 0
}

# 
# add storage network, refresh every time
# intput: null
# output: null, but update system network file and ksvd network file
#
add_storage_net(){
	log "Enter: add_storage_net $@"
	local t_type=$1
	local t_iface=$2
	local t_ip=$3
	local t_prefix=$4
	local t_mode=$5
	local tmp_iface=""

	# need check ?
	if [ "${t_type}" == "share" ];then
		:
	elif [ "${t_type}" == "single" ];then
		[ ! -e "/sys/class/net/${t_iface}" ] && fatal "add_storage_net: error iface: ${t_iface}"
		ipcalc -c -s "${t_ip}" || fatal "add_storage_net: error ip: ${t_ip}"
		[[ ${t_prefix} =~ ^[0-9]+$ ]] || fatal "add_storage_net: error prefix: ${t_prefix}"
		[ ${t_prefix} -lt 1 -o ${t_prefix} -gt 32 ] && fatal "add_storage_net: error prefix: ${t_prefix}"
	elif [ "${t_type}" == "bond" ];then
		for tmp_iface in ${t_iface//,/ };do 
			[ ! -e "/sys/class/net/${tmp_iface}" ] && fatal "add_storage_net: error iface: ${tmp_iface}"
		done
		ipcalc -c -s "${t_ip}" || fatal "add_storage_net: error ip: ${t_ip}"
		[[ ${t_prefix} =~ ^[0-9]+$ ]] || fatal "add_storage_net: error prefix: ${t_prefix}"
		[ ${t_prefix} -lt 1 -o ${t_prefix} -gt 32 ] && fatal "add_storage_net: error prefix: ${t_prefix}"
	else
		fatal "add_storage_net: error type: ${t_type}"
	fi

	# clear storage network configuration
	if [ -e "${STORAGE_NET_CONF}" ];then
		source "${STORAGE_NET_CONF}"
		if [ "${type}" == "single" -o "${type}" == "bond" ];then
			del_storage_net
		fi
	fi

	# add to system network
	if [ "${t_type}" == "single" ];then
		add_system_iface "${t_iface}" "${t_ip}" "${t_prefix}"
	elif [ "${t_type}" == "bond" ];then
		add_system_bond "${STORAGE_BOND}" "${t_iface}" "${t_ip}" "${t_prefix}" "${t_mode}"
	fi

	# create STORAGE_NET_CONF
	add_storage_conf $@

	log "END   : add_storage_net"
	return 0
}

# 
# get available hardware nic name, which is not used, include single manage and storage port
# intput: null
# output: available hardware nic name list, such as p1p1|p1p2
#
list_phy_iface(){
	log "Enter: list_phy_iface $@"
	[ $# -ne 0 ] && fatal "list_phy_iface $@  : the number of args is wrong, so exit"
	local i=0
	local network_iface=""
	local virtual_network_iface=""
	local hardware_iface=()
	local is_virtual_iface=0
	local output=""
	for network_iface in `ls -F /sys/class/net/|grep "@$"|sed s/'@$'//g`;do
		is_virtual_network_iface=0
		for virtual_network_iface in `ls -F /sys/devices/virtual/net/|grep "/$"|sed s/'\/$'//g`;do
			[ "${network_iface}"x == "${virtual_network_iface}"x ] && is_virtual_network_iface=1 && break
		done
		if [ -e "/sys/class/net/${network_iface}/device/physfn/net" ] ; then
			is_virtual_network_iface=1
		fi
		[ ${is_virtual_network_iface} -eq 0 ] && hardware_iface[$i]="${network_iface}" && let i=i+1
	done
	local free_hardware_iface=""
	local tmp_device=""
	for tmp_device in "${hardware_iface[@]}";do
		if [ ! -e "/sys/class/net/${tmp_device}/master" ];then
			if [ ${free_hardware_iface}x == x ];then
				free_hardware_iface="${tmp_device}"
			else
				free_hardware_iface="${free_hardware_iface}|${tmp_device}"
			fi
		fi
	done
	output="${free_hardware_iface}"
	echo "${output}"
	log "END   : list_phy_iface ${output}"
}

#
# get single hardware nic brief
# input: $1 hareward nic name,such as p4p1
# output: one hardware nic information, such as name|type|macaddr|speed|duplex|link_state|master_device 
#
get_phy_iface_brief(){
	log "Enter: get_phy_iface_brief $@"
	local name=$1
	local storage_nic_name=$2
	local manage_nic_name=(${3/,// })
	local type=""
	local speed=""
	local duplex=""
	local macaddr=""
	local link_state=""
	local master_device=""
	local vendorID=`cat /sys/class/net/$name/device/vendor|sed s/0x//g`
	local deviceID=`cat /sys/class/net/$name/device/device|sed s/0x//g`
	local is_support_sriov="0"
	local sriov_totalvfs="0"
	local sriov_numvfs="0"
	local master_type=","
	local output=""
	type=`lspci -d $vendorID:$deviceID|head -1|awk -F 'Ethernet controller: ' '{print $2}'`
	[ -e "/sys/class/net/$name/address" ] && macaddr=`cat /sys/class/net/$name/address`
	link_state=`ethtool $name|grep "Link detected:"|awk -F 'Link detected: ' '{print $2}'`
	if [ "${link_state}" == "yes" ];then
	        speed=`ethtool $name|grep "Speed:"|awk -F 'Speed: ' '{print $2}'`
	        duplex=`ethtool $name|grep "Duplex:"|awk -F 'Duplex: ' '{print $2}'`
	fi
	if [ -e "/sys/class/net/$name/master/" ];then
		master_device=`grep INTERFACE /sys/class/net/$name/master/uevent |head -1|awk -F 'INTERFACE=' '{print $2}'`
		master_type=`grep DEVTYPE /sys/class/net/$name/master/uevent |head -1|awk -F 'DEVTYPE=' '{print $2}'`
		if [ "$master_type" == "bond" ];then
		        master_device="${master_device},bond"
		elif [ "$master_device" == "ovs-system" ];then
		        master_device=`${OVS} iface-to-br ${name}`",ovs-system"
		else
		        master_device="${master_device},"
		fi
	fi
	if [ "$name" == "$manage_nic_name" ];then
		master_device="manage"
	elif [[ ${manage_nic_name[@]/${name}/} != ${manage_nic_name[@]} ]];then
		master_device="${manage_nic_name[0]},manage"
	fi

	if [ "$name" == "$storage_nic_name" ];then
		master_device="storage"
	elif [[ "${storage_nic_name[@]/${name}/}" != "${storage_nic_name[@]}" ]];then
		master_device="${STORAGE_BOND},storage"
	fi
	if [ -e "/sys/class/net/$name/device/sriov_totalvfs" ];then
		is_support_sriov="1"
		sriov_totalvfs=`cat /sys/class/net/$name/device/sriov_totalvfs`
		sriov_numvfs=`cat /sys/class/net/$name/device/sriov_numvfs`
	fi
	output="$name|$type|$macaddr|$speed|$duplex|$link_state|$master_device|$is_support_sriov|$sriov_totalvfs|$sriov_numvfs"
	echo ${output}
	log "END   : get_phy_iface_brief output:${output}"
}

#
# all nic brief
# input: null
# output: list all hardware nic infomation
#
list_phy_iface_brief(){
	log "Enter: list_phy_iface_brief"
	ifaces=`list_all_phy_iface`
	[ $? -ne 0 ] && exit 1
	b_manage_iface=`get_manage_iface "${ifaces}"`
	[ $? -ne 0 ] && exit 1
	[ "${b_manage_iface}" == "" ] && b_manage_iface="${DEFAULT_VALUE}"
	b_storage_iface=`get_storage_iface`
	[ $? -ne 0 ] && exit 1
	[ "${b_storage_iface}" == "" ] && b_storage_iface="${DEFAULT_VALUE}"
	local iface=''
	for iface in ${ifaces};do
		get_phy_iface_brief "$iface" "$b_storage_iface" "$b_manage_iface"
	done
	log "END   : list_phy_iface_brief"
}

#
# all phy nic  
# input: null
# output: list all hardware nic infomation
#
list_network_speed_status(){
    log "Enter: list_network_speed_status"
    ifaces=`list_all_phy_iface`
    [ $? -ne 0 ] && exit 1
    local iface=''
    for iface in ${ifaces};do
        local state=`cat /sys/class/net/${iface}/operstate`
        if  [ ${state} == "up" ]; then
            state='0'
        else
            state='1'
        fi
        local speed='1000M'
        ethtool ${iface} | grep "100000base" >> ${LOGFILE} 2>&1
        if  [ $? -eq 0 ]; then
            speed='100GE'
        else
            ethtool ${iface} | grep "10000base" >> ${LOGFILE} 2>&1
            if  [ $? -eq 0 ]; then
                speed='10GE'
            else
                ethtool ${iface} | grep "1000base" >> ${LOGFILE} 2>&1
                if  [ $? -eq 0 ]; then
                    speed='1000M'
                else
                    ethtool ${iface} | grep "100base" >> ${LOGFILE} 2>&1
                    if  [ $? -eq 0 ]; then
                        speed='100M'
                    fi
                fi
            fi
        fi
        echo "${iface}|${speed}|${state}"
    done
    log "END   : list_network_speed_status"
}

#
# list available uplinks, needed when adding uplink for bridge
# input: null
# output: available uplinks, include physical nic and bond,which does not have master
#
list_available_uplink(){
	log "Enter: list_available_uplink $@"
	available_nics=`list_phy_iface`
	[ $? -ne 0 ] && exit 1
	available_bonds=`list_available_bond`
	[ $? -ne 0 ] && exit 1
	local available_uplink=""
	if [ -z "${available_nics}" ];then
		available_uplink="${available_bonds}"
	elif [ -z "${available_bonds}" ];then
		available_uplink="${available_nics}"
	else
		available_uplink="${available_nics}|${available_bonds}"
	fi
	# remove manage and storage port
	ifaces=`list_all_phy_iface`
	[ $? -ne 0 ] && exit 1
	a_manage_iface=`get_manage_iface ${ifaces}`
	[ $? -ne 0 ] && exit 1
	a_storage_iface=`get_storage_iface`
	[ $? -ne 0 ] && exit 1
	local r_available_uplink_arr=()
	local uplink=""
	for uplink in ${available_uplink//\|/ };do
		if [[ ${a_manage_iface/${uplink}/} != ${a_manage_iface} ]] || [[ ${a_storage_iface/${uplink}/} != ${a_storage_iface} ]];then
			continue
		fi
		r_available_uplink_arr[${#r_available_uplink_arr[@]}]=${uplink}
	done
	r_available_uplink_str="${r_available_uplink_arr[@]}"
	local output="${r_available_uplink_str// /|}"
	echo ${output}
	log "END   : list_available_uplink ${output}"
}

#
# list all ovs bridge names
# input: null
# output: all ovs bridge names in multi lines
#
list_dvs(){
	log "Enter: list_dvs $@"
	[ $# -ne 0 ] && fatal "list_dvs $@  : the number of args is wrong, so exit"

	${OVS} list-br
}

#
# show single uplink name of one bridge
# input: $1 bridge name,called dvsname
# output: uplink name, hareware nic name or bond name, such as p4p1
# note: may more hareware nic name type.
#
list_uplink(){
	log "Enter: list_uplink $@"
	local dvsname_value="$1"
	[ $# -ne 1 ] && fatal "list_uplink $@  : the number of args is wrong, so exit"

	# WARN: if mirror exists, multi nic will print below, but web use data in db, not use list_uplink
	${OVS} list-ports "${dvsname_value}" 2>>"${LOGFILE}" |egrep "^ens|^enp|^eno|^enx|^p[1-9][1-9]?p[0-9][0-9]?|^eth|^bond"
}

#
# list available bond,which is not used
# input: null
# output: available bonds
#
list_available_bond(){
	local available_bonds=""
	local bond_name=""
	local output=""
	if [ -e "${BOND_CONF}" ];then
		for bond_name in `grep "_uplink" "${BOND_CONF}" |awk -F '_uplink' '{print $1}'`;do
			if [ ! -e "/proc/net/bonding/$bond_name" ];then
				warn "bond:${bond_name} is configured, but not active"
			elif [ ! -e "/sys/class/net/${bond_name}/master" ];then
				available_bonds="${available_bonds}${bond_name}|"
			fi
			#others configured bond is in use.
		done
		output=`echo "${available_bonds}"|sed s/'|$'//`
		echo "${output}"
	fi
	log "END   :list_available_bond ${output}"
}

#
# list all bond interface,exclude ovs bond.
# input: null
# output: list all bond information
#
list_active_bond(){
	log "Enter:  list_active_bond"
	local output=""
	if [ -e "/sys/class/net/bonding_masters" ];then
		local output=`cat "/sys/class/net/bonding_masters"`
	fi
	echo "${output}"
	log "END   :list_active_bond ${output}"
}

#
# get a bond information via bond name,exclude ovs bond
# input: $1 bond name
# output: a bond information,such as bond0|storage|192.168.100.100|255.255.255.0|p1p1,p1p2|active-backup|up
#
get_bond_brief(){
	log "Enter:  get_bond_brief $@"
	local name=$1
	local ipaddr=``
	local mask=""
	local mode=""
	local uplink=""
	local conf_state="no"
	local link_state="no"

	if [ -e "/proc/net/bonding/${name}" ];then
		ipaddr=`ifconfig ${name}|grep "\<inet\> "|awk '{print $2}'`
		mask=`ifconfig ${name}|grep "\<inet\> "|awk '{print $4}'`
		mode=`cat /sys/class/net/${name}/bonding/mode|head -1|awk '{print $1}'`
		uplink=`cat /sys/class/net/${name}/bonding/slaves`
		uplink=${uplink// /,}
		conf_state="yes"
		link_state=`ethtool ${name}|grep "Link detected:"|awk -F 'Link detected: ' '{print $2}'`
		# Enter get macvlan0 ip addr
		local manage_iface=`get_manage_iface`
		local manage_iface_tmp=`echo "$manage_iface"|awk '{print $1}'`
		if [ x"${name}" = x"${manage_iface_tmp}" ];then
			ipaddr=`ifconfig macvlan0|grep "\<inet\> "|awk '{print $2}'`
			mask=`ifconfig macvlan0|grep "\<inet\> "|awk '{print $4}'`
		fi
		# End get macvlan0 ip addr

		#Enter judgment bond0 is have slaves
		if [ x"${name}" = x"bond0" ];then
			iface=`cat "/sys/class/net/bond0/bonding/slaves"`
			if [ -z "$iface" ];then
				log "bond0 not have slaves,default remove it"
				ifconfig bond0 down >> ${LOGFILE} 2>&1
				echo "-bond0" > /sys/class/net/bonding_masters 2>/dev/null || log "failed to del bond :bond0 bonding_masters"
				echo "options bonding max_bonds=0" > /etc/modprobe.d/bonding.conf 2>/dev/null
				return 0
			fi
		fi
		#End judgment bond0
	fi
	echo "${name}|${ipaddr}|${mask}|${uplink}|${mode}|${conf_state}|${link_state}"
}

#
# list all bond interface,exclude ovs bond.
# input: null
# output: list all bond information
#
list_bond_brief(){
	log "Enter:  list_bond_brief $@"
	local bonds_in_conf=()
	[ -e "${BOND_CONF}" ] && bonds_in_conf=(`grep "_uplink" "${BOND_CONF}" |awk -F '_uplink' '{print $1}'`)
	local bond_name_list=${bonds_in_conf[@]}
	local bond_name=""
	if [ -e "/sys/class/net/bonding_masters" ];then
		local bonds_in_sys=(`cat "/sys/class/net/bonding_masters"`)
		for bond_name in ${bonds_in_sys[@]};do
			echo "${bonds_in_conf[@]}"|grep -wq "${bond_name}" || bond_name_list[${#bond_name_list[@]}]="${bond_name}"
		done
	fi
	local bond_brief_list=()
	for bond_name in ${bond_name_list[@]};do
		get_bond_brief "${bond_name}"
	done
}
# reback docker net
reback_docker_net(){
	log "Enter: reback docker network"
	local uplink_name=$1
	local docker_parent_name=`docker network inspect kylin-macvlan-net |grep parent|awk -F ":" '{print $2}'|sed 's/"//g'|sed 's/ //g'`
	local uplink_value=""
	if [ -z $uplink_name ];then
		uplink_value="$docker_parent_name"
		log "Get: docker parent name:$uplink_value"
	else
		uplink_value="$uplink_name"
		log "Get: uplink name:$uplink_value"
	fi
	[ -z "$uplink_value" ] && fatal "uplink_value: ${uplink_value} is null, so exit"
	[ ! -e "/sys/class/net/${uplink_value}" ] && fatal "uplink_value: ${uplink_value} is not a nic in this host, so exit"
	if [ ! -e "/var/lib/ksvd/docker-ifcfg.eth0.tmp" ]; then
		rm -f /var/lib/ksvd/docker-ifcfg.eth0.tmp
		cat > /var/lib/ksvd/docker-ifcfg.eth0.tmp <<EOF
NAME=eth0
DEVICE=eth0
TYPE=Ethernet
ONBOOT=yes
PROXY_METHOD=none
BROWSER_ONLY=no
DEFROUTE=no
IPV4_FAILURE_FATAL=yes
IPADDR=
PREFIX=
GATEWAY=
IPV6INIT=no
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
EOF
		echo "/var/lib/ksvd/docker-ifcfg.eth0.tmp is not settings docker address,so exit"
		return 1
    fi
    local docker_ip=`cat /var/lib/ksvd/docker-ifcfg.eth0.tmp |grep IPADDR |awk -F "=" '{print $2}'|sed 's/"//g'`
    local docker_netmask=`cat /var/lib/ksvd/docker-ifcfg.eth0.tmp |grep PREFIX |awk -F "=" '{print $2}'|sed 's/"//g'`
    local docker_gatway=`cat /var/lib/ksvd/docker-ifcfg.eth0.tmp |grep GATEWAY |awk -F "=" '{print $2}'|sed 's/"//g'`
    if [ -z "$docker_gatway" -o -z "$docker_ip" -o -z "$docker_netmask" ]; then
		echo "docker_ip or docker_netmask or docker_gatway is null,so exit"
		return 1
    fi
	docker network disconnect kylin-macvlan-net ksvd-MStorage >> ${LOGFILE} 2>&1
	docker network rm kylin-macvlan-net >> ${LOGFILE} 2>&1

	log "Enter add docker network cmd:docker network create -d macvlan --subnet=${docker_ip}/${docker_netmask} --gateway=${docker_gatway} -o parent=${uplink_value} kylin-macvlan-net"
	docker network create -d macvlan --subnet="${docker_ip}/${docker_netmask}" --gateway="${docker_gatway}" -o parent="${uplink_value}" kylin-macvlan-net >> ${LOGFILE} 2>&1
	docker network connect kylin-macvlan-net ksvd-MStorage >> ${LOGFILE} 2>&1
	docker container cp /var/lib/ksvd/docker-ifcfg.eth0.tmp ksvd-MStorage:/etc/sysconfig/network-scripts/ifcfg-eth0 >> ${LOGFILE} 2>&1
	#sleep 1
	local nic_name=`docker exec ksvd-MStorage ip a s|grep "@if"|awk '{print $2}' |awk -F '@' '{print $1}'`
	log "get docker nic_name:${nic_name}"
	if [ -n "${nic_name}" ];then
		docker exec ksvd-MStorage ifconfig "${nic_name}" "${docker_ip}/${docker_netmask}" >> ${LOGFILE} 2>&1
		docker exec ksvd-MStorage ifconfig "${nic_name}" promisc >> ${LOGFILE} 2>&1
		docker exec ksvd-MStorage ip route add default via "${docker_gatway}" dev ${nic_name} >> ${LOGFILE} 2>&1
		docker exec ksvd-MStorage /sbin/arping -q -A -c 1 -I "${nic_name}" "${docker_ip}" >> ${LOGFILE} 2>&1
		docker exec ksvd-MStorage /sbin/arping -q -U -c 1 -I "${nic_name}" "${docker_ip}" >> ${LOGFILE} 2>&1
	fi
	log "End: reback_docker_net"

}
# set manage bond port, use change_conf to save bond config or not
set_manage_bond(){
	log "Enter: set_manage_bond $@"
	local bond_name=$1
	local uplink=(${2//,/ })
	local bond_mode=$3
	local change_conf=${4-}
	local t_mode=$(get_bond_mode $bond_mode)
	# check manage network configuration
	[ -z "$bond_name" ] && fatal "bond_name: ${bond_name} is null, so exit"
	local manage_iface_tmp1=`get_manage_iface`
	local manage_iface_tmp2=`echo "$manage_iface_tmp1"|awk '{print $1}'`
	if [ "$manage_iface_tmp2" != "$1" ]; then
		log "Enter: set_manage_bond error,$1 is not manage iface"
		return 1
	fi
	for uplink_value in ${uplink[@]};do
		[ ! -e "/sys/class/net/${uplink_value}" ] && fatal "set_manage_bond: ${uplink_value} is not a nic in this host, so exit"
	done
	check_bond_mode "$bond_mode" || fatal "set bond_mode error: ${bond_mode}"
	[ ! -e "/sys/class/net/bonding_masters" ] && modprobe bonding max_bonds=0 && echo "options bonding max_bonds=0" > /etc/modprobe.d/bonding.conf
	if [ -e "/proc/net/bonding/$bond_name" ];then
		local macaddr=`cat /sys/class/net/${bond_name}/address 2>/dev/null`
		local bond_addr=`ip addr show ${bond_name} 2>/dev/null|grep -w inet |awk '{print $2}' |sed 's/ //g'`
		local t_gatway=$(grep GATEWAY /etc/sysconfig/network-scripts/ifcfg-*|awk -F: '$2 !~ /0.0.0.0/' |awk -F "=" 'NR==1 {print $2}'|sed 's/"//g')
		log "Enter: set_manage_bond t_gatway : $t_gatway"
		#get docker network
		local docker_net_parent=`docker network inspect kylin-macvlan-net 2>/dev/null| grep $bond_name`
		local docker_gatway="$t_gatway"
		local docker_netmask=`get_manage_network_prefix`
		local docker_ip=`/usr/lib/ksvd/bin/ksvd-local-cli |jq '.stdout.storage.manageStorage.docker_ip'|sed 's/"//g'`
		ipcalc -c -s "${docker_ip}" || log "get error docker_ip: ${docker_ip}"
		[[ "${docker_netmask}" =~ ^[0-9]+$ ]] || log "get error netmask_1: ${docker_netmask}"
		[ "${docker_netmask}" -lt 1 -o "${docker_netmask}" -gt 32 ] && log "get error netmask_2: ${docker_netmask}"
		ipcalc -c -s "${docker_gatway}" || log "get error docker_gatway: ${docker_gatway}"
		log "End get docker_ip:$docker_ip,docker_netmask:$docker_netmask,docker_gatway:$docker_gatway"

		if [ -n "$docker_net_parent" -a -n "$docker_ip" -a -n "$docker_netmask" ]; then
			log "Enter: get docker network"
			docker network disconnect kylin-macvlan-net ksvd-MStorage >> ${LOGFILE} 2>&1
			rm -f /var/lib/ksvd/docker-ifcfg.eth0.tmp
			cat > /var/lib/ksvd/docker-ifcfg.eth0.tmp <<EOF
NAME=eth0
DEVICE=eth0
TYPE=Ethernet
ONBOOT=yes
PROXY_METHOD=none
BROWSER_ONLY=no
DEFROUTE=no
IPV4_FAILURE_FATAL=yes
IPADDR=${docker_ip}
PREFIX=${docker_netmask}
GATEWAY=${docker_gatway}
IPV6INIT=no
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
EOF
			log "End: get docker network,and Enter delete_macvlan0_from_manage_port"
		fi
		#delete_macvlan0_from_manage_port
		ip link delete macvlan0 >> ${LOGFILE} 2>&1
		log "End delete_macvlan0_from_manage_port"
    
		log "Enter: release bond resoures"
		ifconfig $bond_name down >> ${LOGFILE} 2>&1
		local s=""
		for s in `cat /sys/class/net/$bond_name/bonding/slaves`;do
			reback_iface "${s}"
			echo -$s >/sys/class/net/$bond_name/bonding/slaves || log "failed to del bond slave:$s"
			ifconfig $s up >> ${LOGFILE} 2>&1
		done
		echo -$bond_name > /sys/class/net/bonding_masters || log "failed to del bond :$bond_name from bonding_masters"
		log "End: release bond resoures"
		ifdown ${bond_name} >> ${LOGFILE} 2>&1
		ip link delete $bond_name >> ${LOGFILE} 2>&1
		if [ -e "${SYS_NETWORK_DIR}/ifcfg-$bond_name" ];then
			set_key_value "BONDING_OPTS" "\"mode=${t_mode} miimon=100\"" "${SYS_NETWORK_DIR}/ifcfg-$bond_name"
		fi
        
		log "Enter: update bond resoures"
		echo +$bond_name > /sys/class/net/bonding_masters || fatal "failed create bond"
		echo $bond_mode > /sys/class/net/$bond_name/bonding/mode || fatal "failed to set bond mode:$bond_mode"
		echo 100 > /sys/class/net/$bond_name/bonding/miimon  || fatal "failed to set bond miinon:100"
                
		for uplink_value in ${uplink[@]};do
			rm -f /etc/sysconfig/network-scripts/ifcfg-$uplink_value
			cat<<EOF>> /etc/sysconfig/network-scripts/ifcfg-$uplink_value
TYPE=Ethernet
NAME=$uplink_value
DEVICE=$uplink_value
MASTER=$bond_name
SLAVE=yes
EOF
			ifconfig "${uplink_value}" down >> ${LOGFILE} 2>&1
			echo +${uplink_value} > /sys/class/net/$bond_name/bonding/slaves || log "failed to add bond uplink:${uplink_value}"
		done
		if [ -n "$macaddr" ]; then
			ifconfig $bond_name hw ether $macaddr >> ${LOGFILE} 2>&1
		fi
		ifconfig $bond_name up >> ${LOGFILE} 2>&1
		if [ -n "$bond_addr" ]; then
			ifconfig $bond_name $bond_addr >> ${LOGFILE} 2>&1
		fi
		#set bond3+
		if [ "$t_mode" = "3" ]; then
			log "Get current mode is bond3+"
			if [ -e "/sys/devices/virtual/net/${bond_name}/bonding/gateway_ip" -a -n "$t_gatway" ];then
                echo "$t_gatway" > "/sys/devices/virtual/net/${bond_name}/bonding/gateway_ip"
			fi
		fi
		log "End: update bond resoures"
		if [ -n "$docker_net_parent" -a -n "$docker_ip" -a -n "$docker_netmask" ]; then
			log "Enter: update docker network"
			docker network connect kylin-macvlan-net ksvd-MStorage >> ${LOGFILE} 2>&1
			docker container cp /var/lib/ksvd/docker-ifcfg.eth0.tmp ksvd-MStorage:/etc/sysconfig/network-scripts/ifcfg-eth0 >> ${LOGFILE} 2>&1
			#sleep 1
			local nic_name=`docker exec ksvd-MStorage ip a s|grep "@if"|awk '{print $2}' |awk -F '@' '{print $1}'`
			log "get docker nic_name:${nic_name}"
			if [ -n "${nic_name}" ];then
				docker exec ksvd-MStorage ifconfig "${nic_name}" "${docker_ip}/${docker_netmask}" >> ${LOGFILE} 2>&1
				docker exec ksvd-MStorage ifconfig "${nic_name}" promisc >> ${LOGFILE} 2>&1
				docker exec ksvd-MStorage ip route add default via "${docker_gatway}" dev ${nic_name} >> ${LOGFILE} 2>&1
				docker exec ksvd-MStorage /sbin/arping -q -A -c 1 -I "${nic_name}" "${docker_ip}" >> ${LOGFILE} 2>&1
				docker exec ksvd-MStorage /sbin/arping -q -U -c 1 -I "${nic_name}" "${docker_ip}" >> ${LOGFILE} 2>&1
			fi
			log "End: update docker network"
		fi
		# add macvlan0
		add_macvlan0_to_manage_port
		
		set_key_value "${bond_name}_uplink" "$2" "${MANAGE_BOND_CONF}"
		set_key_value "${bond_name}_mode" "${bond_mode}" "${MANAGE_BOND_CONF}"
		local tmp_manage_iface=`get_key_value "${bond_name}_backup" ${MANAGE_BOND_CONF}`
		if [ x"$tmp_manage_iface" = x"" ];then
			set_key_value "${bond_name}_backup" "${uplink[0]}" "${MANAGE_BOND_CONF}"
		else
			set_key_value "${bond_name}_backup" "${tmp_manage_iface}" "${MANAGE_BOND_CONF}"	
		fi		
	else
		fatal "failed to set bond :$bond_name"
	fi
	log "End: set_manage_bond"
}
# add manage bond port, use change_conf to save bond config or not
add_manage_bond(){
	log "Enter: add_manage_bond $@"
	local bond_name=$1
	local uplink=(${2//,/ })
	local bond_mode=$3
	local change_conf=${4-}
	local manage_iface=`get_manage_iface`
	if [[ -n "${2##*$manage_iface*}" ]]; then
		log "Enter: add_manage_bond error,$2 is not manage iface"
		return 1
	fi
	[ -z "$bond_name" ] && fatal "bond_name: ${bond_name} is null, so exit"
	[ -e "/proc/net/bonding/$bond_name" ] && fatal "add_manage_bond: ${bond_name} already exist, so exit"
	for uplink_value in ${uplink[@]};do
		[ ! -e "/sys/class/net/${uplink_value}" ] && fatal "add_manage_bond: ${uplink_value} is not a nic in this host, so exit"
		[ -e "/sys/class/net/${uplink_value}/master" ] && fatal "add_manage_bond: ${uplink_value} already used, so exit"
	done
	check_bond_mode $bond_mode || fatal "set bond_mode error: ${bond_mode}"
	local t_mode=$(get_bond_mode $bond_mode)
 	local IPADDR=`get_key_value "IPADDR" "${SYS_NETWORK_DIR}/ifcfg-$manage_iface"`
	local PREFIX=`get_key_value "PREFIX" "${SYS_NETWORK_DIR}/ifcfg-$manage_iface"`
	local GATEWAY=`get_key_value "GATEWAY" "${SYS_NETWORK_DIR}/ifcfg-$manage_iface"`
	log "Enter: add_manage_bond get bond_mode:$t_mode,ipaddr:$IPADDR,prefix:${PREFIX},gatway:${GATEWAY}"
    	  
	if [ ${#uplink[@]} -ge 1 ];then
        [ ! -e "/sys/class/net/bonding_masters" ] && modprobe bonding max_bonds=0 && echo "options bonding max_bonds=0" > /etc/modprobe.d/bonding.conf
        rm -f /etc/sysconfig/network-scripts/ifcfg-$bond_name
    	cat<<EOF>> /etc/sysconfig/network-scripts/ifcfg-$bond_name
NAME=$bond_name
DEVICE=$bond_name
ONBOOT=yes
TYPE=Bond
BONDING_MASTER=yes
IPADDR=${IPADDR}
PREFIX=${PREFIX}
GATEWAY=${GATEWAY}
EOF
    	set_key_value "BONDING_OPTS" "\"mode=${t_mode} miimon=100\"" "${SYS_NETWORK_DIR}/ifcfg-$bond_name"
    	#del docker network
    	log "Enter: del docker network"
		local docker_net_parent=`docker network inspect kylin-macvlan-net 2>/dev/null| grep $manage_iface`
		local docker_gatway="$GATEWAY"
		local docker_netmask=`get_manage_network_prefix`
		local docker_ip=`/usr/lib/ksvd/bin/ksvd-local-cli |jq '.stdout.storage.manageStorage.docker_ip'|sed 's/"//g'`
		ipcalc -c -s "${docker_ip}" || log "get error docker_ip: ${docker_ip}"
		[[ "${docker_netmask}" =~ ^[0-9]+$ ]] || log "get error netmask_1: ${docker_netmask}"
		[ "${docker_netmask}" -lt 1 -o "${docker_netmask}" -gt 32 ] && log "get error netmask_2: ${docker_netmask}"
		ipcalc -c -s "${docker_gatway}" || log "get error docker_gatway: ${docker_gatway}"
		log "End get docker_ip:$docker_ip,docker_netmask:$docker_netmask,docker_gatway:$docker_gatway"

		if [ -n "$docker_net_parent" -a -n "$docker_ip" -a -n "$docker_netmask" ]; then
            docker network disconnect kylin-macvlan-net ksvd-MStorage >> ${LOGFILE} 2>&1
            docker network rm kylin-macvlan-net >> ${LOGFILE} 2>&1
            rm -f /var/lib/ksvd/docker-ifcfg.eth0.tmp
            cat > /var/lib/ksvd/docker-ifcfg.eth0.tmp <<EOF
NAME=eth0
DEVICE=eth0
TYPE=Ethernet
ONBOOT=yes
PROXY_METHOD=none
BROWSER_ONLY=no
DEFROUTE=no
IPV4_FAILURE_FATAL=yes
IPADDR=${docker_ip}
PREFIX=${docker_netmask}
GATEWAY=${docker_gatway}
IPV6INIT=no
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
EOF
            log "End: del docker network"
        fi
    		local macaddr=`cat /sys/class/net/${manage_iface}/address 2>/dev/null`
			#delete_macvlan0_from_manage_port
    		log "Enter delete_macvlan0_from_manage_port"
    		ip link delete macvlan0 >> ${LOGFILE} 2>&1
    		log "End delete_macvlan0_from_manage_port"
        
		echo +$bond_name > /sys/class/net/bonding_masters || fatal "failed create bond"
		echo $bond_mode > /sys/class/net/$bond_name/bonding/mode || fatal "failed to set bond mode:$bond_mode"
		echo 100 > /sys/class/net/$bond_name/bonding/miimon  || fatal "failed to set bond miinon:100"
                
		for uplink_value in ${uplink[@]};do
			#ifdown /etc/sysconfig/network-scripts/ifcfg-$uplink_value >> ${LOGFILE} 2>&1
			rm -f /etc/sysconfig/network-scripts/ifcfg-$uplink_value
			cat<<EOF>> /etc/sysconfig/network-scripts/ifcfg-$uplink_value
TYPE=Ethernet
NAME=$uplink_value
DEVICE=$uplink_value
MASTER=$bond_name
SLAVE=yes
EOF
			ifconfig "${uplink_value}" down >> ${LOGFILE} 2>&1
			echo +${uplink_value} > /sys/class/net/$bond_name/bonding/slaves || log "failed to add bond uplink:${uplink_value}"

			#log "Enter add_manage_bond ifup $uplink_value"
			#ifup /etc/sysconfig/network-scripts/ifcfg-$uplink_value >> ${LOGFILE} 2>&1
			#log "End add_manage_bond ifup $uplink_value"
		done
        
		log "Enter add_manage_bond ifup $bond_name"
		#ifup /etc/sysconfig/network-scripts/ifcfg-$bond_name >> ${LOGFILE} 2>&1
		if [ -n "$macaddr" ]; then
			ifconfig $bond_name hw ether $macaddr >> ${LOGFILE} 2>&1
		fi
		ifconfig $bond_name up >> ${LOGFILE} 2>&1
		log "End add_manage_bond ifup $bond_name"
		#set bond3+
		if [ "$t_mode" = "3" ]; then
			log "Get current mode is bond3+"
			if [ -e "/sys/devices/virtual/net/${bond_name}/bonding/gateway_ip" -a -n "$GATEWAY" ];then
                echo "$GATEWAY" > "/sys/devices/virtual/net/${bond_name}/bonding/gateway_ip"
			fi
		fi
		# add macvlan0
		log "add_manage_bond reload macvlan0 from $bond_name"
		add_macvlan0_to_manage_port
		log "End add_manage_bond add_macvlan0_to_manage_port"
        
		#add docker network
        	if [ -n "$docker_net_parent" -a -n "$docker_ip" -a -n "$docker_netmask" ]; then
            		log "Enter add docker network cmd:docker network create -d macvlan --subnet=${docker_ip}/${docker_netmask} --gateway=${docker_gatway} -o parent=${bond_name} kylin-macvlan-net"
            		docker network create -d macvlan --subnet="${docker_ip}/${docker_netmask}" --gateway="${docker_gatway}" -o parent="${bond_name}" kylin-macvlan-net >> ${LOGFILE} 2>&1
            		docker network connect kylin-macvlan-net ksvd-MStorage >> ${LOGFILE} 2>&1
            		docker container cp /var/lib/ksvd/docker-ifcfg.eth0.tmp ksvd-MStorage:/etc/sysconfig/network-scripts/ifcfg-eth0 >> ${LOGFILE} 2>&1
            		#sleep 1
            		local nic_name=`docker exec ksvd-MStorage ip a s|grep "@if"|awk '{print $2}' |awk -F '@' '{print $1}'`
            		log "get docker nic_name:${nic_name}"
            		if [ -n "${nic_name}" ];then
                		docker exec ksvd-MStorage ifconfig "${nic_name}" "${docker_ip}/${docker_netmask}" >> ${LOGFILE} 2>&1
                		docker exec ksvd-MStorage ifconfig "${nic_name}" promisc >> ${LOGFILE} 2>&1
                		docker exec ksvd-MStorage ip route add default via "${docker_gatway}" dev ${nic_name} >> ${LOGFILE} 2>&1
                		docker exec ksvd-MStorage /sbin/arping -q -A -c 1 -I "${nic_name}" "${docker_ip}" >> ${LOGFILE} 2>&1
                		docker exec ksvd-MStorage /sbin/arping -q -U -c 1 -I "${nic_name}" "${docker_ip}" >> ${LOGFILE} 2>&1
            		fi
            		log "End add docker network"
        	fi
		set_key_value "${bond_name}_uplink" "$2" "${MANAGE_BOND_CONF}"
		set_key_value "${bond_name}_mode" "${bond_mode}" "${MANAGE_BOND_CONF}"
		set_key_value "${bond_name}_backup" "${manage_iface}" "${MANAGE_BOND_CONF}"
	fi
	log "End: add_manage_bond"
}
# del manage bond port, use change_conf to save bond config or not
del_manage_bond(){
	log "Enter: del_manage_bond $@"
	local bond_name=$1
	[ -z "$bond_name" ] && fatal "bond_name: ${bond_name} is null, so exit"
	local manage_iface=`get_manage_iface`
	local manage_iface_tmp=`echo "$manage_iface"|awk '{print $1}'`
	if [ "$manage_iface_tmp" != "$1" ]; then
		log "Enter: set_manage_bond error,$1 is not manage iface"
		return 1
	fi
    if [ -e "/proc/net/bonding/$bond_name" ];then
        #del docker network
        local t_gatway=$(grep GATEWAY /etc/sysconfig/network-scripts/ifcfg-*|awk -F: '$2 !~ /0.0.0.0/' |awk -F "=" 'NR==1 {print $2}'|sed 's/"//g')
		log "Enter: del_manage_bond t_gatway : $t_gatway"
		#get docker network
		local docker_net_parent=`docker network inspect kylin-macvlan-net 2>/dev/null| grep $bond_name`
		local docker_gatway="$t_gatway"
		local docker_netmask=`get_manage_network_prefix`
		local docker_ip=`/usr/lib/ksvd/bin/ksvd-local-cli |jq '.stdout.storage.manageStorage.docker_ip'|sed 's/"//g'`
		ipcalc -c -s "${docker_ip}" || log "get error docker_ip: ${docker_ip}"
		[[ "${docker_netmask}" =~ ^[0-9]+$ ]] || log "get error netmask_1: ${docker_netmask}"
		[ "${docker_netmask}" -lt 1 -o "${docker_netmask}" -gt 32 ] && log "get error netmask_2: ${docker_netmask}"
		ipcalc -c -s "${docker_gatway}" || log "get error docker_gatway: ${docker_gatway}"
		log "End get docker_ip:$docker_ip,docker_netmask:$docker_netmask,docker_gatway:$docker_gatway"

        if [ -n "$docker_net_parent" -a -n "$docker_ip" -a -n "$docker_netmask" ]; then
            log "Enter: del docker network"
            docker network disconnect kylin-macvlan-net ksvd-MStorage >> ${LOGFILE} 2>&1
            docker network rm kylin-macvlan-net >> ${LOGFILE} 2>&1
            rm -f /var/lib/ksvd/docker-ifcfg.eth0.tmp >> ${LOGFILE} 2>&1
            cat > /var/lib/ksvd/docker-ifcfg.eth0.tmp <<EOF
NAME=eth0
DEVICE=eth0
TYPE=Ethernet
ONBOOT=yes
PROXY_METHOD=none
BROWSER_ONLY=no
DEFROUTE=no
IPV4_FAILURE_FATAL=yes
IPADDR=${docker_ip}
PREFIX=${docker_netmask}
GATEWAY=${docker_gatway}
IPV6INIT=no
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
EOF

            log "End: del docker network"
        fi
        #delete_macvlan0_from_manage_port
        log "Enter delete_macvlan0_from_manage_port"
        ip link delete macvlan0 >> ${LOGFILE} 2>&1
        log "End delete_macvlan0_from_manage_port"
    
	
		ifconfig $bond_name down >> ${LOGFILE} 2>&1
		local s=""
		local IPADDR=`get_key_value "IPADDR" "${SYS_NETWORK_DIR}/ifcfg-$bond_name"`
		local PREFIX=`get_key_value "PREFIX" "${SYS_NETWORK_DIR}/ifcfg-$bond_name"`
		local GATEWAY=`get_key_value "GATEWAY" "${SYS_NETWORK_DIR}/ifcfg-$bond_name"`
		local tmp_manage_iface=`get_key_value "${bond_name}_backup" ${MANAGE_BOND_CONF}`
		for s in `cat /sys/class/net/$bond_name/bonding/slaves`;do
			echo -$s >/sys/class/net/$bond_name/bonding/slaves || log "failed to del bond slave:$s"
			if [ "$tmp_manage_iface" = "${s}" ]; then
				reback_manage_iface "${s}" "${IPADDR}" "${PREFIX}" "${GATEWAY}"
			else
				reback_iface "${s}"
			fi
			ifconfig $s up >> ${LOGFILE} 2>&1
		done
        if [ -z "$tmp_manage_iface" ];then
        	local tmp_reback_manage_iface=`cat /sys/class/net/${bond_name}/bonding/slaves |awk '{print $1}' |sed 's/ //g'`
        	if [ -n "$tmp_reback_manage_iface" ];then
        		reback_manage_iface "${tmp_reback_manage_iface}" "${IPADDR}" "${PREFIX}" "${GATEWAY}"
        		ifconfig ${tmp_reback_manage_iface} up >> ${LOGFILE} 2>&1
        		tmp_manage_iface="$tmp_reback_manage_iface"
        	fi
        fi
		echo -$bond_name > /sys/class/net/bonding_masters || log "failed to del bond :$bond_name from bonding_masters"
		ifdown ${bond_name} >> ${LOGFILE} 2>&1
        rm -rf "${SYS_NETWORK_DIR}/ifcfg-${bond_name}"
        ip link delete $bond_name >> ${LOGFILE} 2>&1
        # add macvlan0
        log "Enter reback macvlan0 from $tmp_manage_iface"
        add_macvlan0_to_manage_port
        log "End reback macvlan0 from $tmp_manage_iface"
        
        #add docker network
        if [ -n "$docker_net_parent" -a -n "$docker_ip" -a -n "$docker_netmask" ]; then
            log "Enter reback docker network from $tmp_manage_iface"
            docker network create -d macvlan --subnet="${docker_ip}/${docker_netmask}" --gateway="${docker_gatway}" -o parent="${tmp_manage_iface}" kylin-macvlan-net >> ${LOGFILE} 2>&1
            docker network connect kylin-macvlan-net ksvd-MStorage >> ${LOGFILE} 2>&1
            docker container cp /var/lib/ksvd/docker-ifcfg.eth0.tmp ksvd-MStorage:/etc/sysconfig/network-scripts/ifcfg-eth0 >> ${LOGFILE} 2>&1
            #sleep 1
            local nic_name=`docker exec ksvd-MStorage ip a s|grep "@if"|awk '{print $2}' |awk -F '@' '{print $1}'`
            log "get docker nic_name:${nic_name}"
            if [ -n "${nic_name}" ];then
                docker exec ksvd-MStorage ifconfig "${nic_name}" "${docker_ip}/${docker_netmask}" >> ${LOGFILE} 2>&1
                docker exec ksvd-MStorage ifconfig "${nic_name}" promisc >> ${LOGFILE} 2>&1
                docker exec ksvd-MStorage ip route add default via "${docker_gatway}" dev ${nic_name} >> ${LOGFILE} 2>&1
                docker exec ksvd-MStorage /sbin/arping -q -A -c 1 -I "${nic_name}" "${docker_ip}" >> ${LOGFILE} 2>&1
                docker exec ksvd-MStorage /sbin/arping -q -U -c 1 -I "${nic_name}" "${docker_ip}" >> ${LOGFILE} 2>&1
            fi
            log "End reback docker network from $tmp_manage_iface"
        fi
        rm -rf "${MANAGE_BOND_CONF}"
    else
        log "End: del_manage_bond failed,$bond_name is not exist"
        return 1
	fi
    log "End: del_manage_bond"
	return 0
}
# update bond mode, use change_conf to save bond config or not
set_bond(){
	log "Enter: set_bond $@"
	local bond_name=$1
	local uplink=(${2//,/ })
	local bond_mode=$3
	local change_conf=${4-}
	local t_mode=$(get_bond_mode $bond_mode)
	[ -z "$bond_name" ] && fatal "set_bond_name: ${bond_name} is null, so exit"
    local manage_iface=`get_manage_iface`
	local manage_iface_tmp=`echo "$manage_iface"|awk '{print $1}'`
	if [ "$manage_iface_tmp" = "$1" ]; then
		log "Enter: set_bond error,$1 is manage iface"
		return 1
	fi
	for uplink_value in ${uplink[@]};do
		[ ! -e "/sys/class/net/${uplink_value}" ] && fatal "set_bond: ${uplink_value} is not a nic in this host, so exit"
	done
    if [ -e "/proc/net/bonding/$bond_name" ];then
        local macaddr=`cat /sys/class/net/${bond_name}/address 2>/dev/null`
        local bond_addr=`ip addr show ${bond_name} 2>/dev/null|grep -w inet |awk 'NR==1 {print $2}' |sed 's/ //g'`
        local t_gatway=$(grep GATEWAY /etc/sysconfig/network-scripts/ifcfg-*|awk -F: '$2 !~ /0.0.0.0/' |awk -F "=" 'NR==1 {print $2}'|sed 's/"//g')
        log "Enter: add_bond t_gatway:$t_gatway,macaddr:$macaddr,bond_addr:$bond_addr"
        del_bond "$bond_name"
        
        add_bond "$bond_name" "$2" "$3" 
        if [ -n "$bond_addr" ]; then
            ifconfig $bond_name $bond_addr >> ${LOGFILE} 2>&1
        fi
        log "End: update bond resoures"
    else
        fatal "failed to set bond :$bond_name"
	fi
    log "End: set_bond $t_mode"
}
#
# add bond port, use change_conf to save bond config or not
# input: $1 bond name,$2 uplink list,$3 bond mode,$4 change_conf,$4 is private, not public
# output: null, exclude errors
#
add_bond(){
	log "Enter: add_bond $@"
	local bond_name=$1
	local uplink=(${2//,/ })
	local bond_mode=$3
	local change_conf=${4-}
	[ -z "$bond_name" ] && fatal "add_bond_name: ${bond_name} is null, so exit"
	[ -e "/proc/net/bonding/$bond_name" ] && fatal "add_bond: ${bond_name} already used, so exit" 
	local uplink_value=""
	local manage_iface=`get_manage_iface`
	if [[ ! -n "${2##*$manage_iface*}" ]]; then
		log "Enter: add_bond error,$2 is manage iface"
		return 1
	fi
  
	for uplink_value in ${uplink[@]};do
		[ ! -e "/sys/class/net/${uplink_value}" ] && fatal "add_bond: ${uplink_value} is not a nic in this host, so exit"
		[ -e "/sys/class/net/${uplink_value}/master" ] && fatal "add_bond: ${uplink_value} already used, so exit"
	done

	[ -e "/sys/class/net/${uplink[0]}/address" ] || fatal "add_bond: /sys/class/net/${uplink[0]}/address does not exist, so exit"
	local macaddr=`cat /sys/class/net/${uplink[0]}/address 2>/dev/null`
	local t_mode=$(get_bond_mode $bond_mode)
	local t_gatway=$(grep GATEWAY /etc/sysconfig/network-scripts/ifcfg-*|awk -F: '$2 !~ /0.0.0.0/' |awk -F "=" 'NR==1 {print $2}'|sed 's/"//g')
	log "Enter: add_bond t_gatway : $t_gatway"
	if [ ${#uplink[@]} -ge 1 ];then
        [ ! -e "/sys/class/net/bonding_masters" ] && modprobe bonding max_bonds=0 && echo "options bonding max_bonds=0" > /etc/modprobe.d/bonding.conf
        rm -f /etc/sysconfig/network-scripts/ifcfg-$bond_name
        cat<<EOF>> /etc/sysconfig/network-scripts/ifcfg-$bond_name
NAME=$bond_name
DEVICE=$bond_name
ONBOOT=yes
TYPE=Bond
BONDING_MASTER=yes
EOF
		set_key_value "BONDING_OPTS" "\"mode=${t_mode} miimon=100\"" "${SYS_NETWORK_DIR}/ifcfg-$bond_name"
		ifdown /etc/sysconfig/network-scripts/ifcfg-$bond_name >> ${LOGFILE} 2>&1
		ifup /etc/sysconfig/network-scripts/ifcfg-$bond_name >> ${LOGFILE} 2>&1
        
		for uplink_value in ${uplink[@]};do
			ifdown /etc/sysconfig/network-scripts/ifcfg-$uplink_value >> ${LOGFILE} 2>&1
			rm -f /etc/sysconfig/network-scripts/ifcfg-$uplink_value
			cat<<EOF>> /etc/sysconfig/network-scripts/ifcfg-$uplink_value
TYPE=Ethernet
NAME=$uplink_value
DEVICE=$uplink_value
MASTER=$bond_name
SLAVE=yes
EOF

			ifup /etc/sysconfig/network-scripts/ifcfg-$uplink_value >> ${LOGFILE} 2>&1
		done
		set_key_value "${bond_name}_uplink" "$2" "${BOND_CONF}"
		set_key_value "${bond_name}_mode" "${bond_mode}" "${BOND_CONF}"
	fi
	#set bond3+
	if [ "$t_mode" = "3" ]; then
		log "Get current mode is bond3+"
		if [ -e "/sys/devices/virtual/net/${bond_name}/bonding/gateway_ip" -a -n "$t_gatway" ];then
			echo "$t_gatway" > "/sys/devices/virtual/net/${bond_name}/bonding/gateway_ip"
		fi
	fi
	ifconfig ${bond_name} up >> ${LOGFILE} 2>&1
    log "End: add_bond"
}

#
# del bond port, use change_conf to save bond config or not
# input: $1 bond name, $2 change_conf,default "",$2 is private, not public
# output: null, exclude errors
#
del_bond(){
	log "Enter: del_bond $@"
	local bond_name=$1
	local change_conf=${2-}
	[ -z "$bond_name" ] && fatal "del_bond_name: ${bond_name} is null, so exit"
	# check manage network configuration
	local manage_iface=`get_manage_iface`
	local manage_iface_tmp=`echo "$manage_iface"|awk '{print $1}'`
	if [ "$manage_iface_tmp" = "$1" ]; then
		log "END: del_bond error,$1 is manage iface"
		return 1
	fi
	if [ -e "/proc/net/bonding/$bond_name" ];then
		ifconfig $bond_name down >> ${LOGFILE} 2>&1
		local s=""
		for s in `cat /sys/class/net/$bond_name/bonding/slaves`;do
			#echo -$s >/sys/class/net/$bond_name/bonding/slaves || fatal "failed to del bond slave:$s"
			revert_iface "${s}"
			ifconfig $s up >> ${LOGFILE} 2>&1
		done
		if [ -z "$s" ]; then
			log "Enter: del_bond for BOND_CONF"
			local tmp_bond_iface=`grep "${bond_name}_uplink=" ${BOND_CONF} 2>/dev/null|sed s/\"//g |sed s/" "//g |awk -F "=" '{print $2}'`
			local ifaces=(${tmp_bond_iface/\,/ })
			for slave in ${ifaces[@]};do
				log "Enter: revert_iface BOND_CONF to $slave"
				revert_iface "${slave}"
				ifconfig $slave up >> ${LOGFILE} 2>&1
			done
		fi
		#echo -$bond_name > /sys/class/net/bonding_masters || fatal "failed to del bond :$bond_name from bonding_masters"
		ifdown ${bond_name} >> ${LOGFILE} 2>&1
		ip link delete $bond_name >> ${LOGFILE} 2>&1
		rm -rf "${SYS_NETWORK_DIR}/ifcfg-${bond_name}"
	fi
	[ "${change_conf}"x != "no"x ] && del_key_value "${bond_name}" "${BOND_CONF}"
	log "End: del_bond $bond_name"
	return 0
}

#
# add dvs
# input: $1 dvs name,$2 uplink, a hareware nic name or a bond name or "empty"
# output: null, exclude errors
#
add_dvs(){
	log "Enter: add_dvs $@"
	[ $# -lt 2 ] && fatal "add_dvs $@  : the number of args is wrong, so exit"
	local dvs_name=$1
	local uplink=$2

	${OVS} br-exists "${dvs_name}"
	if [ $? -eq 0 ];then
		fatal "dvs:${dvs_name} is already exist, so exit"
	fi

	if [ "${uplink}" == "empty" ];then
		${OVS} --may-exist add-br ${dvs_name} >> ${LOGFILE} 2>&1
	else
		[ ! -e "/sys/class/net/${uplink}" ] && fatal "add_dvs: uplink:$uplink does not exist, so exit"

		${OVS} --may-exist add-br ${dvs_name} >> ${LOGFILE} 2>&1
		# set mac, if have a ipaddress on bond
		#if [ -e "/proc/net/bonding/$uplink" ];then
		#	ip link set ${dvs_name} address $macaddr
		#fi
		${OVS} --may-exist add-port ${dvs_name} $uplink >> ${LOGFILE} 2>&1
	fi
	ifconfig ${dvs_name} up >> ${LOGFILE} 2>&1
	set_key_value "${dvs_name}_uplink" "$uplink" "${DVS_CONF}"
}

#
# del dvs
# input: $1 dvs name
# output: null
#
del_dvs(){
	log "Enter: del_dvs $@"
	[ $# -ne 1 ] && fatal "del_dvs $@  : the number of args is wrong, so exit"

	# if exist, kill all dnsmasq on this dvs
	for dnsmasq_pid in `ps -ef|grep "interface=${1}-"|grep -v "grep"|awk '{print $2}'`;do
		kill "${dnsmasq_pid}"
	done

	${OVS} --if-exists del-br "$1"
	[ $? -eq 0 ] && rm -rf /home/kylin-ksvd/network/${1}-*

	# del dvs from config file
	del_key_value "$1" "${DVS_CONF}"
}


#
# change uplink
# input: $1 dvs name,$2 new uplink name
# output: null
#
update_dvs(){
	log "Enter: update_dvs $@"
	[ $# -ne 2 ] && fatal "update_dvs $@  : the number of args is wrong, so exit"
	if [ "$2" != "empty" ];then
		list_available_uplink|grep -wq $2 || fatal "update_dvs $@  : uplink $2 is in use, can not be added to $1, so exit"
	fi

	old_uplink=`get_key_value "${1}_uplink" "${DVS_CONF}"`
	if [ $? -eq 0 ];then
		if [ "${old_uplink}" != "empty" ];then
			${OVS} --if-exists del-port $1 ${old_uplink} || fatal "fail to del-port $1 ${old_uplink} , so exit"
		fi
		if [ "${2}" != "empty" ];then
			${OVS} --may-exist add-port $1 "${2}" || fatal "fail to add-port $1 ${2} , so exit"
		fi
		del_key_value "${1}_uplink" "${DVS_CONF}"
		set_key_value "$1_uplink" "$2" "${DVS_CONF}"
	else
		fatal "update_dvs can not find old uplink of $1 in dvs.conf, so exit"
	fi
}


#
# load network conf with ksvd service, such as dvs.conf/bond.conf
# input: null
# output: null
#
load_conf(){
	log "Enter: load_conf $@"
	[ $# -ne 0 ] && fatal "load_conf $@  : the number of args is wrong, so exit"

	# add bond
	local bond_name=""

	if [ -e "${BOND_CONF}" ];then
		for bond_name in `grep "_uplink" "${BOND_CONF}" |awk -F '_uplink' '{print $1}'`;do
			local uplink_value=`get_key_value "${bond_name}_uplink" "${BOND_CONF}"`
			local bondmode_value=`get_key_value "${bond_name}_mode" "${BOND_CONF}"`
			if [ ! -e "/proc/net/bonding/$bond_name" ];then
				log "Enter: load_conf add_bond"
				add_bond ${bond_name} ${uplink_value} ${bondmode_value} "no"
			fi
		done
	fi
	
	if [ -e "${MANAGE_BOND_CONF}" ];then
		for bond_name in `grep "_uplink" "${MANAGE_BOND_CONF}" |awk -F '_uplink' '{print $1}'`;do
			local uplink_value=`get_key_value "${bond_name}_uplink" "${MANAGE_BOND_CONF}"`
			local bondmode_value=`get_key_value "${bond_name}_mode" "${MANAGE_BOND_CONF}"`
			if [ -e "/proc/net/bonding/$bond_name" ];then
				local tmp_bond_mode=`cat /sys/class/net/${bond_name}/bonding/mode 2>/dev/null|awk '{print $1}'`
				if [ x"$tmp_bond_mode" != x"$bondmode_value" ];then
					log "Enter: local mode:$tmp_bond_mode,load_conf set_manage_bond $bondmode_value"
					set_manage_bond ${bond_name} ${uplink_value} ${bondmode_value}
				fi
			else
				log "Enter: load_conf add_manage_bond"
				add_manage_bond ${bond_name} ${uplink_value} ${bondmode_value}
			fi
		done
	fi

	if [ -e "${DVS_CONF}" ];then
		for dvs in `grep "_uplink" "${DVS_CONF}"|awk -F '_uplink' '{print $1}'`;do
			local tmp_dvs_uplink=`get_key_value "${dvs}_uplink" ${DVS_CONF}`
			${OVS} br-exists "${dvs}"
			if [ $? -ne 0 ];then
				${OVS} --may-exist add-br "${dvs}"
				[ "${tmp_dvs_uplink}" != "empty" ] && ${OVS} --may-exist add-port "${dvs}" "${tmp_dvs_uplink}"
			fi
			ifconfig "${dvs}" up
			log "${OVS} --may-exist add-port ${dvs} ${tmp_dvs_uplink}" 
		done
	fi
}	

#
# clear all bonds in bond.conf with ksvd service,include rollback
# input: null
# output: null
#
clear_bond(){
	[ ! -e "${BOND_CONF}" ] && return 0

	local bond_name=""
	for bond_name in `grep "_uplink" "${BOND_CONF}" |awk -F '_uplink' '{print $1}'`;do 
		if [ -e "/proc/net/bonding/$bond_name" ];then
			del_bond "${bond_name}" "no" || fatal "failed to del bond: ${bond_name}"
		fi
	done
}

#
# clear all dvs in dvs.conf with ksvd service,include rollback
# input: null
# output: null
#
clear_dvs(){
	for dvs in `grep "_uplink" "${DVS_CONF}"|awk -F '_uplink' '{print $1}'`;do
		${OVS} br-exists "${dvs}"
		if [ $? -eq 0 ];then
			${OVS} del-br "${dvs}"
		fi
		log "${OVS} del-br ${dvs}"
	done
}

#
# clear bond and dvs 
# input: null
# output: null
#
clear_conf(){
	clear_bond
	clear_dvs
}
#
# list all subnet name of a bridge
# input: $1 dvs name
# output: subnet names, if more than one, separated by '|'
#	
list_subnet(){
	log "Enter: list_subnet $@"
	[ $# -ne 1 ] && fatal "list_subnet $@  : the number of args is wrong, so exit"

	local subnet_list=""
	local subnet_name=""
	for subnet_name in `${OVS} list-ports $1|grep $1`;do
		subnet_name=`echo "${subnet_name}"|sed 's/'"$1-"'//g'`
		if [ "${subnet_list}"x == x ];then
			subnet_list="${subnet_name}"
		else
			subnet_list="${subnet_list}|${subnet_name}"
		fi
	done
	echo "${subnet_list}"
}

#
# add a subnet to a bridge or a dvs
# input: $1 dvs name, $2 subnet name,$3 vlan,$4 ip,$5 mask,$6 dhcp range,$7 gateway,$8 dns server
# output: null
#
add_subnet(){
	log "Enter: add_subnet $@"
	local dvsname_value=$1
	local subnetname_value=$2
	local vlan_value=$3
	local ip_value=$4
	local mask_value=$5
	local dhcprange_value=$6
	local gateway_value=$7
	local dnsserver_value=$8

	[ ! -e "/var/lib/ksvd/settings.node" ] && exit 1
	source /var/lib/ksvd/settings.node
	${OVS} --may-exist add-port "${dvsname_value}" "${subnetname_value}" -- set interface "${subnetname_value}" type=internal -- set Port "${subnetname_value}" tag="${vlan_value}" || fatal "add_subnet $@ : failed to add-port"
	ifconfig "${subnetname_value}" ${ip_value} netmask ${mask_value} up
	dns_cmd="dnsmasq --strict-order --except-interface=lo  --bind-dynamic --dhcp-no-override --interface=${subnetname_value} --pid-file=/run/openvswitch/${subnetname_value}.pid --dhcp-leasefile=/home/${UNIQB_MC_USER}/network/${subnetname_value}.leases --dhcp-range=${dhcprange_value} --dhcp-option=option:netmask,${mask_value} --dhcp-option=option:router,${gateway_value} --dhcp-option=option:dns-server,${dnsserver_value}"
	dns_exist=`ps -ef|grep "interface=${subnetname_value} "|grep -v "grep"`
	if [ "${dns_exist}"x == x ];then
		${dns_cmd}
	else
		#for edit, if changed, start a new one, else do nothing.
		exist_same_dns=`ps -ef|egrep "${subnetname_value}\.leases ..dhcp-range=${dhcprange_value} ..dhcp-option=option:netmask,${mask_value} ..dhcp-option=option:router,${gateway_value} ..dhcp-option=option:dns-server,${dnsserver_value}"|grep -v grep`
		if [ "${exist_same_dns}"x == x ];then
			for dns_pid in `ps -ef|grep "interface=${subnetname_value} "|grep -v grep|awk '{print $2}'`;do kill "${dns_pid}";done
			rm -rf "/home/${UNIQB_MC_USER}/network/${subnetname_value}.leases"
			${dns_cmd}
			log "add_subnet edit: restart dnsmasq"
		fi
	fi
}

#
# delete all subnets from a bridge
# input: $1 dvs name, $2 subnet name
# output: null
#
del_subnet(){
	log "Enter: del_subnet $@"
	[ $# -ne 2 ] && fatal "del_subnet $@  : the number of args is wrong, so exit"
	
	local dvsname_value=$1
	local subnetname_value=$2
	[ ! -e "/var/lib/ksvd/settings.node" ] && exit 1
	source /var/lib/ksvd/settings.node
	for dns_pid in `ps -ef|grep "interface=${subnetname_value} "|grep -v grep|awk '{print $2}'`;do
		if [ "${dns_pid}"x != x ];then
			kill "${dns_pid}"
			rm -rf /home/${UNIQB_MC_USER}/network/${subnetname_value}.leases
			log "kill ${dns_pid}"
		fi
	done
	${OVS} del-port "${dvsname_value}" "${subnetname_value}" || fatal "${subnetname_value} may not exist on ${dvsname_value}"	
}

#
# delete all subnets from all bridges
# input: null
# output: null
#
clear_all_subnet(){
	log "Enter: clear_all_subnet"

	local tmp_vs
	local tmp_subnet
	for tmp_vs in `${OVS} list-br|grep -v ksvdinat0`;do
		for tmp_subnet in `${OVS} list-ports ${tmp_vs}|grep ${tmp_vs}|grep -v "${tmp_vs}.b"`;do
			del_subnet ${tmp_vs} ${tmp_subnet}
			log "del_subnet ${tmp_vs} ${tmp_subnet}"
		done
	done
}

#
# add a mirror on a bridge
# input: $1 dvs name,$2 mirror name,$3 source type,$4 source value,$5 dest type,$6 dest value
# output: null
#
add_mirror(){
	log "Enter: add_mirror $@"
	local dvsname_value=$1
	local name=$2
	local src_type=$3
	local src_param=$4
	local dst_type=$5
	local dst_param=$6

	[ $# -ne 6 ] && fatal "add_mirror $@  : the number of args is wrong, so exit"

	if [ "${dvsname_value}"x == x ] || [ "${name}"x == x ] ||  [ "${src_type}"x == x ] || [ "${src_param}"x == x ] || [ "${dst_type}"x == x ] || [ "${dst_param}"x == x ];then
		fatal "invalid params. add_mirror $@ "
	fi

	local tmp_var=""
	local tmp_select=""
	if [ "${src_type}" == "piface" ];then
		${OVS} list-ports ${dvsname_value} |grep ${src_param}
		[ $? -ne 0 ] && fatal "src iface/port ${src_param} is not on ${dvsname_value},so exit 1"

		tmp_var=" -- --id=@${src_param} get Port ${src_param}"
		tmp_select=" select-dst-port=@${src_param} select-src-port=@${src_param}"
	elif [ "${src_type}" == "viface" ];then
		${OVS} list-ports "${dvsname_value}"|grep -q "^${src_param}$" || fatal "src iface/port ${src_param} is not on ${dvsname_value},so exit 1"
		tmp_var=" -- --id=@${src_param} get Port ${src_param}"
		tmp_select=" select-dst-port=@${src_param} select-src-port=@${src_param}"
	elif [ "${src_type}" == "vlan" ];then
		tmp_select=" select-all=true select-vlan=${src_param}"
	else
		fatal "invalid src_type: ${src_type}"
	fi

	if [ "${dst_type}" == "piface" ];then
		${OVS} --may-exist add-port ${dvsname_value} ${dst_param} || fatal "failed add ${dst_param} to ${dvsname_value}"
		tmp_var="${tmp_var} -- --id=@${dst_param} get Port ${dst_param}"
		tmp_select="${tmp_select} output-port=@${dst_param}"
	elif [ "${dst_type}" == "viface" ];then
		${OVS} list-ports "${dvsname_value}"|grep -q "^${dst_param}$" || fatal "dst iface/port ${dst_param} is not on ${dvsname_value},so exit 1"
		tmp_var="${tmp_var} -- --id=@${dst_param} get Port ${dst_param}"
		tmp_select="${tmp_select} output-port=@${dst_param}"
	elif [ "${dst_type}" == "vlan" ];then
		tmp_select="${tmp_select} output-vlan=${dst_param}"
	else
		fatal "invalid dst_type: ${dst_type}"
	fi
	
	local cmd="${OVS} -- set bridge ${dvsname_value} mirrors=@${name} ${tmp_var} -- --id=@${name} create Mirror name=${name} ${tmp_select}"
	log "cmd: $cmd"
	${cmd} || fatal "failed to add mirror ${name} on ${dvsname_value}"
	del_key_value "${dvsname_value}_mirror_cmd" "${DVS_CONF}"
	echo "${dvsname_value}_mirror_cmd=${CMD}" >> ${DVS_CONF}		#warn: avoid writting at the same time
}

#
# delete a mirror on a bridge
# input: $1 dvs name
# output: null
#
del_mirror(){
	log "Enter: del_mirror $@"
	local dvsname_value=$1

	#warn: a switch has only a mirror, fatal error occurs when more than one mirror on a switch.
	#if dst_port = phy iface ,del-port phy-iface from dvs.

	local mirror_uuid=`${OVS} get bridge "${dvsname_value}" mirrors | sed -e s/"\["//g -e s/"\]"//g`
	if [ "${mirror_uuid}"x == x ];then
		log "no mirror active on ${dvsname_value}, if a mirror config of ${dvsname_value} in dvs.conf, delete it and exit 0"
		del_key_value "${dvsname_value}_mirror_cmd" "${DVS_CONF}"
		exit 0
	fi
	local mirror_output_uuid=`${OVS} get mirror "${mirror_uuid}" output-port | sed -e s/"\["//g -e s/"\]"//g | sed s/","/" "/g`
	if [ "${mirror_output_uuid}"x != x ];then
		local mirror_output_port=`${OVS} get port ${mirror_output_uuid} name|sed s/"\""//g`
		echo "${mirror_output_port}" | grep -E "^ens|^enp|^eno|^enx|^p[1-9][1-9]?p[0-9][0-9]?|^eth"
		if [ $? -eq 0 ];then
			${OVS} --if-exists del-port ${dvsname_value} ${mirror_output_port}
		fi
	fi
	$OVS clear bridge $dvsname_value mirror || fatal "failed to clear mirror on ${dvsname_value}"
	del_key_value "${dvsname_value}_mirror_cmd" "${DVS_CONF}"
}

  
 # get all iface ip
 # input: null
 # output: iface ip list ,such as 192.5.1.161|192.5.1.162
 #
 list_all_iface_ip() {
         # list all physical iface and macvlan0
         local ifaces=()
         for network_iface in `ls -F /sys/class/net/|grep "@$"|sed s/'@$'//g`;do
                 local is_virtual_network_iface=0
                 for virtual_network_iface in `ls -F /sys/devices/virtual/net/|grep "/$"|sed s/'\/$'//g`;do
                         [ "${network_iface}"x == "macvlan0"x ] && break
                         [ "${network_iface}"x == "${virtual_network_iface}"x ] && is_virtual_network_iface=1 && break
                 done
                 [ ${is_virtual_network_iface} -eq 0 ] && ifaces[${#ifaces[*]}]=${network_iface}
         done
         # list all bond iface
         if [ -e "/sys/class/net/bonding_masters" ];then
                 local bonds_in_sys=(`cat "/sys/class/net/bonding_masters"`)
                 for bond_name in ${bonds_in_sys[@]};do
                         ifaces[${#ifaces[*]}]=${bond_name}
                 done
         fi
         # if iface set ip, get all iface ip
         local iface_ips=()
         for iface in ${ifaces[@]};do
                 local iface_ip=`ip a s|grep "scope global ${iface}"|awk -F 'inet ' '{print $2}'|awk -F '/' '{print $1}'`
                 [ -n "$iface_ip" ] && iface_ips[${#iface_ips[*]}]="$iface_ip"
         done
         # get result in defined format
         local res=''
         for ip in ${iface_ips[@]};do
                 if [ -z "$res" ];then
                         res="${ip}"
                 else
                         res="${res}|${ip}"
                 fi
         done
         echo ${res}
}

 # get manage network prefix
 # input: null
 # output: mask ,such as 24|20
 #
get_manage_network_prefix() {
	manage_eth=$(grep "GATEWAY="  /etc/sysconfig/network-scripts/ifcfg-*|awk -F ":" '{print $1}')
	gateway=$(grep "GATEWAY="  ${manage_eth}|awk -F "=" 'NR==1 {print $2}'|tr -d " \"")
	prefix=$(grep "PREFIX="  ${manage_eth}|awk -F "=" 'NR==1 {print $2}'|tr -d " \"")
	echo $prefix
}

main(){

	log "Enter: main $@"
	CMD="/usr/lib/ksvd/bin/ksvd_net_vsctl $@"
	### 1 accept args
	ARGS=`getopt -o ha:t:d:I:u:s:v:n:m:p:i::r::D::g:: --long help,action:,type:,dvsname:,iface:,uplink:,subnetname:,vlan:,network:,mask:,prefix:,ip::,dhcprange::,dnsserver::,gateway::,\
mirrorname:,srctype:,srcparam:,dsttype:,dstparam:,\
bondname:,bondmode:,ipset: -n 'ksvd_net_vsctl' -- "$@"`
	if [ $? != 0 ]; then
		fatal "getopt error,then terminating ..."
	fi
	eval set -- "${ARGS}"

	while true ; do
		case "$1" in
			-h|--help)  usage; exit 0;;
			-a|--action)  action_value="$2"; shift 2;;
			-t|--type) type_value=$2; shift 2;;
			-d|--dvsname) dvsname_value=$2; shift 2;;
			-I|--iface) iface_value=$2; shift 2;;
			-u|--uplink) uplink_value=$2; shift 2;;
			-s|--subnetname) subnetname_value=$2; shift 2;;
			-v|--vlan) vlan_value=$2; shift 2;;
			-n|--network) network_value=$2; shift 2;;
			-m|--mask) mask_value=$2; shift 2;;
			-p|--prefix) prefix_value=$2; shift 2;;
			-i|--ip)
				case "$2" in
					"") ip_value="$2"; shift 2 ;;
					*)  ip_value="$2"; shift 2 ;;
				esac;;
			-r|--dhcprange)
				case "$2" in
					"") dhcprange_value="$2"; shift 2 ;;
					*)  dhcprange_value="$2"; shift 2 ;;
				esac;;
			-D|--dnsserver)
				case "$2" in
					"") dnsserver_value="$2"; shift 2 ;;
					*)  dnsserver_value="$2"; shift 2 ;;
				esac;;
			-g|--gateway)
				case "$2" in
					"") gateway_value="$2"; shift 2 ;;
					*)  gateway_value="$2"; shift 2 ;;
				esac;;
			--mirrorname) mirror_name_value=$2; shift 2;;
			--srctype) src_type_value=$2; shift 2;;
			--srcparam) src_param_value=$2; shift 2;;
			--dsttype) dst_type_value=$2; shift 2;;
			--dstparam) dst_param_value=$2; shift 2;;
			--bondname) bondname_value=$2; shift 2;;
			--bondmode) bondmode_value=$2; shift 2;;
			--ipset) ipset_value=$2; shift 2;;
			--) shift ; break ;;
			*) echo "Args unkonw: $1 "; exit 1 ;;
		esac
	done

	### 2 change args
	if [ "${action_value}" == "add-subnet" ];then
		if [ "${dvsname_value}"x == x ] || [ "${subnetname_value}"x == x ] || [ "${network_value}"x == x ] || [ "${mask_value}"x == x ];then
			fatal "$0 $@ , params is not valid, so exit"
		else
			subnetname_value="${dvsname_value}-${subnetname_value}"
			[ "${gateway_value}"x == x ] && gateway_value=`get_default_gateway "${network_value}" "${mask_value}"`
			[ "${dhcprange_value}"x == x ] && dhcprange_value=`get_default_dhcprange "${network_value}" "${mask_value}"`
			[ "${dnsserver_value}"x == x ] && dnsserver_value="114.114.114.114,8.8.8.8"
			ip_value=`echo ${dhcprange_value} | awk -F ',' '{print $2}'`
		fi
	elif [ "${action_value}" == "del-subnet" ];then
		subnetname_value="${dvsname_value}-${subnetname_value}"
	fi
	case "$action_value" in
		add-macvlan0-to-manage-port) add_macvlan0_to_manage_port; exit 0;;
		delete-macvlan0-from-manage-port) delete_macvlan0_from_manage_port; exit 0;;
		reback-docker-net) reback_docker_net ${uplink_value}; exit 0;;
		list-phy-iface) list_phy_iface; exit 0;;
		list-phy-iface-brief) list_phy_iface_brief; exit 0;;
		list-network-speed-status) list_network_speed_status; exit 0;;
		active-phy-iface) active_phy_iface; exit 0;;
		list-bond-brief) list_bond_brief; exit 0;;
		list-available-uplink) list_available_uplink; exit 0;;
		add-bond) add_bond ${bondname_value} ${uplink_value} ${bondmode_value}; exit 0;;
		add-manage-bond) add_manage_bond ${bondname_value} ${uplink_value} ${bondmode_value}; exit 0;;
		set-bond) set_bond ${bondname_value} ${uplink_value} ${bondmode_value}; exit 0;;
		set-manage-bond) set_manage_bond ${bondname_value} ${uplink_value} ${bondmode_value}; exit 0;;
		del-bond) del_bond ${bondname_value}; exit 0;;
		del-manage-bond) del_manage_bond ${bondname_value}; exit 0;;
		clear-bond) clear_bond; exit 0;;
		clear-conf) clear_conf; exit 0;;
		list-all-iface-ip) list_all_iface_ip; exit 0;;
		get-storage-net) get_storage_net; exit 0;;
		add-storage-net) add_storage_net "${type_value}" "${iface_value}" "${ip_value}" "${prefix_value}" "${bondmode_value}"; exit 0;;
		del-storage-net) del_storage_net; exit 0;;
		get-manage-iface) get_manage_iface; exit 0;;
		get-manage-network-prefix) get_manage_network_prefix; exit 0;;
		set-portsec) /usr/bin/flock -xn /var/run/lock/port_security_node.lock -c "/usr/lib/ksvd/bin/port_security_node \"${ipset_value}\""; exit $?;;
		*) ;;
	esac

	### 3 switch actions
	${OVS} show >> ${LOGFILE} 2>&1
	if [ $? != 0 ];then 
		fatal "ovs do not work, so exit"
	fi
	case "$action_value" in
		list-dvs) list_dvs ;;
		list-uplink) list_uplink ${dvsname_value} ;;
		add-dvs) add_dvs ${dvsname_value} ${uplink_value};;
		del-dvs) del_dvs ${dvsname_value} ;;
		update-dvs) update_dvs ${dvsname_value} ${uplink_value} ;;
		load-conf) load_conf ;;
		list-subnet) list_subnet ${dvsname_value} ;;
		add-subnet) add_subnet "${dvsname_value}" "${subnetname_value}" "${vlan_value}" "${ip_value}" "${mask_value}" "${dhcprange_value}" "${gateway_value}" "${dnsserver_value}" ;;
		del-subnet) del_subnet "${dvsname_value}" "${subnetname_value}" ;;
		clear-all-subnet) clear_all_subnet ;;
		add-mirror) add_mirror "${dvsname_value}" "${mirror_name_value}" "${src_type_value}" "${src_param_value}" "${dst_type_value}" "${dst_param_value}" ;;
		del-mirror) del_mirror "${dvsname_value}" ;;
		*) echo "ERROR: unknow action: ${action_value}"; exit 1 ;;
	esac
	find /var/log/ksvd-network/ -name ksvd_net_vsctl.log -size +1G -exec rm -rf {} \;
}

[ ! -e "${LOG_DIR}" ] && mkdir -p "${LOG_DIR}"
#echo "" >> ${LOGFILE}
[ -e "${SYS_NETWORK_DIR}" ] && rm -rf "${SYS_NETWORK_DIR}"/ifcfg*.bak "${SYS_NETWORK_DIR}"/ifcfg*.old
log " ----------------- Begin - $0 $@ ---------------"

main $@

log " ----------------- End - $0 $@ -----------------"
#echo "" >> ${LOGFILE}

