磐维2.0.2VIP高可用实践经验

2.0.2支持通过回调脚本方式在主备切换场景下自动切换VIP。cm_agent在本地数据库角色发生切换为主库时，通过回调脚本或配置参数的形式，绑定VIP地址；当本节点数据库角色切换为备机时，主动解除VIP地址绑定。

1.1 配置omm用户的sudo权限

echo 'omm ALL=(ALL:ALL) NOPASSWD:/sbin/ip,/sbin/arping,/sbin/iptables,/sbin/ifconfig'>>/etc/sudoers 

注意：ip和arping命令需要结合实际操作系统路径进行修改

1.2 omm用户测试sudo权限

su - omm

sudo ip addr add 10.146.xx.xx/24 brd 10.146.xx.xx dev bond0 label bond0:1

sudo arping -q -A -c 1 -I bond0 10.146.xx.xx

sudo ip addr del 10.146.xx.xx/24 dev bond0 label bond0:1

1.3 修改回调脚本

echo '
#!/bin/bash
# -------------------------------------------------------------------------------
# Filename:    cm_callback.sh
# Revision:    1.0
# Date:        2019/10/09
# Description: 
# Notes:       
#      callback 仅实现vip 的 添加和移除        
#              
# -------------------------------------------------------------------------------
# 
# -------------------------------------------------------------------------------

readonly cb_name=$1
readonly role=$2
readonly scope=$3

VIP=10.228.xx.xx  # vip 地址
VIPBRD=10.228.xx.xx # 广播地址
VIPNETMASKBIT=24   # 掩码
VIPDEV=bond1     # 网络接口名称,vip会绑定到该接口
VIPLABEL=1   # 接口标签,默认为1

PING_TIMEOUT=2 # 设置为不高于panweidb.yml配置文件里loop_wait的40%，如果loop_wait设置为10，PING_TIMEOUT建议设置为3，如果loop_wait设置为5，建议设置为2

function usage() {
    echo "Usage: $0   ";
    exit 1;
}

function addvip(){
    echo "`date +%Y-%m-%d\ %H:%M:%S,%3N` INFO: /sbin/ip addr add ${VIP}/${VIPNETMASKBIT} brd ${VIPBRD} dev ${VIPDEV} label ${VIPDEV}:${VIPLABEL}"
    sudo /sbin/ip addr add ${VIP}/${VIPNETMASKBIT} brd ${VIPBRD} dev ${VIPDEV} label ${VIPDEV}:${VIPLABEL}
    sudo /usr/sbin/arping -q -A -c 1 -I ${VIPDEV} ${VIP}
    #sudo /sbin/iptables -F

}

function delvip(){
    echo "`date +%Y-%m-%d\ %H:%M:%S,%3N` INFO: sudo /sbin/ip addr del ${VIP}/${VIPNETMASKBIT} dev ${VIPDEV} label ${VIPDEV}:${VIPLABEL}"
    sudo /sbin/ip addr del ${VIP}/${VIPNETMASKBIT} dev ${VIPDEV} label ${VIPDEV}:${VIPLABEL}
    #sudo /usr/sbin/arping -q -A -c 1 -I ${VIPDEV} ${VIP}
    #sudo /sbin/iptables -F
    
}

#if [[ $cb_name != "on_master_check"  ]]; then
#    echo "`date +%Y-%m-%d\ %H:%M:%S,%3N` WARNING: has callback $cb_name $role $scope"
#fi

case $cb_name in
    on_stop)
        delvip
        ;;
    on_start)
	if [[ $role == "master" ]]; then
	    addvip
	fi
        ;;
    on_role_change)
        if [[ $role == "master" ]]; then
            addvip
        elif [[ $role == "slave" ]]||[[ $role == "replica" ]]||[[ $role == "logical" ]]; then
            delvip
        fi
        ;;
    on_master_check)
        if [[ $role == "master" ]]; then
            vip_status=`/sbin/ip addr|grep ${VIP}`
            if [[ ${vip_status} == "" ]]; then
                echo "`date +%Y-%m-%d\ %H:%M:%S,%3N` WARNING: has callback $cb_name $role $scope"
                echo "`date +%Y-%m-%d\ %H:%M:%S,%3N` WARNING: The vip was lost,bind vip because i am leader"
                /usr/bin/ping -I ${VIPDEV} -q -c 3 -W ${PING_TIMEOUT} ${VIP}
                is_vip_used=$?
                if [[ $is_vip_used -eq 0 ]]; then
                    echo "`date +%Y-%m-%d\ %H:%M:%S,%3N` WARNING: The vip bind failed, ${VIP} is in use"
                else
                    addvip
                fi
            fi
        fi
        ;; 
    on_slave_check)
        if [[ $role == "slave" ]]; then
            vip_status=`/sbin/ip addr|grep ${VIP}`
            if [[ ${vip_status} != "" ]]; then
                delvip
            fi
        fi
        ;;  
    *)
        usage
        ;;
esac
'>/database/panweidb/cm/cm_agent/cm_callback.sh

给脚本文件赋执行权限，用于在本节点数据库主备角色发生变化时，自动由集群管理服务执行绑定和释放VIP的操作。

chmod u+x /database/panweidb/cm/cm_agent/cm_callback.sh

1.4 修改cm_agent.conf脚本

需要将原来的单引号去掉

echo 'callback_bin_path=/database/panweidb/cm/cm_agent/cm_callback.sh' >> /database/panweidb/cm/cm_agent/cm_agent.conf

1.5 修改alarmConfig.conf脚本

需要将原来的单引号去掉

echo 'callback_bin_path=/database/panweidb/cm/cm_agent/cm_callback.sh' >> /database/panweidb/app/bin/alarmConfig.conf

# 脚本授权
chown omm:  /database/panweidb/cm/cm_agent/cm_callback.sh
chmod 755  /database/panweidb/cm/cm_agent/cm_callback.sh

1.6 重启集群

cm_ctl stop && cm_ctl start

1.7 switchover切换，查看vip是否漂移

cm_ctl switchover -n 1 -D /panwei/database/panweidb/data
gs_om -t status --detail && ip a|grep -i 10.228.xx.xx

可以看到，随着主节点切换成功，vip会自动漂移到新的主节点上

1.8 如果未进行漂移，查看agent日志

这个问题是没有找到网卡bon01,实际应该是bond1,修改之后，可以看到恢复正常