负载均衡 on Colben Notes

Lvs 笔记

Wed, 30 Oct 2019 12:55:50 +0800

环境

角色	地址
负载分配服务器	192.168.1.209
tomcat 服务器1	192.168.1.207
tomcat 服务器2	192.168.1.208
VIP	192.168.1.250

LVS-DR

配置负载分配服务器(192.168.1.209)

#在与 tomcat 服务器连通的网卡(eth0)上配置虚拟 IP(192.168.1.250)
ip addr add 192.168.1.250/32 brd 192.168.1.250 dev eth0
ip route add 192.168.1.250 dev eth0
#开启转发
echo 1 > /proc/sys/net/ipv4/ip_forward
#安装 ipvsadm
yum install ipvsadm
ipvsadm -C
ipvsadm -A -t 192.168.1.250:80 -s rr
ipvsadm -a -t 192.168.1.250:80 -r 192.168.1.207:80 -g
ipvsadm -a -t 192.168.1.250:80 -r 192.168.1.208:80 -g

在两台 tomcat 服务器上都做如下配置

#禁用 arp 响应
echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
#添加虚拟 IP
ip addr add 192.168.1.250/32 brd 192.168.1.250 dev lo
ip route add 192.168.1.250 dev lo
#在个人浏览器中访问 http://192.168.1.250/webapp/

Haproxy 笔记

Wed, 30 Oct 2019 11:40:20 +0800

CentOS7 下安装

CentOS7 自带的 haproxy 版本太低，这里通过 cheese 源安装最新版本

安装 cheese repo，详细参考这里

1
2

wget http://www.nosuchhost.net/~cheese/fedora/packages/epel-7/x86_64/cheese-release-7-1.noarch.rpm
rpm -Uvh cheese-release-7-1.noarch.rpm

安装 haproxy
1

yum install haproxy

修改 sysctl.conf

cat >> /etc/sysctl.conf <<-END
net.ipv4.ip_forward=1
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_fin_timeout = 8
END
sysctl -p

禁用 selinux

1
2

sed -i '/^SELINUX=/cSELINUX=disabled' /etc/selinux/config
setenforce 0

全局配置

global
    log         127.0.0.1 local2 info
    chroot      /var/lib/haproxy # 如果需要外部检查脚本，则需注释该行
    #external-check # 如果需要外部检查脚本，则取消注释
    pidfile     /var/run/haproxy.pid
    maxconn     102400
    user        haproxy
    group       haproxy
    daemon
    stats socket /var/lib/haproxy/stats

默认配置

defaults
    log global
    option dontlognull
    option redispatch
    option abortonclose
    timeout check 8s

tcp 连接多个 ceph-radosgw

frontend  ceph-radosgw
    bind *:7480
    timeout client 8s
    default_backend ceph-radosgw
backend ceph-radosgw
    mode tcp
    balance roundrobin
    timeout connect 8s
    timeout server 8s
    retries 2
    #option external-check
    #external-check command /var/lib/haproxy/health_check.sh
    server cpeh240 10.9.10.234:7480 check
    server ceph241 10.9.10.235:7480 check
    server ceph243 10.9.10.236:7480 check

tcp 连接 mysql galera cluster

frontend mysql
    bind *:3306
    timeout client 1800s
    default_backend mysql
backend mysql
    balance source
    option tcpka
    timeout connect    8s
    timeout server     1800s
    retries 2
    server  mysql231   10.9.10.231:3306 check inter 4s
    server  mysql232   10.9.10.232:3306 check inter 4s
    server  mysql233   10.9.10.233:3306 check inter 4s

tcp 连接 redis 主库

frontend redis
    bind *:6379
    timeout client 1800s
    default_backend redis
backend redis
    balance roundrobin
    timeout connect 8s
    timeout server 1800s
    retries 2
    option tcp-check
    tcp-check connect
    tcp-check send PING\r\n
    tcp-check expect string +PONG
    tcp-check send info\ replication\r\n
    tcp-check expect string role:master
    tcp-check send QUIT\r\n
    tcp-check expect string +OK
    server redis87 10.1.14.87:6379 check inter 4s
    server redis88 10.1.14.88:6379 check inter 4s
    server redis89 10.1.14.89:6379 check inter 4s

状态页面

listen admin_stats
    bind 0.0.0.0:10080
    mode http
    maxconn 100
    timeout client 1m
    timeout connect 4s
    timeout server 4s
    stats refresh 30s
    stats uri /
    stats auth username:password
    stats realm haproxy for private user, enter username/password
    stats hide-version

通过 rsyslog 生成日志

sed -i -e '/ModLoad imudp/s/^#//' \
    -e '/UDPServerRun 514/s/^#//' /etc/rsyslog.conf
cat > /etc/rsyslog.d/haproxy.conf <<EOF
## haproxy log
template(name="HaproxyTime" type="list"){
    property(name="timereported" dateformat="year")
    constant(value="-")
    property(name="timereported" dateformat="month")
    constant(value="-")
    property(name="timereported" dateformat="day")
    constant(value=" ")
    property(name="timereported" dateformat="hour")
    constant(value=":")
    property(name="timereported" dateformat="minute")
    constant(value=":")
    property(name="timereported" dateformat="second")
    constant(value=" ")
    property(name="msg" droplastlf="on")
    constant(value="\n")
}
template(name="DynFile" type="list"){
    constant(value="/var/log/haproxy/haproxy-")
    property(name="timereported" dateformat="year")
    property(name="timereported" dateformat="month")
    property(name="timereported" dateformat="day")
    constant(value=".log")
}
local2.* action(type="omfile" dynaFile="DynFile" template="HaproxyTime")
EOF
systemctl restart rsyslog
systemctl restart haproxy

参考

Keepalived 笔记

Wed, 30 Oct 2019 11:22:03 +0800

keepalive 服务器

环境
- 应用服务器
  - ka101: 10.1.1.101
  - ka102: 10.1.1.102
- 虚拟地址
  - ip: 10.1.1.100
安装 keepalived
1

yum install keepalived

配置虚拟 ip

echo "1" >/proc/sys/net/ipv4/conf/lo/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/lo/arp_announce
echo "1" >/proc/sys/net/ipv4/conf/all/arp_ignore
echo "2" >/proc/sys/net/ipv4/conf/all/arp_announce
ip addr add 10.1.1.100/32 brd 10.1.1.100 dev lo
ip route add 10.1.1.100 dev lo

两个 haproxy 不抢占

MASTER/BACKUP 完整配置

global_defs {
    router_id ka101 #BACKUP 这里是 ka102
    script_user root
    enable_script_security
}
vrrp_script chk_haproxy {
    script "/usr/bin/systemctl status haproxy"
    interval 2
    weight 0
    fall 2
    rise 2
}
vrrp_instance VI_1 {
    state BACKUP #MASTER 和 BACKUP 这里都是 BACKUP
    virtual_router_id 1
    priority 150 #BACKUP 这里是 100
    advert_int 2
    nopreempt #BACKUP 优先级低，需注释此行
    interface eth0
    track_script {
        chk_haproxy
    }
    authentication {
        auth_type PASS
        auth_pass 1011100
    }
    virtual_ipaddress {
        10.1.1.100/24 dev eth0
    }
}

两个 LVS-DR 调度器不抢占均衡后端 MySQL 和 Ceph 负载

环境