Re: 503 errors when services available and other connections succeed to same service during failure window

From: Jerry Champlin <jgc#absolute-performance.com>
Date: Sun, 1 Aug 2010 10:26:22 -0600


Some additional information:

We are using conntrac and iptables for firewalling on this box. Our typical environment has a pair of load balancing firewalls in front of the application. We are trying to switch from ipvs to haproxy. root#lb1:/proc/sys# cat ./net/nf_conntrack_max 65536
root#lb1:/proc/sys# cat ./net/netfilter/nf_conntrack_max 65536
root#lb1:/proc/sys# cat ./net/netfilter/nf_conntrack_count 4468
root#lb1:/proc/sys# cat ./net/ipv4/netfilter/ip_conntrack_count 4389
root#lb1:/proc/sys# cat ./net/ipv4/netfilter/ip_conntrack_buckets 16384 

Configuration File ------------------------------------------

# vim:ft=haproxy

global 

        log 127.0.0.1 local0


#       log 127.0.0.1 local1 notice

        node proxy1


#       ulimit-n 65536 this is auto-tuned

        daemon
        maxconn 32000
        stats socket /tmp/haproxy_stats

defaults
        log global
        mode http
        timeout connect 5000ms
        timeout client 50000ms
        timeout server 1800s
        timeout queue  5000ms
        retries 5
        option httplog
        option forwardfor
        option redispatch
        option httpclose


listen smtp
        mode tcp
        option smtpchk
        bind 76.12.34.30:25
        server pn1 172.17.48.31:25 check inter 15s
        server pn2 172.17.48.32:25 check inter 15s

frontend internal
        maxconn 10000
        bind 172.17.48.1:80
        acl hdr.host:alarm hdr_beg(host) alarm
        acl hdr.host:asset hdr_beg(host) asset
        acl hdr.host:config hdr_beg(host) config
        use_backend alarm if hdr.host:alarm
        use_backend configurator if hdr.host:config
        default_backend internal_core

frontend ssbe-alarm
        maxconn 10000
        bind 76.12.34.26:80
        acl hdr.host:alarm hdr_beg(host) alarm
        default_backend alarm

frontend ssbe-assets
        maxconn 4000
        bind 76.12.34.27:80
        acl hdr.host:asset hdr_beg(host) asset
        default_backend servers

frontend ssbe-config
        maxconn 4000
        bind 76.12.34.28:80
        bind 76.12.34.28:443
        acl hdr.host:config hdr_beg(host) config
        default_backend configurator

frontend ssbe-dash
        maxconn 10000
        bind 76.12.34.29:80
        acl hdr.host:dash hdr_beg(host) dash
        use_backend dash if hdr.host:dash
        default_backend dash

frontend ssbe-core
        maxconn 20000
        bind 76.12.34.30:80
        acl hdr.host:core hdr_beg(host) core
        acl src:internal src 172.17.48.0/24
        use_backend internal_core if hdr.host:core src:internal
        default_backend servers



#1-IP#frontend http-in

#1-IP#  bind *:80

#1-IP## stats enable

#1-IP## stats auth dev:dev

#1-IP#

#1-IP## ACLs to filter incoming requests

#1-IP#  # Host: dash*

#1-IP#  acl hdr.host:dash hdr_beg(host) dash

#1-IP#

#1-IP#  # Host: core*

#1-IP#  acl hdr.host:core hdr_beg(host) core

#1-IP#

#1-IP#  # Host: asset*

#1-IP#  acl hdr.host:asset hdr_beg(host) asset

#1-IP#

#1-IP#  # Host: alarm*

#1-IP#  acl hdr.host:alarm hdr_beg(host) alarm

#1-IP#

#1-IP#  # Internal machines

#1-IP#  acl src:internal src 172.16.101.0/24

#1-IP#

#1-IP#  # Map ACLs to backend sections below

#1-IP#  use_backend dash if hdr.host:dash
#1-IP# use_backend internal_core if hdr.host:core src:internal
#1-IP# default_backend servers

backend servers 

        balance roundrobin
        #option allbackups
        option httpchk GET /service_descriptors/kernel HTTP/1.0\r\nHost:\
core.monitoring.hosting.com
        server pn1 172.17.48.31 check inter 5000 rise 2 fall 2 port 80
        server pn2 172.17.48.32 check inter 5000 rise 2 fall 2 port 80

backend dash
        balance roundrobin
        cookie hapnode insert


#       option httpchk HEAD / HTTP/1.0\r\nHost:\

dashboard.monitoring.hosting.com
        server pn1 172.17.48.31 cookie pn1 check inter 5000 rise 2 fall 2
port 80
        server pn2 172.17.48.32 cookie pn2 check inter 5000 rise 2 fall 2
port 80

backend internal_core 

        balance leastconn
        option httpchk GET /service_descriptors/kernel HTTP/1.0\r\nHost:\
core.monitoring.hosting.com
        server pn1 172.17.48.31 track servers/pn1 inter 5000 rise 2 fall 2
port 80
        server pn2 172.17.48.32 track servers/pn2 inter 5000 rise 2 fall 2
port 80

backend alarm 

        balance leastconn
        option httpchk HEAD / HTTP/1.0\r\nHost:\
alarm.monitoring.hosting.com
        server pn1 172.17.48.31 track servers/pn1 inter 5000 rise 2 fall 2
port 80
        server pn2 172.17.48.32 track servers/pn2 inter 5000 rise 2 fall 2
port 80

backend configurator 

        balance leastconn
        option httpchk GET / HTTP/1.0\r\nHost:\
config.monitoring.hosting.com
        server pn1 172.17.48.31 check inter 5000 rise 2 fall 2 port 80
        server pn2 172.17.48.32 check inter 5000 rise 2 fall 2 port 80

End Configuration File -----------------------------------

uname -a: Linux lb1 2.6.32-23-server #37-Ubuntu SMP Fri Jun 11 09:11:11 UTC 2010 x86_64 GNU/Linux
(I was mistaken earlier - 64bit - not 32)

root#lb1:~# haproxy -vv
HA-Proxy version 1.3.22 2009/10/14
Copyright 2000-2009 Willy Tarreau <w#1wt.eu>

Build options : 

  TARGET  = linux26
  CPU     = generic
  CC      = gcc
  CFLAGS  = -O2 -g
  OPTIONS = USE_PCRE=1

Default settings :
  maxconn = 2000, maxpollevents = 200

Available polling systems : 

     sepoll : pref=400,  test result OK
      epoll : pref=300,  test result OK
       poll : pref=200,  test result OK
     select : pref=150,  test result OK

Total: 4 (4 usable), will use sepoll.

Jerry Champlin
Absolute Performance Inc.
Phone: 303-565-4401 

--
Enabling businesses to deliver critical applications at lower cost and
higher value to their customers.
Received on 2010/08/01 18:26

This archive was generated by hypermail 2.2.0 : 2010/08/01 18:30 CEST