Search code examples
haproxy

HAProxy Backend Layer7 Invalid Response


I am trying to load balance two server using HAProxy v1.8 but in my case the backends are domain names instead of IP addresses.

My HAProxy config looks like this:

global
    log         /dev/log    local0
    log         /dev/log    local1 notice
    chroot      /var/lib/haproxy
    pidfile     /var/run/rh-haproxy18-haproxy.pid

    user        haproxy
    group       haproxy
    daemon
    stats       socket /run/haproxy/admin.sock mode 660 level admin expose-fd listeners

    spread-checks  21

    # Default SSL material locations
    ca-base     /etc/ssl/certs
    crt-base    /etc/ssl/private

    # Default ciphers to use on SSL-enabled listening sockets.
    # For more information, see ciphers(1SSL). This list is from:
    #  https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
    # An alternative list with additional directives can be obtained from
    #  https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=haproxy
    ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS
    ssl-default-bind-options no-sslv3

defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option                  http-server-close
    option                  redispatch
    retries                 3

    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 10000
    balance                 roundrobin


frontend https-443
        bind *:443
        mode http
        option httplog
        acl ACL_global.domain.com hdr(host) -i global.domain.com
        use_backend www-443-app if ACL_global.domain.com

backend www-443-app
        balance roundrobin
        mode http
        option httpchk GET /health
        option forwardfor
        http-check expect status 200
        server backendnode1 app1.domain.com:443 check
        server backendnode2 app2.domain.com:443 check

frontend health-443
    bind *:8443
    acl backend_dead nbsrv(www-443-app) lt 1
    monitor-uri /haproxy_status
    monitor fail if backend_dead

listen stats # Define a listen section called "stats"
    bind :9000 # Listen on localhost:9000
    mode http
    stats enable  # Enable stats page
    stats hide-version  # Hide HAProxy version
    stats realm Haproxy\ Statistics  # Title text for popup window
    stats uri /haproxy_stats  # Stats URI
    stats auth haproxy:passwd  # Authentication credentials

However, the health check is not passing. When I checked the stat page it says: Layer7 invalid response.

I checked if I can connect to the backend domains from my HAProxy server and I am successfully able to do so.

curl -X GET -I https://app1.domain.com/health
HTTP/2 200 
cache-control: no-cache, private, max-age=0
content-type: application/json
expires: Thu, 01 Jan 1970 00:00:00 UTC
pragma: no-cache
x-accel-expires: 0
x-content-type-options: nosniff
x-xss-protection: 1; mode=block
date: Wed, 28 Jul 2021 12:05:09 GMT
content-length: 18
x-envoy-upstream-service-time: 0
endpoint: health
version: 1.0.0
server: istio-envoy

Is there something that I am missing in my configuration or something that I need to change to make this work?


Solution

  • You're missing ssl keyword for server lines. You may also want to set sni

    backend foo
        default-server ssl check verify none
        server backendnode1 app1.domain.com:443 sni str('app1.domain.com')
        server backendnode2 app2.domain.com:443 sni str('app2.domain.com')
    

    You should also decide if you want to verify SSL certificates of your backend servers. Can you trust the connection? Is it your network? Haproxy encourages you to verify, but requires supplying CA certificate for them to verify. You can also add verifyhost and check-sni settings if you verify certificate:

    backend foo
        default-server ssl check verify required 
        server backendnode1 app1.domain.com:443 sni str('app1.domain.com') check-sni 'app1.domain.com' verifyhost 'app1.domain.com' ca-file /path/to/CA1.pem
        server backendnode2 app2.domain.com:443 sni str('app2.domain.com') check-sni 'app2.domain.com' verifyhost 'app2.domain.com' ca-file /path/to/CA2.pem