From c0e0ba3562c3dfa23bb7c0a28c41f228e74d1b9b Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Sun, 27 Nov 2022 00:22:26 +0100 Subject: [PATCH] boost-proxy: add some variables/templates examples --- boost-proxy/defaults/examples.yml | 45 +++ boost-proxy/defaults/main.yml | 25 +- .../000-example/haproxy/backend.j2 | 7 + .../000-example/haproxy/frontend_external.j2 | 31 ++ .../000-example/haproxy/frontend_internal.j2 | 5 + .../000-example/haproxy/maintenance.http | 15 + .../000-example/varnish/default.vcl.j2 | 150 +++++++++ .../templates-examples/haproxy/400.http | 15 + .../templates-examples/haproxy/403.http | 15 + .../templates-examples/haproxy/404.http | 15 + .../templates-examples/haproxy/500.http | 15 + .../templates-examples/haproxy/502.http | 15 + .../templates-examples/haproxy/503.http | 15 + .../templates-examples/haproxy/504.http | 15 + .../haproxy/haproxy.default.cfg.j2 | 293 ++++++++++++++++++ .../haproxy/maintenance.http | 15 + .../nginx/evolinux-default.conf.j2 | 59 ++++ .../conf.d/builtin.backend_response.vcl | 12 + .../varnish/conf.d/builtin.complete.vcl | 200 ++++++++++++ .../varnish/conf.d/builtin.recv.vcl | 29 ++ .../conf.d/cleanup_requests_static.recv.vcl | 7 + .../conf.d/default_ttl.backend_response.vcl | 19 ++ .../conf.d/devicedetect.backend_response.vcl | 18 ++ .../varnish/conf.d/devicedetect.deliver.vcl | 5 + .../varnish/conf.d/devicedetect.functions.vcl | 113 +++++++ .../conf.d/prestashop.backend_response.vcl | 48 +++ .../varnish/conf.d/prestashop.recv.vcl | 87 ++++++ .../respect_cache_request_headers.recv.vcl | 4 + .../conf.d/wordpress.backend_response.vcl | 4 + .../varnish/conf.d/wordpress.recv.vcl | 3 + .../varnish/varnish.default.vcl.j2 | 176 +++++++++++ 31 files changed, 1471 insertions(+), 4 deletions(-) create mode 100644 boost-proxy/defaults/examples.yml create mode 100644 boost-proxy/templates-examples/boost-sites/000-example/haproxy/backend.j2 create mode 100644 boost-proxy/templates-examples/boost-sites/000-example/haproxy/frontend_external.j2 create mode 100644 boost-proxy/templates-examples/boost-sites/000-example/haproxy/frontend_internal.j2 create mode 100644 boost-proxy/templates-examples/boost-sites/000-example/haproxy/maintenance.http create mode 100644 boost-proxy/templates-examples/boost-sites/000-example/varnish/default.vcl.j2 create mode 100644 boost-proxy/templates-examples/haproxy/400.http create mode 100644 boost-proxy/templates-examples/haproxy/403.http create mode 100644 boost-proxy/templates-examples/haproxy/404.http create mode 100644 boost-proxy/templates-examples/haproxy/500.http create mode 100644 boost-proxy/templates-examples/haproxy/502.http create mode 100644 boost-proxy/templates-examples/haproxy/503.http create mode 100644 boost-proxy/templates-examples/haproxy/504.http create mode 100644 boost-proxy/templates-examples/haproxy/haproxy.default.cfg.j2 create mode 100644 boost-proxy/templates-examples/haproxy/maintenance.http create mode 100644 boost-proxy/templates-examples/nginx/evolinux-default.conf.j2 create mode 100644 boost-proxy/templates-examples/varnish/conf.d/builtin.backend_response.vcl create mode 100644 boost-proxy/templates-examples/varnish/conf.d/builtin.complete.vcl create mode 100644 boost-proxy/templates-examples/varnish/conf.d/builtin.recv.vcl create mode 100644 boost-proxy/templates-examples/varnish/conf.d/cleanup_requests_static.recv.vcl create mode 100644 boost-proxy/templates-examples/varnish/conf.d/default_ttl.backend_response.vcl create mode 100644 boost-proxy/templates-examples/varnish/conf.d/devicedetect.backend_response.vcl create mode 100644 boost-proxy/templates-examples/varnish/conf.d/devicedetect.deliver.vcl create mode 100644 boost-proxy/templates-examples/varnish/conf.d/devicedetect.functions.vcl create mode 100644 boost-proxy/templates-examples/varnish/conf.d/prestashop.backend_response.vcl create mode 100644 boost-proxy/templates-examples/varnish/conf.d/prestashop.recv.vcl create mode 100644 boost-proxy/templates-examples/varnish/conf.d/respect_cache_request_headers.recv.vcl create mode 100644 boost-proxy/templates-examples/varnish/conf.d/wordpress.backend_response.vcl create mode 100644 boost-proxy/templates-examples/varnish/conf.d/wordpress.recv.vcl create mode 100644 boost-proxy/templates-examples/varnish/varnish.default.vcl.j2 diff --git a/boost-proxy/defaults/examples.yml b/boost-proxy/defaults/examples.yml new file mode 100644 index 00000000..6b3045af --- /dev/null +++ b/boost-proxy/defaults/examples.yml @@ -0,0 +1,45 @@ +### +# This is list of variables that should copy to your inventory when using this role +# Those values are sensible, for a dedicated Boost Proxy server. +# +--- + +# minifirewall_protected_ports_tcp: [] +# minifirewall_protected_ports_udp: [] +# minifirewall_public_ports_tcp: [80, 443] +# minifirewall_public_ports_udp: [] +# minifirewall_semipublic_ports_tcp: [22, "{{ haproxy_stats_port }}"] +# minifirewall_semipublic_ports_udp: [] +# minifirewall_private_ports_tcp: [5666] +# minifirewall_private_ports_udp: [] +# minifirewall_http_sites: ['0.0.0.0/0'] +# minifirewall_https_sites: ['0.0.0.0/0'] + +# haproxy_socket: /run/haproxy/admin.sock +# haproxy_chroot: /var/lib/haproxy + +# haproxy_server_state_file: /var/lib/haproxy/server_state.txt + +# haproxy_stats_access_ips: "{{ trusted_ips | union(['127.0.0.1']) }}" +# haproxy_stats_admin_ips: "{{ trusted_ips }}" +# haproxy_maintenance_ips: [] +# haproxy_deny_ips: [] + +# haproxy_stats_enable: True +# haproxy_stats_port: "8080" + + +# nginx_package_name: "nginx-light" +# nginx_force_default_template: True +# nginx_default_template_regular: "nginx/evolinux-default.conf.j2" + +# varnish_addresses: +# - "{{ haproxy_chroot }}{{ boost_varnish_proxy_socket }},user=vcache,group=varnish,mode=666,PROXY" +# - 127.0.0.1:82 +# varnish_tmp_dir: /var/tmp-vcache + +# boost_sites_enabled_for_all: +# - 000-example + +# Set this to Tur if you have multiple servers that need to communicate with a root SSH access +# boost_allow_root_ssh_between_servers: False \ No newline at end of file diff --git a/boost-proxy/defaults/main.yml b/boost-proxy/defaults/main.yml index 290a8a8f..d0ecefc4 100644 --- a/boost-proxy/defaults/main.yml +++ b/boost-proxy/defaults/main.yml @@ -1,17 +1,34 @@ --- -boost_sysctl_config: [] + boost_sysctl_file_path: /etc/sysctl.d/boost.conf +boost_sysctl_config: + - { key: "net.ipv4.ip_local_port_range", value: "20000 65023" } + - { key: "net.ipv4.tcp_max_syn_backlog", value: "60000" } + - { key: "net.ipv4.tcp_fin_timeout", value: "30" } + - { key: "net.ipv4.ip_nonlocal_bind", value: "1" } + - { key: "net.core.somaxconn", value: "60000" } + - { key: "net.nf_conntrack_max", value: "524288" } + - { key: "net.unix.max_dgram_qlen", value: "100" } + - { key: "net.ipv4.conf.default.rp_filter", value: "0" } + - { key: "net.ipv4.conf.all.rp_filter", value: "0" } + - { key: "net.ipv4.conf.all.arp_ignore", value: "1" } + - { key: "net.ipv4.conf.all.arp_announce", value: "2" } + boost_allow_root_ssh_between_servers: False +other_servers_from_group_ips: [] boost_sites_enabled: [] boost_sites_enabled_for_all: [] boost_sites_enabled_for_group: [] boost_sites_enabled_for_host: [] -other_servers_from_group_ips: [] - boost_validate_haproxy: True boost_validate_varnish: True +boost_nginx_check_url: "/nginxcheck" boost_haproxy_check_url: "/haproxycheck" -boost_varnish_check_url: "/varnishcheck" \ No newline at end of file +boost_varnish_check_url: "/varnishcheck" + +boost_nginx_proxy_port: 81 +boost_haproxy_proxy_socket: /run/haproxy-frontend-default.sock +boost_varnish_proxy_socket: "/run/varnish.sock" \ No newline at end of file diff --git a/boost-proxy/templates-examples/boost-sites/000-example/haproxy/backend.j2 b/boost-proxy/templates-examples/boost-sites/000-example/haproxy/backend.j2 new file mode 100644 index 00000000..f4ad9b32 --- /dev/null +++ b/boost-proxy/templates-examples/boost-sites/000-example/haproxy/backend.j2 @@ -0,0 +1,7 @@ +backend example + errorfile 503 /etc/haproxy/sites/000-example/maintenance.http + server example-hostmane 1.2.3.4:443 check observe layer4 ssl verify none + +backend example_maintenance + http-request set-log-level silent + errorfile 503 /etc/haproxy/sites/000-example/maintenance.http diff --git a/boost-proxy/templates-examples/boost-sites/000-example/haproxy/frontend_external.j2 b/boost-proxy/templates-examples/boost-sites/000-example/haproxy/frontend_external.j2 new file mode 100644 index 00000000..d407faa7 --- /dev/null +++ b/boost-proxy/templates-examples/boost-sites/000-example/haproxy/frontend_external.j2 @@ -0,0 +1,31 @@ + acl example_domains var(req.hdr_host) -m str example.com + acl example_domains2 var(req.hdr_host) -m str example.org www.example.org + ### acl example_maintenance_ips src -f /etc/haproxy/example/maintenance_ips + + # Redirect to HTTPS without Let's Encrypt certificate + ### http-request redirect scheme https code 301 if example_domains !{ ssl_fc } + + # Redirect to HTTPS with Let's Encrypt certificate (exclude LE challenge from redirection) + ### http-request redirect scheme https code 301 if example_domains !{ ssl_fc } !letsencrypt + + # Redirect a domain to another without Let's encrypt certificate + ### http-request redirect prefix https://example-to.org code 301 if { var(req.hdr_host) -m str example-from.org } + + # Redirect a domain to another with a Let's encrypt certificate (exclude LE challenge from redirection) + ### http-request redirect prefix https://example-to.org code 301 if { var(req.hdr_host) -m str example-from.org } !letsencrypt + + # HSTS (31536000 seconds = 1 year) + ### http-response set-header Strict-Transport-Security max-age=31536000 if example_domains + + # custom directives, all conditionned by at least the "example_domains" ACL + ### http-request deny if example_domains { src 31.170.9.129 } + + # routing directives, all conditionned by (at least) the "example_domains" ACL + # Maintenance mode (### -> uncomment BUT define example_maintenance_ips acl before) + ### use_backend example_maintenance if example_domains !example_maintenance_ips !maintenance_ips + # Use Varnish if available + use_backend varnish if example_domains varnish_available varnish_http_verb + + # … or use normal backend + use_backend example if example_domains + diff --git a/boost-proxy/templates-examples/boost-sites/000-example/haproxy/frontend_internal.j2 b/boost-proxy/templates-examples/boost-sites/000-example/haproxy/frontend_internal.j2 new file mode 100644 index 00000000..b53a041a --- /dev/null +++ b/boost-proxy/templates-examples/boost-sites/000-example/haproxy/frontend_internal.j2 @@ -0,0 +1,5 @@ + acl example_domains var(req.hdr_host) -m str example.com + + # routing directives, all conditionned by (at least) the "example_domains" ACL + use_backend example if example_domains + diff --git a/boost-proxy/templates-examples/boost-sites/000-example/haproxy/maintenance.http b/boost-proxy/templates-examples/boost-sites/000-example/haproxy/maintenance.http new file mode 100644 index 00000000..ac4f16d2 --- /dev/null +++ b/boost-proxy/templates-examples/boost-sites/000-example/haproxy/maintenance.http @@ -0,0 +1,15 @@ +HTTP/1.0 503 Service Unavailable +Cache-Control: no-cache +Connection: close +Content-Type: text/html + + + + + Site Maintenance + + + + +

Maintenance en cours

+ diff --git a/boost-proxy/templates-examples/boost-sites/000-example/varnish/default.vcl.j2 b/boost-proxy/templates-examples/boost-sites/000-example/varnish/default.vcl.j2 new file mode 100644 index 00000000..0a217fd9 --- /dev/null +++ b/boost-proxy/templates-examples/boost-sites/000-example/varnish/default.vcl.j2 @@ -0,0 +1,150 @@ +sub vcl_recv { + # Happens before we check if we have this in cache already. + # + # Typically you clean up the request here, removing cookies you don't need, + # rewriting the request, etc. + if (req.http.host == "example.com" || req.http.host == "www.example.com") { + + # Accept PURGE requests from whitelisted IPs + # Uncomment to enable + ### if (req.method == "PURGE") { + ### # Allow from monitoring & hosting08 + ### if (client.ip == "31.170.9.129" || client.ip == "31.170.11.159") { + ### if (req.url == "/_purge_all") { + ### ban("req.http.host == "+req.http.host+" && req.url ~ ."); + ### return (synth(200, "ALL purge cache done")); + ### } + ### ban("req.http.host == "+req.http.host+" && req.url ~ "+req.url); + ### return (synth(200, "purge cache done")); + ### } else { + ### return (synth(403, "permission denied")); + ### } + ### } + + # return (pass) when Cache-Control: no-cache, private etc. from client + include "/etc/varnish/conf.d/respect_cache_request_headers.recv.vcl"; + + # unset cookie and auth headers for static files (jpg, png, pdf...) + include "/etc/varnish/conf.d/cleanup_requests_static.recv.vcl"; + + # Wordpress : return (pass) when WP cookie or "^/wp-(login|admin)" url + ### include "/etc/varnish/conf.d/wordpress.recv.vcl"; + + # Uncomment if your site uses Prestashop + ### include "/etc/varnish/conf.d/prestashop.recv.vcl"; + + # Uncomment to use devide detection + ### call devicedetect; + + # builtin configuration + ### include "/etc/varnish/conf.d/builtin.recv.vcl"; + if (req.method == "PRI") { + /* This will never happen in properly formed traffic (see: RFC7540) */ + return (synth(405)); + } + if (!req.http.host && + req.esi_level == 0 && + req.proto ~ "^(?i)HTTP/1.1") { + /* In HTTP/1.1, Host is required. */ + return (synth(400)); + } + if (req.method != "GET" && + req.method != "HEAD" && + req.method != "PUT" && + req.method != "POST" && + req.method != "TRACE" && + req.method != "OPTIONS" && + req.method != "DELETE" && + req.method != "PATCH") { + /* Non-RFC2616 or CONNECT which is weird. */ + return (pipe); + } + if (req.method != "GET" && req.method != "HEAD") { + /* We only deal with GET and HEAD by default */ + return (pass); + } + if (req.http.Authorization || req.http.Cookie) { + /* Not cacheable by default */ + return (pass); + } + + return (hash); + } +} + +sub vcl_backend_response { + # Happens after we have read the response headers from the backend. + # + # Here you clean the response headers, removing silly Set-Cookie headers + # and other mistakes your backend does.. + if (bereq.http.host == "example.com" || bereq.http.host == "www.example.com") { + + # Low TTL for objects with an error response code. + if (beresp.status == 403 || beresp.status == 404 || beresp.status >= 500) { + set beresp.ttl = 10s; + # mark as "hit_for_pass" for 10s + ### set beresp.uncacheable = false; + return (deliver); + } + + # Default TTL if the backend does not send Expires or max-age/s-max-age headers + if (!beresp.http.expires && beresp.http.cache-control !~ "max-age=") { + set beresp.ttl = 4h; + } + # grace time + ### set beresp.grace = 1d; + + # Exceptions + if (bereq.url ~ "\.(rss|xml|atom)(\?.*|)$") { + set beresp.ttl = 2h; + } + + # Wordpress : no cache when WP cookie or "^/wp-(login|admin)" url + ### include "/etc/varnish/conf.d/wordpress.backend_response.vcl"; + + # Uncomment if your site uses Prestashop + ### include "/etc/varnish/conf.d/prestashop.backend_response.vcl"; + + # Uncomment if you want to do device detection + ### include "/etc/varnish/conf.d/devicedetect.backend_response.vcl"; + + # builtin configuration + ### include "/etc/varnish/conf.d/builtin.backend_response.vcl"; + if (bereq.uncacheable) { + return (deliver); + } else if (beresp.ttl <= 0s || + beresp.http.Set-Cookie || + beresp.http.Surrogate-control ~ "no-store" || + (!beresp.http.Surrogate-Control && + beresp.http.Cache-Control ~ "no-cache|no-store|private") || + beresp.http.Vary == "*") { + # Mark as "Hit-For-Miss" for the next 2 minutes + set beresp.ttl = 120s; + set beresp.uncacheable = true; + } + + return (deliver); + } +} + +sub vcl_deliver { + # Happens when we have all the pieces we need, and are about to send the + # response to the client. + # + # You can do accounting or modifying the final object here. + if (req.http.host == "example.com" || req.http.host == "www.example.com") { + + # Uncomment if you want to do device detection + ### include "/etc/varnish/conf.d/devicedetect.deliver.vcl"; + + # Tell wich config file has been used +{% if 'preprod' in group_names %} + set resp.http.X-Varnish-Config = "{{ site }}"; +{% else %} + # Uncomment to enable + ### set resp.http.X-Varnish-Config = "{{ site }}"; +{% endif %} + + return (deliver); + } +} diff --git a/boost-proxy/templates-examples/haproxy/400.http b/boost-proxy/templates-examples/haproxy/400.http new file mode 100644 index 00000000..c78e77aa --- /dev/null +++ b/boost-proxy/templates-examples/haproxy/400.http @@ -0,0 +1,15 @@ +HTTP/1.0 400 Bad Request +Cache-Control: no-cache, no-store +Connection: close +Content-Type: text/html + + + + + 400 Bad Request + + + + +

400 Bad Request

+ \ No newline at end of file diff --git a/boost-proxy/templates-examples/haproxy/403.http b/boost-proxy/templates-examples/haproxy/403.http new file mode 100644 index 00000000..c6ded402 --- /dev/null +++ b/boost-proxy/templates-examples/haproxy/403.http @@ -0,0 +1,15 @@ +HTTP/1.0 403 Forbidden +Cache-Control: no-cache, no-store +Connection: close +Content-Type: text/html + + + + + 403 Forbidden + + + + +

403 Forbidden

+ \ No newline at end of file diff --git a/boost-proxy/templates-examples/haproxy/404.http b/boost-proxy/templates-examples/haproxy/404.http new file mode 100644 index 00000000..37ca1b5c --- /dev/null +++ b/boost-proxy/templates-examples/haproxy/404.http @@ -0,0 +1,15 @@ +HTTP/1.0 404 Not found +Cache-Control: no-cache, no-store +Connection: close +Content-Type: text/html + + + + + 404 Not found + + + + +

404 Not found

+ \ No newline at end of file diff --git a/boost-proxy/templates-examples/haproxy/500.http b/boost-proxy/templates-examples/haproxy/500.http new file mode 100644 index 00000000..8c186d30 --- /dev/null +++ b/boost-proxy/templates-examples/haproxy/500.http @@ -0,0 +1,15 @@ +HTTP/1.0 500 Internal Server Error +Cache-Control: no-cache, no-store +Connection: close +Content-Type: text/html + + + + + 500 Internal Server Error + + + + +

500 Internal Server Error

+ \ No newline at end of file diff --git a/boost-proxy/templates-examples/haproxy/502.http b/boost-proxy/templates-examples/haproxy/502.http new file mode 100644 index 00000000..e8612906 --- /dev/null +++ b/boost-proxy/templates-examples/haproxy/502.http @@ -0,0 +1,15 @@ +HTTP/1.0 502 Bad Gateway +Cache-Control: no-cache, no-store +Connection: close +Content-Type: text/html + + + + + 502 Bad Gateway + + + + +

502 Bad Gateway

+ \ No newline at end of file diff --git a/boost-proxy/templates-examples/haproxy/503.http b/boost-proxy/templates-examples/haproxy/503.http new file mode 100644 index 00000000..f09aa4da --- /dev/null +++ b/boost-proxy/templates-examples/haproxy/503.http @@ -0,0 +1,15 @@ +HTTP/1.0 503 Service Unavailable +Cache-Control: no-cache, no-store +Connection: close +Content-Type: text/html + + + + + 503 Service Unavailable + + + + +

503 Service Unavailable

+ \ No newline at end of file diff --git a/boost-proxy/templates-examples/haproxy/504.http b/boost-proxy/templates-examples/haproxy/504.http new file mode 100644 index 00000000..925766b1 --- /dev/null +++ b/boost-proxy/templates-examples/haproxy/504.http @@ -0,0 +1,15 @@ +HTTP/1.0 504 Gateway Timeout +Cache-Control: no-cache, no-store +Connection: close +Content-Type: text/html + + + + + 504 Gateway Timeout + + + + +

504 Gateway Timeout

+ \ No newline at end of file diff --git a/boost-proxy/templates-examples/haproxy/haproxy.default.cfg.j2 b/boost-proxy/templates-examples/haproxy/haproxy.default.cfg.j2 new file mode 100644 index 00000000..da9a466a --- /dev/null +++ b/boost-proxy/templates-examples/haproxy/haproxy.default.cfg.j2 @@ -0,0 +1,293 @@ +# {{ ansible_managed }} +# +# Inspired by https://gist.github.com/haproxytechblog/dc5c3b5e2801d36b79e00f07b2309c14 +# +# This is an example HAProxy config for a Boost Proxy. +# You should copy it into your projetct and customize it. +###################################################################################### + +global + log /dev/log local0 + log /dev/log local1 notice + chroot {{ haproxy_chroot }} + stats socket {{ haproxy_socket }} mode 660 level admin + stats timeout 30s + user haproxy + group haproxy + daemon + + node {{ ansible_hostname }} + + server-state-file {{ haproxy_server_state_file }} + + maxconn 60000 + + # Default SSL material locations + ca-base /etc/ssl/certs + crt-base /etc/ssl/private + + # generated 2021-05-03, Mozilla Guideline v5.6, HAProxy 2.2, OpenSSL 1.1.1d, intermediate configuration + # https://ssl-config.mozilla.org/#server=haproxy&version=2.2&config=intermediate&openssl=1.1.1d&guideline=5.6 + ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 + ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 + ssl-default-bind-options prefer-client-ciphers no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets + + ssl-default-server-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 + ssl-default-server-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 + ssl-default-server-options no-sslv3 no-tlsv10 no-tlsv11 no-tls-tickets + + # curl https://ssl-config.mozilla.org/ffdhe2048.txt > /etc/ssl/dhparam-haproxy + ssl-dh-param-file /etc/ssl/dhparam-haproxy + +defaults + log global + + mode http + + option httplog + option dontlognull + option log-separate-errors + option httpclose + option abortonclose + option http-server-close + option redispatch + + timeout connect 5s + + timeout client 300s + timeout server 300s + + timeout queue 60s + timeout http-request 15s + + load-server-state-from-file global + + maxconn 50000 + + # Default options for "server" directive in backends + default-server maxconn 2000 on-error fail-check slowstart 60s inter 5s fastinter 1s downinter 1s weight 100 fall 3 rise 2 + +http-errors boost-default-errors + errorfile 400 /etc/haproxy/errors/400.http + errorfile 403 /etc/haproxy/errors/403.http + errorfile 408 /etc/haproxy/errors/408.http + errorfile 500 /etc/haproxy/errors/500.http + errorfile 502 /etc/haproxy/errors/502.http + errorfile 503 /etc/haproxy/errors/503.http + errorfile 504 /etc/haproxy/errors/504.http + +resolvers local_dns + parse-resolv-conf + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold other 30s + hold refused 30s + hold nx 30s + hold timeout 30s + hold valid 10s + hold obsolete 30s + + +{% if haproxy_stats_enable %} +# BEGIN Stats section +{% if haproxy_stats_users %} +userlist stats_users +{% for user in haproxy_stats_users | default([]) %} + user {{ user.login }} password {{ user.password }} +{% endfor %} + +{% endif %} +listen stats + mode http + bind {{ haproxy_stats_bind_directive }} +{% if haproxy_stats_internal_enable %} + bind {{ haproxy_stats_internal_host }}:{{ haproxy_stats_internal_port }} +{% endif %} + + acl stats_access_ips src -f /etc/haproxy/stats_access_ips + acl stats_admin_ips src -f /etc/haproxy/stats_admin_ips + + stats enable + stats refresh 10s + stats uri {{ haproxy_stats_path }} + stats show-legends + stats show-node + stats admin if stats_admin_ips + +{% if haproxy_stats_users %} + acl stats_users http_auth(stats_users) + stats http-request auth realm "HAProxy admin" if !stats_access_ips !stats_users +{% else %} + stats http-request deny if !stats_access_ips +{% endif %} + + http-request set-log-level silent +# END Stats section +{% endif %} + +# This frontend is the main entry point from "origin" http clients +frontend external + bind 0.0.0.0:80,:::80 + bind 0.0.0.0:443,:::443 ssl strict-sni alpn h2,http/1.1 crt /etc/haproxy/ssl + + option forwardfor + + errorfiles boost-default-errors + + # Reject the request at the TCP level if source is in the denylist + tcp-request connection reject if { src -f /etc/haproxy/deny_ips } + + # Remove a possible x-forwarded-for header already present + http-request del-header x-forwarded-for if { req.hdr(x-forwarded-for) -m found } + + # Store the Host header in lowercase, to speedup ACL later + http-request set-var(req.hdr_host) req.hdr(host),lower + # Capture host header in logs + http-request capture var(req.hdr_host) len 32 + + # Is the request coming for the server itself (stats…) + acl server_hostname var(req.hdr_host) -m str {{ ansible_fqdn }} {{ ansible_hostname }} + acl munin var(req.hdr_host) -m str munin + + # List of IP that will not go the maintenance backend + acl maintenance_ips src -f /etc/haproxy/maintenance_ips + + # Detect Let's Encrypt challenge requests + acl letsencrypt path_dir -i /.well-known/acme-challenge + + # Determine if the request is routable to Varnish + acl varnish_available nbsrv(varnish) gt 0 + acl varnish_detected res.hdr(x-varnish) -m found + acl varnish_http_verb method GET HEAD PURGE + + # Reject the request at the TCP level if source is in the denylist + tcp-request connection reject if { src -f /etc/haproxy/deny_ips } + + http-request set-header x-forwarded-port %[dst_port] + + http-request set-header x-forwarded-proto http if !{ ssl_fc } + http-request set-header x-forwarded-proto https if { ssl_fc } + + # BEGIN marquage HTTP + acl xid_req_exists req.hdr(x-request-id) -m found + http-request set-var(txn.xid) req.hdr(x-request-id) if xid_req_exists + http-request set-var(txn.xid) uuid() unless xid_req_exists + http-request capture var(txn.xid) len 64 + http-request set-header x-request-id %[var(txn.xid)] unless xid_req_exists + + acl xid_res_exists res.hdr(x-request-id) -m found + http-after-response set-header x-request-id %[var(txn.xid)] unless xid_res_exists + + http-request set-header x-boost-step1 "haproxy-external" + + http-response set-header x-boost-step1 "haproxy-external; ssl-frontend" if { ssl_fc } + http-response set-header x-boost-step1 "haproxy-external; no-ssl-frontend" if !{ ssl_fc } + + http-response set-header x-boost-server {{ ansible_hostname }} + + # Full log line added in header to help debug +{% if 'preprod' in group_names %} + http-response set-header x-haproxy-log-external "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r" +{% else %} + # Uncomment to enable + ### http-response set-header x-haproxy-log-external "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r" +{% endif %} + # END marquage HTTP + + # Global maintenance mode + # Uncomment to enable + ### use_backend maintenance unless maintenance_ips + + use_backend local if server_hostname + use_backend local if munin + # "letsencrypt" must stay after "local" + use_backend letsencrypt if letsencrypt + +{% for site in boost_sites_enabled %} +# BEGIN frontend_external section for site '{{ site }}' +{% include "templates/boost-sites/" + site + "/haproxy/frontend_external.j2" %} +{# Ne pas enlever la ligne vide ci-dessous #} + +# END frontend_external section for site '{{ site }}' +{% endfor %} + + default_backend goto_internal + +# This is used as a fallback to go to the internal frontend +# This is not supposed to happen +backend goto_internal + server haproxy-internal {{ boost_haproxy_proxy_socket }} send-proxy-v2 + +# This frontend is only used when returning from varnish +frontend internal + bind {{ boost_haproxy_proxy_socket }} user root mode 666 accept-proxy + + capture request header Host len 32 + + option forwardfor + + # Check URL (used by Varnish) + monitor-uri {{ boost_haproxy_check_url | mandatory }} + + # Store the Host header in lowercase, to speedup ACL later + http-request set-var(req.hdr_host) req.hdr(host),lower + # Capture host header in logs + http-request capture var(req.hdr_host) len 32 + + acl varnish_from req.hdr(x-varnish) -m found + acl forwarded_proto req.hdr(x-forwarded-proto) -m found + + # Keep header if present and coming from Varnish + http-request set-header x-forwarded-proto %[req.hdr(x-forwarded-proto)] if forwarded_proto varnish_from + + # BEGIN marquage HTTP + http-request set-header x-boost-step3 "haproxy-internal" + + http-response set-header x-boost-step3 "haproxy-internal; ssl-backend" if { ssl_bc } + http-response set-header x-boost-step3 "haproxy-internal; no-ssl-backend" if !{ ssl_bc } + + # Full log line added in header to help debug +{% if 'preprod' in group_names %} + http-response set-header x-haproxy-log-internal "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r" +{% else %} + # Uncomment to enable + ### http-response set-header x-haproxy-log-internal "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r" +{% endif %} + # END marquage HTTP + +{% for site in boost_sites_enabled %} +# BEGIN frontend_internal section for site '{{ site }}' +{% include "templates/boost-sites/" + site + "/haproxy/frontend_internal.j2" %} +{# Ne pas enlever la ligne vide ci-dessous #} + +# END frontend_internal section for site '{{ site }}' +{% endfor %} + + +backend varnish + option httpchk HEAD {{ boost_varnish_check_url | mandatory }} + server varnish_sock {{ boost_varnish_proxy_socket }} check observe layer7 maxconn 3000 inter 1s send-proxy-v2 + +{% for site in boost_sites_enabled %} +# BEGIN backend section for site '{{ site }}' +{% include "templates/boost-sites/" + site + "/haproxy/backend.j2" %} +{# Ne pas enlever la ligne vide ci-dessous #} + +# END backend section for site '{{ site }}' +{% endfor %} + +backend letsencrypt +{% if boost_certificates_server is not defined or boost_certificates_server == inventory_hostname %} + server localhost 127.0.0.1:{{ boost_nginx_port }} send-proxy-v2 maxconn 10 +{% else %} + server {{ boost_certificates_server }} {{ hostvars[boost_certificates_server].ansible_host }}:80 maxconn 10 +{% endif %} + +backend local + option httpchk HEAD {{ boost_nginx_check_url | mandatory }} + server localhost 127.0.0.1:81 send-proxy-v2 maxconn 10 + +backend maintenance + http-request set-log-level silent + errorfile 503 /etc/haproxy/errors/maintenance.http diff --git a/boost-proxy/templates-examples/haproxy/maintenance.http b/boost-proxy/templates-examples/haproxy/maintenance.http new file mode 100644 index 00000000..ac4f16d2 --- /dev/null +++ b/boost-proxy/templates-examples/haproxy/maintenance.http @@ -0,0 +1,15 @@ +HTTP/1.0 503 Service Unavailable +Cache-Control: no-cache +Connection: close +Content-Type: text/html + + + + + Site Maintenance + + + + +

Maintenance en cours

+ diff --git a/boost-proxy/templates-examples/nginx/evolinux-default.conf.j2 b/boost-proxy/templates-examples/nginx/evolinux-default.conf.j2 new file mode 100644 index 00000000..6451acb7 --- /dev/null +++ b/boost-proxy/templates-examples/nginx/evolinux-default.conf.j2 @@ -0,0 +1,59 @@ +server { + listen {{ boost_nginx_proxy_port | mandatory }} proxy_protocol default_server; + + set_real_ip_from 127.0.0.1; + real_ip_header proxy_protocol; + + server_name {{ ansible_fqdn }}; + index index.htm index.html; + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + error_page 403 {{ nginx_default_redirect_url }}; + + root /var/www; + + # Auth. + satisfy any; + include /etc/nginx/snippets/ipaddr_whitelist; + deny all; + + auth_basic "Reserved {{ ansible_fqdn }}"; + auth_basic_user_file /etc/nginx/snippets/private_htpasswd; + + location / { + index index.html index.htm; + } + + location /munin/ { + alias /var/cache/munin/www/; + add_header X-Frame-Options "SAMEORIGIN"; + } + + location = {{ boost_nginx_check_url | mandatory }} { + echo_status 200; + echo 'OK'; + } + + location ^~ /munin-cgi/munin-cgi-graph/ { + fastcgi_split_path_info ^(/munin-cgi/munin-cgi-graph)(.*); + fastcgi_param PATH_INFO $fastcgi_path_info; + fastcgi_pass unix:/var/run/munin/spawn-fcgi-munin-graph.sock; + include fastcgi_params; + } + + include /etc/nginx/snippets/letsencrypt.conf; +} + +server { + listen {{ boost_nginx_proxy_port | mandatory }} proxy_protocol; + server_name munin; + + set_real_ip_from 127.0.0.1; + real_ip_header proxy_protocol; + + location /server-status-{{ nginx_serverstatus_suffix | mandatory }} { + stub_status on; + access_log off; + } +} diff --git a/boost-proxy/templates-examples/varnish/conf.d/builtin.backend_response.vcl b/boost-proxy/templates-examples/varnish/conf.d/builtin.backend_response.vcl new file mode 100644 index 00000000..000c3e74 --- /dev/null +++ b/boost-proxy/templates-examples/varnish/conf.d/builtin.backend_response.vcl @@ -0,0 +1,12 @@ +if (bereq.uncacheable) { + return (deliver); +} else if (beresp.ttl <= 0s || + beresp.http.Set-Cookie || + beresp.http.Surrogate-control ~ "no-store" || + (!beresp.http.Surrogate-Control && + beresp.http.Cache-Control ~ "no-cache|no-store|private") || + beresp.http.Vary == "*") { + # Mark as "Hit-For-Miss" for the next 2 minutes + set beresp.ttl = 120s; + set beresp.uncacheable = true; +} diff --git a/boost-proxy/templates-examples/varnish/conf.d/builtin.complete.vcl b/boost-proxy/templates-examples/varnish/conf.d/builtin.complete.vcl new file mode 100644 index 00000000..a578a9c3 --- /dev/null +++ b/boost-proxy/templates-examples/varnish/conf.d/builtin.complete.vcl @@ -0,0 +1,200 @@ +/*- + * Copyright (c) 2006 Verdens Gang AS + * Copyright (c) 2006-2015 Varnish Software AS + * All rights reserved. + * + * Author: Poul-Henning Kamp + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * This is the builtin VCL code + */ + +vcl 4.0; + +####################################################################### +# Client side + +sub vcl_recv { + if (req.method == "PRI") { + /* This will never happen in properly formed traffic (see: RFC7540) */ + return (synth(405)); + } + if (!req.http.host && + req.esi_level == 0 && + req.proto ~ "^(?i)HTTP/1.1") { + /* In HTTP/1.1, Host is required. */ + return (synth(400)); + } + if (req.method != "GET" && + req.method != "HEAD" && + req.method != "PUT" && + req.method != "POST" && + req.method != "TRACE" && + req.method != "OPTIONS" && + req.method != "DELETE" && + req.method != "PATCH") { + /* Non-RFC2616 or CONNECT which is weird. */ + return (pipe); + } + + if (req.method != "GET" && req.method != "HEAD") { + /* We only deal with GET and HEAD by default */ + return (pass); + } + if (req.http.Authorization || req.http.Cookie) { + /* Not cacheable by default */ + return (pass); + } + return (hash); +} + +sub vcl_pipe { + # By default Connection: close is set on all piped requests, to stop + # connection reuse from sending future requests directly to the + # (potentially) wrong backend. If you do want this to happen, you can undo + # it here. + # unset bereq.http.connection; + return (pipe); +} + +sub vcl_pass { + return (fetch); +} + +sub vcl_hash { + hash_data(req.url); + if (req.http.host) { + hash_data(req.http.host); + } else { + hash_data(server.ip); + } + return (lookup); +} + +sub vcl_purge { + return (synth(200, "Purged")); +} + +sub vcl_hit { + if (obj.ttl >= 0s) { + // A pure unadulterated hit, deliver it + return (deliver); + } + if (obj.ttl + obj.grace > 0s) { + // Object is in grace, deliver it + // Automatically triggers a background fetch + return (deliver); + } + // fetch & deliver once we get the result + return (miss); +} + +sub vcl_miss { + return (fetch); +} + +sub vcl_deliver { + return (deliver); +} + +/* + * We can come here "invisibly" with the following errors: 500 & 503 + */ +sub vcl_synth { + set resp.http.Content-Type = "text/html; charset=utf-8"; + set resp.http.Retry-After = "5"; + set resp.body = {" + + + "} + resp.status + " " + resp.reason + {" + + +

Error "} + resp.status + " " + resp.reason + {"

+

"} + resp.reason + {"

+

Guru Meditation:

+

XID: "} + req.xid + {"

+
+

Varnish cache server

+ + +"}; + return (deliver); +} + +####################################################################### +# Backend Fetch + +sub vcl_backend_fetch { + if (bereq.method == "GET") { + unset bereq.body; + } + return (fetch); +} + +sub vcl_backend_response { + if (bereq.uncacheable) { + return (deliver); + } else if (beresp.ttl <= 0s || + beresp.http.Set-Cookie || + beresp.http.Surrogate-control ~ "no-store" || + (!beresp.http.Surrogate-Control && + beresp.http.Cache-Control ~ "no-cache|no-store|private") || + beresp.http.Vary == "*") { + # Mark as "Hit-For-Miss" for the next 2 minutes + set beresp.ttl = 120s; + set beresp.uncacheable = true; + } + return (deliver); +} + +sub vcl_backend_error { + set beresp.http.Content-Type = "text/html; charset=utf-8"; + set beresp.http.Retry-After = "5"; + set beresp.body = {" + + + "} + beresp.status + " " + beresp.reason + {" + + +

Error "} + beresp.status + " " + beresp.reason + {"

+

"} + beresp.reason + {"

+

Guru Meditation:

+

XID: "} + bereq.xid + {"

+
+

Varnish cache server

+ + +"}; + return (deliver); +} + +####################################################################### +# Housekeeping + +sub vcl_init { + return (ok); +} + +sub vcl_fini { + return (ok); +} diff --git a/boost-proxy/templates-examples/varnish/conf.d/builtin.recv.vcl b/boost-proxy/templates-examples/varnish/conf.d/builtin.recv.vcl new file mode 100644 index 00000000..bea4a37c --- /dev/null +++ b/boost-proxy/templates-examples/varnish/conf.d/builtin.recv.vcl @@ -0,0 +1,29 @@ +if (req.method == "PRI") { + /* This will never happen in properly formed traffic (see: RFC7540) */ + return (synth(405)); +} +if (!req.http.host && + req.esi_level == 0 && + req.proto ~ "^(?i)HTTP/1.1") { + /* In HTTP/1.1, Host is required. */ + return (synth(400)); +} +if (req.method != "GET" && + req.method != "HEAD" && + req.method != "PUT" && + req.method != "POST" && + req.method != "TRACE" && + req.method != "OPTIONS" && + req.method != "DELETE" && + req.method != "PATCH") { + /* Non-RFC2616 or CONNECT which is weird. */ + return (pipe); +} +if (req.method != "GET" && req.method != "HEAD") { + /* We only deal with GET and HEAD by default */ + return (pass); +} +if (req.http.Authorization || req.http.Cookie) { + /* Not cacheable by default */ + return (pass); +} diff --git a/boost-proxy/templates-examples/varnish/conf.d/cleanup_requests_static.recv.vcl b/boost-proxy/templates-examples/varnish/conf.d/cleanup_requests_static.recv.vcl new file mode 100644 index 00000000..f63d0001 --- /dev/null +++ b/boost-proxy/templates-examples/varnish/conf.d/cleanup_requests_static.recv.vcl @@ -0,0 +1,7 @@ +# Cleanup requests on static binary files and force serving from cache. +if (req.url ~ "\.(jpe?g|png|gif|ico|swf|gz|zip|rar|bz2|tgz|tbz|pdf|pls|torrent|mp4)(\?.*|)$") { + unset req.http.Authenticate; + unset req.http.POSTDATA; + unset req.http.cookie; + ### set req.method = "GET"; +} diff --git a/boost-proxy/templates-examples/varnish/conf.d/default_ttl.backend_response.vcl b/boost-proxy/templates-examples/varnish/conf.d/default_ttl.backend_response.vcl new file mode 100644 index 00000000..370da9d0 --- /dev/null +++ b/boost-proxy/templates-examples/varnish/conf.d/default_ttl.backend_response.vcl @@ -0,0 +1,19 @@ +# Low TTL for objects with an error response code. +if (beresp.status == 403 || beresp.status == 404 || beresp.status >= 500) { + set beresp.ttl = 10s; + # mark as "hit_for_pass" for 10s + ### set beresp.uncacheable = false; + return (deliver); +} + +set beresp.http.foo-bar "BAZ" + +# Default TTL if the backend does not send any header. +if (!beresp.http.Cache-Control) { + set beresp.ttl = 1d; +} + +# Exceptions +if (bereq.url ~ "\.(rss|xml|atom)(\?.*|)$") { + set beresp.ttl = 2h; +} diff --git a/boost-proxy/templates-examples/varnish/conf.d/devicedetect.backend_response.vcl b/boost-proxy/templates-examples/varnish/conf.d/devicedetect.backend_response.vcl new file mode 100644 index 00000000..f93366a9 --- /dev/null +++ b/boost-proxy/templates-examples/varnish/conf.d/devicedetect.backend_response.vcl @@ -0,0 +1,18 @@ + +# so, this is a bit counterintuitive. The backend creates content based on +# the normalized User-Agent, but we use Vary on X-UA-Device so Varnish will +# use the same cached object for all U-As that map to the same X-UA-Device. +# +# If the backend does not mention in Vary that it has crafted special +# content based on the User-Agent (==X-UA-Device), add it. +# If your backend does set Vary: User-Agent, you may have to remove that here. +if (bereq.http.X-UA-Device) { + if (!beresp.http.Vary) { # no Vary at all + set beresp.http.Vary = "X-UA-Device"; + } elseif (beresp.http.Vary !~ "X-UA-Device") { # add to existing Vary + set beresp.http.Vary = beresp.http.Vary + ", X-UA-Device"; + } +} +# comment this out if you don't want the client to know your +# classification +set beresp.http.X-UA-Device = bereq.http.X-UA-Device; diff --git a/boost-proxy/templates-examples/varnish/conf.d/devicedetect.deliver.vcl b/boost-proxy/templates-examples/varnish/conf.d/devicedetect.deliver.vcl new file mode 100644 index 00000000..79751c3d --- /dev/null +++ b/boost-proxy/templates-examples/varnish/conf.d/devicedetect.deliver.vcl @@ -0,0 +1,5 @@ +# to keep any caches in the wild from serving wrong content to client #2 +# behind them, we need to transform the Vary on the way out. +if ((req.http.X-UA-Device) && (resp.http.Vary)) { + set resp.http.Vary = regsub(resp.http.Vary, "X-UA-Device", "User-Agent"); +} diff --git a/boost-proxy/templates-examples/varnish/conf.d/devicedetect.functions.vcl b/boost-proxy/templates-examples/varnish/conf.d/devicedetect.functions.vcl new file mode 100644 index 00000000..18ace87e --- /dev/null +++ b/boost-proxy/templates-examples/varnish/conf.d/devicedetect.functions.vcl @@ -0,0 +1,113 @@ +# +# Copyright (c) 2016-2018 Varnish Cache project +# Copyright (c) 2012-2016 Varnish Software AS +# +# SPDX-License-Identifier: BSD-2-Clause +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +# detectdevice.vcl - regex based device detection for Varnish +# https://github.com/varnishcache/varnish-devicedetect/ +# +# Original author: Lasse Karstensen + +sub devicedetect { + unset req.http.X-UA-Device; + set req.http.X-UA-Device = "pc"; + + # Handle that a cookie may override the detection alltogether. + if (req.http.Cookie ~ "(?i)X-UA-Device-force") { + /* ;?? means zero or one ;, non-greedy to match the first. */ + set req.http.X-UA-Device = regsub(req.http.Cookie, "(?i).*X-UA-Device-force=([^;]+);??.*", "\1"); + /* Clean up our mess in the cookie header */ + set req.http.Cookie = regsuball(req.http.Cookie, "(^|; ) *X-UA-Device-force=[^;]+;? *", "\1"); + /* If the cookie header is now empty, or just whitespace, unset it. */ + if (req.http.Cookie ~ "^ *$") { unset req.http.Cookie; } + } else { + if (req.http.User-Agent ~ "\(compatible; Googlebot-Mobile/2.1; \+http://www.google.com/bot.html\)" || + (req.http.User-Agent ~ "(Android|iPhone)" && req.http.User-Agent ~ "\(compatible.?; Googlebot/2.1.?; \+http://www.google.com/bot.html") || + (req.http.User-Agent ~ "(iPhone|Windows Phone)" && req.http.User-Agent ~ "\(compatible; bingbot/2.0; \+http://www.bing.com/bingbot.htm")) { + set req.http.X-UA-Device = "mobile-bot"; } + elsif (req.http.User-Agent ~ "(?i)(ads|google|bing|msn|yandex|baidu|ro|career|seznam|)bot" || + req.http.User-Agent ~ "(?i)(baidu|jike|symantec)spider" || + req.http.User-Agent ~ "(?i)pingdom" || + req.http.User-Agent ~ "(?i)facebookexternalhit" || + req.http.User-Agent ~ "(?i)scanner" || + req.http.User-Agent ~ "(?i)slurp" || + req.http.User-Agent ~ "(?i)(web)crawler") { + set req.http.X-UA-Device = "bot"; } + elsif (req.http.User-Agent ~ "(?i)ipad") { set req.http.X-UA-Device = "tablet-ipad"; } + elsif (req.http.User-Agent ~ "(?i)ip(hone|od)") { set req.http.X-UA-Device = "mobile-iphone"; } + /* how do we differ between an android phone and an android tablet? + http://stackoverflow.com/questions/5341637/how-do-detect-android-tablets-in-general-useragent */ + elsif (req.http.User-Agent ~ "(?i)android.*(mobile|mini)") { set req.http.X-UA-Device = "mobile-android"; } + // android 3/honeycomb was just about tablet-only, and any phones will probably handle a bigger page layout. + elsif (req.http.User-Agent ~ "(?i)android 3") { set req.http.X-UA-Device = "tablet-android"; } + /* Opera Mobile */ + elsif (req.http.User-Agent ~ "Opera Mobi") { set req.http.X-UA-Device = "mobile-smartphone"; } + // May very well give false positives towards android tablets. Suggestions welcome. + elsif (req.http.User-Agent ~ "(?i)android") { set req.http.X-UA-Device = "tablet-android"; } + elsif (req.http.User-Agent ~ "PlayBook; U; RIM Tablet") { set req.http.X-UA-Device = "tablet-rim"; } + elsif (req.http.User-Agent ~ "hp-tablet.*TouchPad") { set req.http.X-UA-Device = "tablet-hp"; } + elsif (req.http.User-Agent ~ "Kindle/3") { set req.http.X-UA-Device = "tablet-kindle"; } + elsif (req.http.User-Agent ~ "Touch.+Tablet PC" || + req.http.User-Agent ~ "Windows NT [0-9.]+; ARM;" ) { + set req.http.X-UA-Device = "tablet-microsoft"; + } + elsif (req.http.User-Agent ~ "Mobile.+Firefox") { set req.http.X-UA-Device = "mobile-firefoxos"; } + elsif (req.http.User-Agent ~ "^HTC" || + req.http.User-Agent ~ "Fennec" || + req.http.User-Agent ~ "IEMobile" || + req.http.User-Agent ~ "BlackBerry" || + req.http.User-Agent ~ "BB10.*Mobile" || + req.http.User-Agent ~ "GT-.*Build/GINGERBREAD" || + req.http.User-Agent ~ "SymbianOS.*AppleWebKit") { + set req.http.X-UA-Device = "mobile-smartphone"; + } + elsif (req.http.User-Agent ~ "(?i)symbian" || + req.http.User-Agent ~ "(?i)^sonyericsson" || + req.http.User-Agent ~ "(?i)^nokia" || + req.http.User-Agent ~ "(?i)^samsung" || + req.http.User-Agent ~ "(?i)^lg" || + req.http.User-Agent ~ "(?i)bada" || + req.http.User-Agent ~ "(?i)blazer" || + req.http.User-Agent ~ "(?i)cellphone" || + req.http.User-Agent ~ "(?i)iemobile" || + req.http.User-Agent ~ "(?i)midp-2.0" || + req.http.User-Agent ~ "(?i)u990" || + req.http.User-Agent ~ "(?i)netfront" || + req.http.User-Agent ~ "(?i)opera mini" || + req.http.User-Agent ~ "(?i)palm" || + req.http.User-Agent ~ "(?i)nintendo wii" || + req.http.User-Agent ~ "(?i)playstation portable" || + req.http.User-Agent ~ "(?i)portalmmm" || + req.http.User-Agent ~ "(?i)proxinet" || + req.http.User-Agent ~ "(?i)windows\ ?ce" || + req.http.User-Agent ~ "(?i)winwap" || + req.http.User-Agent ~ "(?i)eudoraweb" || + req.http.User-Agent ~ "(?i)htc" || + req.http.User-Agent ~ "(?i)240x320" || + req.http.User-Agent ~ "(?i)avantgo") { + set req.http.X-UA-Device = "mobile-generic"; + } + } +} diff --git a/boost-proxy/templates-examples/varnish/conf.d/prestashop.backend_response.vcl b/boost-proxy/templates-examples/varnish/conf.d/prestashop.backend_response.vcl new file mode 100644 index 00000000..f86f87ef --- /dev/null +++ b/boost-proxy/templates-examples/varnish/conf.d/prestashop.backend_response.vcl @@ -0,0 +1,48 @@ +## From https://github.com/CleverCloud/varnish-examples/blob/master/prestashop.vcl + +# Remove some headers we never want to see +unset beresp.http.Server; +unset beresp.http.X-Powered-By; +# For static content strip all backend cookies +if (bereq.url ~ "\.(css|js|png|gif|jp(e?)g)|swf|ico|woff") { + unset beresp.http.cookie; +} +# Don't store backend +if (bereq.url ~ "admin70" || bereq.url ~ "preview=true") { + set beresp.uncacheable = true; + set beresp.ttl = 30s; + + return (deliver); +} +if (bereq.method == "GET" && (bereq.url ~ "^/?mylogout=")) { + set beresp.ttl = 0s; + unset beresp.http.Set-Cookie; + set beresp.uncacheable = true; + + return (deliver); +} +# don't cache response to posted requests or those with basic auth +if ( bereq.method == "POST" || bereq.http.Authorization ) { + set beresp.uncacheable = true; + set beresp.ttl = 120s; + + return (deliver); +} + # don't cache search results +if ( bereq.url ~ "\?s=" ){ + set beresp.uncacheable = true; + set beresp.ttl = 120s; + + return (deliver); +} +# only cache status ok +if ( beresp.status != 200 ) { + set beresp.uncacheable = true; + set beresp.ttl = 120s; + + return (deliver); +} +# A TTL of 2h +set beresp.ttl = 2h; +# Define the default grace period to serve cached content +set beresp.grace = 30s; \ No newline at end of file diff --git a/boost-proxy/templates-examples/varnish/conf.d/prestashop.recv.vcl b/boost-proxy/templates-examples/varnish/conf.d/prestashop.recv.vcl new file mode 100644 index 00000000..a869f391 --- /dev/null +++ b/boost-proxy/templates-examples/varnish/conf.d/prestashop.recv.vcl @@ -0,0 +1,87 @@ +## From https://github.com/CleverCloud/varnish-examples/blob/master/prestashop.vcl + +# Normalize the header, remove the port (in case you're testing this on various TCP ports) +set req.http.Host = regsub(req.http.Host, ":[0-9]+", ""); +# Remove has_js and CloudFlare/Google Analytics __* cookies. +set req.http.Cookie = regsuball(req.http.Cookie, "(^|;\s*)(_[_a-z]+|has_js)=[^;]*", ""); +# Remove a ";" prefix, if present. +set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", ""); +# Allow purging from ACL +if (req.method == "PURGE") { + # If not allowed then a error 405 is returned + if (client.ip != "127.0.0.1" ) { + return (synth(405, "This IP is not allowed to send PURGE requests.")); + } + # If allowed, do a cache_lookup -> vlc_hit() or vlc_miss() + return (purge); +} +# Post requests will not be cached +if (req.http.Authorization || req.method == "POST") { + return (pass); +} +if (req.method == "GET" && (req.url ~ "^/?mylogout=")) { + unset req.http.Cookie; + return (pass); +} +#we should not cache any page for Prestashop backend +if (req.method == "GET" && (req.url ~ "^/admin70")) { + return (pass); +} +#we should not cache any page for customers +if (req.method == "GET" && (req.url ~ "^/authentification" || req.url ~ "^/my-account")) { + return (pass); +} +#we should not cache any page for customers +if (req.method == "GET" && (req.url ~ "^/identity" || req.url ~ "^/my-account.php")) { + return (pass); +} +#we should not cache any page for sales +if (req.method == "GET" && (req.url ~ "^/cart.php" || req.url ~ "^/order.php")) { + return (pass); +} +#we should not cache any page for sales +if (req.method == "GET" && (req.url ~ "^/addresses.php" || req.url ~ "^/order-detail.php")) { + return (pass); +} +#we should not cache any page for sales +if (req.method == "GET" && (req.url ~ "^/order-confirmation.php" || req.url ~ "^/order-return.php")) { + return (pass); +} +if (req.method != "GET" && req.method != "HEAD") { + return (pass); +} +# Remove the "has_js" cookie +set req.http.Cookie = regsuball(req.http.Cookie, "has_js=[^;]+(; )?", ""); +# Remove any Google Analytics based cookies +# set req.http.Cookie = regsuball(req.http.Cookie, "__utm.=[^;]+(; )?", ""); +# removes all cookies named __utm? (utma, utmb...) - tracking thing +set req.http.Cookie = regsuball(req.http.Cookie, "(^|(?<=; )) *__utm.=[^;]+;? *", "\1"); +# Remove a ";" prefix, if present. +set req.http.Cookie = regsub(req.http.Cookie, "^;\s*", ""); +# Are there cookies left with only spaces or that are empty? +if (req.http.Cookie ~ "^ *$") { + unset req.http.Cookie; +} +# Cache the following files extensions +if (req.url ~ "\.(css|js|png|gif|jp(e)?g|swf|ico|woff)") { + unset req.http.Cookie; +} +# Normalize Accept-Encoding header and compression +# https://www.varnish-cache.org/docs/3.0/tutorial/vary.html +if (req.http.Accept-Encoding) { + # Do no compress compressed files... + if (req.url ~ "\.(jpg|png|gif|gz|tgz|bz2|tbz|mp3|ogg)$") { + unset req.http.Accept-Encoding; + } elsif (req.http.Accept-Encoding ~ "gzip") { + set req.http.Accept-Encoding = "gzip"; + } elsif (req.http.Accept-Encoding ~ "deflate") { + set req.http.Accept-Encoding = "deflate"; + } else { + unset req.http.Accept-Encoding; + } +} +# Did not cache HTTP authentication and HTTP Cookie +if (req.http.Authorization) { + # Not cacheable by default + return (pass); +} diff --git a/boost-proxy/templates-examples/varnish/conf.d/respect_cache_request_headers.recv.vcl b/boost-proxy/templates-examples/varnish/conf.d/respect_cache_request_headers.recv.vcl new file mode 100644 index 00000000..8213a318 --- /dev/null +++ b/boost-proxy/templates-examples/varnish/conf.d/respect_cache_request_headers.recv.vcl @@ -0,0 +1,4 @@ +# respect Cache-Control from client +if (req.http.Cache-Control ~ "(private|no-cache|no-store)" || req.http.Pragma == "no-cache") { + return (pass); +} diff --git a/boost-proxy/templates-examples/varnish/conf.d/wordpress.backend_response.vcl b/boost-proxy/templates-examples/varnish/conf.d/wordpress.backend_response.vcl new file mode 100644 index 00000000..b226981d --- /dev/null +++ b/boost-proxy/templates-examples/varnish/conf.d/wordpress.backend_response.vcl @@ -0,0 +1,4 @@ +if (bereq.url ~ "wp-(login|admin)" || bereq.url ~ "preview=true" || bereq.http.Cookie ~ "wordpress_logged_in_" ) { + set beresp.uncacheable = true; + set beresp.ttl = 0s; +} diff --git a/boost-proxy/templates-examples/varnish/conf.d/wordpress.recv.vcl b/boost-proxy/templates-examples/varnish/conf.d/wordpress.recv.vcl new file mode 100644 index 00000000..0dfd50da --- /dev/null +++ b/boost-proxy/templates-examples/varnish/conf.d/wordpress.recv.vcl @@ -0,0 +1,3 @@ +if (req.url ~ "^/wp-(login|admin)" || req.url ~ "preview=true" || req.http.Cookie ~ "wordpress_logged_in_" ) { + return (pass); +} diff --git a/boost-proxy/templates-examples/varnish/varnish.default.vcl.j2 b/boost-proxy/templates-examples/varnish/varnish.default.vcl.j2 new file mode 100644 index 00000000..fe5c8152 --- /dev/null +++ b/boost-proxy/templates-examples/varnish/varnish.default.vcl.j2 @@ -0,0 +1,176 @@ +vcl 4.1; + +probe haproxycheck { + .request = + "HEAD {{ boost_haproxy_check_url | mandatory }} HTTP/1.1" + "Connection: close"; + .timeout = 1s; + .interval = 3s; + .window = 3; + .threshold = 2; +} + +backend default { + .path = "{{ boost_haproxy_proxy_socket }}"; + .proxy_header = 1; + .connect_timeout = 3s; + .first_byte_timeout = 300s; + .between_bytes_timeout = 300s; + .probe = haproxycheck; +} + +# Uncomment if you want to do device detection +# cf. https://varnish-cache.org/docs/6.0/users-guide/devicedetection.html +### include "/etc/varnish/conf.d/devicedetect.functions.vcl"; + +# Routines appliquées dans tous les cas +# A modifier avec précaution +# Ne pas mettre de "return" dans ces routines communes + +sub vcl_recv { + # Health check from HAProxy + if (req.url == "{{ boost_varnish_check_url | mandatory }}") { + return (synth(200, "Hi HAProxy, I'm fine!")); + } + + # Normalize encoding, and unset it on yet-compressed formats. + if (req.http.Accept-Encoding) { + if (req.url ~ "\.(jpg|jpeg|png|gif|gz|tgz|bz2|lzma|tbz|zip|rar)(\?.*|)$") { + unset req.http.Accept-Encoding; + } + # use gzip when possible, otherwise use deflate + if (req.http.Accept-Encoding ~ "gzip") { + set req.http.Accept-Encoding = "gzip"; + } + elsif (req.http.Accept-Encoding ~ "deflate") { + set req.http.Accept-Encoding = "deflate"; + } + else { + # unknown algorithm, unset accept-encoding header + unset req.http.Accept-Encoding; + } + } + + # Remove known cookies used only on client side (by JavaScript). + if (req.http.cookie) { + set req.http.Cookie = regsuball(req.http.Cookie, "_gat=[^;]+(; )?", ""); # Google Analytics + set req.http.Cookie = regsuball(req.http.Cookie, "_ga=[^;]+(; )?", ""); # Google Analytics + set req.http.Cookie = regsuball(req.http.Cookie, "_gaq=[^;]+(; )?", ""); # Google Analytics + set req.http.Cookie = regsuball(req.http.Cookie, "__utm[^=]*=[^;]+(; )?", ""); # Google Analytics + set req.http.Cookie = regsuball(req.http.Cookie, "__gads=[^;]+(; )?", ""); # Google Doubleclick + set req.http.Cookie = regsuball(req.http.Cookie, "__auc=[^;]+(; )?", ""); # Alexa Analytics + + if (req.http.cookie ~ "^ *$") { + unset req.http.cookie; + } + } + + # BEGIN marquage HTTP + # TODO: ajouter un en-tête pour marquer le passage dans les 3 composants + # avec une artie permanente et une partie optionnelle commentée par défaut + set req.http.X-Boost-Layer = "varnish"; + # END marquage HTTP +} + +sub vcl_backend_response { + if (beresp.uncacheable) { + set beresp.http.X-Cacheable = "FALSE"; + } else { + set beresp.http.X-Cacheable = "TRUE"; + } + + # our default TTL is 60s instead of 86400s + if (beresp.http.cache-control !~ "max-age=") { + set beresp.ttl = 60s; + } + + # Grace mode (Stale content delivery) + # abandon when 5xx errors to keep grace mode even 503 from HAProxy + if (beresp.status >= 500 && bereq.is_bgfetch) { + return (abandon); + } + set beresp.grace = 4h; +} + +sub vcl_deliver { + + unset resp.http.Via; + + # BEGIN marquage HTTP + # TODO: ajouter un en-tête pour marquer le passage dans les 3 composants + # avec une partie permanente et une partie optionnelle commentée par défaut + if (resp.http.Set-Cookie && resp.http.Cache-Control) { + set resp.http.X-Boost-Step2 = "varnish; set-cookie; cache-control"; + } elseif (resp.http.Set-Cookie) { + set resp.http.X-Boost-Step2 = "varnish; set-cookie; no-cache-control"; + } elseif (resp.http.Cache-Control) { + set resp.http.X-Boost-Step2 = "varnish; no-set-cookie; cache-control"; + } else { + set resp.http.X-Boost-Step2 = "varnish; no-set-cookie; no-cache-control"; + } + # END marquage HTTP + + if (resp.http.X-Varnish ~ "[0-9]+ +[0-9]+") { + set resp.http.X-Cache = "HIT"; + unset resp.http.X-Boost-Step3; + } else { + set resp.http.X-Cache = "MISS"; + } + + # DEBUG infos + ## désactivation pour test + ## https://github.com/varnishcache/varnish-cache/issues/3765 + # set resp.http.X-Varnish-Client-Ip = client.ip; + set resp.http.X-Varnish-Client-Method = req.method; + set resp.http.X-Varnish-Client-Url = req.url; + set resp.http.X-Varnish-Client-Proto = req.proto; + set resp.http.X-Varnish-Object-Ttl = obj.ttl; +{% if 'preprod' in group_names %} + set resp.http.X-Varnish-Client-Cache-Control = req.http.cache-control; + set resp.http.X-Varnish-Client-Cookie = req.http.cookie; + set resp.http.X-Varnish-Client-Ua = req.http.user-agent; + set resp.http.X-Varnish-Object-Grace = obj.grace; + set resp.http.X-Varnish-Object-Keep = obj.keep; + set resp.http.X-Varnish-Object-Storage = obj.storage; +{% else %} + ### set resp.http.X-Varnish-Client-Cache-Control = req.http.cache-control; + ### set resp.http.X-Varnish-Client-Cookie = req.http.cookie; + ### set resp.http.X-Varnish-Client-Ua = req.http.user-agent; + ### set resp.http.X-Varnish-Object-Grace = obj.grace; + ### set resp.http.X-Varnish-Object-Keep = obj.keep; + ### set resp.http.X-Varnish-Object-Storage = obj.storage; +{% endif %} +} + +# BEGIN sites +{% for site_name in boost_sites_enabled %} +include "/etc/varnish/sites/{{ site_name }}.vcl"; +{% endfor %} +# END sites + +# Routines personnalisées, appliquées en "fallback" +# TODO: mieux expliquer ! + +sub vcl_backend_error { + set beresp.http.Content-Type = "text/html; charset=utf-8"; + set beresp.http.Retry-After = "5"; + set beresp.body = {" + + + "} + beresp.status + " " + beresp.reason + {" + + +

Error "} + beresp.status + " " + beresp.reason + {"

+

"} + beresp.reason + {"

+

EvoGuru Meditation:

+

XID: "} + bereq.xid + {"

+
+

Varnish cache server

+ + +"}; + return (deliver); +} + +# Les routines internes de Varnish sont visibles avec la commande +# varnishd -x builtin