We are setting up varnish cache system for WordPress and Ghost blogging platform with nginx as webserver/proxy.
Wordpress Vcl(Default)
vcl 4.0;
import std;
backend default {
.host = "127.0.0.1";
.port = "8080";
.first_byte_timeout = 600s;
}
acl purger {
"localhost";
"127.0.0.1";
"172.17.0.1";
}
sub vcl_recv {
if (req.restarts \> 0) {
set req.hash_always_miss = true;
}
#return (pass);
if (req.method == "PURGE") {
if (client.ip !~ purger) {
return (synth(405, "Method not allowed"));
}
if (req.http.X-Cache-Tags) {
ban("obj.http.X-Cache-Tags ~ " + req.http.X-Cache-Tags);
} else {
ban("req.http.host == " +req.http.host+" && req.url ~ "+req.url);
return (synth(200, "Purged"));
}
return (synth(200, "Purged"));
}
if (req.method != "GET" &&
req.method != "HEAD" &&
req.method != "PUT" &&
req.method != "POST" &&
req.method != "TRACE" &&
req.method != "OPTIONS" &&
req.method != "DELETE") {
/* Non-RFC2616 or CONNECT which is weird. */
return (pipe);
}
# We only deal with GET and HEAD by default
if (req.method != "GET" && req.method != "HEAD") {
return (pass);
}
# Set initial grace period usage status
set req.http.grace = "none";
# normalize url in case of leading HTTP scheme and domain
set req.url = regsub(req.url, "^http[s]?://", "");
# collect all cookies
std.collect(req.http.Cookie);
if (req.url ~ "^/admin/" || req.url ~ "/paypal/") {
return (pass);
}
if (req.http.cookie ~ "wordpress_logged_in_") {
return (pass);
}
if (req.http.Accept-Encoding) {
if (req.url ~ "\.(jpg|jpeg|png|gif|gz|tgz|bz2|tbz|mp3|ogg|swf|flv)$") {
# No point in compressing these
unset req.http.Accept-Encoding;
} elsif (req.http.Accept-Encoding ~ "gzip") {
set req.http.Accept-Encoding = "gzip";
} elsif (req.http.Accept-Encoding ~ "deflate" && req.http.user-agent !~ "MSIE") {
set req.http.Accept-Encoding = "deflate";
} else {
# unknown algorithm
unset req.http.Accept-Encoding;
}
}
if (req.url ~ "(\?|&)(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=") {
set req.url = regsuball(req.url, "(gclid|cx|ie|cof|siteurl|zanpid|origin|fbclid|mc_[a-z]+|utm_[a-z]+|_bta_[a-z]+)=[-_A-z0-9+()%.]+&?", "");
set req.url = regsub(req.url, "[?|&]+$", "");
}
if (req.http.Authorization ~ "^Bearer") {
return (pass);
}
return (hash);
}
sub vcl_hash {
if (req.http.host) {
hash_data(req.http.host);
} else {
hash_data(server.ip);
}
}
sub vcl_backend_response {
set beresp.grace = 3d;
if (beresp.http.content-type ~ "text") {
set beresp.do_esi = true;
}
if (beresp.http.content-type ~ "text") {
set beresp.do_gzip = true;
}
# cache only successfully responses and 404s that are not marked as private
if (beresp.status != 200 && beresp.status != 404 && beresp.http.Cache-Control ~ "private") {
set beresp.uncacheable = true;
set beresp.ttl = 86400s;
return (deliver);
}
# validate if we need to cache it and prevent from setting cookie
if (beresp.ttl > 0s && (bereq.method == "GET" || bereq.method == "HEAD")) {
unset beresp.http.set-cookie;
}
if (!beresp.http.cache-control) {
set beresp.ttl = 0s;
set beresp.uncacheable = true;
}
return (deliver);
}
sub vcl_deliver {
set resp.http.X-Cache-Age = resp.http.Age;
unset resp.http.Age;
# Avoid being cached by the browser.
if (resp.http.Cache-Control !~ "private") {
set resp.http.Pragma = "no-cache";
set resp.http.Expires = "-1";
set resp.http.Cache-Control = "no-store, no-cache, must-revalidate, max-age=0";
}
unset resp.http.X-Powered-By;
unset resp.http.Server;
unset resp.http.X-Varnish;
unset resp.http.Via;
unset resp.http.Link;
unset resp.http.X-Frame-Options;
unset resp.http.X-Content-Type-Options;
unset resp.http.X-Xss-Protection;
unset resp.http.Referer-Policy;
unset resp.http.X-Permitted-cross-domain-policies;
}
sub vcl_hit {
if (obj.ttl \>= 0s) {
return (deliver);
}
set req.http.grace = "unlimited (unhealthy server)";
return (deliver);
}
include "all-vhosts.vcl";
Nginx Vhost for all WordPress sites
server {
listen 80;
listen \[::\]:80;
listen 443 ssl http2;
listen \[::\]:443 ssl http2;
{{ssl_certificate_key}}
{{ssl_certificate}}
server_name www.test.in;
return 301 https://test.in$request_uri;
}
server {
listen 8080;
listen \[::\]:8080;
server_name yourdomain.in www1.yourdomain.in;
{{root}}
try_files $uri $uri/ /index.php?$args;
index index.php index.html;
location \~ .php$ {
include fastcgi_params;
fastcgi_intercept_errors on;
fastcgi_index index.php;
fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
try_files $uri =404;
fastcgi_read_timeout 3600;
fastcgi_send_timeout 3600;
fastcgi_param HTTPS "on";
fastcgi_param SERVER_PORT 443;
fastcgi_pass 127.0.0.1:{{php_fpm_port}};
fastcgi_param PHP_VALUE "{{php_settings}}";
}
if (-f $request_filename) {
break;
}
}
server {
listen 80;
listen \[::\]:80;
listen 443 ssl http2;
listen \[::\]:443 ssl http2;
{{ssl_certificate_key}}
{{ssl_certificate}}
server_name yourdomain.in www1.yourdomain.in;
{{root}}
{{nginx_access_log}}
{{nginx_error_log}}
if ($scheme != "https") {
rewrite ^ https://$host$uri permanent;
}
location \~ /.well-known {
auth_basic off;
allow all;
}
{{settings}}
location \~/.git {
deny all;
}
location \~/(wp-admin/|wp-login.php) {
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_set_header X-Forwarded-Host $http_host;
proxy_set_header Host $host;
proxy_pass http://127.0.0.1:8080;
proxy_max_temp_file_size 0;
proxy_connect_timeout 7200;
proxy_send_timeout 7200;
proxy_read_timeout 7200;
proxy_buffer_size 128k;
proxy_buffers 4 256k;
proxy_busy_buffers_size 256k;
proxy_temp_file_write_size 256k;
}
location / {
{{varnish_proxy_pass}}
proxy_set_header Host $http_host;
proxy_set_header X-Forwarded-Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_hide_header X-Varnish;
proxy_redirect off;
proxy_max_temp_file_size 0;
proxy_connect_timeout 720;
proxy_send_timeout 720;
proxy_read_timeout 720;
proxy_buffer_size 128k;
proxy_buffers 4 256k;
proxy_busy_buffers_size 256k;
proxy_temp_file_write_size 256k;
}
location \~\* ^.+.(css|js|jpg|jpeg|gif|png|ico|gz|svg|svgz|ttf|otf|woff|woff2|eot|mp4|ogg|ogv|webm|webp|zip|swf|map)$ {
add_header Access-Control-Allow-Origin "\*";
expires max;
access_log off;
}
if (-f $request_filename) {
break;
}
}
this works perfect as soon as we add another vcl file for Ghost we get 503 backend fetch error for ghost cms.(wordpress still works fine).Ghost is running on 3021 port. varnish port 6081
Ghost Vcl
vcl 4.0;
import std;
backend ghost {
.host = "127.0.0.1";
.port = "3021";
}
acl purge {
"127.0.0.1";
}
# first vcl_recv block handles the cache purging
sub vcl_recv {
set req.backend_hint = ghost;
return (hash);
if (req.url \~ "/rebuild/purge") {
if (client.ip !\~ purge) {
return (synth(405, "Method Not Allowed"));
}
ban("req.http.host == yourdomain.you");
return(synth(200, "Cache cleared"));
}
}
# second vcl_recv block handles the actual caching
sub vcl_recv {
if (req.url \~ "/assets" || req.url \~ "/content/images") {
return (hash);
} else {
return (pass);
}
}
sub vcl_backend_response {
if (bereq.url \~ "/assets" || bereq.url \~ "/content/images") {
set beresp.http.cache-control = "public, max-age=259200";
set beresp.ttl = 3d;
return (deliver);
}
}
sub vcl_deliver {
# nothing here
}
GHOST Vhost
proxy_cache_path /tmp/nginx_ghost levels=1:2 keys_zone=ghostcache:600m max_size=100m inactive=24h;
server {
listen 80;
listen \[::\]:80;
listen 443 ssl http2;
listen \[::\]:443 ssl http2;
{{ssl_certificate_key}}
{{ssl_certificate}}
server_name $yourdomain;
{{root}}
{{nginx_access_log}}
{{nginx_error_log}}
if ($scheme != "https") {
rewrite ^ https://$host$uri permanent;
}
location \~ /.well-known {
auth_basic off;
allow all;
}
{{settings}}
index index.html;
location / {
proxy_pass http://127.0.0.1:6081/;
proxy_http_version 1.1;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Server $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Host $http_host;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "Upgrade";
\# Remove cookies which are useless for anonymous visitor and prevent caching
proxy_ignore_headers Set-Cookie Cache-Control;
proxy_hide_header Set-Cookie;
\# Add header for cache status (miss or hit)
add_header X-Cache-Status $upstream_cache_status;
proxy_cache ghostcache;
\# Default TTL: 1 day
proxy_cache_valid 1d;
\# Cache 404 pages for 1h
proxy_cache_valid 404 1h;
\# use conditional GET requests to refresh the content from origin servers
proxy_cache_revalidate on;
proxy_buffering on;
\# Allows starting a background subrequest to update an expired cache item,
\# while a stale cached response is returned to the client.
proxy_cache_background_update on;
\# Bypass cache for errors
proxy_cache_use_stale error timeout invalid_header updating http_500 http_502 http_503 http_504;
proxy_pass_request_headers on;
proxy_max_temp_file_size 0;
proxy_connect_timeout 900;
proxy_send_timeout 900;
proxy_read_timeout 900;
proxy_buffer_size 128k;
proxy_buffers 4 256k;
proxy_busy_buffers_size 256k;
proxy_temp_file_write_size 256k;
}
# No cache + keep cookies for admin and previews
location \~ ^/(ghost/|p/) {
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto https;
proxy_pass http://127.0.0.1:{{app_port}};
}
# Bypass ghost for static assets
location ^\~ /assets/ {
root /home/$yourpath/htdocs/$yourpath/content/themes/$yourtheme;
}
# Bypass ghost for original images but not resized ones
location ^\~ /content/images/(!size) {
root /home/$yourpath/htdocs/$yourpath;
}
}
we need help on how to deal with multiple vcl files when it comes multiple applications. we have tried everything, but nothing seems to work.
Varnish Version (varnish-7.1.1)
There are two ways you can keep things organized:
See https://varnish-cache.org/docs/trunk/users-guide/vcl-separate.html for tutorial about VCL labels.
The idea is to load multiple VCL files into the varnishd program and assign a label to each of these VCL files. Then there is a master VCL configuration that has awareness of the various criteria and will load the right sub-configuration.
Here's an example implementation:
vcl 4.1;
import std;
# We have to have a backend, even if we do not use it
backend default { .host = "127.0.0.1"; }
sub vcl_recv {
# Normalize host header
set req.http.host = std.tolower(req.http.host);
if (req.http.host ~ "^(www.)?domain.ext$") {
return (vcl(wordpress));
}
if (req.http.host == "ghost.domain.ext") {
return (vcl(ghost));
}
return (synth(302, "http://www.domain.ext"));
}
sub vcl_synth {
if (resp.status == 301 || resp.status == 302) {
set resp.http.location = resp.reason;
set resp.reason = "Moved";
return (deliver);
}
}
With the following commands you can load the 2 separate VCL files for WordPress and Ghost, while labeling them at the same time:
varnishadm vcl.load vc_wordpress /path/to/wordpress.vcl
varnishadm vcl.load vc_ghost /path/to/ghost.vcl
varnishadm vcl.label wordpress vc_wordpress
varnishadm vcl.label ghost vc_ghost
As long as there's a master VCL configuration that has awareness of the labels, you can host VCL configurations for multiple websites on a single Varnish instance
Another way to do it is through VCL includes, which literally pastes the code from the include files into the placeholders upon startup.
Here's that same example, but refactored with includes instead of labels:
vcl 4.1;
import std;
# We have to have a backend, even if we do not use it
backend default { .host = "127.0.0.1"; }
sub vcl_recv {
# Normalize host header
set req.http.host = std.tolower(req.http.host);
if (req.http.host ~ "^(www.)?domain.ext$") {
include "wordpress.vcl";
}elseif (req.http.host == "ghost.domain.ext") {
include "ghost.vcl";
} else {
return (synth(302, "http://www.domain.ext"));
}
}
sub vcl_synth {
if (resp.status == 301 || resp.status == 302) {
set resp.http.location = resp.reason;
set resp.reason = "Moved";
return (deliver);
}
}
I personally prefer labels, because there's better isolation between the various VCL configurations.
When changes are required, you can alter a single VCL configuration, rather than having to reload the config for all websites.
The downside is that you have to ensure the sub-configurations are there at all times. A restart of varnishd erases these configs.
A way to circumvent this is by specifying the -I runtime parameter in varnishd. -I points to a file that contains CLI commands which will be executed on started. That way you can populate your labeled sub-configurations without the risk of losing them after a restart.
Using includes is a lot simpler, but offers less isolation.