diff --git a/CHANGELOG.md b/CHANGELOG.md index 62590672..3f413ee4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,10 +33,11 @@ The **patch** part changes is incremented if multiple releases happen the same m * rbenv: install Ruby 3.1.0 by default * evolinux-base: backup-server-state: add "force" mode + ### Fixed * evolinux-base: backup-server-state: fix systemctl invocation - +* varnish: update munin plugin to work with recent varnish versions ## [22.01.2] 2022-01-27 diff --git a/varnish/files/munin/varnish4_ b/varnish/files/munin/varnish4_ deleted file mode 100644 index 60dc8212..00000000 --- a/varnish/files/munin/varnish4_ +++ /dev/null @@ -1,1165 +0,0 @@ -#!/usr/bin/perl -# -*- perl -*- -# -# varnish4_ - Munin plugin to for Varnish 4.x -# Copyright (C) 2009 Redpill Linpro AS -# -# Author: Kristian Lyngstol -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -=head1 NAME - -varnish4_ - Munin plugin to monitor various aspects of varnish - -=head1 APPLICABLE SYSTEMS - -Varnish 4.x with varnishstat - -=head1 CONFIGURATION - -The plugin needs to be able to execute varnishstat. - -The configuration section shows the defaults - [varnish4_*] - env.varnishstat varnishstat - env.name - -env.varnishstat can be a full path to varnishstat if it's -not in the path already. - -env.name is blank (undefined) by default and can be used to specify a -n -name argument to varnish if multiple instances are running on the same -server. - -A few aspects are not linked by default. They are marked as -'DEBUG' => 'yes' (or any other value). They are: - -vcl, bans, bans_lurker, lru, objects_per_objhead, -losthdr, esi, hcb, shm, shm_writes, overflow, -session, session_herd, gzip - -You can link them yourself with something like this: - - ln -s @@LIBDIR@@/plugins/varnish4_ \ - @@CONFDIR@@/plugins/varnish4_data_structures - -=head1 INTERPRETATION - -Each graph uses data from varnishstat. - -=head1 MAGIC MARKERS - - #%# family=auto - #%# capabilities=autoconf suggest - -=head1 VERSION - - $Id$ - -=head1 BUGS - -The hit_rate graph requires munin r2040 or newer to display -correctly. - -=head1 PATCHES-TO - -Please send patches to Kristian Lyngstol -and/or varnish-misc@varnish-cache.org for significant changes. Munin SVN -is the authoritative repository for this plugin. - -=head1 AUTHOR - -Kristian Lyngstol - -=head1 MODIFICATIONS - -Ingo Oppermann - -=head1 LICENSE - -GPLv2 - -=cut - - -use XML::Parser; -use strict; - -# Set to 1 to enable output when a variable is defined in a graph but -# omitted because it doesn't exist in varnishstat. -my $DEBUG = 0; - -# Set to 1 to ignore 'DEBUG' and suggest all available aspects. -my $FULL_SUGGEST = 0; - -# Varnishstat executable. Include full path if it's not in your path. -my $varnishstatexec = exists $ENV{'varnishstat'} ? $ENV{'varnishstat'} : "varnishstat"; - -# For multiple instances -my $varnishname = exists $ENV{'name'} ? $ENV{'name'} : undef; - -my $self; # Haha, myself, what a clever pun. - -# Parameters that can be defined on top level of a graph. Config will print -# them as "graph_$foo $value\n" -my @graph_parameters = ('title','total','order','scale','vlabel','args'); - -# Parameters that can be defined on a value-to-value basis and will be -# blindly passed to config. Printed as "$fieldname.$param $value\n". -# -# 'label' is hardcoded as it defaults to a varnishstat-description if not -# set. -my @field_parameters = ('graph', 'min', 'max', 'draw', 'cdef', 'warning', - 'colour', 'info', 'type'); - -# Varnishstat data is stored here. Example -# ('n_vbe' => { 'value' => '124', 'description'=>...,'flag'=>... }, SMA => -# { s0 => { 'value' => '...', 'flag'=> '...' },'Transient' => ...}) -# Both dynamic and static counters are kept here. -# -# Notes: -# - The 'flag' field for a counter is in RRD-dialect, not varnishstat -my %data; - -# Data structure that defines all possible graphs (aspects) and how they -# are to be plotted. Every top-level entry is a graph/aspect. Each -# top-level graph MUST have title set and 'values'. -# -# The 'values' hash must have at least one value definition. The actual -# value used is either fetched from varnishstat based on the value-name, or -# if 'rpn' is defined: calculated. 'type' SHOULD be set. -# -# Graphs with 'DEBUG' set to anything is omitted from 'suggest'. -# -# 'rpn' on values allows easy access to graphs consisting of multiple -# values from varnishstat. (Reverse polish notation). The RPN -# implementation only accepts +-*/ and varnishstat-values. -# -# With the exception of 'label', which is filled with the -# varnishstat-description if left undefined, any value left undefined will -# be left up to Munin to define/ignore/yell about. -# -# For dynamic counters, the values listed need to specify a counter and -# family. This will plot the specified counter for each identity within -# that family. Example: family of SMA, counter c_fail. This will create a -# c_fail-counter for each of the SMA-identities (e.g: Transient, s0, etc). -# For dynamic graphs, the value-name is only used to identify the data -# point, and does not relate to any varnishstat data as that is set by -# family/counter. -# -# Note that dynamic counters fetch the type from the XML and things like -# min/max are currently not supported (and silently ignored). -# -# See munin documentation or rrdgraph/rrdtool for more information. -my %ASPECTS = ( - 'request_rate' => { - 'title' => 'Request rates', - 'order' => 'cache_hit cache_hitpass cache_miss ' - . 'backend_conn backend_unhealthy ' - . 'client_req client_conn' , - 'values' => { - 'sess_conn' => { - 'type' => 'DERIVE', - 'min' => '0', - 'colour' => '444444', - 'graph' => 'ON' - }, - 'client_req' => { - 'type' => 'DERIVE', - 'colour' => '111111', - 'min' => '0' - }, - 'cache_hit' => { - 'type' => 'DERIVE', - 'draw' => 'AREA', - 'colour' => '00FF00', - 'min' => '0' - }, - 'cache_hitpass' => { - 'info' => 'Hitpass are cached passes: An ' - . 'entry in the cache instructing ' - . 'Varnish to pass. Typically ' - . 'achieved after a pass in ' - . 'vcl_fetch.', - 'type' => 'DERIVE', - 'draw' => 'STACK', - 'colour' => 'FFFF00', - 'min' => '0' - }, - 'cache_miss' => { - 'type' => 'DERIVE', - 'colour' => 'FF0000', - 'draw' => 'STACK', - 'min' => '0' - }, - 'backend_conn' => { - 'type' => 'DERIVE', - 'colour' => '995599', - 'min' => '0' - }, - 'backend_unhealthy' => { - 'type' => 'DERIVE', - 'min' => '0', - 'colour' => 'FF55FF' - }, - 's_pipe' => { - 'type' => 'DERIVE', - 'min' => '0', - 'colour' => '1d2bdf' - }, - 's_pass' => { - 'type' => 'DERIVE', - 'min' => '0', - 'colour' => '785d0d' - } - } - }, - 'hit_rate' => { - 'title' => 'Hit rates', - 'order' => 'client_req cache_hit cache_miss ' - . 'cache_hitpass' , - 'vlabel' => '%', - 'args' => '-u 100 --rigid', - 'scale' => 'no', - 'values' => { - 'client_req' => { - 'type' => 'DERIVE', - 'min' => '0', - 'graph' => 'off', - 'rpn' => [ 'cache_hit' , 'cache_miss' , 'cache_hitpass' , '+' , '+' ] - }, - 'cache_hit' => { - 'type' => 'DERIVE', - 'min' => '0', - 'draw' => 'AREA', - 'cdef' => 'cache_hit,client_req,/,100,*' - }, - 'cache_miss' => { - 'type' => 'DERIVE', - 'draw' => 'STACK', - 'min' => '0', - 'cdef' => 'cache_miss,client_req,/,100,*' - }, - 'cache_hitpass' => { - 'type' => 'DERIVE', - 'draw' => 'STACK', - 'min' => '0', - 'cdef' => 'cache_hitpass,client_req,/,100,*' - }, - } - }, - 'backend_traffic' => { - 'title' => 'Backend traffic', - 'values' => { - 'backend_conn' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'backend_unhealthy' => { - 'type' => 'DERIVE', - 'min' => '0', - 'warning' => ':1' - }, - 'backend_busy' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'backend_fail' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'backend_reuse' => { - 'type' => 'DERIVE', - 'min' => 0 - }, - 'backend_recycle' => { - 'type' => 'DERIVE', - 'min' => 0 - }, - 'backend_toolate' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'backend_retry' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'backend_req' => { - 'type' => 'DERIVE', - 'min' => '0' - } - } - }, - 'objects' => { - 'title' => 'Number of objects', - 'order' => 'n_object n_objectcore n_vampireobject n_objecthead', - 'values' => { - 'n_object' => { - 'type' => 'GAUGE', - 'label' => 'Number of objects' - }, - 'n_objectcore' => { - 'type' => 'GAUGE', - 'label' => 'Number of object cores' - }, - 'n_vampireobject' => { - 'type' => 'GAUGE', - 'label' => 'Number of unresurrected objects' - }, - 'n_objecthead' => { - 'type' => 'GAUGE', - 'label' => 'Number of object heads', - 'info' => 'Each object head can have one ' - . 'or more object attached, ' - . 'typically based on the Vary: header' - } - } - }, - 'transfer_rates' => { - 'title' => 'Transfer rates', - 'order' => 's_resp_bodybytes s_resp_hdrbytes', - 'args' => '-l 0', - 'vlabel' => 'bit/s', - 'values' => { - 's_resp_hdrbytes' => { - 'type' => 'DERIVE', - 'label' => 'Header traffic', - 'draw' => 'STACK', - 'min' => '0', - 'info' => 'HTTP Header traffic. TCP/IP ' - . 'overhead is not included.', - 'cdef' => 's_resp_hdrbytes,8,*' - }, - 's_resp_bodybytes' => { - 'type' => 'DERIVE', - 'draw' => 'AREA', - 'label' => 'Body traffic', - 'min' => '0', - 'cdef' => 's_resp_bodybytes,8,*' - } - } - }, - 'threads' => { - 'title' => 'Thread status', - 'values' => { - 'threads' => { - 'type' => 'GAUGE', - 'min' => '0', - 'warning' => '1:' - }, - 'threads_created' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'threads_failed' => { - 'type' => 'DERIVE', - 'min' => '0', - 'warning' => ':1' - }, - 'threads_limited' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'threads_destroyed' => { - 'type' => 'DERIVE', - 'min' => '0', - 'warning' => ':1' - } - } - }, - 'memory_usage' => { - 'title' => 'Memory usage', - 'args' => '--base 1024', - 'vlabel' => 'bytes', - 'values' => { - 'sms_balloc' => { - 'type' => 'GAUGE', - }, - 'sms_nbytes' => { - 'type' => 'GAUGE', - }, - 'SMA_1' => { - 'counter' => 'g_bytes', - 'family' => 'SMA', - }, - 'SMA_2' => { - 'counter' => 'g_space', - 'family' => 'SMA', - }, - 'SMA_3' => { - 'counter' => 'c_bytes', - 'family' => 'SMA' - }, - 'SMF_1' => { - 'counter' => 'g_bytes', - 'family' => 'SMF', - }, - 'SMF_2' => { - 'counter' => 'g_space', - 'family' => 'SMF', - } - } - }, - 'uptime' => { - 'title' => 'Varnish uptime', - 'vlabel' => 'days', - 'scale' => 'no', - 'values' => { - 'uptime' => { - 'type' => 'GAUGE', - 'cdef' => 'uptime,86400,/' - } - } - }, - 'objects_per_objhead' => { - 'title' => 'Objects per objecthead', - 'DEBUG' => 'yes', - 'values' => { - 'obj_per_objhead' => { - 'type' => 'GAUGE', - 'label' => 'Objects per object heads', - 'rpn' => [ 'n_object','n_objecthead','/' ] - } - } - }, - 'losthdr' => { - 'title' => 'HTTP Header overflows', - 'DEBUG' => 'yes', - 'values' => { - 'losthdr' => { - 'type' => 'DERIVE', - 'min' => '0' - } - } - }, - 'hcb' => { - 'title' => 'Critbit data', - 'DEBUG' => 'yes', - 'values' => { - 'hcb_nolock' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'hcb_lock' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'hcb_insert' => { - 'type' => 'DERIVE', - 'min' => '0' - } - } - }, - 'esi' => { - 'title' => 'ESI', - 'DEBUG' => 'yes', - 'values' => { - 'esi_parse' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'esi_errors' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'esi_warnings' => { - 'type' => 'DERIVE', - 'min' => '0' - } - } - }, - 'session' => { - 'title' => 'Sessions', - 'DEBUG' => 'yes', - 'values' => { - 'sess_conn' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'sess_drop' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'sess_fail' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'sess_pipe_overflow' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'sess_queued' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'sess_dropped' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'sess_closed' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'sess_pipeline' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'sess_readahead' => { - 'type' => 'DERIVE', - 'min' => '0' - } - } - }, - 'session_herd' => { - 'title' => 'Session herd', - 'DEBUG' => 'yes', - 'values' => { - 'sess_herd' => { - 'type' => 'DERIVE', - 'min' => '0' - } - } - }, - 'shm_writes' => { - 'title' => 'SHM writes and records', - 'DEBUG' => 'yes', - 'values' => { - 'shm_records' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'shm_writes' => { - 'type' => 'DERIVE', - 'min' => '0' - } - } - }, - 'shm' => { - 'title' => 'Shared memory activity', - 'DEBUG' => 'yes', - 'values' => { - 'shm_flushes' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'shm_cont' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'shm_cycles' => { - 'type' => 'DERIVE', - 'min' => '0' - } - } - }, - 'allocations' => { - 'title' => 'Memory allocation requests', - 'DEBUG' => 'yes', - 'values' => { - 'sm_nreq' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'sma_nreq' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'sms_nreq' => { - 'type' => 'DERIVE', - 'min' => '0' - } - } - }, - 'vcl' => { - 'title' => 'VCL', - 'DEBUG' => 'yes', - 'values' => { - 'n_backend' => { - 'type' => 'GAUGE' - }, - 'n_vcl' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'n_vcl_avail' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'n_vcl_discard' => { - 'type' => 'DERIVE', - 'min' => '0' - } - } - }, - 'bans' => { - 'title' => 'Bans', - 'DEBUG' => 'yes', - 'values' => { - 'bans' => { - 'type' => 'GAUGE' - }, - 'bans_added' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'bans_deleted' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'bans_completed' => { - 'type' => 'GAUGE' - }, - 'bans_obj' => { - 'type' => 'GAUGE' - }, - 'bans_req' => { - 'type' => 'GAUGE' - }, - 'bans_tested' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'bans_obj_killed' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'bans_tests_tested' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'bans_dups' => { - 'type' => 'GAUGE' - }, - 'bans_persisted_bytes' => { - 'type' => 'GAUGE' - }, - 'bans_persisted_fragmentation' => { - 'type' => 'GAUGE' - } - } - }, - 'bans_lurker' => { - 'title' => 'Ban Lurker', - 'DEBUG' => 'yes', - 'values' => { - 'bans_lurker_tested' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'bans_lurker_tests_tested' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'bans_lurker_obj_killed' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'bans_lurker_contention' => { - 'type' => 'DERIVE', - 'min' => '0' - } - } - }, - 'expunge' => { - 'title' => 'Object expunging', - 'order' => 'n_expired n_lru_nuked', - 'values' => { - 'n_expired' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'n_lru_nuked' => { - 'type' => 'DERIVE', - 'min' => '0' - } - } - }, - 'lru' => { - 'title' => 'LRU activity', - 'DEBUG' => 'yes', - 'values' => { - 'n_lru_nuked' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'n_lru_moved' => { - 'type' => 'DERIVE', - 'min' => '0' - } - } - }, - 'bad' => { - 'title' => 'Misbehavior', - 'values' => { - 'SMA_1' => { - 'counter' => 'c_fail', - 'family' => 'SMA', - }, - 'SMF_1' => { - 'counter' => 'c_fail', - 'family' => 'SMF', - }, - 'sess_drop' => { - 'type' => 'DERIVE' - }, - 'backend_unhealthy' => { - 'type' => 'DERIVE' - }, - 'fetch_failed' => { - 'type' => 'DERIVE' - }, - 'backend_busy' => { - 'type' => 'DERIVE' - }, - 'threads_failed' => { - 'type' => 'DERIVE' - }, - 'threads_limited' => { - 'type' => 'DERIVE' - }, - 'threads_destroyed' => { - 'type' => 'DERIVE' - }, - 'thread_queue_len' => { - 'type' => 'GAUGE' - }, - 'losthdr' => { - 'type' => 'DERIVE' - }, - 'esi_errors' => { - 'type' => 'DERIVE' - }, - 'esi_warnings' => { - 'type' => 'DERIVE' - }, - 'sess_fail' => { - 'type' => 'DERIVE' - }, - 'sess_pipe_overflow' => { - 'type' => 'DERIVE' - } - } - }, - 'gzip' => { - 'title' => 'GZIP activity', - 'DEBUG' => 'yes', - 'values' => { - 'n_gzip' => { - 'type' => 'DERIVE', - 'min' => '0' - }, - 'n_gunzip' => { - 'type' => 'DERIVE', - 'min' => '0' - } - } - } -); - -################################ -# Various helper functions # -################################ - -# Translate $_[0] from varnish' internal types (flags) to munin/rrd -# variants (e.g: from 'i' to GAUGE). Returns the result. -sub translate_type -{ - my $d = $_[0]; - if ($d eq "i") { - $d = "GAUGE"; - } elsif ($d eq "a") { - $d = "DERIVE"; - } - return $d; -} - -# Print the value of a two-dimensional hash if it exist. -# Returns false if non-existent. -# -# Output is formatted for plugins if arg4 is blank, otherwise arg4 is used -# as the title/name of the field (ie: arg4=graph_title). -sub print_if_exist -{ - my %values = %{$_[0]}; - my $value = $_[1]; - my $field = $_[2]; - my $title = "$value.$field"; - if (defined($_[3])) { - $title = $_[3]; - } - if (defined($values{$value}{$field})) { - print "$title $values{$value}{$field}\n"; - } else { - return 0; - } -} - -# Create a output-friendly name -sub normalize_name -{ - my $name = $_[0]; - $name =~ s/[^a-zA-Z0-9]/_/g; - return $name; -} - -# Braindead RPN: +,-,/,* will pop two items from @stack, and perform -# the relevant operation on the items. If the item in the array isn't one -# of the 4 basic math operations, a value from varnishstat is pushed on to -# the stack. IE: 'client_req','client_conn','/' will leave the value of -# "client_req/client_conn" on the stack. -# -# If only one item is left on the stack, it is printed. Otherwise, an error -# message is printed. -sub rpn -{ - my @stack; - my $left; - my $right; - foreach my $item (@{$_[0]}) { - if ($item eq "+") { - $right = pop(@stack); - $left = pop(@stack); - push(@stack,$left+$right); - } elsif ($item eq "-") { - $right = pop(@stack); - $left = pop(@stack); - push(@stack,$left-$right); - } elsif ($item eq "/") { - $right = pop(@stack); - $left = pop(@stack); - push(@stack,$left/$right); - } elsif ($item eq "*") { - $right = pop(@stack); - $left = pop(@stack); - push(@stack,$left*$right); - } else { - push(@stack,int($data{$item}{'value'})); - } - } - if (@stack > 1) - { - print STDERR "RPN error: Stack has more than one item left.\n"; - print STDERR "@stack\n"; - exit 255; - } - print "@stack"; - print "\n"; -} - -# Bail-function. -sub usage -{ - if (@_) { - print STDERR "@_" . "\n\n"; - } - print STDERR "Known arguments: suggest, config, autoconf.\n"; - print STDERR "Run with suggest to get a list of known aspects.\n"; - exit 1; -} - -################################ -# XML Parsing # -################################ -# The following code is for parsing varnishstat -x. While %data should be -# stable, the following bits can easily be replaced with anything (json, an -# other xml-parser, magic, etc) -# -# The basic concept is simple enough. Only worry about stuff inside -# . Updating %state on each new data field, and commit it to %data -# when is seen. -# -# We do use translate_type() on the 'flag' field. - - -# Internal state for the XML parsing -my %state = ( - 'stat' => 0, # inside or not - 'field' => 'none', # , , , etc. -); - -# Reset the state of XML, mainly used for end-elements. -sub xml_reset_state() { - $state{'stat'} = '0'; - $state{'field'} = 'none'; - $state{'values'} = (); -} - -# Callback for data entry. Cleans leading whitespace and updates state. -sub xml_characters { - my $d = $_[1]; - if ($state{'stat'} == 0) { - return; - } - if ($state{'field'} eq "type" && $d eq "MAIN") { - return; - } - $d =~ s/^\s*$//g; - if ($d eq "") { - return; - } - $state{'values'}{$state{'field'}} = $d; -} - -# Store the current state in %data. Issued at -# Note that 'flag' is translated to RRD-equivalents here. -sub xml_commit_state -{ - my $name = $state{'values'}{'name'}; - my $type = $state{'values'}{'type'}; - my $ident = $state{'values'}{'ident'}; - - foreach my $key (keys %{$state{'values'}}) { - my $data = $state{'values'}{$key}; - if ($key eq 'flag') { - $data = translate_type($data); - } - if (defined($type) and $type ne '' and defined($ident) and $ident ne '') { - $data{$type}{$ident}{$name}{$key} = $data; - } else { - $data{$name}{$key} = $data - } - } -} - -# Callback for end tag. E.g: -sub xml_end_elem { - my $element = $_[1]; - if ($element ne "stat") { - return; - } - - xml_commit_state(); - xml_reset_state(); -} - -# Callback for opening tag. E.g: -sub xml_start_elem { - $state{'field'} = $_[1]; - if ($state{'field'} eq "stat") { - $state{'stat'} = 1; - } -} - -################################ -# Internal API # -################################ - - -# Populate %data, includes both values and descriptions and more. -# Currently driven by XML, but that could change. -sub populate_stats -{ - my $arg = "-x"; - my $parser = new XML::Parser(Handlers => {Start => \&xml_start_elem, - End => \&xml_end_elem, - Char => \&xml_characters} ); - - if ($varnishname) { - $arg .= " -n $varnishname"; - } - - open (XMLDATA, "$varnishstatexec $arg|") or die "meh"; - $parser->parse(*XMLDATA, ProtocolEncoding => 'ISO-8859-1'); - close(XMLDATA); -} - -# Prints the fields in the list in $_[2] (e.g: 'value'/'description') for -# each identity of the varnish counter/family combination as defined by -# the $_[0]-counter on the aspect definition. Err, that's jibberish, so -# an example: -# -# e.g: dynamic_print('SMA_1','',('value')) -# e.g: dynamic_print('SMA_2','.label',('ident','description')) -# SMA_1 is the counter-value. If it is a dynamic counter, it has a counter -# and family-member (e.g: counter: c_req, family: SMA) and print_dynamic -# will print c_req for each SMA-identity. -# -# Note that the variables to print is a list. This is to allow printing a -# single item with multiple fields. Typically for identity+description so -# you can distinguish between different data points. -# -# Returns true if it was a dynamic counter. -sub print_dynamic -{ - my $name = $_[0]; - shift; - my $suffix = $_[0]; - shift; - my @field = @_; - if (!defined($ASPECTS{$self}{'values'}{$name}{'counter'})) { - return 0; - } - if (!defined($ASPECTS{$self}{'values'}{$name}{'family'})) { - return 0; - } - my $counter = $ASPECTS{$self}{'values'}{$name}{'counter'}; - my $type = $ASPECTS{$self}{'values'}{$name}{'family'}; - - foreach my $key (keys %{$data{$type}}) { - my $pname = normalize_name($type . "_" . $key . "_" . $counter); - print $pname . $suffix . " "; - my $i = 0; - foreach my $f (@field) { - if ($i != 0) { - print " "; - } - $i += 1; - print $data{$type}{$key}{$counter}{$f}; - } - print "\n"; - } - return 1; -} - -# Read and verify the aspect ($self). -sub set_aspect -{ - $self = $0; - $self =~ s/^.*\/varnish[0-9]?_//; - if (!defined($ASPECTS{$self}) && @ARGV == 0) { - usage "No such aspect"; - } -} - -# Print 'yes' if it's reasonable to use this plugin, or 'no' with a -# human-readable error message. Always exit true, even if the response -# is 'no'. -sub autoconf -{ - # XXX: Solaris outputs errors to stderr and always returns true. - # XXX: See #873 - if (`which $varnishstatexec 2>/dev/null` =~ m{^/}) { - print "yes\n"; - } else { - print "no ($varnishstatexec could not be found)\n"; - } - exit 0; -} - -# Suggest relevant aspects/values of $self. -# 'DEBUG'-graphs are excluded. -sub suggest -{ - foreach my $key (keys %ASPECTS) { - if (defined($ASPECTS{$key}{'DEBUG'}) && $FULL_SUGGEST != 1) { - next; - } - print "$key\n"; - } -} - -# Walk through the relevant aspect and print all top-level configuration -# values and value-definitions. -sub get_config -{ - my $graph = $_[0]; - - # Need to double-check since set_aspect only checks this if there - # is no argument (suggest/autoconf doesn't require a valid aspect) - if (!defined($ASPECTS{$graph})) { - usage "No such aspect"; - } - my %values = %{$ASPECTS{$graph}{'values'}}; - - print "graph_category Varnish\n"; - foreach my $field (@graph_parameters) { - print_if_exist(\%ASPECTS,$graph,$field,"graph_$field"); - } - - foreach my $value (sort keys %values) { - # Just print the description/type if it's a dynamic - # counter. It'll be silent if it isn't. - if(print_dynamic($value,'.label',('description','type','ident'))) { - print_dynamic($value,'.type',('flag')); - next; - } - - # Need either RPN definition or a varnishstat value. - if (!defined($data{$value}{'value'}) && - !defined($values{$value}{'rpn'})) { - if ($DEBUG) { - print "ERROR: $value not part of varnishstat.\n" - } - next; - } - - if (!print_if_exist(\%values,$value,'label')) { - print "$value.label $data{$value}{'description'}\n"; - } - foreach my $field (@field_parameters) { - print_if_exist(\%values,$value,$field); - } - } -} - -# Handle arguments (config, autoconf, suggest) -# Populate stats for config is necessary, but we want to avoid it for -# autoconf as it would generate a nasty error. -sub check_args -{ - if (@ARGV && $ARGV[0] eq '') { - shift @ARGV; - } - if (@ARGV == 1) { - if ($ARGV[0] eq "config") { - populate_stats; - get_config($self); - exit 0; - } elsif ($ARGV[0] eq "autoconf") { - autoconf($self); - exit 0; - } elsif ($ARGV[0] eq "suggest") { - suggest; - exit 0; - } - usage "Unknown argument"; - } -} - -################################ -# Execution starts here # -################################ - -set_aspect; -check_args; -populate_stats; - -# We only get here if we're supposed to. -# Walks through the relevant values and either prints the varnishstat, or -# if the 'rpn' variable is set, calls rpn() to execute ... the rpn. -# -# NOTE: Due to differences in varnish-versions, this checks if the value -# actually exist before using it. -foreach my $value (sort keys %{$ASPECTS{$self}{'values'}}) { - if (defined($ASPECTS{$self}{'values'}{$value}{'rpn'})) { - print "$value.value "; - rpn($ASPECTS{$self}{'values'}{$value}{'rpn'}); - } else { - if (print_dynamic($value,'.value',('value'))) { - next; - } - - if (!defined($data{$value}{'value'})) { - if ($DEBUG) { - print STDERR "Error: $value not part of " - . "varnishstat.\n"; - } - next; - } - print "$value.value "; - print "$data{$value}{'value'}\n"; - } -} diff --git a/varnish/files/munin/varnish5.conf b/varnish/files/munin/varnish5.conf new file mode 100644 index 00000000..bb367caf --- /dev/null +++ b/varnish/files/munin/varnish5.conf @@ -0,0 +1,2 @@ +[varnish5_*] +user vcache diff --git a/varnish/files/munin/varnish5_ b/varnish/files/munin/varnish5_ new file mode 100644 index 00000000..279805eb --- /dev/null +++ b/varnish/files/munin/varnish5_ @@ -0,0 +1,1221 @@ +#!/usr/bin/perl +# -*- perl -*- +# +# varnish5_ - Munin plugin to for Varnish 5.x and 6.x +# Copyright (C) 2009,2018 Redpill Linpro AS +# +# Author: Kristian Lyngstøl +# Pål-Eivind Johnsen +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +=head1 NAME + +varnish5_ - Munin plugin to monitor various aspects of varnish + +=head1 APPLICABLE SYSTEMS + +Varnish 5.x with varnishstat + +=head1 CONFIGURATION + +The plugin needs to be able to execute varnishstat. + +The configuration section shows the defaults + [varnish5_*] + group varnish + env.varnishstat varnishstat + env.name + +env.varnishstat can be a full path to varnishstat if it's +not in the path already. + +env.name is blank (undefined) by default and can be used to specify a -n +name argument to varnish if multiple instances are running on the same +server. + +A few aspects are not linked by default. They are marked as +'DEBUG' => 'yes' (or any other value). They are: + +vcl, bans, bans_lurker, lru, objects_per_objhead, +losthdr, esi, hcb, shm, shm_writes, overflow, +session, session_herd, gzip + +You can link them yourself with something like this: + + ln -s @@LIBDIR@@/plugins/varnish5_ \ + @@CONFDIR@@/plugins/varnish5_data_structures + +=head1 INTERPRETATION + +Each graph uses data from varnishstat. + +=head1 MAGIC MARKERS + + #%# family=auto + #%# capabilities=autoconf suggest + +=head1 VERSION + + $Id$ + +=head1 BUGS + +The hit_rate graph requires munin r2040 or newer to display +correctly. + +=head1 PATCHES-TO + +Please send patches to Kristian Lyngstøl +and/or varnish-misc@varnish-cache.org for significant changes. The +munin-contrib Git repo is the authoritative repository for this plugin. + +=head1 AUTHOR + +Kristian Lyngstøl + +=head1 MODIFICATIONS + +Ingo Oppermann +Pål-Eivind Johnsen + +=head1 LICENSE + +GPLv2 + +=cut + + +use XML::Parser; +use strict; + +# Set to 1 to enable output when a variable is defined in a graph but +# omitted because it doesn't exist in varnishstat. +my $DEBUG = 0; + +# Set to 1 to ignore 'DEBUG' and suggest all available aspects. +my $FULL_SUGGEST = 0; + +# Varnishstat executable. Include full path if it's not in your path. +my $varnishstatexec = exists $ENV{'varnishstat'} ? $ENV{'varnishstat'} : "varnishstat"; + +# For multiple instances +my $varnishname = exists $ENV{'name'} ? $ENV{'name'} : undef; + +my $self; # Haha, myself, what a clever pun. + +# Parameters that can be defined on top level of a graph. Config will print +# them as "graph_$foo $value\n" +my @graph_parameters = ('title','total','order','scale','vlabel','args'); + +# Parameters that can be defined on a value-to-value basis and will be +# blindly passed to config. Printed as "$fieldname.$param $value\n". +# +# 'label' is hardcoded as it defaults to a varnishstat-description if not +# set. +my @field_parameters = ('graph', 'min', 'max', 'draw', 'cdef', 'warning', + 'colour', 'info', 'type'); + +# Varnishstat data is stored here. Example +# ('n_vbe' => { 'value' => '124', 'description'=>...,'flag'=>... }, SMA => +# { s0 => { 'value' => '...', 'flag'=> '...' },'Transient' => ...}) +# Both dynamic and static counters are kept here. +# +# Notes: +# - The 'flag' field for a counter is in RRD-dialect, not varnishstat +my %data; + +# Data structure that defines all possible graphs (aspects) and how they +# are to be plotted. Every top-level entry is a graph/aspect. Each +# top-level graph MUST have title set and 'values'. +# +# The 'values' hash must have at least one value definition. The actual +# value used is either fetched from varnishstat based on the value-name, or +# if 'rpn' is defined: calculated. 'type' SHOULD be set. +# +# Graphs with 'DEBUG' set to anything is omitted from 'suggest'. +# +# 'rpn' on values allows easy access to graphs consisting of multiple +# values from varnishstat. (Reverse polish notation). The RPN +# implementation only accepts +-*/ and varnishstat-values. +# +# With the exception of 'label', which is filled with the +# varnishstat-description if left undefined, any value left undefined will +# be left up to Munin to define/ignore/yell about. +# +# For dynamic counters, the values listed need to specify a counter and +# family. This will plot the specified counter for each identity within +# that family. Example: family of SMA, counter c_fail. This will create a +# c_fail-counter for each of the SMA-identities (e.g: Transient, s0, etc). +# For dynamic graphs, the value-name is only used to identify the data +# point, and does not relate to any varnishstat data as that is set by +# family/counter. +# +# Note that dynamic counters fetch the type from the XML and things like +# min/max are currently not supported (and silently ignored). +# +# See munin documentation or rrdgraph/rrdtool for more information. +my %ASPECTS = ( + 'request_rate' => { + 'title' => 'Request rates', + 'order' => 'cache_hit cache_hitpass cache_miss cache_hitmiss' + . 'backend_conn backend_unhealthy ' + . 'client_req client_conn' , + 'values' => { + 'sess_conn' => { + 'type' => 'DERIVE', + 'min' => '0', + 'colour' => '444444', + 'graph' => 'ON' + }, + 'client_req' => { + 'type' => 'DERIVE', + 'colour' => '111111', + 'min' => '0' + }, + 'cache_hit' => { + 'type' => 'DERIVE', + 'draw' => 'AREA', + 'colour' => '00FF00', + 'min' => '0' + }, + 'cache_hitpass' => { + 'info' => 'Hitpass are cached passes: An ' + . 'entry in the cache instructing ' + . 'Varnish to pass. Typically ' + . 'achieved after a pass in ' + . 'vcl_fetch.', + 'type' => 'DERIVE', + 'draw' => 'STACK', + 'colour' => 'FFFF00', + 'min' => '0' + }, + 'cache_miss' => { + 'type' => 'DERIVE', + 'colour' => 'FF0000', + 'draw' => 'STACK', + 'min' => '0' + }, + 'cache_hitmiss' => { + 'info' => 'Hitmiss are cached missing: An ' + . 'entry in the cache instructing ' + . 'Varnish to pass. Typically ' + . 'achieved after a pass in ' + . 'vcl_fetch.', + 'type' => 'DERIVE', + 'draw' => 'STACK', + 'colour' => 'FFFF00', + 'min' => '0' + }, + 'backend_conn' => { + 'type' => 'DERIVE', + 'colour' => '995599', + 'min' => '0' + }, + 'backend_unhealthy' => { + 'type' => 'DERIVE', + 'min' => '0', + 'colour' => 'FF55FF' + }, + 's_pipe' => { + 'type' => 'DERIVE', + 'min' => '0', + 'colour' => '1d2bdf' + }, + 's_pass' => { + 'type' => 'DERIVE', + 'min' => '0', + 'colour' => '785d0d' + } + } + }, + 'hit_rate' => { + 'title' => 'Hit rates', + 'order' => 'client_req cache_hit cache_miss ' + . 'cache_hitpass cache_hitmiss' , + 'vlabel' => '%', + 'args' => '-l 0 -u 100 --rigid', + 'scale' => 'no', + 'values' => { + 'client_req' => { + 'type' => 'DERIVE', + 'min' => '0', + 'graph' => 'off', + 'rpn' => [ 'cache_hit' , 'cache_miss' , 'cache_hitpass' ,'cache_hitmiss', '+' , '+' ,'+' ] + }, + 'cache_hit' => { + 'type' => 'DERIVE', + 'min' => '0', + 'draw' => 'AREA', + 'cdef' => 'cache_hit,client_req,/,100,*' + }, + 'cache_miss' => { + 'type' => 'DERIVE', + 'draw' => 'STACK', + 'min' => '0', + 'cdef' => 'cache_miss,client_req,/,100,*' + }, + 'cache_hitpass' => { + 'type' => 'DERIVE', + 'draw' => 'STACK', + 'min' => '0', + 'cdef' => 'cache_hitpass,client_req,/,100,*' + }, + 'cache_hitmiss' => { + 'type' => 'DERIVE', + 'draw' => 'STACK', + 'min' => '0', + 'cdef' => 'cache_hitmiss,client_req,/,100,*' + } + } + }, + 'backend_traffic' => { + 'title' => 'Backend traffic', + 'values' => { + 'backend_conn' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'backend_unhealthy' => { + 'type' => 'DERIVE', + 'min' => '0', + 'warning' => ':1' + }, + 'backend_busy' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'backend_fail' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'backend_reuse' => { + 'type' => 'DERIVE', + 'min' => 0 + }, + 'backend_recycle' => { + 'type' => 'DERIVE', + 'min' => 0 + }, + 'backend_retry' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'backend_req' => { + 'type' => 'DERIVE', + 'min' => '0' + } + } + }, + 'objects' => { + 'title' => 'Number of objects', + 'order' => 'n_object n_objectcore n_vampireobject n_objecthead', + 'values' => { + 'n_object' => { + 'type' => 'GAUGE', + 'label' => 'Number of objects' + }, + 'n_objectcore' => { + 'type' => 'GAUGE', + 'label' => 'Number of object cores' + }, + 'n_vampireobject' => { + 'type' => 'GAUGE', + 'label' => 'Number of unresurrected objects' + }, + 'n_objecthead' => { + 'type' => 'GAUGE', + 'label' => 'Number of object heads', + 'info' => 'Each object head can have one ' + . 'or more object attached, ' + . 'typically based on the Vary: header' + } + } + }, + 'transfer_rates' => { + 'title' => 'Transfer rates', + 'order' => 's_resp_bodybytes s_resp_hdrbytes', + 'args' => '-l 0', + 'vlabel' => 'bit/s', + 'values' => { + 's_resp_hdrbytes' => { + 'type' => 'DERIVE', + 'label' => 'Header traffic', + 'draw' => 'STACK', + 'min' => '0', + 'info' => 'HTTP Header traffic. TCP/IP ' + . 'overhead is not included.', + 'cdef' => 's_resp_hdrbytes,8,*' + }, + 's_resp_bodybytes' => { + 'type' => 'DERIVE', + 'draw' => 'AREA', + 'label' => 'Body traffic', + 'min' => '0', + 'cdef' => 's_resp_bodybytes,8,*' + } + } + }, + 'threads' => { + 'title' => 'Thread status', + 'values' => { + 'threads' => { + 'type' => 'GAUGE', + 'min' => '0', + 'warning' => '1:' + }, + 'threads_created' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'threads_failed' => { + 'type' => 'DERIVE', + 'min' => '0', + 'warning' => ':1' + }, + 'threads_limited' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'threads_destroyed' => { + 'type' => 'DERIVE', + 'min' => '0', + 'warning' => ':1' + } + } + }, + 'memory_usage' => { + 'title' => 'Memory usage', + 'args' => '--base 1024', + 'vlabel' => 'bytes', + 'values' => { + 'SMA_1' => { + 'counter' => 'g_bytes', + 'family' => 'SMA', + }, + 'SMA_2' => { + 'counter' => 'g_space', + 'family' => 'SMA', + }, + 'SMA_3' => { + 'counter' => 'c_bytes', + 'family' => 'SMA' + }, + 'SMF_1' => { + 'counter' => 'g_bytes', + 'family' => 'SMF', + }, + 'SMF_2' => { + 'counter' => 'g_space', + 'family' => 'SMF', + }, + 'SMF_3' => { + 'counter' => 'c_bytes', + 'family' => 'SMF', + } + } + }, + 'main_uptime' => { + 'type' => 'MAIN', + 'title' => 'Varnish Child uptime', + 'vlabel' => 'days', + 'scale' => 'no', + 'values' => { + 'uptime' => { + 'type' => 'GAUGE', + 'cdef' => 'uptime,86400,/' + }, + } + }, + 'mgt_uptime' => { + 'type' => 'MGT', + 'title' => 'Varnish Management uptime', + 'vlabel' => 'days', + 'scale' => 'no', + 'values' => { + 'uptime' => { + 'type' => 'GAUGE', + 'cdef' => 'uptime,86400,/' + }, + } + }, + 'objects_per_objhead' => { + 'title' => 'Objects per objecthead', + 'DEBUG' => 'yes', + 'values' => { + 'n_objecthead' => { + 'type' => 'GAUGE', + 'label' => 'Objects per object heads', + 'rpn' => [ 'n_object','n_objecthead','/' ] + } + } + }, + 'losthdr' => { + 'title' => 'HTTP Header overflows', + 'DEBUG' => 'yes', + 'values' => { + 'losthdr' => { + 'type' => 'DERIVE', + 'min' => '0' + } + } + }, + 'hcb' => { + 'title' => 'Critbit data', + 'DEBUG' => 'yes', + 'values' => { + 'hcb_nolock' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'hcb_lock' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'hcb_insert' => { + 'type' => 'DERIVE', + 'min' => '0' + } + } + }, + 'esi' => { + 'title' => 'ESI', + 'DEBUG' => 'yes', + 'values' => { + 'esi_errors' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'esi_warnings' => { + 'type' => 'DERIVE', + 'min' => '0' + } + } + }, + 'session' => { + 'title' => 'Sessions', + 'DEBUG' => 'yes', + 'values' => { + 'sess_conn' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'sess_drop' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'sess_fail' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'sess_pipe_overflow' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'sess_queued' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'sess_dropped' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'sess_closed' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'sess_pipeline' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'sess_readahead' => { + 'type' => 'DERIVE', + 'min' => '0' + } + } + }, + 'session_herd' => { + 'title' => 'Session herd', + 'DEBUG' => 'yes', + 'values' => { + 'sess_herd' => { + 'type' => 'DERIVE', + 'min' => '0' + } + } + }, + 'shm_writes' => { + 'title' => 'SHM writes and records', + 'DEBUG' => 'yes', + 'values' => { + 'shm_records' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'shm_writes' => { + 'type' => 'DERIVE', + 'min' => '0' + } + } + }, + 'shm' => { + 'title' => 'Shared memory activity', + 'DEBUG' => 'yes', + 'values' => { + 'shm_flushes' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'shm_cont' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'shm_cycles' => { + 'type' => 'DERIVE', + 'min' => '0' + } + } + }, + 'allocations' => { + 'title' => 'Memory allocation requests', + 'DEBUG' => 'yes', + 'values' => { + 'sm_nreq' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'sma_nreq' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'sms_nreq' => { + 'type' => 'DERIVE', + 'min' => '0' + } + } + }, + 'vcl' => { + 'title' => 'VCL', + 'DEBUG' => 'yes', + 'values' => { + 'n_backend' => { + 'type' => 'GAUGE' + }, + 'n_vcl' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'n_vcl_avail' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'n_vcl_discard' => { + 'type' => 'DERIVE', + 'min' => '0' + } + } + }, + 'bans' => { + 'title' => 'Bans', + 'DEBUG' => 'yes', + 'values' => { + 'bans' => { + 'type' => 'GAUGE' + }, + 'bans_added' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'bans_deleted' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'bans_completed' => { + 'type' => 'GAUGE' + }, + 'bans_obj' => { + 'type' => 'GAUGE' + }, + 'bans_req' => { + 'type' => 'GAUGE' + }, + 'bans_tested' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'bans_obj_killed' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'bans_tests_tested' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'bans_dups' => { + 'type' => 'GAUGE' + }, + 'bans_persisted_bytes' => { + 'type' => 'GAUGE' + }, + 'bans_persisted_fragmentation' => { + 'type' => 'GAUGE' + } + } + }, + 'bans_lurker' => { + 'title' => 'Ban Lurker', + 'DEBUG' => 'yes', + 'values' => { + 'bans_lurker_tested' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'bans_lurker_tests_tested' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'bans_lurker_obj_killed' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'bans_lurker_contention' => { + 'type' => 'DERIVE', + 'min' => '0' + } + } + }, + 'expunge' => { + 'title' => 'Object expunging', + 'order' => 'n_expired n_lru_nuked', + 'values' => { + 'n_expired' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'n_lru_nuked' => { + 'type' => 'DERIVE', + 'min' => '0' + } + } + }, + 'lru' => { + 'title' => 'LRU activity', + 'DEBUG' => 'yes', + 'values' => { + 'n_lru_nuked' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'n_lru_moved' => { + 'type' => 'DERIVE', + 'min' => '0' + } + } + }, + 'bad' => { + 'title' => 'Misbehavior', + 'values' => { + 'SMA_1' => { + 'counter' => 'c_fail', + 'family' => 'SMA', + }, + 'SMF_1' => { + 'counter' => 'c_fail', + 'family' => 'SMF', + }, + 'sess_drop' => { + 'type' => 'DERIVE' + }, + 'backend_unhealthy' => { + 'type' => 'DERIVE' + }, + 'fetch_failed' => { + 'type' => 'DERIVE' + }, + 'backend_busy' => { + 'type' => 'DERIVE' + }, + 'threads_failed' => { + 'type' => 'DERIVE' + }, + 'threads_limited' => { + 'type' => 'DERIVE' + }, + 'threads_destroyed' => { + 'type' => 'DERIVE' + }, + 'thread_queue_len' => { + 'type' => 'GAUGE' + }, + 'losthdr' => { + 'type' => 'DERIVE' + }, + 'esi_errors' => { + 'type' => 'DERIVE' + }, + 'esi_warnings' => { + 'type' => 'DERIVE' + }, + 'sess_fail' => { + 'type' => 'DERIVE' + }, + 'sess_pipe_overflow' => { + 'type' => 'DERIVE' + + } + } + }, + 'gzip' => { + 'title' => 'GZIP activity', + 'DEBUG' => 'yes', + 'values' => { + 'n_gzip' => { + 'type' => 'DERIVE', + 'min' => '0' + }, + 'n_gunzip' => { + 'type' => 'DERIVE', + 'min' => '0' + } + } + }, + 'backend' => { + 'title' => 'Backend Status', + 'DEBUG' => 'yes', + 'values' => { + 'VBE.boot_1' => { + 'counter' => 'happy', + 'family' => 'VBE', + }, + + } + } + +); + +################################ +# Various helper functions # +################################ + +# Translate $_[0] from varnish' internal types (flags) to munin/rrd +# variants (e.g: from 'i' to GAUGE). Returns the result. +sub translate_type +{ + my $d = $_[0]; + if ($d eq "i" or $d eq "g") { + $d = "GAUGE"; + } elsif ($d eq "a" or $d eq "c") { + $d = "DERIVE"; + } + return $d; +} + +# Print the value of a two-dimensional hash if it exist. +# Returns false if non-existent. +# +# Output is formatted for plugins if arg4 is blank, otherwise arg4 is used +# as the title/name of the field (ie: arg4=graph_title). +sub print_if_exist +{ + my %values = %{$_[0]}; + my $value = $_[1]; + my $field = $_[2]; + my $pvalue = normalize_name($value); + + my $title = "$pvalue.$field"; + if (defined($_[3])) { + $title = $_[3]; + } + if (defined($values{$value}{$field})) { + print "$title $values{$value}{$field}\n"; + } else { + return 0; + } +} + +# Create a output-friendly name +sub normalize_name +{ + my $name = $_[0]; + $name =~ s/[^a-zA-Z0-9]/_/g; + return $name; +} + +# Braindead RPN: +,-,/,* will pop two items from @stack, and perform +# the relevant operation on the items. If the item in the array isn't one +# of the 4 basic math operations, a value from varnishstat is pushed on to +# the stack. IE: 'client_req','client_conn','/' will leave the value of +# "client_req/client_conn" on the stack. +# +# If only one item is left on the stack, it is printed. Otherwise, an error +# message is printed. +sub rpn +{ + my @stack; + my $left; + my $right; + foreach my $item (@{$_[0]}) { + if ($item eq "+") { + $right = pop(@stack); + $left = pop(@stack); + push(@stack,$left+$right); + } elsif ($item eq "-") { + $right = pop(@stack); + $left = pop(@stack); + push(@stack,$left-$right); + } elsif ($item eq "/") { + $right = pop(@stack); + $left = pop(@stack); + push(@stack,$left/$right); + } elsif ($item eq "*") { + $right = pop(@stack); + $left = pop(@stack); + push(@stack,$left*$right); + } else { + push(@stack,int($data{$item}{'value'})); + } + } + if (@stack > 1) + { + print STDERR "RPN error: Stack has more than one item left.\n"; + print STDERR "@stack\n"; + exit 255; + } + print "@stack"; + print "\n"; +} + +# Bail-function. +sub usage +{ + if (@_) { + print STDERR "@_" . "\n\n"; + } + print STDERR "Known arguments: suggest, config, autoconf.\n"; + print STDERR "Run with suggest to get a list of known aspects.\n"; + exit 1; +} + +################################ +# XML Parsing # +################################ +# The following code is for parsing varnishstat -x. While %data should be +# stable, the following bits can easily be replaced with anything (json, an +# other xml-parser, magic, etc) +# +# The basic concept is simple enough. Only worry about stuff inside +# . Updating %state on each new data field, and commit it to %data +# when is seen. +# +# We do use translate_type() on the 'flag' field. + + +# Internal state for the XML parsing +my %state = ( + 'stat' => 0, # inside or not + 'field' => 'none', # , , , etc. +); + +# Reset the state of XML, mainly used for end-elements. +sub xml_reset_state() { + $state{'stat'} = '0'; + $state{'field'} = 'none'; + $state{'values'} = (); +} + +# Callback for data entry. Cleans leading whitespace and updates state. +sub xml_characters { + my $d = $_[1]; + if ($state{'stat'} == 0) { + return; + } + if ($state{'field'} eq "type" && $d eq "MAIN") { + return; + } + $d =~ s/^\s*$//g; + if ($d eq "") { + return; + } + $state{'values'}{$state{'field'}} = $d; +} + +# Store the current state in %data. Issued at +# Note that 'flag' is translated to RRD-equivalents here. +sub xml_commit_state +{ + my $configtype = $ASPECTS{$self}{'type'}; + my $name = $state{'values'}{'name'}; + my @namelist = split(/\./,$name); + my $type = shift @namelist; + if ($configtype eq '' || $configtype eq $type) { + my $name = ""; + my $ident = ""; + if (scalar(@namelist) == 1) { + $name = shift @namelist; + } elsif (scalar(@namelist) == 2) { + $ident = shift @namelist; + $name = shift @namelist; + } else { + $name = pop @namelist; + $ident = join('_', @namelist); + } + foreach my $key (keys %{$state{'values'}}) { + my $data = $state{'values'}{$key}; + if ($key eq 'flag') { + $data = translate_type($data); + } + if (defined($type) and $type ne '' and defined($ident) and $ident ne '') { + $data{$type}{$ident}{$name}{$key} = $data; + } else { + $data{$name}{$key} = $data; + } + } + } +} + +# Callback for end tag. E.g: +sub xml_end_elem { + my $element = $_[1]; + if ($element ne "stat") { + return; + } + + xml_commit_state(); + xml_reset_state(); +} + +# Callback for opening tag. E.g: +sub xml_start_elem { + $state{'field'} = $_[1]; + if ($state{'field'} eq "stat") { + $state{'stat'} = 1; + } +} + +################################ +# Internal API # +################################ + + +# Populate %data, includes both values and descriptions and more. +# Currently driven by XML, but that could change. +sub populate_stats +{ + my $arg = "-x"; + my $parser = new XML::Parser(Handlers => {Start => \&xml_start_elem, + End => \&xml_end_elem, + Char => \&xml_characters} ); + + if ($varnishname) { + $arg .= " -n $varnishname"; + } + + open (XMLDATA, "$varnishstatexec $arg|") or die "meh"; + $parser->parse(*XMLDATA, ProtocolEncoding => 'ISO-8859-1'); + close(XMLDATA); +} + +# Prints the fields in the list in $_[2] (e.g: 'value'/'description') for +# each identity of the varnish counter/family combination as defined by +# the $_[0]-counter on the aspect definition. Err, that's jibberish, so +# an example: +# +# e.g: dynamic_print('SMA_1','',('value')) +# e.g: dynamic_print('SMA_2','.label',('ident','description')) +# SMA_1 is the counter-value. If it is a dynamic counter, it has a counter +# and family-member (e.g: counter: c_req, family: SMA) and print_dynamic +# will print c_req for each SMA-identity. +# +# Note that the variables to print is a list. This is to allow printing a +# single item with multiple fields. Typically for identity+description so +# you can distinguish between different data points. +# +# Returns true if it was a dynamic counter. +sub print_dynamic +{ + my $name = $_[0]; + shift; + my $suffix = $_[0]; + shift; + my @field = @_; + if (!defined($ASPECTS{$self}{'values'}{$name}{'counter'})) { + return 0; + } + if (!defined($ASPECTS{$self}{'values'}{$name}{'family'})) { + return 0; + } + my $counter = $ASPECTS{$self}{'values'}{$name}{'counter'}; + my $type = $ASPECTS{$self}{'values'}{$name}{'family'}; + + foreach my $key (keys %{$data{$type}}) { + my $pname = normalize_name($type . "_" . $key . "_" . $counter); + print $pname . $suffix . " "; + my $i = 0; + foreach my $f (@field) { + if ($i != 0) { + print " "; + } + $i += 1; + print $data{$type}{$key}{$counter}{$f}; + } + print "\n"; + } + return 1; +} + +# Read and verify the aspect ($self). +sub set_aspect +{ + $self = $0; + $self =~ s/^.*\/varnish[0-9]?_//; + return if defined($ASPECTS{$self}); + # remove instance name and try again + $self =~ s/^.*?_//; + if (!defined($ASPECTS{$self}) && @ARGV == 0) { + usage "No such aspect"; + } +} + +# Print 'yes' if it's reasonable to use this plugin, or 'no' with a +# human-readable error message. Always exit true, even if the response +# is 'no'. +sub autoconf +{ + # XXX: Solaris outputs errors to stderr and always returns true. + # XXX: See #873 + if (`which $varnishstatexec 2>/dev/null` =~ m{^/}) { + print "yes\n"; + } else { + print "no ($varnishstatexec could not be found)\n"; + } + exit 0; +} + +# Suggest relevant aspects/values of $self. +# 'DEBUG'-graphs are excluded. +sub suggest +{ + foreach my $key (keys %ASPECTS) { + if (defined($ASPECTS{$key}{'DEBUG'}) && $FULL_SUGGEST != 1) { + next; + } + print "$key\n"; + } +} + +# Walk through the relevant aspect and print all top-level configuration +# values and value-definitions. +sub get_config +{ + my $graph = $_[0]; + + # Need to double-check since set_aspect only checks this if there + # is no argument (suggest/autoconf doesn't require a valid aspect) + if (!defined($ASPECTS{$graph})) { + usage "No such aspect"; + } + my %values = %{$ASPECTS{$graph}{'values'}}; + + print "graph_category varnish\n"; + foreach my $field (@graph_parameters) { + print_if_exist(\%ASPECTS,$graph,$field,"graph_$field"); + } + + foreach my $value (sort keys %values) { + # Just print the description/type if it's a dynamic + # counter. It'll be silent if it isn't. + if(print_dynamic($value,'.label',('description','type','ident'))) { + print_dynamic($value,'.type',('flag')); + next; + } + + # Need either RPN definition or a varnishstat value. + if (!defined($data{$value}{'value'}) && + !defined($values{$value}{'rpn'})) { + if ($DEBUG) { + print STDERR "ERROR: $value not part of varnishstat.\n" + } + next; + } + + if (!print_if_exist(\%values,$value,'label')) { + my $pvalue = normalize_name($value); + print "$pvalue.label $data{$value}{'description'}\n"; + } + foreach my $field (@field_parameters) { + print_if_exist(\%values,$value,$field); + } + } +} + +# Handle arguments (config, autoconf, suggest) +# Populate stats for config is necessary, but we want to avoid it for +# autoconf as it would generate a nasty error. +sub check_args +{ + if (@ARGV && $ARGV[0] eq '') { + shift @ARGV; + } + if (@ARGV == 1) { + if ($ARGV[0] eq "config") { + populate_stats; + get_config($self); + exit 0; + } elsif ($ARGV[0] eq "autoconf") { + autoconf($self); + exit 0; + } elsif ($ARGV[0] eq "suggest") { + suggest; + exit 0; + } + usage "Unknown argument"; + } +} + +################################ +# Execution starts here # +################################ + +set_aspect; +check_args; +populate_stats; + +# We only get here if we're supposed to. +# Walks through the relevant values and either prints the varnishstat, or +# if the 'rpn' variable is set, calls rpn() to execute ... the rpn. +# +# NOTE: Due to differences in varnish-versions, this checks if the value +# actually exist before using it. +foreach my $value (keys %{$ASPECTS{$self}{'values'}}) { + my $pvalue = normalize_name($value); + if (defined($ASPECTS{$self}{'values'}{$value}{'rpn'})) { + print "$pvalue.value "; + rpn($ASPECTS{$self}{'values'}{$value}{'rpn'}); + } else { + if (print_dynamic($value,'.value',('value'))) { + next; + } + + if (!defined($data{$value}{'value'})) { + if ($DEBUG) { + print STDERR "Error: $value not part of " + . "varnishstat.\n"; + } + next; + } + print "$pvalue.value "; + print "$data{$value}{'value'}\n"; + } +} diff --git a/varnish/tasks/munin.yml b/varnish/tasks/munin.yml index 1d58aee6..1ccf5f88 100644 --- a/varnish/tasks/munin.yml +++ b/varnish/tasks/munin.yml @@ -22,29 +22,38 @@ mode: "0755" tags: varnish -- name: Copy varnish4 munin plugin +- name: Copy varnish5 munin plugin copy: - src: munin/varnish4_ + src: munin/varnish5_ dest: /usr/local/share/munin/plugins/ mode: "0755" notify: restart munin-node tags: varnish -- name: Enable varnish4 munin plugin +- name: Enable varnish5 munin plugin file: - src: /usr/local/share/munin/plugins/varnish4_ - dest: "/etc/munin/plugins/varnish4_{{item}}" + src: /usr/local/share/munin/plugins/varnish5_ + dest: "/etc/munin/plugins/varnish5_{{item}}" state: link loop: - - backend_traffic - - bad - - expunge - - hit_rate - memory_usage + - expunge - objects - request_rate + - mgt_uptime - threads + - backend_traffic + - hit_rate + - main_uptime - transfer_rates - - uptime + - bad + notify: restart munin-node + tags: varnish + +- name: Copy varnish5 munin plugin config + copy: + src: munin/varnish5.conf + dest: /etc/munin/plugin-conf.d/varnish5 + mode: "0644" notify: restart munin-node tags: varnish