metricbeat: allow using a template

CHANGELOG.md

@@ -20,6 +20,7 @@ The **patch** part changes incrementally at each release.
 * haproxy: preconfigure SSL with defaults
 * lxc-php: Install php-sqlite by default
 * lxc-php: Don't disable putenv() by default in PHP settings
+* metricbeat: allow using a template
 * mysql: activate binary logs by specifying log_bin path
 * mysql: specify a custom server_id
 * mysql: option to define as read only

metricbeat/README.md

@@ -1,4 +1,4 @@
-# filebeat
+# metricbeat
 
 Install Metricbeat.
 

metricbeat/defaults/main.yml

@@ -1,10 +1,25 @@
 ---
 elastic_stack_version: "6.x"
 
-metricbeat_elasticsearch_protocol: ""
 metricbeat_elasticsearch_hosts:
   - "localhost:9200"
+metricbeat_elasticsearch_protocol: ""
+metricbeat_elasticsearch_auth_api_key: ""
 metricbeat_elasticsearch_auth_username: ""
 metricbeat_elasticsearch_auth_password: ""
 
 metricbeat_processors_cloud_metadata: False
+
+metricbeat_use_config_template: False
+metricbeat_update_config: True
+metricbeat_force_config: True
+
+# Example :
+# metricbeat_tags:
+#   - "service-X"
+#   - "web-tier"
+metricbeat_tags: Null
+# Example :
+# metricbeat_fields:
+#   - "env: staging"
+metricbeat_fields: Null

metricbeat/tasks/main.yml

@@ -40,51 +40,79 @@
     name: metricbeat
     enabled: yes
 
-- name: Metricbeat knows where to find Elasticsearch
-  lineinfile:
-    dest: /etc/metricbeat/metricbeat.yml
-    regexp: '^  hosts: .*'
-    line: "  hosts: [\"{{ metricbeat_elasticsearch_hosts | join('\", \"') }}\"]"
-    insertafter: "output.elasticsearch:"
-  notify: restart metricbeat
-  when:
-    - metricbeat_elasticsearch_hosts
+# When we don't use a config template (default)
+- block:
+  - name: Metricbeat knows where to find Elasticsearch
+    lineinfile:
+      dest: /etc/metricbeat/metricbeat.yml
+      regexp: '^  hosts: .*'
+      line: "  hosts: [\"{{ metricbeat_elasticsearch_hosts | join('\", \"') }}\"]"
+      insertafter: "output.elasticsearch:"
+    notify: restart metricbeat
+    when:
+      - metricbeat_elasticsearch_hosts
 
-- name: Metricbeat protocol for Elasticsearch
-  lineinfile:
-    dest: /etc/metricbeat/metricbeat.yml
-    regexp: '^  #?protocol: .*'
-    line: "  protocol: \"{{ metricbeat_elasticsearch_protocol }}\""
-    insertafter: "output.elasticsearch:"
-  notify: restart metricbeat
-  when: metricbeat_elasticsearch_protocol == "http" or metricbeat_elasticsearch_protocol == "https"
+  - name: Metricbeat protocol for Elasticsearch
+    lineinfile:
+      dest: /etc/metricbeat/metricbeat.yml
+      regexp: '^  #?protocol: .*'
+      line: "  protocol: \"{{ metricbeat_elasticsearch_protocol }}\""
+      insertafter: "output.elasticsearch:"
+    notify: restart metricbeat
+    when: metricbeat_elasticsearch_protocol == "http" or metricbeat_elasticsearch_protocol == "https"
 
-- name: Metricbeat auth/username for Elasticsearch are configured
-  lineinfile:
-    dest: /etc/metricbeat/metricbeat.yml
-    regexp: '{{ item.regexp }}'
-    line: '{{ item.line }}'
-    insertafter: "output.elasticsearch:"
-  with_items:
-    - { regexp: '^  #?username: .*', line: '  username: "{{ metricbeat_elasticsearch_auth_username }}"' }
-    - { regexp: '^  #?password: .*', line: '  password: "{{ metricbeat_elasticsearch_auth_password }}"' }
-  notify: restart metricbeat
-  when:
-    - metricbeat_elasticsearch_auth_username != ""
-    - metricbeat_elasticsearch_auth_password != ""
+  - name: Metricbeat auth/username for Elasticsearch are configured
+    lineinfile:
+      dest: /etc/metricbeat/metricbeat.yml
+      regexp: '{{ item.regexp }}'
+      line: '{{ item.line }}'
+      insertafter: "output.elasticsearch:"
+    with_items:
+      - { regexp: '^  #?username: .*', line: '  username: "{{ metricbeat_elasticsearch_auth_username }}"' }
+      - { regexp: '^  #?password: .*', line: '  password: "{{ metricbeat_elasticsearch_auth_password }}"' }
+    notify: restart metricbeat
+    when:
+      - metricbeat_elasticsearch_auth_username
+      - metricbeat_elasticsearch_auth_password
 
-- name: disable cloud_metadata
-  replace:
-    dest: /etc/metricbeat/metricbeat.yml
-    regexp: '^(\s+)(- add_cloud_metadata:)'
-    replace: '\1# \2'
-  notify: restart metricbeat
-  when: not metricbeat_processors_cloud_metadata
+  - name: Metricbeat api_key for Elasticsearch are configured
+    lineinfile:
+      dest: /etc/metricbeat/metricbeat.yml
+      regexp: '^  #?api_key: .*'
+      line: '  api_key: "{{ metricbeat_elasticsearch_auth_api_key }}"'
+      insertafter: "output.elasticsearch:"
+    notify: restart metricbeat
+    when: metricbeat_elasticsearch_auth_api_key
 
-- name: cloud_metadata processor is disabled
-  lineinfile:
-    dest: /etc/metricbeat/metricbeat.yml
-    line: "  - add_cloud_metadata: ~"
-    insert_after: '^processors:'
-  notify: restart metricbeat
-  when: metricbeat_processors_cloud_metadata
+  - name: disable cloud_metadata
+    replace:
+      dest: /etc/metricbeat/metricbeat.yml
+      regexp: '^(\s+)(- add_cloud_metadata:)'
+      replace: '\1# \2'
+    notify: restart metricbeat
+    when: not metricbeat_processors_cloud_metadata
+
+  - name: cloud_metadata processor is disabled
+    lineinfile:
+      dest: /etc/metricbeat/metricbeat.yml
+      line: "  - add_cloud_metadata: ~"
+      insert_after: '^processors:'
+    notify: restart metricbeat
+    when: metricbeat_processors_cloud_metadata
+  when: not metricbeat_use_config_template
+
+# When we use a config template
+- block:
+  - name: Configuration is up-to-date
+    template:
+      src: "{{ item }}"
+      dest: /etc/metricbeat/metricbeat.yml
+      force: "{{ metricbeat_force_config }}"
+    with_first_found:
+      - "templates/metricbeat/metricbeat.{{ inventory_hostname }}.yml.j2"
+      - "templates/metricbeat/metricbeat.{{ host_group }}.yml.j2"
+      - "templates/metricbeat/metricbeat.default.yml.j2"
+      - "metricbeat.default.yml.j2"
+    notify: restart metricbeat
+    when: metricbeat_update_config
+  when: metricbeat_use_config_template

metricbeat/templates/metricbeat.default.yml.j2

@@ -0,0 +1,180 @@
+###################### Metricbeat Configuration Example #######################
+
+# This file is an example configuration file highlighting only the most common
+# options. The metricbeat.reference.yml file from the same directory contains all the
+# supported options with more comments. You can use it as a reference.
+#
+# You can find the full configuration reference here:
+# https://www.elastic.co/guide/en/beats/metricbeat/index.html
+
+# =========================== Modules configuration ============================
+
+metricbeat.config.modules:
+  # Glob pattern for configuration loading
+  path: ${path.config}/modules.d/*.yml
+
+  # Set to true to enable config reloading
+  reload.enabled: false
+
+  # Period on which files under path should be checked for changes
+  #reload.period: 10s
+
+# ======================= Elasticsearch template setting =======================
+
+setup.template.settings:
+  index.number_of_shards: 1
+  index.codec: best_compression
+  #_source.enabled: false
+
+
+# ================================== General ===================================
+
+# The name of the shipper that publishes the network data. It can be used to group
+# all the transactions sent by a single shipper in the web interface.
+#name:
+
+# The tags of the shipper are included in their own field with each
+# transaction published.
+{% if metricbeat_tags %}
+tags: ["{{ metricbeat_tags | join('", "') }}"]
+{% endif %}
+
+# Optional fields that you can specify to add additional information to the
+# output.
+{% if metricbeat_fields %}
+fields:
+{% for field in metricbeat_fields %}
+  {{ field }}
+{% endfor %}
+{% endif %}
+
+# ================================= Dashboards =================================
+# These settings control loading the sample dashboards to the Kibana index. Loading
+# the dashboards is disabled by default and can be enabled either by setting the
+# options here or by using the `setup` command.
+#setup.dashboards.enabled: false
+
+# The URL from where to download the dashboards archive. By default this URL
+# has a value which is computed based on the Beat name and version. For released
+# versions, this URL points to the dashboard archive on the artifacts.elastic.co
+# website.
+#setup.dashboards.url:
+
+# =================================== Kibana ===================================
+
+# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.
+# This requires a Kibana endpoint configuration.
+setup.kibana:
+
+  # Kibana Host
+  # Scheme and port can be left out and will be set to the default (http and 5601)
+  # In case you specify and additional path, the scheme is required: http://localhost:5601/path
+  # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
+  #host: "localhost:5601"
+
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
+# =============================== Elastic Cloud ================================
+
+# These settings simplify using Metricbeat with the Elastic Cloud (https://cloud.elastic.co/).
+
+# The cloud.id setting overwrites the `output.elasticsearch.hosts` and
+# `setup.kibana.host` options.
+# You can find the `cloud.id` in the Elastic Cloud web UI.
+#cloud.id:
+
+# The cloud.auth setting overwrites the `output.elasticsearch.username` and
+# `output.elasticsearch.password` settings. The format is `<user>:<pass>`.
+#cloud.auth:
+
+# ================================== Outputs ===================================
+
+# Configure what output to use when sending the data collected by the beat.
+
+# ---------------------------- Elasticsearch Output ----------------------------
+output.elasticsearch:
+  # Array of hosts to connect to.
+  hosts: ["{{ metricbeat_elasticsearch_hosts | join('", "') }}"]
+
+  # Protocol - either `http` (default) or `https`.
+  protocol: "{{ metricbeat_elasticsearch_protocol | default('http') }}"
+
+  # Authentication credentials - either API key or username/password.
+{% if metricbeat_elasticsearch_auth_api_key %}
+  api_key: "{{ metricbeat_elasticsearch_auth_api_key }}"
+{% endif %}
+{% if metricbeat_elasticsearch_auth_username %}
+  username: "{{ metricbeat_elasticsearch_auth_username }}"
+{% endif %}
+{% if metricbeat_elasticsearch_auth_password %}
+  password: "{{ metricbeat_elasticsearch_auth_password }}"
+{% endif %}
+
+# ------------------------------ Logstash Output -------------------------------
+#output.logstash:
+  # The Logstash hosts
+  #hosts: ["localhost:5044"]
+
+  # Optional SSL. By default is off.
+  # List of root certificates for HTTPS server verifications
+  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+  # Certificate for SSL client authentication
+  #ssl.certificate: "/etc/pki/client/cert.pem"
+
+  # Client Certificate Key
+  #ssl.key: "/etc/pki/client/cert.key"
+
+# ================================= Processors =================================
+
+# Configure processors to enhance or manipulate events generated by the beat.
+
+processors:
+  - add_host_metadata: ~
+{% if metricbeat_processors_cloud_metadata %}
+  - add_cloud_metadata: ~
+{% endif %}
+  - add_docker_metadata: ~
+  - add_kubernetes_metadata: ~
+
+
+# ================================== Logging ===================================
+
+# Sets log level. The default log level is info.
+# Available log levels are: error, warning, info, debug
+#logging.level: debug
+
+# At debug level, you can selectively enable logging only for some components.
+# To enable all selectors use ["*"]. Examples of other selectors are "beat",
+# "publish", "service".
+#logging.selectors: ["*"]
+
+# ============================= X-Pack Monitoring ==============================
+# Metricbeat can export internal metrics to a central Elasticsearch monitoring
+# cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The
+# reporting is disabled by default.
+
+# Set to true to enable the monitoring reporter.
+#monitoring.enabled: false
+
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Metricbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.cluster_uuid:
+
+# Uncomment to send the metrics to Elasticsearch. Most settings from the
+# Elasticsearch output are accepted here as well.
+# Note that the settings should point to your Elasticsearch *monitoring* cluster.
+# Any setting that is not set is automatically inherited from the Elasticsearch
+# output configuration, so if you have the Elasticsearch output configured such
+# that it is pointing to your Elasticsearch monitoring cluster, you can simply
+# uncomment the following line.
+#monitoring.elasticsearch:
+
+# ================================= Migration ==================================
+
+# This allows to enable 6.7 migration aliases
+#migration.6_to_7.enabled: true