From 23cd57e9aa4951d33a229990e84a2ecbdaea6485 Mon Sep 17 00:00:00 2001 From: Jeremy Lecour Date: Tue, 18 Aug 2020 14:01:09 +0200 Subject: [PATCH] metricbeat: allow using a template --- CHANGELOG.md | 1 + metricbeat/README.md | 2 +- metricbeat/defaults/main.yml | 17 +- metricbeat/tasks/main.yml | 116 ++++++----- .../templates/metricbeat.default.yml.j2 | 180 ++++++++++++++++++ 5 files changed, 270 insertions(+), 46 deletions(-) create mode 100644 metricbeat/templates/metricbeat.default.yml.j2 diff --git a/CHANGELOG.md b/CHANGELOG.md index 874df943..b1c38ed1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The **patch** part changes incrementally at each release. * haproxy: preconfigure SSL with defaults * lxc-php: Install php-sqlite by default * lxc-php: Don't disable putenv() by default in PHP settings +* metricbeat: allow using a template * mysql: activate binary logs by specifying log_bin path * mysql: specify a custom server_id * mysql: option to define as read only diff --git a/metricbeat/README.md b/metricbeat/README.md index 443e1107..ad348053 100644 --- a/metricbeat/README.md +++ b/metricbeat/README.md @@ -1,4 +1,4 @@ -# filebeat +# metricbeat Install Metricbeat. diff --git a/metricbeat/defaults/main.yml b/metricbeat/defaults/main.yml index df2d9649..51113271 100644 --- a/metricbeat/defaults/main.yml +++ b/metricbeat/defaults/main.yml @@ -1,10 +1,25 @@ --- elastic_stack_version: "6.x" -metricbeat_elasticsearch_protocol: "" metricbeat_elasticsearch_hosts: - "localhost:9200" +metricbeat_elasticsearch_protocol: "" +metricbeat_elasticsearch_auth_api_key: "" metricbeat_elasticsearch_auth_username: "" metricbeat_elasticsearch_auth_password: "" metricbeat_processors_cloud_metadata: False + +metricbeat_use_config_template: False +metricbeat_update_config: True +metricbeat_force_config: True + +# Example : +# metricbeat_tags: +# - "service-X" +# - "web-tier" +metricbeat_tags: Null +# Example : +# metricbeat_fields: +# - "env: staging" +metricbeat_fields: Null diff --git a/metricbeat/tasks/main.yml b/metricbeat/tasks/main.yml index 535037ec..1ae3a2cd 100644 --- a/metricbeat/tasks/main.yml +++ b/metricbeat/tasks/main.yml @@ -40,51 +40,79 @@ name: metricbeat enabled: yes -- name: Metricbeat knows where to find Elasticsearch - lineinfile: - dest: /etc/metricbeat/metricbeat.yml - regexp: '^ hosts: .*' - line: " hosts: [\"{{ metricbeat_elasticsearch_hosts | join('\", \"') }}\"]" - insertafter: "output.elasticsearch:" - notify: restart metricbeat - when: - - metricbeat_elasticsearch_hosts +# When we don't use a config template (default) +- block: + - name: Metricbeat knows where to find Elasticsearch + lineinfile: + dest: /etc/metricbeat/metricbeat.yml + regexp: '^ hosts: .*' + line: " hosts: [\"{{ metricbeat_elasticsearch_hosts | join('\", \"') }}\"]" + insertafter: "output.elasticsearch:" + notify: restart metricbeat + when: + - metricbeat_elasticsearch_hosts -- name: Metricbeat protocol for Elasticsearch - lineinfile: - dest: /etc/metricbeat/metricbeat.yml - regexp: '^ #?protocol: .*' - line: " protocol: \"{{ metricbeat_elasticsearch_protocol }}\"" - insertafter: "output.elasticsearch:" - notify: restart metricbeat - when: metricbeat_elasticsearch_protocol == "http" or metricbeat_elasticsearch_protocol == "https" + - name: Metricbeat protocol for Elasticsearch + lineinfile: + dest: /etc/metricbeat/metricbeat.yml + regexp: '^ #?protocol: .*' + line: " protocol: \"{{ metricbeat_elasticsearch_protocol }}\"" + insertafter: "output.elasticsearch:" + notify: restart metricbeat + when: metricbeat_elasticsearch_protocol == "http" or metricbeat_elasticsearch_protocol == "https" -- name: Metricbeat auth/username for Elasticsearch are configured - lineinfile: - dest: /etc/metricbeat/metricbeat.yml - regexp: '{{ item.regexp }}' - line: '{{ item.line }}' - insertafter: "output.elasticsearch:" - with_items: - - { regexp: '^ #?username: .*', line: ' username: "{{ metricbeat_elasticsearch_auth_username }}"' } - - { regexp: '^ #?password: .*', line: ' password: "{{ metricbeat_elasticsearch_auth_password }}"' } - notify: restart metricbeat - when: - - metricbeat_elasticsearch_auth_username != "" - - metricbeat_elasticsearch_auth_password != "" + - name: Metricbeat auth/username for Elasticsearch are configured + lineinfile: + dest: /etc/metricbeat/metricbeat.yml + regexp: '{{ item.regexp }}' + line: '{{ item.line }}' + insertafter: "output.elasticsearch:" + with_items: + - { regexp: '^ #?username: .*', line: ' username: "{{ metricbeat_elasticsearch_auth_username }}"' } + - { regexp: '^ #?password: .*', line: ' password: "{{ metricbeat_elasticsearch_auth_password }}"' } + notify: restart metricbeat + when: + - metricbeat_elasticsearch_auth_username + - metricbeat_elasticsearch_auth_password -- name: disable cloud_metadata - replace: - dest: /etc/metricbeat/metricbeat.yml - regexp: '^(\s+)(- add_cloud_metadata:)' - replace: '\1# \2' - notify: restart metricbeat - when: not metricbeat_processors_cloud_metadata + - name: Metricbeat api_key for Elasticsearch are configured + lineinfile: + dest: /etc/metricbeat/metricbeat.yml + regexp: '^ #?api_key: .*' + line: ' api_key: "{{ metricbeat_elasticsearch_auth_api_key }}"' + insertafter: "output.elasticsearch:" + notify: restart metricbeat + when: metricbeat_elasticsearch_auth_api_key -- name: cloud_metadata processor is disabled - lineinfile: - dest: /etc/metricbeat/metricbeat.yml - line: " - add_cloud_metadata: ~" - insert_after: '^processors:' - notify: restart metricbeat - when: metricbeat_processors_cloud_metadata + - name: disable cloud_metadata + replace: + dest: /etc/metricbeat/metricbeat.yml + regexp: '^(\s+)(- add_cloud_metadata:)' + replace: '\1# \2' + notify: restart metricbeat + when: not metricbeat_processors_cloud_metadata + + - name: cloud_metadata processor is disabled + lineinfile: + dest: /etc/metricbeat/metricbeat.yml + line: " - add_cloud_metadata: ~" + insert_after: '^processors:' + notify: restart metricbeat + when: metricbeat_processors_cloud_metadata + when: not metricbeat_use_config_template + +# When we use a config template +- block: + - name: Configuration is up-to-date + template: + src: "{{ item }}" + dest: /etc/metricbeat/metricbeat.yml + force: "{{ metricbeat_force_config }}" + with_first_found: + - "templates/metricbeat/metricbeat.{{ inventory_hostname }}.yml.j2" + - "templates/metricbeat/metricbeat.{{ host_group }}.yml.j2" + - "templates/metricbeat/metricbeat.default.yml.j2" + - "metricbeat.default.yml.j2" + notify: restart metricbeat + when: metricbeat_update_config + when: metricbeat_use_config_template diff --git a/metricbeat/templates/metricbeat.default.yml.j2 b/metricbeat/templates/metricbeat.default.yml.j2 new file mode 100644 index 00000000..bd88cdc6 --- /dev/null +++ b/metricbeat/templates/metricbeat.default.yml.j2 @@ -0,0 +1,180 @@ +###################### Metricbeat Configuration Example ####################### + +# This file is an example configuration file highlighting only the most common +# options. The metricbeat.reference.yml file from the same directory contains all the +# supported options with more comments. You can use it as a reference. +# +# You can find the full configuration reference here: +# https://www.elastic.co/guide/en/beats/metricbeat/index.html + +# =========================== Modules configuration ============================ + +metricbeat.config.modules: + # Glob pattern for configuration loading + path: ${path.config}/modules.d/*.yml + + # Set to true to enable config reloading + reload.enabled: false + + # Period on which files under path should be checked for changes + #reload.period: 10s + +# ======================= Elasticsearch template setting ======================= + +setup.template.settings: + index.number_of_shards: 1 + index.codec: best_compression + #_source.enabled: false + + +# ================================== General =================================== + +# The name of the shipper that publishes the network data. It can be used to group +# all the transactions sent by a single shipper in the web interface. +#name: + +# The tags of the shipper are included in their own field with each +# transaction published. +{% if metricbeat_tags %} +tags: ["{{ metricbeat_tags | join('", "') }}"] +{% endif %} + +# Optional fields that you can specify to add additional information to the +# output. +{% if metricbeat_fields %} +fields: +{% for field in metricbeat_fields %} + {{ field }} +{% endfor %} +{% endif %} + +# ================================= Dashboards ================================= +# These settings control loading the sample dashboards to the Kibana index. Loading +# the dashboards is disabled by default and can be enabled either by setting the +# options here or by using the `setup` command. +#setup.dashboards.enabled: false + +# The URL from where to download the dashboards archive. By default this URL +# has a value which is computed based on the Beat name and version. For released +# versions, this URL points to the dashboard archive on the artifacts.elastic.co +# website. +#setup.dashboards.url: + +# =================================== Kibana =================================== + +# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API. +# This requires a Kibana endpoint configuration. +setup.kibana: + + # Kibana Host + # Scheme and port can be left out and will be set to the default (http and 5601) + # In case you specify and additional path, the scheme is required: http://localhost:5601/path + # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601 + #host: "localhost:5601" + + # Kibana Space ID + # ID of the Kibana Space into which the dashboards should be loaded. By default, + # the Default Space will be used. + #space.id: + +# =============================== Elastic Cloud ================================ + +# These settings simplify using Metricbeat with the Elastic Cloud (https://cloud.elastic.co/). + +# The cloud.id setting overwrites the `output.elasticsearch.hosts` and +# `setup.kibana.host` options. +# You can find the `cloud.id` in the Elastic Cloud web UI. +#cloud.id: + +# The cloud.auth setting overwrites the `output.elasticsearch.username` and +# `output.elasticsearch.password` settings. The format is `:`. +#cloud.auth: + +# ================================== Outputs =================================== + +# Configure what output to use when sending the data collected by the beat. + +# ---------------------------- Elasticsearch Output ---------------------------- +output.elasticsearch: + # Array of hosts to connect to. + hosts: ["{{ metricbeat_elasticsearch_hosts | join('", "') }}"] + + # Protocol - either `http` (default) or `https`. + protocol: "{{ metricbeat_elasticsearch_protocol | default('http') }}" + + # Authentication credentials - either API key or username/password. +{% if metricbeat_elasticsearch_auth_api_key %} + api_key: "{{ metricbeat_elasticsearch_auth_api_key }}" +{% endif %} +{% if metricbeat_elasticsearch_auth_username %} + username: "{{ metricbeat_elasticsearch_auth_username }}" +{% endif %} +{% if metricbeat_elasticsearch_auth_password %} + password: "{{ metricbeat_elasticsearch_auth_password }}" +{% endif %} + +# ------------------------------ Logstash Output ------------------------------- +#output.logstash: + # The Logstash hosts + #hosts: ["localhost:5044"] + + # Optional SSL. By default is off. + # List of root certificates for HTTPS server verifications + #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"] + + # Certificate for SSL client authentication + #ssl.certificate: "/etc/pki/client/cert.pem" + + # Client Certificate Key + #ssl.key: "/etc/pki/client/cert.key" + +# ================================= Processors ================================= + +# Configure processors to enhance or manipulate events generated by the beat. + +processors: + - add_host_metadata: ~ +{% if metricbeat_processors_cloud_metadata %} + - add_cloud_metadata: ~ +{% endif %} + - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ + + +# ================================== Logging =================================== + +# Sets log level. The default log level is info. +# Available log levels are: error, warning, info, debug +#logging.level: debug + +# At debug level, you can selectively enable logging only for some components. +# To enable all selectors use ["*"]. Examples of other selectors are "beat", +# "publish", "service". +#logging.selectors: ["*"] + +# ============================= X-Pack Monitoring ============================== +# Metricbeat can export internal metrics to a central Elasticsearch monitoring +# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The +# reporting is disabled by default. + +# Set to true to enable the monitoring reporter. +#monitoring.enabled: false + +# Sets the UUID of the Elasticsearch cluster under which monitoring data for this +# Metricbeat instance will appear in the Stack Monitoring UI. If output.elasticsearch +# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch. +#monitoring.cluster_uuid: + +# Uncomment to send the metrics to Elasticsearch. Most settings from the +# Elasticsearch output are accepted here as well. +# Note that the settings should point to your Elasticsearch *monitoring* cluster. +# Any setting that is not set is automatically inherited from the Elasticsearch +# output configuration, so if you have the Elasticsearch output configured such +# that it is pointing to your Elasticsearch monitoring cluster, you can simply +# uncomment the following line. +#monitoring.elasticsearch: + +# ================================= Migration ================================== + +# This allows to enable 6.7 migration aliases +#migration.6_to_7.enabled: true