From e851b8cbfe3f0d9d569187a4c03d44aa349b9ec8 Mon Sep 17 00:00:00 2001 From: Tristan PILAT Date: Tue, 23 Apr 2019 17:13:56 +0200 Subject: [PATCH] Add fluentd installation role --- fluentd/.kitchen.yml | 28 +++++++ fluentd/README.md | 17 ++++ fluentd/defaults/main.yml | 12 +++ fluentd/handlers/main.yml | 10 +++ fluentd/meta/main.yml | 20 +++++ fluentd/tasks/main.yml | 52 ++++++++++++ fluentd/templates/td-agent.conf.j2 | 128 +++++++++++++++++++++++++++++ 7 files changed, 267 insertions(+) create mode 100644 fluentd/.kitchen.yml create mode 100644 fluentd/README.md create mode 100644 fluentd/defaults/main.yml create mode 100644 fluentd/handlers/main.yml create mode 100644 fluentd/meta/main.yml create mode 100644 fluentd/tasks/main.yml create mode 100644 fluentd/templates/td-agent.conf.j2 diff --git a/fluentd/.kitchen.yml b/fluentd/.kitchen.yml new file mode 100644 index 00000000..b21cc3db --- /dev/null +++ b/fluentd/.kitchen.yml @@ -0,0 +1,28 @@ +--- +driver: + name: docker + privileged: true + use_sudo: false + +provisioner: + name: ansible_playbook + hosts: test-kitchen + roles_path: ../ + ansible_verbose: true + require_ansible_source: false + require_chef_for_busser: false + idempotency_test: true + +platforms: + - name: debian + driver_config: + image: evolix/ansible:2.2.1 + +suites: + - name: default + provisioner: + name: ansible_playbook + playbook: ./tests/test.yml + +transport: + max_ssh_sessions: 6 diff --git a/fluentd/README.md b/fluentd/README.md new file mode 100644 index 00000000..1bf47e7d --- /dev/null +++ b/fluentd/README.md @@ -0,0 +1,17 @@ +# Fluentd + +Installation and basic configuration of Fluentd. + +This role is based on https://docs.fluentd.org/v1.0/articles/install-by-deb + +## Tasks + +Everything is in the `tasks/main.yml` file. + +## Available variables + +Main variables are : + +* `fluentd_conf_path`: config file location ; + +The full list of variables (with default values) can be found in `defaults/main.yml`. diff --git a/fluentd/defaults/main.yml b/fluentd/defaults/main.yml new file mode 100644 index 00000000..c17cb312 --- /dev/null +++ b/fluentd/defaults/main.yml @@ -0,0 +1,12 @@ +--- +fluentd_daemon: td-agent +fluentd_conf_path: /etc/td-agent/td-agent.conf + +fluentd_port: 24230 +fluentd_bind_interface: + +fluentd_host: +fluentd_host_port: + +fluentd_flush_interval: +fluentd_heartbeat_type: diff --git a/fluentd/handlers/main.yml b/fluentd/handlers/main.yml new file mode 100644 index 00000000..2468cef3 --- /dev/null +++ b/fluentd/handlers/main.yml @@ -0,0 +1,10 @@ +--- +- name: restart fluentd + systemd: + name: td-agent + state: restarted + +- name: restart nagios-nrpe-server + service: + name: nagios-nrpe-server + state: restarted diff --git a/fluentd/meta/main.yml b/fluentd/meta/main.yml new file mode 100644 index 00000000..3bbff0c4 --- /dev/null +++ b/fluentd/meta/main.yml @@ -0,0 +1,20 @@ +galaxy_info: + author: Evolix + description: Installation and basic configuration of Fluentd. + + issue_tracker_url: https://gitea.evolix.org/evolix/ansible-roles/issues + + license: GPLv2 + + min_ansible_version: 2.2 + + platforms: + - name: Debian + versions: + - jessie + - stretch + +dependencies: [] + # List your role dependencies here, one per line. + # Be sure to remove the '[]' above if you add dependencies + # to this list. diff --git a/fluentd/tasks/main.yml b/fluentd/tasks/main.yml new file mode 100644 index 00000000..118b78b0 --- /dev/null +++ b/fluentd/tasks/main.yml @@ -0,0 +1,52 @@ +--- + +- name: Fluentd GPG key is installed + apt_key: + url: https://packages.treasuredata.com/GPG-KEY-td-agent + tags: + - packages + - fluentd + +- name: Fluentd sources list is available + apt_repository: + repo: "deb http://packages.treasuredata.com/3/debian/{{ ansible_distribution_release }}/ {{ ansible_distribution_release }} contrib" + filename: treasuredata + update_cache: yes + state: present + tags: + - packages + - fluentd + +- name: Fluentd is installed. + apt: + name: td-agent + state: present + tags: + - fluentd + - packages + +- name: Fluentd is configured. + template: + src: td-agent.conf.j2 + dest: "{{ fluentd_conf_path }}" + mode: "0644" + notify: "restart fluentd" + tags: + - fluentd + +- name: Fluentd is running and enabled on boot. + systemd: + name: td-agent + enabled: yes + state: started + tags: + - fluentd + +- name: NRPE check is configured + lineinfile: + path: /etc/nagios/nrpe.d/evolix.cfg + line: 'command[check_fluentd]=/usr/lib/nagios/plugins/check_tcp -p {{ fluentd_port }}' + notify: "restart nagios-nrpe-server" + tags: + - fluentd + - nrpe diff --git a/fluentd/templates/td-agent.conf.j2 b/fluentd/templates/td-agent.conf.j2 new file mode 100644 index 00000000..23f922fc --- /dev/null +++ b/fluentd/templates/td-agent.conf.j2 @@ -0,0 +1,128 @@ +#### +## Output descriptions: +## + +# Treasure Data (http://www.treasure-data.com/) provides cloud based data +# analytics platform, which easily stores and processes data from td-agent. +# FREE plan is also provided. +# @see http://docs.fluentd.org/articles/http-to-td +# +# This section matches events whose tag is td.DATABASE.TABLE + + @type tdlog + @id output_td + apikey YOUR_API_KEY + + auto_create_table + + @type file + path /var/log/td-agent/buffer/td + + + + @type file + path /var/log/td-agent/failed_records + + + +## match tag=debug.** and dump to console + + @type stdout + @id output_stdout + + +#### +## Source descriptions: +## + +## built-in TCP input +## @see http://docs.fluentd.org/articles/in_forward + + @type forward + @id input_forward + + +## built-in UNIX socket input +# +# type unix +# + +# HTTP input +# POST http://localhost:8888/?json= +# POST http://localhost:8888/td.myapp.login?json={"user"%3A"me"} +# @see http://docs.fluentd.org/articles/in_http + + @type http + @id input_http + port 8888 + + +## live debugging agent + + @type debug_agent + @id input_debug_agent + bind 127.0.0.1 + port 24230 + + +#### +## Examples: +## + +## File input +## read apache logs continuously and tags td.apache.access +# +# @type tail +# @id input_tail +# +# @type apache2 +# +# path /var/log/httpd-access.log +# tag td.apache.access +# + +## File output +## match tag=local.** and write to file +# +# @type file +# @id output_file +# path /var/log/td-agent/access +# + +## Forwarding +## match tag=system.** and forward to another td-agent server +# +# @type forward +# @id output_system_forward +# +# +# host 192.168.0.11 +# +# # secondary host is optional +# +# +# host 192.168.0.12 +# +# +# + +## Multiple output +## match tag=td.*.* and output to Treasure Data AND file +# +# @type copy +# @id output_copy +# +# @type tdlog +# apikey API_KEY +# auto_create_table +# +# @type file +# path /var/log/td-agent/buffer/td +# +# +# +# @type file +# path /var/log/td-agent/td-%Y-%m-%d/%H.log +# +# +