From afc33a400265dbf8bc5c3e6c8489a402ddb7ebc3 Mon Sep 17 00:00:00 2001 From: Alexis Ben Miloud--Josselin Date: Fri, 9 Mar 2018 10:39:59 +0100 Subject: [PATCH 1/6] Update elasticsearch snapshot --- zzz_evobackup | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index ab99514..5cbfc97 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -122,13 +122,20 @@ mkdir -p -m 700 /home/backup # cp /var/lib/redis/dump.rdb /home/backup/ ## ElasticSearch : example with rsync (warning: don't forget to use NFS if you have a cluster) +# DATE=$(date +%F) ## Disable ES translog flush # curl -s -XPUT 'localhost:9200/_settings' -d '{"index.translog.disable_flush": true}' >/dev/null -## Flushes translog -# curl -s 'localhost:9200/_flush' | grep -qe '"ok":true' -## If it succeed, do an rsync of the datadir +## Flushes translog and check if snapshot has not already been made today +# curl -s 'localhost:9200/_flush' | grep -qe '"ok":true' && +# curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot_${DATE}" | grep -qv 404 +## If it succeed, do a snapshot and remove old ones # if [ $? -eq 0 ]; then -# rsync -a /var/lib/elasticsearch /home/backup/ +# curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${DATE}?wait_for_completion=true" -o /tmp/es_snapshot_${DATE}.log +# cd /home/backup-elasticsearch/snaprepo +# for i in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep '^ *"snapshot" : "snapshot_.*",$' | cut -d_ -f2 | sed 's/",$//' | head -n-10); do +# curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot_${i}" +# done +# cd - # else # echo "Error when flushing ES translog indexes." # fi From 135c1df22af50ee04696507f6eb25760d77e39c0 Mon Sep 17 00:00:00 2001 From: Alexis Ben Miloud--Josselin Date: Wed, 14 Mar 2018 14:49:25 +0100 Subject: [PATCH 2/6] Remove unneeded lines --- zzz_evobackup | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index 5cbfc97..09ec4ea 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -123,24 +123,11 @@ mkdir -p -m 700 /home/backup ## ElasticSearch : example with rsync (warning: don't forget to use NFS if you have a cluster) # DATE=$(date +%F) -## Disable ES translog flush -# curl -s -XPUT 'localhost:9200/_settings' -d '{"index.translog.disable_flush": true}' >/dev/null -## Flushes translog and check if snapshot has not already been made today -# curl -s 'localhost:9200/_flush' | grep -qe '"ok":true' && -# curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot_${DATE}" | grep -qv 404 -## If it succeed, do a snapshot and remove old ones -# if [ $? -eq 0 ]; then -# curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${DATE}?wait_for_completion=true" -o /tmp/es_snapshot_${DATE}.log -# cd /home/backup-elasticsearch/snaprepo -# for i in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep '^ *"snapshot" : "snapshot_.*",$' | cut -d_ -f2 | sed 's/",$//' | head -n-10); do -# curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot_${i}" -# done -# cd - -# else -# echo "Error when flushing ES translog indexes." -# fi -## In any case re-enable translog flush -# curl -s -XPUT 'localhost:9200/_settings' -d '{"index.translog.disable_flush": false}' > /dev/null +## If it succeed, do a snapshot and remove old ones +# curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${DATE}?wait_for_completion=true" -o /tmp/es_snapshot_${DATE}.log +# for i in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep '^ *"snapshot" : "snapshot_.*",$' | cut -d_ -f2 | sed 's/",$//' | head -n-10); do +# curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot_${i}" +# done ## RabbitMQ : export config #rabbitmqadmin export /home/backup/rabbitmq.config >> /var/log/evobackup.log From f449ea9b9f20b9ce90269aef2effa2b22e44ba1e Mon Sep 17 00:00:00 2001 From: Alexis Ben Miloud--Josselin Date: Wed, 21 Mar 2018 18:00:25 +0100 Subject: [PATCH 3/6] Removed useless (old) comment --- zzz_evobackup | 1 - 1 file changed, 1 deletion(-) diff --git a/zzz_evobackup b/zzz_evobackup index 09ec4ea..289fb55 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -123,7 +123,6 @@ mkdir -p -m 700 /home/backup ## ElasticSearch : example with rsync (warning: don't forget to use NFS if you have a cluster) # DATE=$(date +%F) -## If it succeed, do a snapshot and remove old ones # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${DATE}?wait_for_completion=true" -o /tmp/es_snapshot_${DATE}.log # for i in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep '^ *"snapshot" : "snapshot_.*",$' | cut -d_ -f2 | sed 's/",$//' | head -n-10); do # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot_${i}" From 6548c5a1d86595af6a1139bbb0a2eac8b23a3eb8 Mon Sep 17 00:00:00 2001 From: Alexis Ben Miloud--Josselin Date: Wed, 21 Mar 2018 18:06:00 +0100 Subject: [PATCH 4/6] Remove old logs --- zzz_evobackup | 1 + 1 file changed, 1 insertion(+) diff --git a/zzz_evobackup b/zzz_evobackup index 289fb55..4f9e3f8 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -126,6 +126,7 @@ mkdir -p -m 700 /home/backup # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${DATE}?wait_for_completion=true" -o /tmp/es_snapshot_${DATE}.log # for i in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep '^ *"snapshot" : "snapshot_.*",$' | cut -d_ -f2 | sed 's/",$//' | head -n-10); do # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot_${i}" +# rm /tmp/es_snapshot_${i}.log # done ## RabbitMQ : export config From 7a9e4c61682607acd40456ac1b05de8188ab313e Mon Sep 17 00:00:00 2001 From: Alexis Ben Miloud--Josselin Date: Thu, 5 Apr 2018 16:02:07 +0200 Subject: [PATCH 5/6] Simplify and store deletion logs in /tmp --- zzz_evobackup | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index 4f9e3f8..7dd5d09 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -122,12 +122,9 @@ mkdir -p -m 700 /home/backup # cp /var/lib/redis/dump.rdb /home/backup/ ## ElasticSearch : example with rsync (warning: don't forget to use NFS if you have a cluster) -# DATE=$(date +%F) -# curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot_${DATE}?wait_for_completion=true" -o /tmp/es_snapshot_${DATE}.log -# for i in $(curl -s -XGET "localhost:9200/_snapshot/snaprepo/_all?pretty=true" | grep '^ *"snapshot" : "snapshot_.*",$' | cut -d_ -f2 | sed 's/",$//' | head -n-10); do -# curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot_${i}" -# rm /tmp/es_snapshot_${i}.log -# done +## Here we have one snapshot a day. +# curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" -o /tmp/es_delete_snapshot.daily.log +# curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" -o /tmp/es_snapshot.daily.log ## RabbitMQ : export config #rabbitmqadmin export /home/backup/rabbitmq.config >> /var/log/evobackup.log From bc1012b091eb18dbca2795eac66fc7f04647e301 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20S=C3=89RIE?= Date: Thu, 5 Apr 2018 16:24:18 +0200 Subject: [PATCH 6/6] Modification commentaire --- zzz_evobackup | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/zzz_evobackup b/zzz_evobackup index 7dd5d09..a6ef7dc 100755 --- a/zzz_evobackup +++ b/zzz_evobackup @@ -121,8 +121,9 @@ mkdir -p -m 700 /home/backup ## Redis : example with copy .rdb file # cp /var/lib/redis/dump.rdb /home/backup/ -## ElasticSearch : example with rsync (warning: don't forget to use NFS if you have a cluster) -## Here we have one snapshot a day. +## ElasticSearch, take a snapshot as a backup. +## Warning: You need to have a path.repo configured. +## See: https://wiki.evolix.org/HowtoElasticsearch#snapshots-et-sauvegardes # curl -s -XDELETE "localhost:9200/_snapshot/snaprepo/snapshot.daily" -o /tmp/es_delete_snapshot.daily.log # curl -s -XPUT "localhost:9200/_snapshot/snaprepo/snapshot.daily?wait_for_completion=true" -o /tmp/es_snapshot.daily.log