Rename errors file if it's big

when renamed it's not sent to logstash and not indexed, which
prevents logstash OOM failures.
Change-Id: I6389f6700a401f494187a283ca8fbc3b69784541
This commit is contained in:
Sagi Shnaidman 2019-02-25 21:03:57 +02:00
parent 95f6287755
commit 8a6b1ae251
1 changed files with 7 additions and 2 deletions

View File

@ -324,11 +324,16 @@
- name: Collect text version of the journal from last four hours
shell: journalctl --since=-4h --lines=100000 > /var/log/journal.txt
- name: Collect errors
- name: Collect errors and rename if more than 10 MB
shell: >
grep -rE '^[-0-9]+ [0-9:\.]+ [0-9 ]*ERROR ' /var/log/ |
sed "s/\(.*\)\(20[0-9][0-9]-[0-9][0-9]-[0-9][0-9] [0-9][0-9]:[0-9][0-9]:[0-9][0-9]\.[0-9]\+\)\(.*\)/\2 ERROR \1\3/g" > /tmp/errors.txt;
mv /tmp/errors.txt /var/log/extra/errors.txt
if (( $(stat -c "%s" /tmp/errors.txt) > 10485760 )); then
ERR_NAME=big-errors.txt;
else
ERR_NAME=errors.txt;
fi;
mv /tmp/errors.txt /var/log/extra/${ERR_NAME}.txt
- name: Create a index file for logstash
shell: >