Make Heka collect Ceph logs

Fix in stable/liberty
Closes-Bug: #1570529

Change-Id: Ia5d31c5a9bf102bf1f3c22cdae4959ad29cf9578

ansible/roles/ceph/tasks/bootstrap_osds.yml

@@ -50,6 +50,7 @@
     volumes:
       - "{{ node_config_directory }}/ceph-osd/:{{ container_config_directory }}/:ro"
       - "/dev/:/dev/"
+      - "kolla_logs:/var/log/kolla/"
   with_indexed_items: osds_bootstrap|default([])
 
 - name: Bootstrapping Ceph Cache OSDs
@@ -80,4 +81,5 @@
     volumes:
       - "{{ node_config_directory }}/ceph-osd/:{{ container_config_directory }}/:ro"
       - "/dev/:/dev/"
+      - "kolla_logs:/var/log/kolla/"
   with_indexed_items: osds_cache_bootstrap|default([])

ansible/roles/ceph/tasks/generate_cluster.yml

@@ -18,6 +18,7 @@
       - "{{ node_config_directory }}/ceph-mon/:{{ container_config_directory }}/:ro"
       - "ceph_mon:/var/lib/ceph"
       - "ceph_mon_config:/etc/ceph"
+      - "kolla_logs:/var/log/kolla/"
 
 - include: start_mons.yml
 

ansible/roles/ceph/tasks/start_mons.yml

@@ -12,3 +12,4 @@
       - "{{ node_config_directory }}/ceph-mon/:{{ container_config_directory }}/:ro"
       - "ceph_mon:/var/lib/ceph"
       - "ceph_mon_config:/etc/ceph"
+      - "kolla_logs:/var/log/kolla/"

ansible/roles/ceph/tasks/start_osds.yml

@@ -43,6 +43,7 @@
       - "{{ node_config_directory }}/ceph-osd/:{{ container_config_directory }}/:ro"
       - "/dev/:/dev/"
       - "/var/lib/ceph/osd/{{ item.1['fs_uuid'] }}:/var/lib/ceph/osd/ceph-{{ item.0.stdout }}"
+      - "kolla_logs:/var/log/kolla/"
   with_together:
     - id.results
     - osds

ansible/roles/ceph/tasks/start_rgws.yml

@@ -5,4 +5,6 @@
     common_options: "{{ docker_common_options }}"
     image: "{{ ceph_rgw_image_full }}"
     name: "ceph_rgw"
-    volumes: "{{ node_config_directory }}/ceph-rgw/:{{ container_config_directory }}/:ro"
+    volumes:
+      - "{{ node_config_directory }}/ceph-rgw/:{{ container_config_directory }}/:ro"
+      - "kolla_logs:/var/log/kolla/"

ansible/roles/ceph/templates/ceph.conf.j2

@@ -1,4 +1,8 @@
 [global]
+log file = /var/log/kolla/ceph/$cluster-$name.log
+log to syslog = true
+err to syslog = true
+
 fsid = {{ ceph_cluster_fsid }}
 mon initial members = {% for host in groups['ceph-mon'] %}{{ hostvars[host]['ansible_' + hostvars[host]['storage_interface']]['ipv4']['address'] }}{% if not loop.last %}, {% endif %}{% endfor %}
 
@@ -19,6 +23,6 @@ mon compact on start = true
 [client.radosgw.gateway]
 host = {{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}
 keyring = /etc/ceph/ceph.client.radosgw.keyring
-log file = /var/log/radosgw/client.radosgw.gateway.log
+log file = /var/log/kolla/ceph/client.radosgw.gateway.log
 rgw frontends = civetweb port={{ hostvars[inventory_hostname]['ansible_' + storage_interface]['ipv4']['address'] }}:{{ rgw_port }}
 {% endif %}

ansible/roles/common/templates/heka-openstack.toml.j2

@@ -6,6 +6,6 @@ filename = "lua_decoders/os_openstack_log.lua"
 type = "LogstreamerInput"
 decoder = "openstack_log_decoder"
 log_directory = "/var/log/kolla"
-file_match = '(?P<Service>nova|glance|keystone|neutron|cinder|heat|murano|magnum|mistral|manila)/(?P<Program>.*)\.log\.?(?P<Seq>\d*)$'
+file_match = '(?P<Service>nova|glance|keystone|neutron|ceph|cinder|heat|murano|magnum|mistral|manila)/(?P<Program>.*)\.log\.?(?P<Seq>\d*)$'
 priority = ["^Seq"]
 differentiator = ["Service", "_", "Program"]

docker/ceph/ceph-base/Dockerfile.j2

@@ -23,6 +23,9 @@ RUN apt-get install -y --no-install-recommends \
 
 {% endif %}
 
+COPY extend_start.sh /usr/local/bin/kolla_extend_start
+RUN chmod 755 /usr/local/bin/kolla_extend_start
+
 RUN useradd --user-group ceph \
     && mkdir -p /home/ceph \
     && chown -R ceph: /home/ceph

docker/ceph/ceph-base/extend_start.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [[ ! -d "/var/log/kolla/ceph" ]]; then
+    mkdir -p /var/log/kolla/ceph
+fi
+if [[ $(stat -c %a /var/log/kolla/ceph) != "755" ]]; then
+    chmod 755 /var/log/kolla/ceph
+fi

docker/ceph/ceph-mon/extend_start.sh

@@ -7,6 +7,13 @@ KEYRING_RGW="/etc/ceph/ceph.client.radosgw.keyring"
 MONMAP="/etc/ceph/ceph.monmap"
 MON_DIR="/var/lib/ceph/mon/ceph-${HOSTNAME}"
 
+if [[ ! -d "/var/log/kolla/ceph" ]]; then
+    mkdir -p /var/log/kolla/ceph
+fi
+if [[ $(stat -c %a /var/log/kolla/ceph) != "755" ]]; then
+    chmod 755 /var/log/kolla/ceph
+fi
+
 # Bootstrap and exit if KOLLA_BOOTSTRAP variable is set. This catches all cases
 # of the KOLLA_BOOTSTRAP variable being set, including empty.
 if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then

docker/ceph/ceph-osd/extend_start.sh

@@ -1,5 +1,12 @@
 #!/bin/bash
 
+if [[ ! -d "/var/log/kolla/ceph" ]]; then
+    mkdir -p /var/log/kolla/ceph
+fi
+if [[ $(stat -c %a /var/log/kolla/ceph) != "755" ]]; then
+    chmod 755 /var/log/kolla/ceph
+fi
+
 # Bootstrap and exit if KOLLA_BOOTSTRAP variable is set. This catches all cases
 # of the KOLLA_BOOTSTRAP variable being set, including empty.
 if [[ "${!KOLLA_BOOTSTRAP[@]}" ]]; then