summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSpyros Trigazis <spyridon.trigazis@cern.ch>2017-12-15 16:57:52 +0000
committerSpyros Trigazis (strigazi) <strigazi@gmail.com>2017-12-17 16:35:50 +0000
commit273405cb5a8cdbcec816f646cfe82f13784b3bf1 (patch)
treeaa71e031e7a2b29579c61494120606c784e477a2
parent84006f63d721e0e0048a09b79745625f08a42aae (diff)
Leverage heat-container-agent for monitoringHEADmaster
Use the heat-container-agent from a system container. It means that the docker daemon can be started later. Pass as a software deployment with the heat-agent the following software-configurations: * prometheus-monitoring ** pin prometheus to v1.8.2 since its config is not 2.0.0 compatible Add heat-container-agent container image. Implements: blueprint heat-agent Related-Bug: #1680900 Change-Id: I084b7fe51eddb7b36c74f9fe76cda37e8b48f646
Notes
Notes (review): Code-Review+2: yatin <ykarel@redhat.com> Code-Review+1: Rico Lin <rico.lin@easystack.cn> Workflow+1: Spyros Trigazis (strigazi) <strigazi@gmail.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Mon, 18 Dec 2017 10:00:14 +0000 Reviewed-on: https://review.openstack.org/468816 Project: openstack/magnum Branch: refs/heads/master
-rw-r--r--magnum/drivers/common/image/heat-container-agent/Dockerfile49
-rw-r--r--magnum/drivers/common/image/heat-container-agent/config.json.template373
-rwxr-xr-xmagnum/drivers/common/image/heat-container-agent/launch5
-rw-r--r--magnum/drivers/common/image/heat-container-agent/manifest.json4
-rwxr-xr-xmagnum/drivers/common/image/heat-container-agent/scripts/50-heat-config-docker-compose116
-rwxr-xr-xmagnum/drivers/common/image/heat-container-agent/scripts/55-heat-config194
-rw-r--r--magnum/drivers/common/image/heat-container-agent/scripts/configure_container_agent.sh36
-rwxr-xr-xmagnum/drivers/common/image/heat-container-agent/scripts/heat-config-notify163
-rwxr-xr-xmagnum/drivers/common/image/heat-container-agent/scripts/hooks/atomic115
-rwxr-xr-xmagnum/drivers/common/image/heat-container-agent/scripts/hooks/docker-compose127
-rwxr-xr-xmagnum/drivers/common/image/heat-container-agent/scripts/hooks/script96
-rw-r--r--magnum/drivers/common/image/heat-container-agent/scripts/write-os-apply-config-templates.sh71
-rw-r--r--magnum/drivers/common/image/heat-container-agent/service.template11
-rw-r--r--magnum/drivers/common/image/heat-container-agent/tmpfiles.template10
-rw-r--r--magnum/drivers/common/templates/kubernetes/fragments/enable-monitoring.sh139
-rw-r--r--magnum/drivers/common/templates/kubernetes/fragments/enable-prometheus-monitoring434
-rw-r--r--magnum/drivers/common/templates/kubernetes/fragments/start-container-agent.sh16
-rw-r--r--magnum/drivers/common/templates/kubernetes/fragments/write-grafana-service.yaml67
-rw-r--r--magnum/drivers/common/templates/kubernetes/fragments/write-prometheus-configmap.yaml163
-rw-r--r--magnum/drivers/common/templates/kubernetes/fragments/write-prometheus-service.yaml60
-rw-r--r--magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml65
-rw-r--r--magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml65
-rwxr-xr-xmagnum/tests/contrib/copy_instance_logs.sh4
23 files changed, 1878 insertions, 505 deletions
diff --git a/magnum/drivers/common/image/heat-container-agent/Dockerfile b/magnum/drivers/common/image/heat-container-agent/Dockerfile
new file mode 100644
index 0000000..eea7e42
--- /dev/null
+++ b/magnum/drivers/common/image/heat-container-agent/Dockerfile
@@ -0,0 +1,49 @@
1FROM registry.fedoraproject.org/fedora:rawhide
2
3# Fill out the labels
4LABEL name="heat-container-agent" \
5 maintainer="Spyros Trigazis <strigazi@gmail.com>" \
6 license="UNKNOWN" \
7 summary="Heat Container Agent system image" \
8 version="1.0" \
9 help="No help" \
10 architecture="x86_64" \
11 atomic.type="system" \
12 distribution-scope="public"
13
14RUN dnf -y --setopt=tsflags=nodocs install \
15 findutils os-collect-config os-apply-config \
16 os-refresh-config dib-utils python-pip python-docker-py \
17 python-yaml python-zaqarclient python2-oslo-log \
18 python-psutil && dnf clean all
19
20# pip installing dpath as python-dpath is an older version of dpath
21# install docker-compose
22RUN pip install --no-cache dpath docker-compose
23
24ADD ./scripts/55-heat-config \
25 /opt/heat-container-agent/scripts/
26
27ADD ./scripts/50-heat-config-docker-compose \
28 /opt/heat-container-agent/scripts/
29
30ADD ./scripts/hooks/* \
31 /opt/heat-container-agent/hooks/
32
33ADD ./scripts/heat-config-notify \
34 /usr/bin/heat-config-notify
35RUN chmod 755 /usr/bin/heat-config-notify
36
37ADD ./scripts/configure_container_agent.sh /opt/heat-container-agent/
38RUN chmod 700 /opt/heat-container-agent/configure_container_agent.sh
39
40ADD ./scripts/write-os-apply-config-templates.sh /tmp
41RUN chmod 700 /tmp/write-os-apply-config-templates.sh
42RUN /tmp/write-os-apply-config-templates.sh
43
44COPY manifest.json service.template config.json.template tmpfiles.template /exports/
45
46COPY launch /usr/bin/start-heat-container-agent
47
48# Execution
49CMD ["/usr/bin/start-heat-container-agent"]
diff --git a/magnum/drivers/common/image/heat-container-agent/config.json.template b/magnum/drivers/common/image/heat-container-agent/config.json.template
new file mode 100644
index 0000000..c49d938
--- /dev/null
+++ b/magnum/drivers/common/image/heat-container-agent/config.json.template
@@ -0,0 +1,373 @@
1{
2 "hooks": {},
3 "hostname": "acme",
4 "linux": {
5 "namespaces": [
6 {
7 "type": "mount"
8 },
9 {
10 "type": "ipc"
11 },
12 {
13 "type": "uts"
14 }
15 ],
16 "resources": {
17 "devices": [
18 {
19 "access": "rwm",
20 "allow": false
21 }
22 ]
23 }
24 },
25 "mounts": [
26 {
27 "type": "bind",
28 "source": "/srv/magnum",
29 "destination": "/srv/magnum",
30 "options": [
31 "rbind",
32 "rw",
33 "rprivate"
34 ]
35 },
36 {
37 "type": "bind",
38 "source": "/opt/stack/os-config-refresh",
39 "destination": "/opt/stack/os-config-refresh",
40 "options": [
41 "rbind",
42 "rw",
43 "rprivate"
44 ]
45 },
46 {
47 "type": "bind",
48 "source": "/run/systemd",
49 "destination": "/run/systemd",
50 "options": [
51 "rbind",
52 "ro",
53 "rprivate"
54 ]
55 },
56 {
57 "type": "bind",
58 "source": "/etc/",
59 "destination": "/etc/",
60 "options": [
61 "rbind",
62 "rw",
63 "rprivate"
64 ]
65 },
66 {
67 "type": "bind",
68 "source": "/var/lib",
69 "destination": "/var/lib",
70 "options": [
71 "rbind",
72 "rw",
73 "rprivate"
74 ]
75 },
76 {
77 "type": "bind",
78 "source": "/var/run",
79 "destination": "/var/run",
80 "options": [
81 "rbind",
82 "rw",
83 "rprivate"
84 ]
85 },
86 {
87 "type": "bind",
88 "source": "/var/log",
89 "destination": "/var/log",
90 "options": [
91 "rbind",
92 "rw",
93 "rprivate"
94 ]
95 },
96 {
97 "type": "bind",
98 "source": "/tmp",
99 "destination": "/tmp",
100 "options": [
101 "rbind",
102 "rw",
103 "rprivate"
104 ]
105 },
106 {
107 "destination": "/proc",
108 "source": "proc",
109 "type": "proc"
110 },
111 {
112 "destination": "/dev",
113 "options": [
114 "nosuid",
115 "strictatime",
116 "mode=755",
117 "size=65536k"
118 ],
119 "source": "tmpfs",
120 "type": "tmpfs"
121 },
122 {
123 "destination": "/dev/pts",
124 "options": [
125 "nosuid",
126 "noexec",
127 "newinstance",
128 "ptmxmode=0666",
129 "mode=0620",
130 "gid=5"
131 ],
132 "source": "devpts",
133 "type": "devpts"
134 },
135 {
136 "destination": "/dev/shm",
137 "options": [
138 "nosuid",
139 "noexec",
140 "nodev",
141 "mode=1777",
142 "size=65536k"
143 ],
144 "source": "shm",
145 "type": "tmpfs"
146 },
147 {
148 "destination": "/dev/mqueue",
149 "options": [
150 "nosuid",
151 "noexec",
152 "nodev"
153 ],
154 "source": "mqueue",
155 "type": "mqueue"
156 },
157 {
158 "destination": "/sys",
159 "options": [
160 "nosuid",
161 "noexec",
162 "nodev",
163 "ro"
164 ],
165 "source": "sysfs",
166 "type": "sysfs"
167 },
168 {
169 "destination": "/sys/fs/cgroup",
170 "options": [
171 "nosuid",
172 "noexec",
173 "nodev",
174 "relatime",
175 "ro"
176 ],
177 "source": "cgroup",
178 "type": "cgroup"
179 }
180 ],
181 "ociVersion": "0.6.0-dev",
182 "platform": {
183 "arch": "amd64",
184 "os": "linux"
185 },
186 "process": {
187 "args": [
188 "/usr/bin/start-heat-container-agent"
189 ],
190 "capabilities": {
191 "bounding": [
192 "CAP_CHOWN",
193 "CAP_DAC_OVERRIDE",
194 "CAP_DAC_READ_SEARCH",
195 "CAP_FOWNER",
196 "CAP_FSETID",
197 "CAP_KILL",
198 "CAP_SETGID",
199 "CAP_SETUID",
200 "CAP_SETPCAP",
201 "CAP_LINUX_IMMUTABLE",
202 "CAP_NET_BIND_SERVICE",
203 "CAP_NET_BROADCAST",
204 "CAP_NET_ADMIN",
205 "CAP_NET_RAW",
206 "CAP_IPC_LOCK",
207 "CAP_IPC_OWNER",
208 "CAP_SYS_MODULE",
209 "CAP_SYS_RAWIO",
210 "CAP_SYS_CHROOT",
211 "CAP_SYS_PTRACE",
212 "CAP_SYS_PACCT",
213 "CAP_SYS_ADMIN",
214 "CAP_SYS_BOOT",
215 "CAP_SYS_NICE",
216 "CAP_SYS_RESOURCE",
217 "CAP_SYS_TIME",
218 "CAP_SYS_TTY_CONFIG",
219 "CAP_MKNOD",
220 "CAP_LEASE",
221 "CAP_AUDIT_WRITE",
222 "CAP_AUDIT_CONTROL",
223 "CAP_SETFCAP",
224 "CAP_MAC_OVERRIDE",
225 "CAP_MAC_ADMIN",
226 "CAP_SYSLOG",
227 "CAP_WAKE_ALARM",
228 "CAP_BLOCK_SUSPEND",
229 "CAP_AUDIT_READ"
230 ],
231 "permitted": [
232 "CAP_CHOWN",
233 "CAP_DAC_OVERRIDE",
234 "CAP_DAC_READ_SEARCH",
235 "CAP_FOWNER",
236 "CAP_FSETID",
237 "CAP_KILL",
238 "CAP_SETGID",
239 "CAP_SETUID",
240 "CAP_SETPCAP",
241 "CAP_LINUX_IMMUTABLE",
242 "CAP_NET_BIND_SERVICE",
243 "CAP_NET_BROADCAST",
244 "CAP_NET_ADMIN",
245 "CAP_NET_RAW",
246 "CAP_IPC_LOCK",
247 "CAP_IPC_OWNER",
248 "CAP_SYS_MODULE",
249 "CAP_SYS_RAWIO",
250 "CAP_SYS_CHROOT",
251 "CAP_SYS_PTRACE",
252 "CAP_SYS_PACCT",
253 "CAP_SYS_ADMIN",
254 "CAP_SYS_BOOT",
255 "CAP_SYS_NICE",
256 "CAP_SYS_RESOURCE",
257 "CAP_SYS_TIME",
258 "CAP_SYS_TTY_CONFIG",
259 "CAP_MKNOD",
260 "CAP_LEASE",
261 "CAP_AUDIT_WRITE",
262 "CAP_AUDIT_CONTROL",
263 "CAP_SETFCAP",
264 "CAP_MAC_OVERRIDE",
265 "CAP_MAC_ADMIN",
266 "CAP_SYSLOG",
267 "CAP_WAKE_ALARM",
268 "CAP_BLOCK_SUSPEND",
269 "CAP_AUDIT_READ"
270 ],
271 "inheritable": [
272 "CAP_CHOWN",
273 "CAP_DAC_OVERRIDE",
274 "CAP_DAC_READ_SEARCH",
275 "CAP_FOWNER",
276 "CAP_FSETID",
277 "CAP_KILL",
278 "CAP_SETGID",
279 "CAP_SETUID",
280 "CAP_SETPCAP",
281 "CAP_LINUX_IMMUTABLE",
282 "CAP_NET_BIND_SERVICE",
283 "CAP_NET_BROADCAST",
284 "CAP_NET_ADMIN",
285 "CAP_NET_RAW",
286 "CAP_IPC_LOCK",
287 "CAP_IPC_OWNER",
288 "CAP_SYS_MODULE",
289 "CAP_SYS_RAWIO",
290 "CAP_SYS_CHROOT",
291 "CAP_SYS_PTRACE",
292 "CAP_SYS_PACCT",
293 "CAP_SYS_ADMIN",
294 "CAP_SYS_BOOT",
295 "CAP_SYS_NICE",
296 "CAP_SYS_RESOURCE",
297 "CAP_SYS_TIME",
298 "CAP_SYS_TTY_CONFIG",
299 "CAP_MKNOD",
300 "CAP_LEASE",
301 "CAP_AUDIT_WRITE",
302 "CAP_AUDIT_CONTROL",
303 "CAP_SETFCAP",
304 "CAP_MAC_OVERRIDE",
305 "CAP_MAC_ADMIN",
306 "CAP_SYSLOG",
307 "CAP_WAKE_ALARM",
308 "CAP_BLOCK_SUSPEND",
309 "CAP_AUDIT_READ"
310 ],
311 "effective": [
312 "CAP_CHOWN",
313 "CAP_DAC_OVERRIDE",
314 "CAP_DAC_READ_SEARCH",
315 "CAP_FOWNER",
316 "CAP_FSETID",
317 "CAP_KILL",
318 "CAP_SETGID",
319 "CAP_SETUID",
320 "CAP_SETPCAP",
321 "CAP_LINUX_IMMUTABLE",
322 "CAP_NET_BIND_SERVICE",
323 "CAP_NET_BROADCAST",
324 "CAP_NET_ADMIN",
325 "CAP_NET_RAW",
326 "CAP_IPC_LOCK",
327 "CAP_IPC_OWNER",
328 "CAP_SYS_MODULE",
329 "CAP_SYS_RAWIO",
330 "CAP_SYS_CHROOT",
331 "CAP_SYS_PTRACE",
332 "CAP_SYS_PACCT",
333 "CAP_SYS_ADMIN",
334 "CAP_SYS_BOOT",
335 "CAP_SYS_NICE",
336 "CAP_SYS_RESOURCE",
337 "CAP_SYS_TIME",
338 "CAP_SYS_TTY_CONFIG",
339 "CAP_MKNOD",
340 "CAP_LEASE",
341 "CAP_AUDIT_WRITE",
342 "CAP_AUDIT_CONTROL",
343 "CAP_SETFCAP",
344 "CAP_MAC_OVERRIDE",
345 "CAP_MAC_ADMIN",
346 "CAP_SYSLOG",
347 "CAP_WAKE_ALARM",
348 "CAP_BLOCK_SUSPEND",
349 "CAP_AUDIT_READ"
350 ]
351 },
352 "cwd": "/",
353 "env": [
354 "REQUESTS_CA_BUNDLE=$REQUESTS_CA_BUNDLE",
355 "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
356 "SYSTEMD_IGNORE_CHROOT=1",
357 "TERM=xterm"
358 ],
359 "rlimits": [
360 {
361 "hard": 1024,
362 "soft": 1024,
363 "type": "RLIMIT_NOFILE"
364 }
365 ],
366 "terminal": false,
367 "user": {}
368 },
369 "root": {
370 "path": "rootfs",
371 "readonly": true
372 }
373}
diff --git a/magnum/drivers/common/image/heat-container-agent/launch b/magnum/drivers/common/image/heat-container-agent/launch
new file mode 100755
index 0000000..a910694
--- /dev/null
+++ b/magnum/drivers/common/image/heat-container-agent/launch
@@ -0,0 +1,5 @@
1#!/bin/bash
2
3/opt/heat-container-agent/configure_container_agent.sh
4
5exec /usr/bin/os-collect-config --debug
diff --git a/magnum/drivers/common/image/heat-container-agent/manifest.json b/magnum/drivers/common/image/heat-container-agent/manifest.json
new file mode 100644
index 0000000..eb3c75b
--- /dev/null
+++ b/magnum/drivers/common/image/heat-container-agent/manifest.json
@@ -0,0 +1,4 @@
1{
2 "defaultValues": {},
3 "version": "1.0"
4} \ No newline at end of file
diff --git a/magnum/drivers/common/image/heat-container-agent/scripts/50-heat-config-docker-compose b/magnum/drivers/common/image/heat-container-agent/scripts/50-heat-config-docker-compose
new file mode 100755
index 0000000..017c08e
--- /dev/null
+++ b/magnum/drivers/common/image/heat-container-agent/scripts/50-heat-config-docker-compose
@@ -0,0 +1,116 @@
1#!/usr/bin/env python
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15import json
16import logging
17import os
18import subprocess
19import sys
20
21import yaml
22
23
24CONF_FILE = os.environ.get('HEAT_SHELL_CONFIG',
25 '/var/run/heat-config/heat-config')
26
27DOCKER_COMPOSE_DIR = os.environ.get(
28 'HEAT_DOCKER_COMPOSE_WORKING',
29 '/var/lib/heat-config/heat-config-docker-compose')
30
31DOCKER_COMPOSE_CMD = os.environ.get('HEAT_DOCKER_COMPOSE_CMD',
32 'docker-compose')
33
34
35def main(argv=sys.argv):
36 log = logging.getLogger('heat-config')
37 handler = logging.StreamHandler(sys.stderr)
38 handler.setFormatter(
39 logging.Formatter(
40 '[%(asctime)s] (%(name)s) [%(levelname)s] %(message)s'))
41 log.addHandler(handler)
42 log.setLevel('DEBUG')
43
44 if not os.path.exists(CONF_FILE):
45 log.error('No config file %s' % CONF_FILE)
46 return 1
47
48 if not os.path.isdir(DOCKER_COMPOSE_DIR):
49 os.makedirs(DOCKER_COMPOSE_DIR, 0o700)
50
51 try:
52 configs = json.load(open(CONF_FILE))
53 except ValueError:
54 pass
55
56 try:
57 cleanup_stale_projects(configs)
58 for c in configs:
59 write_compose_config(c)
60 except Exception as e:
61 log.exception(e)
62
63
64def cleanup_stale_projects(configs):
65 def deployments(configs):
66 for c in configs:
67 yield c['name']
68
69 def compose_projects(compose_dir):
70 for proj in os.listdir(compose_dir):
71 if os.path.isfile(
72 os.path.join(DOCKER_COMPOSE_DIR,
73 '%s/docker-compose.yml' % proj)):
74 yield proj
75
76 def cleanup_containers(project):
77 cmd = [
78 DOCKER_COMPOSE_CMD,
79 'kill'
80 ]
81 subproc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
82 stderr=subprocess.PIPE)
83 stdout, stderr = subproc.communicate()
84
85 for proj in compose_projects(DOCKER_COMPOSE_DIR):
86 if proj not in deployments(configs):
87 proj_dir = os.path.join(DOCKER_COMPOSE_DIR, proj)
88 os.chdir(proj_dir)
89 cleanup_containers(proj)
90 os.remove('%s/docker-compose.yml' % proj_dir)
91
92
93def write_compose_config(c):
94 group = c.get('group')
95 if group != 'docker-compose':
96 return
97
98 def prepare_dir(path):
99 if not os.path.isdir(path):
100 os.makedirs(path, 0o700)
101
102 compose_conf = c.get('config', '')
103 if isinstance(compose_conf, dict):
104 yaml_config = yaml.safe_dump(compose_conf, default_flow_style=False)
105 else:
106 yaml_config = compose_conf
107 proj_dir = os.path.join(DOCKER_COMPOSE_DIR, c['name'])
108 prepare_dir(proj_dir)
109 fn = os.path.join(proj_dir, 'docker-compose.yml')
110 with os.fdopen(os.open(fn, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600),
111 'w') as f:
112 f.write(yaml_config.encode('utf-8'))
113
114
115if __name__ == '__main__':
116 sys.exit(main(sys.argv))
diff --git a/magnum/drivers/common/image/heat-container-agent/scripts/55-heat-config b/magnum/drivers/common/image/heat-container-agent/scripts/55-heat-config
new file mode 100755
index 0000000..9cc0df1
--- /dev/null
+++ b/magnum/drivers/common/image/heat-container-agent/scripts/55-heat-config
@@ -0,0 +1,194 @@
1#!/usr/bin/env python
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15import json
16import logging
17import os
18import shutil
19import stat
20import subprocess
21import sys
22
23import requests
24
25HOOKS_DIR_PATHS = (
26 os.environ.get('HEAT_CONFIG_HOOKS'),
27 '/usr/libexec/heat-config/hooks',
28 '/var/lib/heat-config/hooks',
29)
30CONF_FILE = os.environ.get('HEAT_SHELL_CONFIG',
31 '/var/run/heat-config/heat-config')
32DEPLOYED_DIR = os.environ.get('HEAT_CONFIG_DEPLOYED',
33 '/var/lib/heat-config/deployed')
34OLD_DEPLOYED_DIR = os.environ.get('HEAT_CONFIG_DEPLOYED_OLD',
35 '/var/run/heat-config/deployed')
36HEAT_CONFIG_NOTIFY = os.environ.get('HEAT_CONFIG_NOTIFY',
37 'heat-config-notify')
38
39
40def main(argv=sys.argv):
41 log = logging.getLogger('heat-config')
42 handler = logging.StreamHandler(sys.stderr)
43 handler.setFormatter(
44 logging.Formatter(
45 '[%(asctime)s] (%(name)s) [%(levelname)s] %(message)s'))
46 log.addHandler(handler)
47 log.setLevel('DEBUG')
48
49 if not os.path.exists(CONF_FILE):
50 log.error('No config file %s' % CONF_FILE)
51 return 1
52
53 conf_mode = stat.S_IMODE(os.lstat(CONF_FILE).st_mode)
54 if conf_mode != 0o600:
55 os.chmod(CONF_FILE, 0o600)
56
57 if not os.path.isdir(DEPLOYED_DIR):
58 if DEPLOYED_DIR != OLD_DEPLOYED_DIR and os.path.isdir(OLD_DEPLOYED_DIR):
59 log.debug('Migrating deployed state from %s to %s' %
60 (OLD_DEPLOYED_DIR, DEPLOYED_DIR))
61 shutil.move(OLD_DEPLOYED_DIR, DEPLOYED_DIR)
62 else:
63 os.makedirs(DEPLOYED_DIR, 0o700)
64
65 try:
66 configs = json.load(open(CONF_FILE))
67 except ValueError:
68 pass
69 else:
70 for c in configs:
71 try:
72 invoke_hook(c, log)
73 except Exception as e:
74 log.exception(e)
75
76
77def find_hook_path(group):
78 # sanitise the group to get an alphanumeric hook file name
79 hook = "".join(
80 x for x in group if x == '-' or x == '_' or x.isalnum())
81
82 for h in HOOKS_DIR_PATHS:
83 if not h or not os.path.exists(h):
84 continue
85 hook_path = os.path.join(h, hook)
86 if os.path.exists(hook_path):
87 return hook_path
88
89
90def invoke_hook(c, log):
91 # Sanitize input values (bug 1333992). Convert all String
92 # inputs to strings if they're not already
93 hot_inputs = c.get('inputs', [])
94 for hot_input in hot_inputs:
95 if hot_input.get('type', None) == 'String' and \
96 not isinstance(hot_input['value'], basestring):
97 hot_input['value'] = str(hot_input['value'])
98 iv = dict((i['name'], i['value']) for i in c['inputs'])
99 # The group property indicates whether it is softwarecomponent or
100 # plain softwareconfig
101 # If it is softwarecomponent, pick up a property config to invoke
102 # according to deploy_action
103 group = c.get('group')
104 if group == 'component':
105 found = False
106 action = iv.get('deploy_action')
107 config = c.get('config')
108 configs = config.get('configs')
109 if configs:
110 for cfg in configs:
111 if action in cfg['actions']:
112 c['config'] = cfg['config']
113 c['group'] = cfg['tool']
114 found = True
115 break
116 if not found:
117 log.warn('Skipping group %s, no valid script is defined'
118 ' for deploy action %s' % (group, action))
119 return
120
121 # check to see if this config is already deployed
122 deployed_path = os.path.join(DEPLOYED_DIR, '%s.json' % c['id'])
123
124 if os.path.exists(deployed_path):
125 log.warn('Skipping config %s, already deployed' % c['id'])
126 log.warn('To force-deploy, rm %s' % deployed_path)
127 return
128
129 signal_data = {}
130 hook_path = find_hook_path(c['group'])
131
132 if not hook_path:
133 log.warn('Skipping group %s with no hook script %s' % (
134 c['group'], hook_path))
135 return
136
137 # write out config, which indicates it is deployed regardless of
138 # subsequent hook success
139 with os.fdopen(os.open(
140 deployed_path, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f:
141 json.dump(c, f, indent=2)
142
143 log.debug('Running %s < %s' % (hook_path, deployed_path))
144 subproc = subprocess.Popen([hook_path],
145 stdin=subprocess.PIPE,
146 stdout=subprocess.PIPE,
147 stderr=subprocess.PIPE)
148 stdout, stderr = subproc.communicate(input=json.dumps(c))
149
150 log.info(stdout)
151 log.debug(stderr)
152
153 if subproc.returncode:
154 log.error("Error running %s. [%s]\n" % (
155 hook_path, subproc.returncode))
156 else:
157 log.info('Completed %s' % hook_path)
158
159 try:
160 if stdout:
161 signal_data = json.loads(stdout)
162 except ValueError:
163 signal_data = {
164 'deploy_stdout': stdout,
165 'deploy_stderr': stderr,
166 'deploy_status_code': subproc.returncode,
167 }
168
169 signal_data_path = os.path.join(DEPLOYED_DIR, '%s.notify.json' % c['id'])
170 # write out notify data for debugging
171 with os.fdopen(os.open(
172 signal_data_path, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f:
173 json.dump(signal_data, f, indent=2)
174
175 log.debug('Running %s %s < %s' % (
176 HEAT_CONFIG_NOTIFY, deployed_path, signal_data_path))
177 subproc = subprocess.Popen([HEAT_CONFIG_NOTIFY, deployed_path],
178 stdin=subprocess.PIPE,
179 stdout=subprocess.PIPE,
180 stderr=subprocess.PIPE)
181 stdout, stderr = subproc.communicate(input=json.dumps(signal_data))
182
183 log.info(stdout)
184
185 if subproc.returncode:
186 log.error(
187 "Error running heat-config-notify. [%s]\n" % subproc.returncode)
188 log.error(stderr)
189 else:
190 log.debug(stderr)
191
192
193if __name__ == '__main__':
194 sys.exit(main(sys.argv))
diff --git a/magnum/drivers/common/image/heat-container-agent/scripts/configure_container_agent.sh b/magnum/drivers/common/image/heat-container-agent/scripts/configure_container_agent.sh
new file mode 100644
index 0000000..688a36d
--- /dev/null
+++ b/magnum/drivers/common/image/heat-container-agent/scripts/configure_container_agent.sh
@@ -0,0 +1,36 @@
1#!/bin/bash
2set -eux
3
4# initial /etc/os-collect-config.conf
5cat <<EOF >/etc/os-collect-config.conf
6[DEFAULT]
7command = os-refresh-config
8EOF
9
10# os-refresh-config scripts directory
11# This moves to /usr/libexec/os-refresh-config in later releases
12# Be sure to have this dir mounted and created by config.json and tmpfiles
13orc_scripts=/opt/stack/os-config-refresh
14for d in pre-configure.d configure.d migration.d post-configure.d; do
15 install -m 0755 -o root -g root -d $orc_scripts/$d
16done
17
18# os-refresh-config script for running os-apply-config
19cat <<EOF >$orc_scripts/configure.d/20-os-apply-config
20#!/bin/bash
21set -ue
22
23exec os-apply-config
24EOF
25
26chmod 700 $orc_scripts/configure.d/20-os-apply-config
27cp /opt/heat-container-agent/scripts/55-heat-config $orc_scripts/configure.d/55-heat-config
28chmod 700 $orc_scripts/configure.d/55-heat-config
29cp /opt/heat-container-agent/scripts/50-heat-config-docker-compose $orc_scripts/configure.d/50-heat-config-docker-compose
30chmod 700 $orc_scripts/configure.d/50-heat-config-docker-compose
31
32mkdir -p /var/lib/heat-config/hooks
33cp /opt/heat-container-agent/hooks/* /var/lib/heat-config/hooks/
34chmod 755 /var/lib/heat-config/hooks/atomic
35chmod 755 /var/lib/heat-config/hooks/docker-compose
36chmod 755 /var/lib/heat-config/hooks/script
diff --git a/magnum/drivers/common/image/heat-container-agent/scripts/heat-config-notify b/magnum/drivers/common/image/heat-container-agent/scripts/heat-config-notify
new file mode 100755
index 0000000..eba1258
--- /dev/null
+++ b/magnum/drivers/common/image/heat-container-agent/scripts/heat-config-notify
@@ -0,0 +1,163 @@
1#!/usr/bin/env python
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15import json
16import logging
17import os
18import sys
19
20import requests
21
22try:
23 from heatclient import client as heatclient
24except ImportError:
25 heatclient = None
26
27try:
28 from keystoneclient.v3 import client as ksclient
29except ImportError:
30 ksclient = None
31
32try:
33 from zaqarclient.queues.v1 import client as zaqarclient
34except ImportError:
35 zaqarclient = None
36
37
38MAX_RESPONSE_SIZE = 950000
39
40
41def init_logging():
42 log = logging.getLogger('heat-config-notify')
43 handler = logging.StreamHandler(sys.stderr)
44 handler.setFormatter(
45 logging.Formatter(
46 '[%(asctime)s] (%(name)s) [%(levelname)s] %(message)s'))
47 log.addHandler(handler)
48 log.setLevel('DEBUG')
49 return log
50
51
52def trim_response(response, trimmed_values=None):
53 """Trim selected values from response.
54
55 Makes given response smaller or the same size as MAX_RESPONSE_SIZE by
56 trimming given trimmed_values from response dict from the left side
57 (beginning). Returns trimmed and serialized JSON response itself.
58 """
59
60 trimmed_values = trimmed_values or ('deploy_stdout', 'deploy_stderr')
61 str_response = json.dumps(response, ensure_ascii=True, encoding='utf-8')
62 len_total = len(str_response)
63 offset = MAX_RESPONSE_SIZE - len_total
64 if offset >= 0:
65 return str_response
66 offset = abs(offset)
67 for key in trimmed_values:
68 len_value = len(response[key])
69 cut = int(round(float(len_value) / len_total * offset))
70 response[key] = response[key][cut:]
71 str_response = json.dumps(response, ensure_ascii=True, encoding='utf-8')
72 return str_response
73
74
75def main(argv=sys.argv, stdin=sys.stdin):
76
77 log = init_logging()
78 usage = ('Usage:\n heat-config-notify /path/to/config.json '
79 '< /path/to/signal_data.json')
80
81 if len(argv) < 2:
82 log.error(usage)
83 return 1
84
85 try:
86 signal_data = json.load(stdin)
87 except ValueError:
88 log.warn('No valid json found on stdin')
89 signal_data = {}
90
91 conf_file = argv[1]
92 if not os.path.exists(conf_file):
93 log.error('No config file %s' % conf_file)
94 log.error(usage)
95 return 1
96
97 c = json.load(open(conf_file))
98
99 iv = dict((i['name'], i['value']) for i in c['inputs'])
100
101 if 'deploy_signal_id' in iv:
102 sigurl = iv.get('deploy_signal_id')
103 sigverb = iv.get('deploy_signal_verb', 'POST')
104 log.debug('Signaling to %s via %s' % (sigurl, sigverb))
105 # we need to trim log content because Heat response size is limited
106 # by max_json_body_size = 1048576
107 str_signal_data = trim_response(signal_data)
108 if sigverb == 'PUT':
109 r = requests.put(sigurl, data=str_signal_data,
110 headers={'content-type': 'application/json'})
111 else:
112 r = requests.post(sigurl, data=str_signal_data,
113 headers={'content-type': 'application/json'})
114 log.debug('Response %s ' % r)
115
116 if 'deploy_queue_id' in iv:
117 queue_id = iv.get('deploy_queue_id')
118 log.debug('Signaling to queue %s' % (queue_id,))
119
120 ks = ksclient.Client(
121 auth_url=iv['deploy_auth_url'],
122 user_id=iv['deploy_user_id'],
123 password=iv['deploy_password'],
124 project_id=iv['deploy_project_id'])
125 endpoint = ks.service_catalog.url_for(
126 service_type='messaging', endpoint_type='publicURL')
127
128 conf = {
129 'auth_opts': {
130 'backend': 'keystone',
131 'options': {
132 'os_auth_token': ks.auth_token,
133 'os_project_id': iv['deploy_project_id'],
134 }
135 }
136 }
137 cli = zaqarclient.Client(endpoint, conf=conf, version=1.1)
138 queue = cli.queue(queue_id)
139 r = queue.post({'body': signal_data, 'ttl': 600})
140 log.debug('Response %s ' % r)
141
142 elif 'deploy_auth_url' in iv:
143 ks = ksclient.Client(
144 auth_url=iv['deploy_auth_url'],
145 user_id=iv['deploy_user_id'],
146 password=iv['deploy_password'],
147 project_id=iv['deploy_project_id'])
148 endpoint = ks.service_catalog.url_for(
149 service_type='orchestration', endpoint_type='publicURL')
150 log.debug('Signalling to %s' % endpoint)
151 heat = heatclient.Client(
152 '1', endpoint, token=ks.auth_token)
153 r = heat.resources.signal(
154 iv.get('deploy_stack_id'),
155 iv.get('deploy_resource_name'),
156 data=signal_data)
157 log.debug('Response %s ' % r)
158
159 return 0
160
161
162if __name__ == '__main__':
163 sys.exit(main(sys.argv, sys.stdin))
diff --git a/magnum/drivers/common/image/heat-container-agent/scripts/hooks/atomic b/magnum/drivers/common/image/heat-container-agent/scripts/hooks/atomic
new file mode 100755
index 0000000..bba5e4f
--- /dev/null
+++ b/magnum/drivers/common/image/heat-container-agent/scripts/hooks/atomic
@@ -0,0 +1,115 @@
1#!/usr/bin/env python
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15import json
16import logging
17import os
18import subprocess
19import sys
20
21WORKING_DIR = os.environ.get('HEAT_ATOMIC_WORKING',
22 '/var/lib/heat-config/heat-config-atomic')
23ATOMIC_CMD = os.environ.get('HEAT_ATOMIC_CMD', 'atomic')
24
25
26def prepare_dir(path):
27 if not os.path.isdir(path):
28 os.makedirs(path, 0o700)
29
30
31def build_response(deploy_stdout, deploy_stderr, deploy_status_code):
32 return {
33 'deploy_stdout': deploy_stdout,
34 'deploy_stderr': deploy_stderr,
35 'deploy_status_code': deploy_status_code,
36 }
37
38
39def main(argv=sys.argv):
40 log = logging.getLogger('heat-config')
41 handler = logging.StreamHandler(sys.stderr)
42 handler.setFormatter(
43 logging.Formatter(
44 '[%(asctime)s] (%(name)s) [%(levelname)s] %(message)s'))
45 log.addHandler(handler)
46 log.setLevel('DEBUG')
47
48 c = json.load(sys.stdin)
49
50 prepare_dir(WORKING_DIR)
51 os.chdir(WORKING_DIR)
52
53 env = os.environ.copy()
54
55 input_values = dict((i['name'], i['value']) for i in c['inputs'])
56
57 stdout, stderr = {}, {}
58 config = c.get('config', '')
59 if not config:
60 log.debug("No 'config' input found, nothing to do.")
61 json.dump(build_response(stdout, stderr, 0), sys.stdout)
62 return
63
64 atomic_subcmd = config.get('command', 'install')
65 image = config.get('image')
66
67 if input_values.get('deploy_action') == 'DELETE':
68 cmd = [
69 'uninstall',
70 atomic_subcmd,
71 image
72 ]
73 subproc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
74 stderr=subprocess.PIPE, env=env)
75 stdout, stderr = subproc.communicate()
76
77 json.dump(build_response(stdout, stderr, subproc.returncode), sys.stdout)
78 return
79
80 install_cmd = config.get('installcmd', '')
81 name = config.get('name', c.get('id'))
82
83 cmd = [
84 ATOMIC_CMD,
85 atomic_subcmd,
86 image,
87 '-n %s' % name
88 ]
89
90 if atomic_subcmd == 'install':
91 cmd.extend([install_cmd])
92
93 privileged = config.get('privileged', False)
94
95 if atomic_subcmd == 'run' and privileged:
96 cmd.extend(['--spc'])
97
98 log.debug('Running %s' % cmd)
99
100 subproc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
101 stderr=subprocess.PIPE)
102 stdout, stderr = subproc.communicate()
103
104 log.debug(stdout)
105 log.debug(stderr)
106
107 if subproc.returncode:
108 log.error("Error running %s. [%s]\n" % (cmd, subproc.returncode))
109 else:
110 log.debug('Completed %s' % cmd)
111
112 json.dump(build_response(stdout, stderr, subproc.returncode), sys.stdout)
113
114if __name__ == '__main__':
115 sys.exit(main(sys.argv))
diff --git a/magnum/drivers/common/image/heat-container-agent/scripts/hooks/docker-compose b/magnum/drivers/common/image/heat-container-agent/scripts/hooks/docker-compose
new file mode 100755
index 0000000..41219ef
--- /dev/null
+++ b/magnum/drivers/common/image/heat-container-agent/scripts/hooks/docker-compose
@@ -0,0 +1,127 @@
1#!/usr/bin/env python
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15import ast
16import dpath
17import json
18import logging
19import os
20import subprocess
21import sys
22import yaml
23
24
25WORKING_DIR = os.environ.get('HEAT_DOCKER_COMPOSE_WORKING',
26 '/var/lib/heat-config/heat-config-docker-compose')
27
28DOCKER_COMPOSE_CMD = os.environ.get('HEAT_DOCKER_COMPOSE_CMD',
29 'docker-compose')
30
31
32def prepare_dir(path):
33 if not os.path.isdir(path):
34 os.makedirs(path, 0o700)
35
36
37def write_input_file(file_path, content):
38 prepare_dir(os.path.dirname(file_path))
39 with os.fdopen(os.open(
40 file_path, os.O_CREAT | os.O_WRONLY, 0o600), 'w') as f:
41 f.write(content.encode('utf-8'))
42
43
44def build_response(deploy_stdout, deploy_stderr, deploy_status_code):
45 return {
46 'deploy_stdout': deploy_stdout,
47 'deploy_stderr': deploy_stderr,
48 'deploy_status_code': deploy_status_code,
49 }
50
51
52def main(argv=sys.argv):
53 log = logging.getLogger('heat-config')
54 handler = logging.StreamHandler(sys.stderr)
55 handler.setFormatter(
56 logging.Formatter(
57 '[%(asctime)s] (%(name)s) [%(levelname)s] %(message)s'))
58 log.addHandler(handler)
59 log.setLevel('DEBUG')
60
61 c = json.load(sys.stdin)
62
63 input_values = dict((i['name'], i['value']) for i in c['inputs'])
64
65 proj = os.path.join(WORKING_DIR, c.get('name'))
66 prepare_dir(proj)
67
68 stdout, stderr = {}, {}
69
70 if input_values.get('deploy_action') == 'DELETE':
71 json.dump(build_response(stdout, stderr, 0), sys.stdout)
72 return
73
74 config = c.get('config', '')
75 if not config:
76 log.debug("No 'config' input found, nothing to do.")
77 json.dump(build_response(stdout, stderr, 0), sys.stdout)
78 return
79
80 # convert config to dict
81 if not isinstance(config, dict):
82 config = ast.literal_eval(json.dumps(yaml.safe_load(config)))
83
84 os.chdir(proj)
85
86 compose_env_files = []
87 for value in dpath.util.values(config, '*/env_file'):
88 if isinstance(value, list):
89 compose_env_files.extend(value)
90 elif isinstance(value, basestring):
91 compose_env_files.extend([value])
92
93 input_env_files = {}
94 if input_values.get('env_files'):
95 input_env_files = dict(
96 (i['file_name'], i['content'])
97 for i in ast.literal_eval(input_values.get('env_files')))
98
99 for file in compose_env_files:
100 if file in input_env_files.keys():
101 write_input_file(file, input_env_files.get(file))
102
103 cmd = [
104 DOCKER_COMPOSE_CMD,
105 'up',
106 '-d',
107 '--no-build',
108 ]
109
110 log.debug('Running %s' % cmd)
111
112 subproc = subprocess.Popen(cmd, stdout=subprocess.PIPE,
113 stderr=subprocess.PIPE)
114 stdout, stderr = subproc.communicate()
115
116 log.debug(stdout)
117 log.debug(stderr)
118
119 if subproc.returncode:
120 log.error("Error running %s. [%s]\n" % (cmd, subproc.returncode))
121 else:
122 log.debug('Completed %s' % cmd)
123
124 json.dump(build_response(stdout, stderr, subproc.returncode), sys.stdout)
125
126if __name__ == '__main__':
127 sys.exit(main(sys.argv))
diff --git a/magnum/drivers/common/image/heat-container-agent/scripts/hooks/script b/magnum/drivers/common/image/heat-container-agent/scripts/hooks/script
new file mode 100755
index 0000000..5c9cb4d
--- /dev/null
+++ b/magnum/drivers/common/image/heat-container-agent/scripts/hooks/script
@@ -0,0 +1,96 @@
1#!/usr/bin/env python
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15import json
16import logging
17import os
18import subprocess
19import sys
20
21WORKING_DIR = os.environ.get('HEAT_SCRIPT_WORKING',
22 '/var/lib/heat-config/heat-config-script')
23OUTPUTS_DIR = os.environ.get('HEAT_SCRIPT_OUTPUTS',
24 '/var/run/heat-config/heat-config-script')
25
26
27def prepare_dir(path):
28 if not os.path.isdir(path):
29 os.makedirs(path, 0o700)
30
31
32def main(argv=sys.argv):
33 log = logging.getLogger('heat-config')
34 handler = logging.StreamHandler(sys.stderr)
35 handler.setFormatter(
36 logging.Formatter(
37 '[%(asctime)s] (%(name)s) [%(levelname)s] %(message)s'))
38 log.addHandler(handler)
39 log.setLevel('DEBUG')
40
41 prepare_dir(OUTPUTS_DIR)
42 prepare_dir(WORKING_DIR)
43 os.chdir(WORKING_DIR)
44
45 c = json.load(sys.stdin)
46
47 env = os.environ.copy()
48 for input in c['inputs']:
49 input_name = input['name']
50 value = input.get('value', '')
51 if isinstance(value, dict) or isinstance(value, list):
52 env[input_name] = json.dumps(value)
53 else:
54 env[input_name] = value
55 log.info('%s=%s' % (input_name, env[input_name]))
56
57 fn = os.path.join(WORKING_DIR, c['id'])
58 heat_outputs_path = os.path.join(OUTPUTS_DIR, c['id'])
59 env['heat_outputs_path'] = heat_outputs_path
60
61 with os.fdopen(os.open(fn, os.O_CREAT | os.O_WRONLY, 0o700), 'w') as f:
62 f.write(c.get('config', '').encode('utf-8'))
63
64 log.debug('Running %s' % fn)
65 subproc = subprocess.Popen([fn], stdout=subprocess.PIPE,
66 stderr=subprocess.PIPE, env=env)
67 stdout, stderr = subproc.communicate()
68
69 log.info(stdout)
70 log.debug(stderr)
71
72 if subproc.returncode:
73 log.error("Error running %s. [%s]\n" % (fn, subproc.returncode))
74 else:
75 log.info('Completed %s' % fn)
76
77 response = {}
78
79 for output in c.get('outputs') or []:
80 output_name = output['name']
81 try:
82 with open('%s.%s' % (heat_outputs_path, output_name)) as out:
83 response[output_name] = out.read()
84 except IOError:
85 pass
86
87 response.update({
88 'deploy_stdout': stdout,
89 'deploy_stderr': stderr,
90 'deploy_status_code': subproc.returncode,
91 })
92
93 json.dump(response, sys.stdout)
94
95if __name__ == '__main__':
96 sys.exit(main(sys.argv))
diff --git a/magnum/drivers/common/image/heat-container-agent/scripts/write-os-apply-config-templates.sh b/magnum/drivers/common/image/heat-container-agent/scripts/write-os-apply-config-templates.sh
new file mode 100644
index 0000000..15b3aa5
--- /dev/null
+++ b/magnum/drivers/common/image/heat-container-agent/scripts/write-os-apply-config-templates.sh
@@ -0,0 +1,71 @@
1#!/bin/bash
2set -eux
3
4# os-apply-config templates directory
5oac_templates=/usr/libexec/os-apply-config/templates
6mkdir -p $oac_templates/etc
7
8
9# template for building os-collect-config.conf for polling heat
10cat <<EOF >$oac_templates/etc/os-collect-config.conf
11[DEFAULT]
12{{^os-collect-config.command}}
13command = os-refresh-config
14{{/os-collect-config.command}}
15{{#os-collect-config}}
16{{#command}}
17command = {{command}}
18{{/command}}
19{{#polling_interval}}
20polling_interval = {{polling_interval}}
21{{/polling_interval}}
22{{#cachedir}}
23cachedir = {{cachedir}}
24{{/cachedir}}
25{{#collectors}}
26collectors = {{.}}
27{{/collectors}}
28
29{{#cfn}}
30[cfn]
31{{#metadata_url}}
32metadata_url = {{metadata_url}}
33{{/metadata_url}}
34stack_name = {{stack_name}}
35secret_access_key = {{secret_access_key}}
36access_key_id = {{access_key_id}}
37path = {{path}}
38{{/cfn}}
39
40{{#heat}}
41[heat]
42auth_url = {{auth_url}}
43user_id = {{user_id}}
44password = {{password}}
45project_id = {{project_id}}
46stack_id = {{stack_id}}
47resource_name = {{resource_name}}
48{{/heat}}
49
50{{#zaqar}}
51[zaqar]
52auth_url = {{auth_url}}
53user_id = {{user_id}}
54password = {{password}}
55project_id = {{project_id}}
56queue_id = {{queue_id}}
57{{/zaqar}}
58
59{{#request}}
60[request]
61{{#metadata_url}}
62metadata_url = {{metadata_url}}
63{{/metadata_url}}
64{{/request}}
65
66{{/os-collect-config}}
67EOF
68mkdir -p $oac_templates/var/run/heat-config
69
70# template for writing heat deployments data to a file
71echo "{{deployments}}" > $oac_templates/var/run/heat-config/heat-config
diff --git a/magnum/drivers/common/image/heat-container-agent/service.template b/magnum/drivers/common/image/heat-container-agent/service.template
new file mode 100644
index 0000000..23c7bc2
--- /dev/null
+++ b/magnum/drivers/common/image/heat-container-agent/service.template
@@ -0,0 +1,11 @@
1[Unit]
2Description=Heat Container Agent system image
3
4[Service]
5ExecStart=$EXEC_START
6ExecStop=$EXEC_STOP
7Restart=on-failure
8WorkingDirectory=$DESTDIR
9
10[Install]
11WantedBy=multi-user.target \ No newline at end of file
diff --git a/magnum/drivers/common/image/heat-container-agent/tmpfiles.template b/magnum/drivers/common/image/heat-container-agent/tmpfiles.template
new file mode 100644
index 0000000..a41e35c
--- /dev/null
+++ b/magnum/drivers/common/image/heat-container-agent/tmpfiles.template
@@ -0,0 +1,10 @@
1d /var/lib/heat-container-agent - - - - -
2Z /var/lib/heat-container-agent - - - - -
3d /var/run/heat-config - - - - -
4Z /var/run/heat-config - - - - -
5d /var/run/os-collect-config - - - - -
6Z /var/run/os-collect-config - - - - -
7d /opt/stack/os-config-refresh - - - - -
8Z /opt/stack/os-config-refresh - - - - -
9d /srv/magnum - - - - -
10Z /srv/magnum - - - - -
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-monitoring.sh b/magnum/drivers/common/templates/kubernetes/fragments/enable-monitoring.sh
deleted file mode 100644
index 282f732..0000000
--- a/magnum/drivers/common/templates/kubernetes/fragments/enable-monitoring.sh
+++ /dev/null
@@ -1,139 +0,0 @@
1#!/bin/bash
2
3. /etc/sysconfig/heat-params
4
5if [ "$(echo $PROMETHEUS_MONITORING | tr '[:upper:]' '[:lower:]')" = "false" ]; then
6 exit 0
7fi
8
9function writeFile {
10 # $1 is filename
11 # $2 is file content
12
13 [ -f ${1} ] || {
14 echo "Writing File: $1"
15 mkdir -p $(dirname ${1})
16 cat << EOF > ${1}
17$2
18EOF
19 }
20}
21
22KUBE_MON_BIN=/usr/local/bin/kube-enable-monitoring
23KUBE_MON_SERVICE=/etc/systemd/system/kube-enable-monitoring.service
24GRAFANA_DEF_DASHBOARDS="/var/lib/grafana/dashboards"
25GRAFANA_DEF_DASHBOARD_FILE=$GRAFANA_DEF_DASHBOARDS"/default.json"
26
27# Write the binary for enable-monitoring
28KUBE_MON_BIN_CONTENT='''#!/bin/sh
29until curl -sf "http://127.0.0.1:8080/healthz"
30do
31 echo "Waiting for Kubernetes API..."
32 sleep 5
33done
34
35# Check if all resources exist already before creating them
36# Check if configmap Prometheus exists
37kubectl get configmap prometheus -n kube-system
38if [ "$?" != "0" ] && \
39 [ -f "/srv/kubernetes/monitoring/prometheusConfigMap.yaml" ]; then
40 kubectl create -f /srv/kubernetes/monitoring/prometheusConfigMap.yaml
41fi
42
43# Check if deployment and service Prometheus exist
44kubectl get service prometheus -n kube-system | kubectl get deployment prometheus -n kube-system
45if [ "${PIPESTATUS[0]}" != "0" ] && [ "${PIPESTATUS[1]}" != "0" ] && \
46 [ -f "/srv/kubernetes/monitoring/prometheusService.yaml" ]; then
47 kubectl create -f /srv/kubernetes/monitoring/prometheusService.yaml
48fi
49
50# Check if configmap graf-dash exists
51kubectl get configmap graf-dash -n kube-system
52if [ "$?" != "0" ] && \
53 [ -f '''$GRAFANA_DEF_DASHBOARD_FILE''' ]; then
54 kubectl create configmap graf-dash --from-file='''$GRAFANA_DEF_DASHBOARD_FILE''' -n kube-system
55fi
56
57# Check if deployment and service Grafana exist
58kubectl get service grafana -n kube-system | kubectl get deployment grafana -n kube-system
59if [ "${PIPESTATUS[0]}" != "0" ] && [ "${PIPESTATUS[1]}" != "0" ] && \
60 [ -f "/srv/kubernetes/monitoring/grafanaService.yaml" ]; then
61 kubectl create -f /srv/kubernetes/monitoring/grafanaService.yaml
62fi
63
64# Wait for Grafana pod and then inject data source
65while true
66do
67 echo "Waiting for Grafana pod to be up and Running"
68 if [ "$(kubectl get po -n kube-system -l name=grafana -o jsonpath={..phase})" = "Running" ]; then
69 break
70 fi
71 sleep 2
72done
73
74# Which node is running Grafana
75NODE_IP=`kubectl get po -n kube-system -o jsonpath={.items[0].status.hostIP} -l name=grafana`
76PROM_SERVICE_IP=`kubectl get svc prometheus --namespace kube-system -o jsonpath={..clusterIP}`
77
78# The Grafana pod might be running but the app might still be initiating
79echo "Check if Grafana is ready..."
80curl --user admin:$ADMIN_PASSWD -X GET http://$NODE_IP:3000/api/datasources/1
81until [ $? -eq 0 ]
82do
83 sleep 2
84 curl --user admin:$ADMIN_PASSWD -X GET http://$NODE_IP:3000/api/datasources/1
85done
86
87# Inject Prometheus datasource into Grafana
88while true
89do
90 INJECT=`curl --user admin:$ADMIN_PASSWD -X POST \
91 -H "Content-Type: application/json;charset=UTF-8" \
92 --data-binary '''"'"'''{"name":"k8sPrometheus","isDefault":true,
93 "type":"prometheus","url":"http://'''"'"'''$PROM_SERVICE_IP'''"'"''':9090","access":"proxy"}'''"'"'''\
94 "http://$NODE_IP:3000/api/datasources/"`
95
96 if [[ "$INJECT" = *"Datasource added"* ]]; then
97 echo "Prometheus datasource injected into Grafana"
98 break
99 fi
100 echo "Trying to inject Prometheus datasource into Grafana - "$INJECT
101done
102'''
103writeFile $KUBE_MON_BIN "$KUBE_MON_BIN_CONTENT"
104
105
106# Write the monitoring service
107KUBE_MON_SERVICE_CONTENT='''[Unit]
108Requires=kubelet.service
109
110[Service]
111Type=oneshot
112Environment=HOME=/root
113EnvironmentFile=-/etc/kubernetes/config
114ExecStart='''${KUBE_MON_BIN}'''
115
116[Install]
117WantedBy=multi-user.target
118'''
119writeFile $KUBE_MON_SERVICE "$KUBE_MON_SERVICE_CONTENT"
120
121chown root:root ${KUBE_MON_BIN}
122chmod 0755 ${KUBE_MON_BIN}
123
124chown root:root ${KUBE_MON_SERVICE}
125chmod 0644 ${KUBE_MON_SERVICE}
126
127# Download the default JSON Grafana dashboard
128# Not a crucial step, so allow it to fail
129# TODO: this JSON should be passed into the minions as gzip in cloud-init
130GRAFANA_DASHB_URL="https://grafana.net/api/dashboards/1621/revisions/1/download"
131mkdir -p $GRAFANA_DEF_DASHBOARDS
132curl $GRAFANA_DASHB_URL -o $GRAFANA_DEF_DASHBOARD_FILE || echo "Failed to fetch default Grafana dashboard"
133if [ -f $GRAFANA_DEF_DASHBOARD_FILE ]; then
134 sed -i -- 's|${DS_PROMETHEUS}|k8sPrometheus|g' $GRAFANA_DEF_DASHBOARD_FILE
135fi
136
137# Launch the monitoring service
138systemctl enable kube-enable-monitoring
139systemctl start --no-block kube-enable-monitoring
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/enable-prometheus-monitoring b/magnum/drivers/common/templates/kubernetes/fragments/enable-prometheus-monitoring
new file mode 100644
index 0000000..c037c9f
--- /dev/null
+++ b/magnum/drivers/common/templates/kubernetes/fragments/enable-prometheus-monitoring
@@ -0,0 +1,434 @@
1#!/bin/bash
2
3. /etc/sysconfig/heat-params
4
5function writeFile {
6 # $1 is filename
7 # $2 is file content
8
9 [ -f ${1} ] || {
10 echo "Writing File: $1"
11 mkdir -p $(dirname ${1})
12 cat << EOF > ${1}
13$2
14EOF
15 }
16}
17
18prometheusConfigMap_file=/srv/magnum/kubernetes/monitoring/prometheusConfigMap.yaml
19[ -f ${prometheusConfigMap_file} ] || {
20 echo "Writing File: $prometheusConfigMap_file"
21 mkdir -p $(dirname ${prometheusConfigMap_file})
22 # NOTE: EOF needs to be in quotes in order to not escape the $ characters
23 cat << 'EOF' > ${prometheusConfigMap_file}
24apiVersion: v1
25kind: ConfigMap
26metadata:
27 name: prometheus
28 namespace: kube-system
29data:
30 prometheus.yml: |
31 global:
32 scrape_interval: 10s
33 scrape_timeout: 10s
34 evaluation_interval: 10s
35
36 scrape_configs:
37 - job_name: 'kubernetes-apiservers'
38
39 kubernetes_sd_configs:
40 - role: endpoints
41
42 scheme: https
43
44 tls_config:
45 ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
46 bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
47 relabel_configs:
48 - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
49 action: keep
50 regex: default;kubernetes;https
51 - job_name: 'kubernetes-nodes'
52 scheme: https
53 tls_config:
54 ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
55 bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
56 kubernetes_sd_configs:
57 - role: node
58 relabel_configs:
59 - action: labelmap
60 regex: __meta_kubernetes_node_label_(.+)
61 - target_label: __address__
62 replacement: kubernetes.default.svc:443
63 - source_labels: [__meta_kubernetes_node_name]
64 regex: (.+)
65 target_label: __metrics_path__
66 replacement: /api/v1/nodes/${1}/proxy/metrics
67
68 - job_name: 'kubernetes-cadvisor'
69 scheme: https
70 tls_config:
71 ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
72 bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
73 kubernetes_sd_configs:
74 - role: node
75 relabel_configs:
76 - action: labelmap
77 regex: __meta_kubernetes_node_label_(.+)
78 - target_label: __address__
79 replacement: kubernetes.default.svc:443
80 - source_labels: [__meta_kubernetes_node_name]
81 regex: (.+)
82 target_label: __metrics_path__
83 replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
84
85 - job_name: 'kubernetes-service-endpoints'
86
87 kubernetes_sd_configs:
88 - role: endpoints
89
90 relabel_configs:
91 - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
92 action: keep
93 regex: true
94 - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
95 action: replace
96 target_label: __scheme__
97 regex: (https?)
98 - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
99 action: replace
100 target_label: __metrics_path__
101 regex: (.+)
102 - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
103 action: replace
104 target_label: __address__
105 regex: ([^:]+)(?::\d+)?;(\d+)
106 replacement: $1:$2
107 - action: labelmap
108 regex: __meta_kubernetes_service_label_(.+)
109 - source_labels: [__meta_kubernetes_namespace]
110 action: replace
111 target_label: kubernetes_namespace
112 - source_labels: [__meta_kubernetes_service_name]
113 action: replace
114 target_label: kubernetes_name
115
116 - job_name: 'kubernetes-services'
117 metrics_path: /probe
118 params:
119 module: [http_2xx]
120 kubernetes_sd_configs:
121 - role: service
122 relabel_configs:
123 - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
124 action: keep
125 regex: true
126 - source_labels: [__address__]
127 target_label: __param_target
128 - target_label: __address__
129 replacement: blackbox
130 - source_labels: [__param_target]
131 target_label: instance
132 - action: labelmap
133 regex: __meta_kubernetes_service_label_(.+)
134 - source_labels: [__meta_kubernetes_namespace]
135 target_label: kubernetes_namespace
136 - source_labels: [__meta_kubernetes_service_name]
137 target_label: kubernetes_name
138
139 - job_name: 'kubernetes-pods'
140 kubernetes_sd_configs:
141 - role: pod
142 relabel_configs:
143 - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
144 action: keep
145 regex: true
146 - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
147 action: replace
148 target_label: __metrics_path__
149 regex: (.+)
150 - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
151 action: replace
152 regex: ([^:]+)(?::\d+)?;(\d+)
153 replacement: $1:$2
154 target_label: __address__
155 - action: labelmap
156 regex: __meta_kubernetes_pod_label_(.+)
157 - source_labels: [__meta_kubernetes_namespace]
158 action: replace
159 target_label: kubernetes_namespace
160 - source_labels: [__meta_kubernetes_pod_name]
161 action: replace
162 target_label: kubernetes_pod_name
163
164 - job_name: 'kubernetes-node-exporter'
165 tls_config:
166 ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
167 bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
168 kubernetes_sd_configs:
169 - role: node
170 relabel_configs:
171 - action: labelmap
172 regex: __meta_kubernetes_node_label_(.+)
173 - source_labels: [__meta_kubernetes_role]
174 action: replace
175 target_label: kubernetes_role
176 - source_labels: [__address__]
177 regex: '(.*):10250'
178 replacement: '${1}:9100'
179 target_label: __address__
180EOF
181}
182
183prometheusService_file=/srv/magnum/kubernetes/monitoring/prometheusService.yaml
184prometheusService_content=$(cat <<EOF
185apiVersion: v1
186kind: Service
187metadata:
188 annotations:
189 prometheus.io/scrape: 'true'
190 labels:
191 name: prometheus
192 name: prometheus
193 namespace: kube-system
194spec:
195 selector:
196 app: prometheus
197 type: NodePort
198 ports:
199 - name: prometheus
200 protocol: TCP
201 port: 9090
202 nodePort: 30900
203---
204apiVersion: extensions/v1beta1
205kind: Deployment
206metadata:
207 name: prometheus
208 namespace: kube-system
209spec:
210 replicas: 1
211 selector:
212 matchLabels:
213 app: prometheus
214 template:
215 metadata:
216 name: prometheus
217 labels:
218 app: prometheus
219 spec:
220 containers:
221 - name: prometheus
222 image: ${CONTAINER_INFRA_PREFIX:-docker.io/prom/}prometheus:v1.8.2
223 args:
224 - '-storage.local.retention=6h'
225 - '-storage.local.memory-chunks=500000'
226 - '-config.file=/etc/prometheus/prometheus.yml'
227 ports:
228 - name: web
229 containerPort: 9090
230 hostPort: 9090
231 volumeMounts:
232 - name: config-volume
233 mountPath: /etc/prometheus
234 volumes:
235 - name: config-volume
236 configMap:
237 name: prometheus
238EOF
239)
240writeFile $prometheusService_file "$prometheusService_content"
241
242grafanaService_file=/srv/magnum/kubernetes/monitoring/grafanaService.yaml
243grafanaService_content=$(cat <<EOF
244apiVersion: v1
245kind: Service
246metadata:
247 labels:
248 name: node
249 role: service
250 name: grafana
251 namespace: kube-system
252spec:
253 type: "NodePort"
254 ports:
255 - port: 3000
256 targetPort: 3000
257 nodePort: 30603
258 selector:
259 grafana: "true"
260---
261apiVersion: extensions/v1beta1
262kind: Deployment
263metadata:
264 name: grafana
265 namespace: kube-system
266spec:
267 replicas: 1
268 template:
269 metadata:
270 labels:
271 name: grafana
272 grafana: "true"
273 role: db
274 spec:
275 containers:
276 - image: ${CONTAINER_INFRA_PREFIX:-docker.io/grafana/}grafana
277 imagePullPolicy: Always
278 name: grafana
279 env:
280 - name: GF_SECURITY_ADMIN_PASSWORD
281 value: $ADMIN_PASSWD
282 - name: GF_DASHBOARDS_JSON_ENABLED
283 value: "true"
284 - name: GF_DASHBOARDS_JSON_PATH
285 value: /var/lib/grafana/dashboards
286 resources:
287 # keep request = limit to keep this container in guaranteed class
288 limits:
289 cpu: 100m
290 memory: 200Mi
291 requests:
292 cpu: 100m
293 memory: 200Mi
294 volumeMounts:
295 - name: default-dashboard
296 mountPath: /var/lib/grafana/dashboards
297 ports:
298 - containerPort: 3000
299 hostPort: 3000
300 volumes:
301 - name: default-dashboard
302 configMap:
303 name: graf-dash
304EOF
305)
306writeFile $grafanaService_file "$grafanaService_content"
307
308. /etc/sysconfig/heat-params
309
310if [ "$(echo $PROMETHEUS_MONITORING | tr '[:upper:]' '[:lower:]')" = "false" ]; then
311 exit 0
312fi
313
314PROMETHEUS_MON_BASE_DIR="/srv/magnum/kubernetes/monitoring"
315KUBE_MON_BIN=${PROMETHEUS_MON_BASE_DIR}"/bin/kube-enable-monitoring"
316KUBE_MON_SERVICE="/etc/systemd/system/kube-enable-monitoring.service"
317GRAFANA_DEF_DASHBOARDS=${PROMETHEUS_MON_BASE_DIR}"/dashboards"
318GRAFANA_DEF_DASHBOARD_FILE=$GRAFANA_DEF_DASHBOARDS"/default.json"
319
320# Write the binary for enable-monitoring
321KUBE_MON_BIN_CONTENT='''#!/bin/sh
322until curl -sf "http://127.0.0.1:8080/healthz"
323do
324 echo "Waiting for Kubernetes API..."
325 sleep 5
326done
327
328# Check if all resources exist already before creating them
329# Check if configmap Prometheus exists
330kubectl get configmap prometheus -n kube-system
331if [ "$?" != "0" ] && \
332 [ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/prometheusConfigMap.yaml" ]; then
333 kubectl create -f '''${PROMETHEUS_MON_BASE_DIR}'''/prometheusConfigMap.yaml
334fi
335
336# Check if deployment and service Prometheus exist
337kubectl get service prometheus -n kube-system | kubectl get deployment prometheus -n kube-system
338if [ "${PIPESTATUS[0]}" != "0" ] && [ "${PIPESTATUS[1]}" != "0" ] && \
339 [ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/prometheusService.yaml" ]; then
340 kubectl create -f '''${PROMETHEUS_MON_BASE_DIR}'''/prometheusService.yaml
341fi
342
343# Check if configmap graf-dash exists
344kubectl get configmap graf-dash -n kube-system
345if [ "$?" != "0" ] && \
346 [ -f '''$GRAFANA_DEF_DASHBOARD_FILE''' ]; then
347 kubectl create configmap graf-dash --from-file='''$GRAFANA_DEF_DASHBOARD_FILE''' -n kube-system
348fi
349
350# Check if deployment and service Grafana exist
351kubectl get service grafana -n kube-system | kubectl get deployment grafana -n kube-system
352if [ "${PIPESTATUS[0]}" != "0" ] && [ "${PIPESTATUS[1]}" != "0" ] && \
353 [ -f "'''${PROMETHEUS_MON_BASE_DIR}'''/grafanaService.yaml" ]; then
354 kubectl create -f '''${PROMETHEUS_MON_BASE_DIR}'''/grafanaService.yaml
355fi
356
357# Wait for Grafana pod and then inject data source
358while true
359do
360 echo "Waiting for Grafana pod to be up and Running"
361 if [ "$(kubectl get po -n kube-system -l name=grafana -o jsonpath={..phase})" = "Running" ]; then
362 break
363 fi
364 sleep 2
365done
366
367# Which node is running Grafana
368NODE_IP=`kubectl get po -n kube-system -o jsonpath={.items[0].status.hostIP} -l name=grafana`
369PROM_SERVICE_IP=`kubectl get svc prometheus --namespace kube-system -o jsonpath={..clusterIP}`
370
371# The Grafana pod might be running but the app might still be initiating
372echo "Check if Grafana is ready..."
373curl --user admin:$ADMIN_PASSWD -X GET http://$NODE_IP:3000/api/datasources/1
374until [ $? -eq 0 ]
375do
376 sleep 2
377 curl --user admin:$ADMIN_PASSWD -X GET http://$NODE_IP:3000/api/datasources/1
378done
379
380# Inject Prometheus datasource into Grafana
381while true
382do
383 INJECT=`curl --user admin:$ADMIN_PASSWD -X POST \
384 -H "Content-Type: application/json;charset=UTF-8" \
385 --data-binary '''"'"'''{"name":"k8sPrometheus","isDefault":true,
386 "type":"prometheus","url":"http://'''"'"'''$PROM_SERVICE_IP'''"'"''':9090","access":"proxy"}'''"'"'''\
387 "http://$NODE_IP:3000/api/datasources/"`
388
389 if [[ "$INJECT" = *"Datasource added"* ]]; then
390 echo "Prometheus datasource injected into Grafana"
391 break
392 fi
393 echo "Trying to inject Prometheus datasource into Grafana - "$INJECT
394done
395'''
396writeFile $KUBE_MON_BIN "$KUBE_MON_BIN_CONTENT"
397
398
399# Write the monitoring service
400KUBE_MON_SERVICE_CONTENT='''[Unit]
401Requires=kubelet.service
402
403[Service]
404Type=oneshot
405Environment=HOME=/root
406EnvironmentFile=-/etc/kubernetes/config
407ExecStart='''${KUBE_MON_BIN}'''
408
409[Install]
410WantedBy=multi-user.target
411'''
412writeFile $KUBE_MON_SERVICE "$KUBE_MON_SERVICE_CONTENT"
413
414chown root:root ${KUBE_MON_BIN}
415chmod 0755 ${KUBE_MON_BIN}
416
417chown root:root ${KUBE_MON_SERVICE}
418chmod 0644 ${KUBE_MON_SERVICE}
419
420# Download the default JSON Grafana dashboard
421# Not a crucial step, so allow it to fail
422# TODO: this JSON should be passed into the minions as gzip in cloud-init
423GRAFANA_DASHB_URL="https://grafana.net/api/dashboards/1621/revisions/1/download"
424mkdir -p $GRAFANA_DEF_DASHBOARDS
425curl $GRAFANA_DASHB_URL -o $GRAFANA_DEF_DASHBOARD_FILE || echo "Failed to fetch default Grafana dashboard"
426if [ -f $GRAFANA_DEF_DASHBOARD_FILE ]; then
427 sed -i -- 's|${DS_PROMETHEUS}|k8sPrometheus|g' $GRAFANA_DEF_DASHBOARD_FILE
428fi
429
430# Launch the monitoring service
431set -x
432systemctl daemon-reload
433systemctl enable kube-enable-monitoring.service
434systemctl start --no-block kube-enable-monitoring.service
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/start-container-agent.sh b/magnum/drivers/common/templates/kubernetes/fragments/start-container-agent.sh
new file mode 100644
index 0000000..ebf4a87
--- /dev/null
+++ b/magnum/drivers/common/templates/kubernetes/fragments/start-container-agent.sh
@@ -0,0 +1,16 @@
1#!/bin/bash
2
3. /etc/sysconfig/heat-params
4
5set -ux
6
7_prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}
8atomic install \
9--storage ostree \
10--system \
11--system-package no \
12--set REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt \
13--name heat-container-agent \
14${_prefix}heat-container-agent:rawhide
15
16systemctl start heat-container-agent
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-grafana-service.yaml b/magnum/drivers/common/templates/kubernetes/fragments/write-grafana-service.yaml
deleted file mode 100644
index 17fe45a..0000000
--- a/magnum/drivers/common/templates/kubernetes/fragments/write-grafana-service.yaml
+++ /dev/null
@@ -1,67 +0,0 @@
1#cloud-config
2merge_how: dict(recurse_array)+list(append)
3write_files:
4 - path: /srv/kubernetes/monitoring/grafanaService.yaml
5 owner: "root:root"
6 permissions: "0644"
7 content: |
8 apiVersion: v1
9 kind: Service
10 metadata:
11 labels:
12 name: node
13 role: service
14 name: grafana
15 namespace: kube-system
16 spec:
17 type: "NodePort"
18 ports:
19 - port: 3000
20 targetPort: 3000
21 nodePort: 30603
22 selector:
23 grafana: "true"
24 ---
25 apiVersion: extensions/v1beta1
26 kind: Deployment
27 metadata:
28 name: grafana
29 namespace: kube-system
30 spec:
31 replicas: 1
32 template:
33 metadata:
34 labels:
35 name: grafana
36 grafana: "true"
37 role: db
38 spec:
39 containers:
40 - image: grafana/grafana
41 imagePullPolicy: Always
42 name: grafana
43 env:
44 - name: GF_SECURITY_ADMIN_PASSWORD
45 value: $ADMIN_PASSWD
46 - name: GF_DASHBOARDS_JSON_ENABLED
47 value: "true"
48 - name: GF_DASHBOARDS_JSON_PATH
49 value: /var/lib/grafana/dashboards
50 resources:
51 # keep request = limit to keep this container in guaranteed class
52 limits:
53 cpu: 100m
54 memory: 200Mi
55 requests:
56 cpu: 100m
57 memory: 200Mi
58 volumeMounts:
59 - name: default-dashboard
60 mountPath: /var/lib/grafana/dashboards
61 ports:
62 - containerPort: 3000
63 hostPort: 3000
64 volumes:
65 - name: default-dashboard
66 configMap:
67 name: graf-dash
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-prometheus-configmap.yaml b/magnum/drivers/common/templates/kubernetes/fragments/write-prometheus-configmap.yaml
deleted file mode 100644
index 0098024..0000000
--- a/magnum/drivers/common/templates/kubernetes/fragments/write-prometheus-configmap.yaml
+++ /dev/null
@@ -1,163 +0,0 @@
1#cloud-config
2merge_how: dict(recurse_array)+list(append)
3write_files:
4 - path: /srv/kubernetes/monitoring/prometheusConfigMap.yaml
5 owner: "root:root"
6 permissions: "0644"
7 content: |
8 apiVersion: v1
9 kind: ConfigMap
10 metadata:
11 name: prometheus
12 namespace: kube-system
13 data:
14 prometheus.yml: |
15 global:
16 scrape_interval: 10s
17 scrape_timeout: 10s
18 evaluation_interval: 10s
19
20 scrape_configs:
21 - job_name: 'kubernetes-apiservers'
22
23 kubernetes_sd_configs:
24 - role: endpoints
25
26 scheme: https
27
28 tls_config:
29 ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
30 bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
31 relabel_configs:
32 - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
33 action: keep
34 regex: default;kubernetes;https
35 - job_name: 'kubernetes-nodes'
36 scheme: https
37 tls_config:
38 ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
39 bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
40 kubernetes_sd_configs:
41 - role: node
42 relabel_configs:
43 - action: labelmap
44 regex: __meta_kubernetes_node_label_(.+)
45 - target_label: __address__
46 replacement: kubernetes.default.svc:443
47 - source_labels: [__meta_kubernetes_node_name]
48 regex: (.+)
49 target_label: __metrics_path__
50 replacement: /api/v1/nodes/${1}/proxy/metrics
51
52 - job_name: 'kubernetes-cadvisor'
53 scheme: https
54 tls_config:
55 ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
56 bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
57 kubernetes_sd_configs:
58 - role: node
59 relabel_configs:
60 - action: labelmap
61 regex: __meta_kubernetes_node_label_(.+)
62 - target_label: __address__
63 replacement: kubernetes.default.svc:443
64 - source_labels: [__meta_kubernetes_node_name]
65 regex: (.+)
66 target_label: __metrics_path__
67 replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor
68
69 - job_name: 'kubernetes-service-endpoints'
70
71 kubernetes_sd_configs:
72 - role: endpoints
73
74 relabel_configs:
75 - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
76 action: keep
77 regex: true
78 - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
79 action: replace
80 target_label: __scheme__
81 regex: (https?)
82 - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
83 action: replace
84 target_label: __metrics_path__
85 regex: (.+)
86 - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
87 action: replace
88 target_label: __address__
89 regex: ([^:]+)(?::\d+)?;(\d+)
90 replacement: $1:$2
91 - action: labelmap
92 regex: __meta_kubernetes_service_label_(.+)
93 - source_labels: [__meta_kubernetes_namespace]
94 action: replace
95 target_label: kubernetes_namespace
96 - source_labels: [__meta_kubernetes_service_name]
97 action: replace
98 target_label: kubernetes_name
99
100 - job_name: 'kubernetes-services'
101 metrics_path: /probe
102 params:
103 module: [http_2xx]
104 kubernetes_sd_configs:
105 - role: service
106 relabel_configs:
107 - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
108 action: keep
109 regex: true
110 - source_labels: [__address__]
111 target_label: __param_target
112 - target_label: __address__
113 replacement: blackbox
114 - source_labels: [__param_target]
115 target_label: instance
116 - action: labelmap
117 regex: __meta_kubernetes_service_label_(.+)
118 - source_labels: [__meta_kubernetes_namespace]
119 target_label: kubernetes_namespace
120 - source_labels: [__meta_kubernetes_service_name]
121 target_label: kubernetes_name
122
123 - job_name: 'kubernetes-pods'
124 kubernetes_sd_configs:
125 - role: pod
126 relabel_configs:
127 - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
128 action: keep
129 regex: true
130 - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
131 action: replace
132 target_label: __metrics_path__
133 regex: (.+)
134 - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
135 action: replace
136 regex: ([^:]+)(?::\d+)?;(\d+)
137 replacement: $1:$2
138 target_label: __address__
139 - action: labelmap
140 regex: __meta_kubernetes_pod_label_(.+)
141 - source_labels: [__meta_kubernetes_namespace]
142 action: replace
143 target_label: kubernetes_namespace
144 - source_labels: [__meta_kubernetes_pod_name]
145 action: replace
146 target_label: kubernetes_pod_name
147
148 - job_name: 'kubernetes-node-exporter'
149 tls_config:
150 ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
151 bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
152 kubernetes_sd_configs:
153 - role: node
154 relabel_configs:
155 - action: labelmap
156 regex: __meta_kubernetes_node_label_(.+)
157 - source_labels: [__meta_kubernetes_role]
158 action: replace
159 target_label: kubernetes_role
160 - source_labels: [__address__]
161 regex: '(.*):10250'
162 replacement: '${1}:9100'
163 target_label: __address__
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-prometheus-service.yaml b/magnum/drivers/common/templates/kubernetes/fragments/write-prometheus-service.yaml
deleted file mode 100644
index 7375a30..0000000
--- a/magnum/drivers/common/templates/kubernetes/fragments/write-prometheus-service.yaml
+++ /dev/null
@@ -1,60 +0,0 @@
1#cloud-config
2merge_how: dict(recurse_array)+list(append)
3write_files:
4 - path: /srv/kubernetes/monitoring/prometheusService.yaml
5 owner: "root:root"
6 permissions: "0644"
7 content: |
8 apiVersion: v1
9 kind: Service
10 metadata:
11 annotations:
12 prometheus.io/scrape: 'true'
13 labels:
14 name: prometheus
15 name: prometheus
16 namespace: kube-system
17 spec:
18 selector:
19 app: prometheus
20 type: NodePort
21 ports:
22 - name: prometheus
23 protocol: TCP
24 port: 9090
25 nodePort: 30900
26 ---
27 apiVersion: extensions/v1beta1
28 kind: Deployment
29 metadata:
30 name: prometheus
31 namespace: kube-system
32 spec:
33 replicas: 1
34 selector:
35 matchLabels:
36 app: prometheus
37 template:
38 metadata:
39 name: prometheus
40 labels:
41 app: prometheus
42 spec:
43 containers:
44 - name: prometheus
45 image: prom/prometheus
46 args:
47 - '-storage.local.retention=6h'
48 - '-storage.local.memory-chunks=500000'
49 - '-config.file=/etc/prometheus/prometheus.yml'
50 ports:
51 - name: web
52 containerPort: 9090
53 hostPort: 9090
54 volumeMounts:
55 - name: config-volume
56 mountPath: /etc/prometheus
57 volumes:
58 - name: config-volume
59 configMap:
60 name: prometheus
diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml
index 335fae7..b1b4a1f 100644
--- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml
+++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml
@@ -386,39 +386,6 @@ resources:
386 group: ungrouped 386 group: ungrouped
387 config: {get_file: ../../common/templates/kubernetes/fragments/write-network-config.sh} 387 config: {get_file: ../../common/templates/kubernetes/fragments/write-network-config.sh}
388 388
389 write_prometheus_configmap:
390 type: OS::Heat::SoftwareConfig
391 properties:
392 group: ungrouped
393 config: {get_file: ../../common/templates/kubernetes/fragments/write-prometheus-configmap.yaml}
394
395
396 write_prometheus_service:
397 type: OS::Heat::SoftwareConfig
398 properties:
399 group: ungrouped
400 config: {get_file: ../../common/templates/kubernetes/fragments/write-prometheus-service.yaml}
401
402 write_grafana_service:
403 type: OS::Heat::SoftwareConfig
404 properties:
405 group: ungrouped
406 config:
407 str_replace:
408 template: {get_file: ../../common/templates/kubernetes/fragments/write-grafana-service.yaml}
409 params:
410 "$ADMIN_PASSWD": {get_param: grafana_admin_passwd}
411
412 enable_monitoring:
413 type: OS::Heat::SoftwareConfig
414 properties:
415 group: ungrouped
416 config:
417 str_replace:
418 template: {get_file: ../../common/templates/kubernetes/fragments/enable-monitoring.sh}
419 params:
420 "$ADMIN_PASSWD": {get_param: grafana_admin_passwd}
421
422 network_config_service: 389 network_config_service:
423 type: OS::Heat::SoftwareConfig 390 type: OS::Heat::SoftwareConfig
424 properties: 391 properties:
@@ -473,6 +440,12 @@ resources:
473 group: ungrouped 440 group: ungrouped
474 config: {get_file: ../../common/templates/kubernetes/fragments/add-proxy.sh} 441 config: {get_file: ../../common/templates/kubernetes/fragments/add-proxy.sh}
475 442
443 start_container_agent:
444 type: OS::Heat::SoftwareConfig
445 properties:
446 group: ungrouped
447 config: {get_file: ../../common/templates/kubernetes/fragments/start-container-agent.sh}
448
476 kube_master_init: 449 kube_master_init:
477 type: OS::Heat::MultipartMime 450 type: OS::Heat::MultipartMime
478 properties: 451 properties:
@@ -487,17 +460,32 @@ resources:
487 - config: {get_resource: add_proxy} 460 - config: {get_resource: add_proxy}
488 - config: {get_resource: enable_services} 461 - config: {get_resource: enable_services}
489 - config: {get_resource: write_network_config} 462 - config: {get_resource: write_network_config}
490 - config: {get_resource: write_prometheus_configmap}
491 - config: {get_resource: write_prometheus_service}
492 - config: {get_resource: write_grafana_service}
493 - config: {get_resource: network_config_service} 463 - config: {get_resource: network_config_service}
494 - config: {get_resource: network_service} 464 - config: {get_resource: network_service}
495 - config: {get_resource: kube_system_namespace_service} 465 - config: {get_resource: kube_system_namespace_service}
496 - config: {get_resource: core_dns_service} 466 - config: {get_resource: core_dns_service}
497 - config: {get_resource: kube_ui_service} 467 - config: {get_resource: kube_ui_service}
498 - config: {get_resource: enable_monitoring} 468 - config: {get_resource: start_container_agent}
499 - config: {get_resource: master_wc_notify} 469 - config: {get_resource: master_wc_notify}
500 470
471 enable_prometheus_monitoring:
472 type: OS::Heat::SoftwareConfig
473 properties:
474 group: script
475 config:
476 str_replace:
477 template: {get_file: ../../common/templates/kubernetes/fragments/enable-prometheus-monitoring}
478 params:
479 "$ADMIN_PASSWD": {get_param: grafana_admin_passwd}
480
481 enable_prometheus_monitoring_deployment:
482 type: OS::Heat::SoftwareDeployment
483 properties:
484 signal_transport: HEAT_SIGNAL
485 config: {get_resource: enable_prometheus_monitoring}
486 server: {get_resource: kube-master}
487 actions: ['CREATE']
488
501 ###################################################################### 489 ######################################################################
502 # 490 #
503 # a single kubernetes master. 491 # a single kubernetes master.
@@ -513,7 +501,8 @@ resources:
513 image: {get_param: server_image} 501 image: {get_param: server_image}
514 flavor: {get_param: master_flavor} 502 flavor: {get_param: master_flavor}
515 key_name: {get_param: ssh_key_name} 503 key_name: {get_param: ssh_key_name}
516 user_data_format: RAW 504 user_data_format: SOFTWARE_CONFIG
505 software_config_transport: POLL_SERVER_HEAT
517 user_data: {get_resource: kube_master_init} 506 user_data: {get_resource: kube_master_init}
518 networks: 507 networks:
519 - port: {get_resource: kube_master_eth0} 508 - port: {get_resource: kube_master_eth0}
diff --git a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml
index e496523..ce99133 100644
--- a/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml
+++ b/magnum/drivers/k8s_fedora_ironic_v1/templates/kubemaster.yaml
@@ -349,39 +349,6 @@ resources:
349 group: ungrouped 349 group: ungrouped
350 config: {get_file: ../../common/templates/kubernetes/fragments/write-network-config.sh} 350 config: {get_file: ../../common/templates/kubernetes/fragments/write-network-config.sh}
351 351
352 write_prometheus_configmap:
353 type: OS::Heat::SoftwareConfig
354 properties:
355 group: ungrouped
356 config: {get_file: ../../common/templates/kubernetes/fragments/write-prometheus-configmap.yaml}
357
358
359 write_prometheus_service:
360 type: OS::Heat::SoftwareConfig
361 properties:
362 group: ungrouped
363 config: {get_file: ../../common/templates/kubernetes/fragments/write-prometheus-service.yaml}
364
365 write_grafana_service:
366 type: OS::Heat::SoftwareConfig
367 properties:
368 group: ungrouped
369 config:
370 str_replace:
371 template: {get_file: ../../common/templates/kubernetes/fragments/write-grafana-service.yaml}
372 params:
373 "$ADMIN_PASSWD": {get_param: grafana_admin_passwd}
374
375 enable_monitoring:
376 type: OS::Heat::SoftwareConfig
377 properties:
378 group: ungrouped
379 config:
380 str_replace:
381 template: {get_file: ../../common/templates/kubernetes/fragments/enable-monitoring.sh}
382 params:
383 "$ADMIN_PASSWD": {get_param: grafana_admin_passwd}
384
385 network_config_service: 352 network_config_service:
386 type: OS::Heat::SoftwareConfig 353 type: OS::Heat::SoftwareConfig
387 properties: 354 properties:
@@ -442,6 +409,12 @@ resources:
442 group: ungrouped 409 group: ungrouped
443 config: {get_file: ../../common/templates/kubernetes/fragments/add-proxy.sh} 410 config: {get_file: ../../common/templates/kubernetes/fragments/add-proxy.sh}
444 411
412 start_container_agent:
413 type: OS::Heat::SoftwareConfig
414 properties:
415 group: ungrouped
416 config: {get_file: ../../common/templates/kubernetes/fragments/start-container-agent.sh}
417
445 kube_master_init: 418 kube_master_init:
446 type: OS::Heat::MultipartMime 419 type: OS::Heat::MultipartMime
447 properties: 420 properties:
@@ -456,18 +429,33 @@ resources:
456 - config: {get_resource: add_proxy} 429 - config: {get_resource: add_proxy}
457 - config: {get_resource: enable_services} 430 - config: {get_resource: enable_services}
458 - config: {get_resource: write_network_config} 431 - config: {get_resource: write_network_config}
459 - config: {get_resource: write_prometheus_configmap}
460 - config: {get_resource: write_prometheus_service}
461 - config: {get_resource: write_grafana_service}
462 - config: {get_resource: network_config_service} 432 - config: {get_resource: network_config_service}
463 - config: {get_resource: network_service} 433 - config: {get_resource: network_service}
464 - config: {get_resource: kube_system_namespace_service} 434 - config: {get_resource: kube_system_namespace_service}
465 - config: {get_resource: enable_kube_controller_manager_scheduler} 435 - config: {get_resource: enable_kube_controller_manager_scheduler}
466 - config: {get_resource: enable_kube_proxy} 436 - config: {get_resource: enable_kube_proxy}
467 - config: {get_resource: kube_ui_service} 437 - config: {get_resource: kube_ui_service}
468 - config: {get_resource: enable_monitoring} 438 - config: {get_resource: start_container_agent}
469 - config: {get_resource: master_wc_notify} 439 - config: {get_resource: master_wc_notify}
470 440
441 enable_prometheus_monitoring:
442 type: OS::Heat::SoftwareConfig
443 properties:
444 group: script
445 config:
446 str_replace:
447 template: {get_file: ../../common/templates/kubernetes/fragments/enable-prometheus-monitoring}
448 params:
449 "$ADMIN_PASSWD": {get_param: grafana_admin_passwd}
450
451 enable_prometheus_monitoring_deployment:
452 type: OS::Heat::SoftwareDeployment
453 properties:
454 signal_transport: HEAT_SIGNAL
455 config: {get_resource: enable_prometheus_monitoring}
456 server: {get_resource: kube-master}
457 actions: ['CREATE']
458
471 ###################################################################### 459 ######################################################################
472 # 460 #
473 # a single kubernetes master. 461 # a single kubernetes master.
@@ -483,7 +471,8 @@ resources:
483 image: {get_param: server_image} 471 image: {get_param: server_image}
484 flavor: {get_param: master_flavor} 472 flavor: {get_param: master_flavor}
485 key_name: {get_param: ssh_key_name} 473 key_name: {get_param: ssh_key_name}
486 user_data_format: RAW 474 user_data_format: SOFTWARE_CONFIG
475 software_config_transport: POLL_SERVER_HEAT
487 user_data: {get_resource: kube_master_init} 476 user_data: {get_resource: kube_master_init}
488 networks: 477 networks:
489 - port: {get_resource: kube_master_eth0} 478 - port: {get_resource: kube_master_eth0}
diff --git a/magnum/tests/contrib/copy_instance_logs.sh b/magnum/tests/contrib/copy_instance_logs.sh
index c3c275d..e8a8845 100755
--- a/magnum/tests/contrib/copy_instance_logs.sh
+++ b/magnum/tests/contrib/copy_instance_logs.sh
@@ -95,6 +95,10 @@ if [[ "$COE" == "kubernetes" ]]; then
95 remote_exec $SSH_USER "sudo cat /etc/systemd/system/flanneld.service.d/flannel-docker-bridge.conf" flannel-docker-bridge.conf 95 remote_exec $SSH_USER "sudo cat /etc/systemd/system/flanneld.service.d/flannel-docker-bridge.conf" flannel-docker-bridge.conf
96 remote_exec $SSH_USER "sudo cat /etc/systemd/system/flannel-docker-bridge.service" flannel-docker-bridge.service 96 remote_exec $SSH_USER "sudo cat /etc/systemd/system/flannel-docker-bridge.service" flannel-docker-bridge.service
97 remote_exec $SSH_USER "sudo cat /etc/systemd/system/flannel-config.service" flannel-config.service 97 remote_exec $SSH_USER "sudo cat /etc/systemd/system/flannel-config.service" flannel-config.service
98 remote_exec $SSH_USER "sudo journalctl -u heat-container-agent --no-pager" heat-container-agent.log
99 remote_exec $SSH_USER "sudo journalctl -u kube-enable-monitoring --no-pager" kube-enable-monitoring.service.log
100 remote_exec $SSH_USER "sudo atomic containers list" atomic-containers-list.log
101 remote_exec $SSH_USER "sudo atomic images list" atomic-images-list.log
98elif [[ "$COE" == "swarm" || "$COE" == "swarm-mode" ]]; then 102elif [[ "$COE" == "swarm" || "$COE" == "swarm-mode" ]]; then
99 SSH_USER=fedora 103 SSH_USER=fedora
100 remote_exec $SSH_USER "sudo systemctl --full list-units --no-pager" systemctl_list_units.log 104 remote_exec $SSH_USER "sudo systemctl --full list-units --no-pager" systemctl_list_units.log