Add TCP publisher
This commit adds a TCP publisher. The publisher works similarly to the UDP publisher. Change-Id: Iac662018039e74ad59ac9c7fa4db994da540ef2f
This commit is contained in:
parent
e496b5d212
commit
ed55b5f4a4
|
@ -0,0 +1,94 @@
|
||||||
|
#
|
||||||
|
# Copyright 2022 Red Hat, Inc
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
"""Publish a sample using a TCP mechanism
|
||||||
|
"""
|
||||||
|
|
||||||
|
import socket
|
||||||
|
|
||||||
|
import msgpack
|
||||||
|
from oslo_log import log
|
||||||
|
from oslo_utils import netutils
|
||||||
|
|
||||||
|
import ceilometer
|
||||||
|
from ceilometer.i18n import _
|
||||||
|
from ceilometer import publisher
|
||||||
|
from ceilometer.publisher import utils
|
||||||
|
|
||||||
|
LOG = log.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class TCPPublisher(publisher.ConfigPublisherBase):
|
||||||
|
def __init__(self, conf, parsed_url):
|
||||||
|
super(TCPPublisher, self).__init__(conf, parsed_url)
|
||||||
|
self.host, self.port = netutils.parse_host_port(
|
||||||
|
parsed_url.netloc, default_port=4952)
|
||||||
|
addrinfo = None
|
||||||
|
try:
|
||||||
|
addrinfo = socket.getaddrinfo(self.host, None, socket.AF_INET6,
|
||||||
|
socket.SOCK_STREAM)[0]
|
||||||
|
except socket.gaierror:
|
||||||
|
try:
|
||||||
|
addrinfo = socket.getaddrinfo(self.host, None, socket.AF_INET,
|
||||||
|
socket.SOCK_STREAM)[0]
|
||||||
|
except socket.gaierror:
|
||||||
|
pass
|
||||||
|
if addrinfo:
|
||||||
|
self.addr_family = addrinfo[0]
|
||||||
|
else:
|
||||||
|
LOG.warning(
|
||||||
|
"Cannot resolve host %s, creating AF_INET socket...",
|
||||||
|
self.host)
|
||||||
|
self.addr_family = socket.AF_INET
|
||||||
|
self.create_and_connect()
|
||||||
|
|
||||||
|
def create_and_connect(self):
|
||||||
|
self.socket = socket.socket(self.addr_family,
|
||||||
|
socket.SOCK_STREAM)
|
||||||
|
self.socket.connect((self.host, self.port))
|
||||||
|
|
||||||
|
def publish_samples(self, samples):
|
||||||
|
"""Send a metering message for publishing
|
||||||
|
|
||||||
|
:param samples: Samples from pipeline after transformation
|
||||||
|
"""
|
||||||
|
|
||||||
|
for sample in samples:
|
||||||
|
msg = utils.meter_message_from_counter(
|
||||||
|
sample, self.conf.publisher.telemetry_secret, self.conf.host)
|
||||||
|
host = self.host
|
||||||
|
port = self.port
|
||||||
|
LOG.debug("Publishing sample %(msg)s over TCP to "
|
||||||
|
"%(host)s:%(port)d", {'msg': msg, 'host': host,
|
||||||
|
'port': port})
|
||||||
|
encoded_msg = msgpack.dumps(msg, use_bin_type=True)
|
||||||
|
msg_len = len(encoded_msg).to_bytes(8, 'little')
|
||||||
|
try:
|
||||||
|
self.socket.send(msg_len + encoded_msg)
|
||||||
|
except Exception:
|
||||||
|
LOG.warning(_("Unable to send sample over TCP,"
|
||||||
|
"trying to reconnect and resend the message"))
|
||||||
|
self.create_and_connect()
|
||||||
|
try:
|
||||||
|
self.socket.send(msg_len + encoded_msg)
|
||||||
|
except Exception:
|
||||||
|
LOG.exception(_("Unable to reconnect and resend"
|
||||||
|
"sample over TCP"))
|
||||||
|
|
||||||
|
def publish_events(self, events):
|
||||||
|
"""Send an event message for publishing
|
||||||
|
|
||||||
|
:param events: events from pipeline after transformation
|
||||||
|
"""
|
||||||
|
raise ceilometer.NotImplementedError
|
|
@ -114,7 +114,7 @@ def verify_signature(message, secret):
|
||||||
return secretutils.constant_time_compare(new_sig, old_sig)
|
return secretutils.constant_time_compare(new_sig, old_sig)
|
||||||
|
|
||||||
|
|
||||||
def meter_message_from_counter(sample, secret):
|
def meter_message_from_counter(sample, secret, publisher_id=None):
|
||||||
"""Make a metering message ready to be published or stored.
|
"""Make a metering message ready to be published or stored.
|
||||||
|
|
||||||
Returns a dictionary containing a metering message
|
Returns a dictionary containing a metering message
|
||||||
|
@ -135,6 +135,8 @@ def meter_message_from_counter(sample, secret):
|
||||||
'message_id': sample.id,
|
'message_id': sample.id,
|
||||||
'monotonic_time': sample.monotonic_time,
|
'monotonic_time': sample.monotonic_time,
|
||||||
}
|
}
|
||||||
|
if publisher_id is not None:
|
||||||
|
msg['publisher_id'] = publisher_id
|
||||||
msg['message_signature'] = compute_signature(msg, secret)
|
msg['message_signature'] = compute_signature(msg, secret)
|
||||||
return msg
|
return msg
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,223 @@
|
||||||
|
#
|
||||||
|
# Copyright 2022 Red Hat, Inc
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
"""Tests for ceilometer/publisher/tcp.py"""
|
||||||
|
|
||||||
|
import datetime
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
import msgpack
|
||||||
|
from oslo_utils import netutils
|
||||||
|
from oslotest import base
|
||||||
|
|
||||||
|
from ceilometer.publisher import tcp
|
||||||
|
from ceilometer.publisher import utils
|
||||||
|
from ceilometer import sample
|
||||||
|
from ceilometer import service
|
||||||
|
|
||||||
|
|
||||||
|
COUNTER_SOURCE = 'testsource'
|
||||||
|
|
||||||
|
|
||||||
|
class TestTCPPublisher(base.BaseTestCase):
|
||||||
|
test_data = [
|
||||||
|
sample.Sample(
|
||||||
|
name='test',
|
||||||
|
type=sample.TYPE_CUMULATIVE,
|
||||||
|
unit='',
|
||||||
|
volume=1,
|
||||||
|
user_id='test',
|
||||||
|
project_id='test',
|
||||||
|
resource_id='test_run_tasks',
|
||||||
|
timestamp=datetime.datetime.utcnow().isoformat(),
|
||||||
|
resource_metadata={'name': 'TestPublish'},
|
||||||
|
source=COUNTER_SOURCE,
|
||||||
|
),
|
||||||
|
sample.Sample(
|
||||||
|
name='test',
|
||||||
|
type=sample.TYPE_CUMULATIVE,
|
||||||
|
unit='',
|
||||||
|
volume=1,
|
||||||
|
user_id='test',
|
||||||
|
project_id='test',
|
||||||
|
resource_id='test_run_tasks',
|
||||||
|
timestamp=datetime.datetime.utcnow().isoformat(),
|
||||||
|
resource_metadata={'name': 'TestPublish'},
|
||||||
|
source=COUNTER_SOURCE,
|
||||||
|
),
|
||||||
|
sample.Sample(
|
||||||
|
name='test2',
|
||||||
|
type=sample.TYPE_CUMULATIVE,
|
||||||
|
unit='',
|
||||||
|
volume=1,
|
||||||
|
user_id='test',
|
||||||
|
project_id='test',
|
||||||
|
resource_id='test_run_tasks',
|
||||||
|
timestamp=datetime.datetime.utcnow().isoformat(),
|
||||||
|
resource_metadata={'name': 'TestPublish'},
|
||||||
|
source=COUNTER_SOURCE,
|
||||||
|
),
|
||||||
|
sample.Sample(
|
||||||
|
name='test2',
|
||||||
|
type=sample.TYPE_CUMULATIVE,
|
||||||
|
unit='',
|
||||||
|
volume=1,
|
||||||
|
user_id='test',
|
||||||
|
project_id='test',
|
||||||
|
resource_id='test_run_tasks',
|
||||||
|
timestamp=datetime.datetime.utcnow().isoformat(),
|
||||||
|
resource_metadata={'name': 'TestPublish'},
|
||||||
|
source=COUNTER_SOURCE,
|
||||||
|
),
|
||||||
|
sample.Sample(
|
||||||
|
name='test3',
|
||||||
|
type=sample.TYPE_CUMULATIVE,
|
||||||
|
unit='',
|
||||||
|
volume=1,
|
||||||
|
user_id='test',
|
||||||
|
project_id='test',
|
||||||
|
resource_id='test_run_tasks',
|
||||||
|
timestamp=datetime.datetime.utcnow().isoformat(),
|
||||||
|
resource_metadata={'name': 'TestPublish'},
|
||||||
|
source=COUNTER_SOURCE,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _make_fake_socket(published):
|
||||||
|
def _fake_socket_socket(family, type):
|
||||||
|
def record_data(msg):
|
||||||
|
msg_length = int.from_bytes(msg[0:8], "little")
|
||||||
|
published.append(msg[8:msg_length + 8])
|
||||||
|
|
||||||
|
def connect(dst):
|
||||||
|
pass
|
||||||
|
|
||||||
|
tcp_socket = mock.Mock()
|
||||||
|
tcp_socket.send = record_data
|
||||||
|
tcp_socket.connect = connect
|
||||||
|
return tcp_socket
|
||||||
|
|
||||||
|
return _fake_socket_socket
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super(TestTCPPublisher, self).setUp()
|
||||||
|
self.CONF = service.prepare_service([], [])
|
||||||
|
self.CONF.publisher.telemetry_secret = 'not-so-secret'
|
||||||
|
|
||||||
|
def test_published(self):
|
||||||
|
self.data_sent = []
|
||||||
|
with mock.patch('socket.socket',
|
||||||
|
self._make_fake_socket(self.data_sent)):
|
||||||
|
publisher = tcp.TCPPublisher(
|
||||||
|
self.CONF,
|
||||||
|
netutils.urlsplit('tcp://somehost'))
|
||||||
|
publisher.publish_samples(self.test_data)
|
||||||
|
|
||||||
|
self.assertEqual(5, len(self.data_sent))
|
||||||
|
|
||||||
|
sent_counters = []
|
||||||
|
|
||||||
|
for data in self.data_sent:
|
||||||
|
counter = msgpack.loads(data, raw=False)
|
||||||
|
sent_counters.append(counter)
|
||||||
|
|
||||||
|
# Check that counters are equal
|
||||||
|
def sort_func(counter):
|
||||||
|
return counter['counter_name']
|
||||||
|
|
||||||
|
counters = [utils.meter_message_from_counter(d,
|
||||||
|
"not-so-secret",
|
||||||
|
publisher.conf.host)
|
||||||
|
for d in self.test_data]
|
||||||
|
counters.sort(key=sort_func)
|
||||||
|
sent_counters.sort(key=sort_func)
|
||||||
|
self.assertEqual(counters, sent_counters)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _make_disconnecting_socket(published, connections):
|
||||||
|
def _fake_socket_socket(family, type):
|
||||||
|
def record_data(msg):
|
||||||
|
if len(published) == len(connections) - 1:
|
||||||
|
# Raise for every each first send attempt to
|
||||||
|
# trigger a reconnection attempt and send the data
|
||||||
|
# correctly after reconnecting
|
||||||
|
raise IOError
|
||||||
|
msg_length = int.from_bytes(msg[0:8], "little")
|
||||||
|
published.append(msg[8:msg_length + 8])
|
||||||
|
|
||||||
|
def record_connection(dest):
|
||||||
|
connections.append(dest)
|
||||||
|
|
||||||
|
tcp_socket = mock.Mock()
|
||||||
|
tcp_socket.send = record_data
|
||||||
|
tcp_socket.connect = record_connection
|
||||||
|
return tcp_socket
|
||||||
|
|
||||||
|
return _fake_socket_socket
|
||||||
|
|
||||||
|
def test_reconnect(self):
|
||||||
|
self.data_sent = []
|
||||||
|
self.connections = []
|
||||||
|
with mock.patch('socket.socket',
|
||||||
|
self._make_disconnecting_socket(self.data_sent,
|
||||||
|
self.connections)):
|
||||||
|
publisher = tcp.TCPPublisher(
|
||||||
|
self.CONF,
|
||||||
|
netutils.urlsplit('tcp://somehost'))
|
||||||
|
publisher.publish_samples(self.test_data)
|
||||||
|
|
||||||
|
sent_counters = []
|
||||||
|
|
||||||
|
for data in self.data_sent:
|
||||||
|
counter = msgpack.loads(data, raw=False)
|
||||||
|
sent_counters.append(counter)
|
||||||
|
|
||||||
|
for connection in self.connections:
|
||||||
|
# Check destination
|
||||||
|
self.assertEqual(('somehost', 4952), connection)
|
||||||
|
self.assertEqual(len(self.connections) - 1, len(self.data_sent))
|
||||||
|
|
||||||
|
# Check that counters are equal
|
||||||
|
def sort_func(counter):
|
||||||
|
return counter['counter_name']
|
||||||
|
|
||||||
|
counters = [utils.meter_message_from_counter(d,
|
||||||
|
"not-so-secret",
|
||||||
|
publisher.conf.host)
|
||||||
|
for d in self.test_data]
|
||||||
|
counters.sort(key=sort_func)
|
||||||
|
sent_counters.sort(key=sort_func)
|
||||||
|
self.assertEqual(counters, sent_counters)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _raise_ioerror(*args):
|
||||||
|
raise IOError
|
||||||
|
|
||||||
|
def _make_broken_socket(self, family, type):
|
||||||
|
def connect(dst):
|
||||||
|
pass
|
||||||
|
|
||||||
|
tcp_socket = mock.Mock()
|
||||||
|
tcp_socket.send = self._raise_ioerror
|
||||||
|
tcp_socket.connect = connect
|
||||||
|
return tcp_socket
|
||||||
|
|
||||||
|
def test_publish_error(self):
|
||||||
|
with mock.patch('socket.socket',
|
||||||
|
self._make_broken_socket):
|
||||||
|
publisher = tcp.TCPPublisher(
|
||||||
|
self.CONF,
|
||||||
|
netutils.urlsplit('tcp://localhost'))
|
||||||
|
publisher.publish_samples(self.test_data)
|
|
@ -180,6 +180,7 @@ ceilometer.sample.publisher =
|
||||||
test = ceilometer.publisher.test:TestPublisher
|
test = ceilometer.publisher.test:TestPublisher
|
||||||
notifier = ceilometer.publisher.messaging:SampleNotifierPublisher
|
notifier = ceilometer.publisher.messaging:SampleNotifierPublisher
|
||||||
udp = ceilometer.publisher.udp:UDPPublisher
|
udp = ceilometer.publisher.udp:UDPPublisher
|
||||||
|
tcp = ceilometer.publisher.tcp:TCPPublisher
|
||||||
file = ceilometer.publisher.file:FilePublisher
|
file = ceilometer.publisher.file:FilePublisher
|
||||||
http = ceilometer.publisher.http:HttpPublisher
|
http = ceilometer.publisher.http:HttpPublisher
|
||||||
prometheus = ceilometer.publisher.prometheus:PrometheusPublisher
|
prometheus = ceilometer.publisher.prometheus:PrometheusPublisher
|
||||||
|
|
Loading…
Reference in New Issue