openstack
/
monasca-persister
Code Issues Proposed changes

Create deb.

This commit is contained in:
Deklan Dieterly 2014-02-26 10:28:31 -07:00
parent 50cc1e57af
commit 28d73219e2
3 changed files with 151 additions and 316 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
pom.xml
View File

@ -15,6 +15,20 @@
</repository>
</repositories>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<artifactNamedVersion>${project.name}-${project.version}-${timestamp}-${buildNumber}
</artifactNamedVersion>
</properties>
<!--Needed for buildnumber-maven-plugin-->
<scm>
<connection>scm:git:git@keg.dev.uswest.hpcloud.net:som-persister</connection>
<developerConnection>scm:git:git@keg.dev.uswest.hpcloud.net:som-persister
</developerConnection>
</scm>
<dependencies>
<dependency>
<groupId>org.apache.kafka</groupId>
@ -75,6 +89,16 @@
<artifactId>vertica-jdbc</artifactId>
<version>6.1.0</version>
</dependency>
<dependency>
<groupId>org.codehaus.mojo</groupId>
<artifactId>buildnumber-maven-plugin</artifactId>
<version>1.2</version>
</dependency>
<dependency>
<groupId>org.vafer</groupId>
<artifactId>jdeb</artifactId>
<version>1.0.1</version>
</dependency>
</dependencies>
@ -96,6 +120,7 @@
<artifactId>maven-shade-plugin</artifactId>
<version>2.2</version>
<configuration>
<finalName>${artifactNamedVersion}</finalName>
<createDependencyReducedPom>true</createDependencyReducedPom>
<filters>
<filter>
@ -150,10 +175,27 @@
</filesets>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>buildnumber-maven-plugin</artifactId>
<version>1.2</version>
<executions>
<execution>
<phase>validate</phase>
<goals>
<goal>create</goal>
</goals>
</execution>
</executions>
<configuration>
<doCheck>false</doCheck>
<shortRevisionLength>6</shortRevisionLength>
</configuration>
</plugin>
<plugin>
<artifactId>jdeb</artifactId>
<groupId>org.vafer</groupId>
<version>1.0</version>
<version>1.0.1</version>
<executions>
<execution>
<phase>package</phase>
@ -161,11 +203,12 @@
<goal>jdeb</goal>
</goals>
<configuration>
<deb>${project.basedir}/../debs/binaries/${artifactNamedVersion}.deb</deb>
<deb>${project.basedir}/debs/binaries/${artifactNamedVersion}.deb</deb>
<dataSet>
<data>
<type>file</type>
<src>${project.build.directory}/${artifactNamedVersion}.jar</src>
<src>${project.build.directory}/${artifactNamedVersion}.jar
</src>
<dst>/opt/mon/mon-persister.jar</dst>
</data>
<data>
@ -175,7 +218,9 @@
</data>
<data>
<type>file</type>
<src>${project.basedir}/src/deb/etc/persister-config.yml-sample</src>
<src>
${project.basedir}/src/deb/etc/persister-config.yml-sample
</src>
<dst>/etc/mon/persister-config.yml-sample</dst>
</data>
</dataSet>
@ -186,10 +231,5 @@
</plugins>
</build>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
</properties>
</project>

407
src/deb/etc/persister-config.yml-sample
View File

@ -1,169 +1,46 @@
#Application Config
name: mon-persister
metricsPersister:
enabled: true
csvConsumerEnabled: true
udpConsumerEnabled: true
disruptorConfig:
claimBufferSize: 262144
numBatches: 8
numOutputProcessors: 4
repositoryType: "vertica"
schemaName: Metrics
enableJournaling: false
enableAmqpOutputProcessor: false
#Kafka settings.
kafkaConfiguration:
# See http://kafka.apache.org/documentation.html#api for semantics and defaults.
topic: test
numThreads: 2
groupId: 1
zookeeperConnect: localhost:2181
consumerId: 1
socketTimeoutMs: 30000
socketReceiveBufferBytes : 65536
fetchMessageMaxBytes: 1048576
autoCommitEnable: true
autoCommitIntervalMs: 60000
queuedMaxMessageChunks: 10
rebalanceMaxRetries: 4
fetchMinBytes: 1
fetchWaitMaxMs: 100
rebalanceBackoffMs: 2000
refreshLeaderBackoffMs: 200
autoOffsetReset: largest
consumerTimeoutMs: -1
clientId : 1
zookeeperSessionTimeoutMs : 6000
zookeeperConnectionTimeoutMs : 6000
zookeeperSyncTimeMs: 2000
udpConsumerConfig:
useMulticast: false
port: 25826
ipAddress: 0.0.0.0
datagramPacketBufferSize: 4096
numDatagramPacketTranslatorProcessors: 4
csvConsumerConfig:
inputFileDir: "/var/som/csv/toProcess"
processedFileDir : "/var/som/csv/processed"
disruptorConfiguration:
bufferSize: 1048576
numProcessors: 2
somMaasConfig:
rabbitExchange: "metrics.rnd"
rabbit:
hosts: [15.185.167.30,15.185.102.80,15.185.224.94]
port: 5672
username: som
password: password
vhost: /
consumerThreadPoolSize: 1
maxWaitForConnection: 1s
connectRetryMilliseconds: 2000
filterType: "plugin"
filterMatch: "libvirt,bock"
amqpMetricsMsgBufferSize: 65536
numAmqpPublishers: 1
verticaOutputProcessorConfiguration:
batchSize: 10
csPersister:
enabled: true
amqpConsumerEnabled: true
disruptorConfig:
claimBufferSize: 512
numBatches: 1
numOutputProcessors: 1
repositoryType: "vertica"
schemaName: CS
enableJournaling: false
enableAmqpOutputProcessor: false
amqpConsumerConfig:
exchangeName: "cs_entities"
queueName: "control-services-sync"
routingKey: "#"
declareQueue: false
prefetchCount: 8192
connection:
hosts:
- som-rabbit.rndd.aw1.hpcloud.net
port: 5672
username: cspersister
password: <password>
vhost: /
consumerThreadPoolSize: 8
nagiosPersister:
enabled: true
amqpConsumerEnabled: true
disruptorConfig:
claimBufferSize: 512
numBatches: 4
numOutputProcessors: 1
repositoryType: "vertica"
schemaName: Nagios
enableJournaling: false
enableAmqpOutputProcessor: false
amqpConsumerConfig:
exchangeName: "icinga_events"
queueName: "som-nagios-consumer"
routingKey: "#"
declareQueue: false
prefetchCount: 1024
connection:
hosts:
- som-rabbit.rndd.aw1.hpcloud.net
port: 5672
username: nagiospersister
password: <password>
vhost: /
consumerThreadPoolSize: 8
salesforcePersister:
enabled: true
httpConsumerEnabled: true
disruptorConfig:
claimBufferSize: 512
numBatches: 1
numOutputProcessors: 0
repositoryType: "vertica"
schemaName: SalesForce
enableJournaling: false
enableAmqpOutputProcessor: false
httpConsumerConfig:
username: hpcs-mon-som@hp.com
password: <password>
securityToken: <token>
endPoint: https://salesforce.com/services/Soap/c/22.0/0DFV00000008OKG
setTrace: false
proxy: web-proxy.dev.uswest.hpcloud.net
port: 8080
apiVersion: 28.0
customMetricsPersister:
enabled: true
amqpConsumerEnabled: true
disruptorConfig:
claimBufferSize: 512
numBatches : 1
numOutputProcessors: 16
repositoryType: http
schemaName: CustomMetrics
enableJournaling: false
enableAmqpOutputProcessor: false
amqpConsumerConfig:
exchangeName: "custom_metrics"
queueName: "som-custommetrics-consumer"
routingKey: "#"
declareQueue: true
prefetchCount: 512
connection:
hosts:
- som-rabbit.rndd.aw1.hpcloud.net
port: 5672
username: cmpersister
password: <password>
vhost: /custom_metrics
consumerThreadPoolSize: 8
httpProducerConfig:
endPoints:
- https://mon-aw1rdd1-somapi0001.rndd.aw1.hpcloud.net/v1/metrics/custom
- https://mon-aw1rdd1-somapi0002.rndd.aw1.hpcloud.net/v1/metrics/custom
maxConnections: 64
# Platform config
database:
databaseConfiguration:
driverClass: com.vertica.jdbc.Driver
url: jdbc:vertica://mon-aw1rdd1-vertica0001.rndd.aw1.hpcloud.net:5433/som
# url: jdbc:vertica://mon-aw1rdd1-vertica0001.rndd.aw1.hpcloud.net:5433/som
url: jdbc:vertica://15.185.94.245:5433/som
user: persister
password: <password>
password: password
properties:
ssl: true
ssl: false
# the maximum amount of time to wait on an empty pool before throwing an exception
maxWaitForConnection: 1s
@ -174,8 +51,8 @@ database:
minSize: 8
# the maximum number of connections to keep open
# If you add a DB connection, this value will
# need to increase
maxSize: 41
# whether or not idle connections should be validated
@ -187,172 +64,90 @@ database:
# the maximum lifetime of an idle connection
closeConnectionIfIdleFor: 1 minute
# HTTP-specific options.
http:
# The port on which the HTTP server listens for service requests.
port: 8080
# The port on which the HTTP server listens for administrative requests.
adminPort: 8081
# Maximum number of threads.
maxThreads: 100
# Minimum number of thread to keep alive.
minThreads: 10
# The type of connector to use. Other valid values are "nonblocking" or "legacy". In general, the
# blocking connector should be used for low-latency services with short request durations. The
# nonblocking connector should be used for services with long request durations or which
# specifically take advantage of Jetty's continuation support.
# If you need SSL support, you can either choose from "nonblocking+ssl" or "legacy+ssl".
connectorType: blocking
# The maximum amount of time a connection is allowed to be idle before being closed.
maxIdleTime: 1s
# The number of threads dedicated to accepting connections. If omitted, this defaults to the
# number of logical CPUs on the current machine.
acceptorThreads: 3
# The offset of the acceptor threads' priorities. Can be [-5...5], with -5 dropping the acceptor
# threads to the lowest possible priority and with 5 raising them to the highest priority.
acceptorThreadPriorityOffset: 0
# The number of unaccepted requests to keep in the accept queue before refusing connections. If
# set to -1 or omitted, the system default is used.
acceptQueueSize: 100
# The maximum number of buffers to keep in memory.
maxBufferCount: 1024
# The initial buffer size for reading requests.
requestBufferSize: 32KB
# The initial buffer size for reading request headers.
requestHeaderBufferSize: 6KB
# The initial buffer size for writing responses.
responseBufferSize: 32KB
# The initial buffer size for writing response headers.
responseHeaderBufferSize: 6KB
# Enables SO_REUSEADDR on the server socket.
reuseAddress: true
# Enables SO_LINGER on the server socket with the specified linger time.
soLingerTime: 1s
# The number of open connections at which the server transitions to a "low-resources" mode.
lowResourcesConnectionThreshold: 25000
# When in low-resources mode, the maximum amount of time a connection is allowed to be idle before
# being closed. Overrides maxIdleTime.
lowResourcesMaxIdleTime: 5s
# If non-zero, the server will allow worker threads to finish processing requests after the server
# socket has been closed for the given amount of time.
shutdownGracePeriod: 2s
# If true, the HTTP server will prefer X-Forwarded headers over their non-forwarded equivalents.
useForwardedHeaders: true
# If true, forces the HTTP connector to use off-heap, direct buffers.
useDirectBuffers: true
# The hostname of the interface to which the HTTP server socket wil be found. If omitted, the
# socket will listen on all interfaces.
# bindHost: app1.example.com
# Trust store for SSL trusted Certificate Authorities
# ssl:
# trustStore: /etc/ssl/hpcs-keystore.jks
# trustStorePassword: <%= @keystore_pass %>
# HTTP request log settings
requestLog:
# Settings for logging to stdout.
console:
# If true, addToBatch log statements to stdout.
enabled: false
# Settings for logging to a file. Possibly to move to this and off syslog if we end up with man http requests
file:
# If true, addToBatch log statements to a file.
enabled: false
# The file to which statements will be logged.
currentLogFilename: ./logs/som-metrics-persister-requests.log
# When the log file rolls over, the file will be archived to example-2012-03-15.log.gz,
# example.log will be truncated, and new statements written to it.
archivedLogFilenamePattern: ./logs/som-metrics-persister-requests-%d.log.gz
# The maximum number of log files to archive.
archivedFileCount: 5
# Settings for logging to syslog.
syslog:
# If true, addToBatch log statements to syslog.
enabled: true
# The hostname of the syslog server to which statements will be sent.
# N.B.: If this is the local host, the local syslog instance will need to be configured to
# listen on an inet socket, not just a Unix socket.
host: localhost
# The syslog facility to which statements will be sent.
facility: local0
# Logging settings.
logging:
# The default level of all loggers. Can be OFF, ERROR, WARN, INFO, DEBUG, TRACE, or ALL.
level: INFO
# The default level of all loggers. Can be OFF, ERROR, WARN, INFO,
# DEBUG, TRACE, or ALL.
level: DEBUG
# Logger-specific levels.
loggers:
com.hpcloud: INFO
com.hpcloud.som: INFO
com.yammer.metrics.reporting.JmxReporter: ERROR
# Settings for logging to a file.
file:
# If true, addToBatch log statements to a file.
enabled: true
# Do not addToBatch log statements below this threshold to the file.
threshold: ALL
# The file to which statements will be logged.
currentLogFilename: /var/log/som/som-persister.log
# When the log file rolls over, the file will be archived to example-2012-03-15.log.gz,
# example.log will be truncated, and new statements written to it.
archivedLogFilenamePattern: /var/log/som/som-persister%d.log.gz
# The maximum number of log files to archive.
archivedFileCount: 10
# Sets the level for 'com.example.app' to DEBUG.
com.example.app: DEBUG
# Settings for logging to stdout.
console:
# If true, addToBatch log statements to stdout.
enabled: false
# If true, write log statements to stdout.
enabled: true
# Do not display log statements below this threshold to stdout.
threshold: ALL
# The time zone in which dates should be displayed.
timeZone: UTC
# A custom Logback format string.
logFormat: null
# Settings for logging to a file.
file:
# If true, write log statements to a file.
enabled: true
# Do not write log statements below this threshold to the file.
threshold: ALL
# The time zone in which dates should be displayed.
timeZone: UTC
# A custom Logback format string.
logFormat: null
# The file to which statements will be logged.
#
# If enabled is true, this must be specified.
currentLogFilename: ./logs/mon-persister.log
# If true, log files are rotated and archived.
archive: true
# When the log file rolls over, the file will be archived to
# app-2012-03-15.log.gz, example.log will be truncated,
# and new statements written to it.
#
# If archive is true, this must be specified.
archivedLogFilenamePattern: ./logs/mon-persister-%d.log.gz
# The maximum number of log files to archive.
archivedFileCount: 5
# Settings for logging to syslog.
syslog:
# If true, addToBatch log statements to syslog.
# If true, write log statements to syslog.
enabled: false
# The hostname of the syslog server to which statements will be sent.
# N.B.: If this is the local host, the local syslog instance will need to be configured to
# listen on an inet socket, not just a Unix socket.
# The hostname of the syslog server to which statements will be
# sent.
#
# N.B.: If this is the local host, the local syslog instance
# will need to be configured to listen on an inet socket, not just
# a Unix socket.
host: localhost
# The syslog facility to which statements will be sent.
#
# Can be one of: {AUTH, AUTHPRIV, DAEMON, CRON, FTP, LPR, KERN,
# MAIL, NEWS, SYSLOG, USER, UUCP, LOCAL0, LOCAL1, LOCAL2, LOCAL3,
# LOCAL4, LOCAL5, LOCAL6, LOCAL7}.
facility: local0
# The time zone in which dates should be displayed.
timeZone: UTC
# A custom Logback format string.
logFormat: null

2
src/deb/init/mon-persister.conf
View File

@ -8,4 +8,4 @@ respawn
setgid persister
setuid persister
exec /usr/bin/java -Xmx8g -cp /opt/mon/mon-persister.jar com.hpcloud.mon.persister.PersisterApp --config /etc/mon/persister-config.yml
exec /usr/bin/java -Xmx8g -cp /opt/mon/mon-persister.jar com.hpcloud.MonPersisterService --config /etc/mon/persister-config.yml