From 28d73219e2f5415da58ad44d0d925e0a7753f5fb Mon Sep 17 00:00:00 2001 From: Deklan Dieterly Date: Wed, 26 Feb 2014 10:28:31 -0700 Subject: [PATCH] Create deb. --- pom.xml | 58 +++- src/deb/etc/persister-config.yml-sample | 407 ++++++------------------ src/deb/init/mon-persister.conf | 2 +- 3 files changed, 151 insertions(+), 316 deletions(-) diff --git a/pom.xml b/pom.xml index 1684b412..173fc55f 100644 --- a/pom.xml +++ b/pom.xml @@ -15,6 +15,20 @@ + + UTF-8 + UTF-8 + ${project.name}-${project.version}-${timestamp}-${buildNumber} + + + + + + scm:git:git@keg.dev.uswest.hpcloud.net:som-persister + scm:git:git@keg.dev.uswest.hpcloud.net:som-persister + + + org.apache.kafka @@ -75,6 +89,16 @@ vertica-jdbc 6.1.0 + + org.codehaus.mojo + buildnumber-maven-plugin + 1.2 + + + org.vafer + jdeb + 1.0.1 + @@ -96,6 +120,7 @@ maven-shade-plugin 2.2 + ${artifactNamedVersion} true @@ -150,10 +175,27 @@ + + org.codehaus.mojo + buildnumber-maven-plugin + 1.2 + + + validate + + create + + + + + false + 6 + + jdeb org.vafer - 1.0 + 1.0.1 package @@ -161,11 +203,12 @@ jdeb - ${project.basedir}/../debs/binaries/${artifactNamedVersion}.deb + ${project.basedir}/debs/binaries/${artifactNamedVersion}.deb file - ${project.build.directory}/${artifactNamedVersion}.jar + ${project.build.directory}/${artifactNamedVersion}.jar + /opt/mon/mon-persister.jar @@ -175,7 +218,9 @@ file - ${project.basedir}/src/deb/etc/persister-config.yml-sample + + ${project.basedir}/src/deb/etc/persister-config.yml-sample + /etc/mon/persister-config.yml-sample @@ -186,10 +231,5 @@ - - UTF-8 - UTF-8 - - diff --git a/src/deb/etc/persister-config.yml-sample b/src/deb/etc/persister-config.yml-sample index c76985a4..70b9d600 100755 --- a/src/deb/etc/persister-config.yml-sample +++ b/src/deb/etc/persister-config.yml-sample @@ -1,169 +1,46 @@ -#Application Config +name: mon-persister -metricsPersister: - enabled: true - csvConsumerEnabled: true - udpConsumerEnabled: true - - disruptorConfig: - claimBufferSize: 262144 - numBatches: 8 - numOutputProcessors: 4 - repositoryType: "vertica" - schemaName: Metrics - enableJournaling: false - enableAmqpOutputProcessor: false +#Kafka settings. +kafkaConfiguration: +# See http://kafka.apache.org/documentation.html#api for semantics and defaults. + topic: test + numThreads: 2 + groupId: 1 + zookeeperConnect: localhost:2181 + consumerId: 1 + socketTimeoutMs: 30000 + socketReceiveBufferBytes : 65536 + fetchMessageMaxBytes: 1048576 + autoCommitEnable: true + autoCommitIntervalMs: 60000 + queuedMaxMessageChunks: 10 + rebalanceMaxRetries: 4 + fetchMinBytes: 1 + fetchWaitMaxMs: 100 + rebalanceBackoffMs: 2000 + refreshLeaderBackoffMs: 200 + autoOffsetReset: largest + consumerTimeoutMs: -1 + clientId : 1 + zookeeperSessionTimeoutMs : 6000 + zookeeperConnectionTimeoutMs : 6000 + zookeeperSyncTimeMs: 2000 - udpConsumerConfig: - useMulticast: false - port: 25826 - ipAddress: 0.0.0.0 - datagramPacketBufferSize: 4096 - numDatagramPacketTranslatorProcessors: 4 - - csvConsumerConfig: - inputFileDir: "/var/som/csv/toProcess" - processedFileDir : "/var/som/csv/processed" +disruptorConfiguration: + bufferSize: 1048576 + numProcessors: 2 - somMaasConfig: - rabbitExchange: "metrics.rnd" - rabbit: - hosts: [15.185.167.30,15.185.102.80,15.185.224.94] - port: 5672 - username: som - password: password - vhost: / - consumerThreadPoolSize: 1 - maxWaitForConnection: 1s - connectRetryMilliseconds: 2000 - filterType: "plugin" - filterMatch: "libvirt,bock" - amqpMetricsMsgBufferSize: 65536 - numAmqpPublishers: 1 +verticaOutputProcessorConfiguration: + batchSize: 10 -csPersister: - enabled: true - amqpConsumerEnabled: true - - disruptorConfig: - claimBufferSize: 512 - numBatches: 1 - numOutputProcessors: 1 - repositoryType: "vertica" - schemaName: CS - enableJournaling: false - enableAmqpOutputProcessor: false - - amqpConsumerConfig: - exchangeName: "cs_entities" - queueName: "control-services-sync" - routingKey: "#" - declareQueue: false - prefetchCount: 8192 - connection: - hosts: - - som-rabbit.rndd.aw1.hpcloud.net - port: 5672 - username: cspersister - password: - vhost: / - consumerThreadPoolSize: 8 - -nagiosPersister: - enabled: true - amqpConsumerEnabled: true - - disruptorConfig: - claimBufferSize: 512 - numBatches: 4 - numOutputProcessors: 1 - repositoryType: "vertica" - schemaName: Nagios - enableJournaling: false - enableAmqpOutputProcessor: false - - amqpConsumerConfig: - exchangeName: "icinga_events" - queueName: "som-nagios-consumer" - routingKey: "#" - declareQueue: false - prefetchCount: 1024 - - connection: - hosts: - - som-rabbit.rndd.aw1.hpcloud.net - port: 5672 - username: nagiospersister - password: - vhost: / - consumerThreadPoolSize: 8 - -salesforcePersister: - enabled: true - httpConsumerEnabled: true - - disruptorConfig: - claimBufferSize: 512 - numBatches: 1 - numOutputProcessors: 0 - repositoryType: "vertica" - schemaName: SalesForce - enableJournaling: false - enableAmqpOutputProcessor: false - - httpConsumerConfig: - username: hpcs-mon-som@hp.com - password: - securityToken: - endPoint: https://salesforce.com/services/Soap/c/22.0/0DFV00000008OKG - setTrace: false - proxy: web-proxy.dev.uswest.hpcloud.net - port: 8080 - apiVersion: 28.0 - -customMetricsPersister: - enabled: true - amqpConsumerEnabled: true - - disruptorConfig: - claimBufferSize: 512 - numBatches : 1 - numOutputProcessors: 16 - repositoryType: http - schemaName: CustomMetrics - enableJournaling: false - enableAmqpOutputProcessor: false - - amqpConsumerConfig: - exchangeName: "custom_metrics" - queueName: "som-custommetrics-consumer" - routingKey: "#" - declareQueue: true - prefetchCount: 512 - connection: - hosts: - - som-rabbit.rndd.aw1.hpcloud.net - port: 5672 - username: cmpersister - password: - vhost: /custom_metrics - consumerThreadPoolSize: 8 - - httpProducerConfig: - endPoints: - - https://mon-aw1rdd1-somapi0001.rndd.aw1.hpcloud.net/v1/metrics/custom - - https://mon-aw1rdd1-somapi0002.rndd.aw1.hpcloud.net/v1/metrics/custom - maxConnections: 64 - -# Platform config - -database: +databaseConfiguration: driverClass: com.vertica.jdbc.Driver - url: jdbc:vertica://mon-aw1rdd1-vertica0001.rndd.aw1.hpcloud.net:5433/som +# url: jdbc:vertica://mon-aw1rdd1-vertica0001.rndd.aw1.hpcloud.net:5433/som + url: jdbc:vertica://15.185.94.245:5433/som user: persister - password: + password: password properties: - ssl: true + ssl: false # the maximum amount of time to wait on an empty pool before throwing an exception maxWaitForConnection: 1s @@ -174,8 +51,8 @@ database: minSize: 8 # the maximum number of connections to keep open - # If you add a DB connection, this value will - # need to increase + + maxSize: 41 # whether or not idle connections should be validated @@ -187,172 +64,90 @@ database: # the maximum lifetime of an idle connection closeConnectionIfIdleFor: 1 minute -# HTTP-specific options. -http: - - # The port on which the HTTP server listens for service requests. - port: 8080 - - # The port on which the HTTP server listens for administrative requests. - adminPort: 8081 - - # Maximum number of threads. - maxThreads: 100 - - # Minimum number of thread to keep alive. - minThreads: 10 - - # The type of connector to use. Other valid values are "nonblocking" or "legacy". In general, the - # blocking connector should be used for low-latency services with short request durations. The - # nonblocking connector should be used for services with long request durations or which - # specifically take advantage of Jetty's continuation support. - # If you need SSL support, you can either choose from "nonblocking+ssl" or "legacy+ssl". - connectorType: blocking - - # The maximum amount of time a connection is allowed to be idle before being closed. - maxIdleTime: 1s - - # The number of threads dedicated to accepting connections. If omitted, this defaults to the - # number of logical CPUs on the current machine. - acceptorThreads: 3 - - # The offset of the acceptor threads' priorities. Can be [-5...5], with -5 dropping the acceptor - # threads to the lowest possible priority and with 5 raising them to the highest priority. - acceptorThreadPriorityOffset: 0 - - # The number of unaccepted requests to keep in the accept queue before refusing connections. If - # set to -1 or omitted, the system default is used. - acceptQueueSize: 100 - - # The maximum number of buffers to keep in memory. - maxBufferCount: 1024 - - # The initial buffer size for reading requests. - requestBufferSize: 32KB - - # The initial buffer size for reading request headers. - requestHeaderBufferSize: 6KB - - # The initial buffer size for writing responses. - responseBufferSize: 32KB - - # The initial buffer size for writing response headers. - responseHeaderBufferSize: 6KB - - # Enables SO_REUSEADDR on the server socket. - reuseAddress: true - - # Enables SO_LINGER on the server socket with the specified linger time. - soLingerTime: 1s - - # The number of open connections at which the server transitions to a "low-resources" mode. - lowResourcesConnectionThreshold: 25000 - - # When in low-resources mode, the maximum amount of time a connection is allowed to be idle before - # being closed. Overrides maxIdleTime. - lowResourcesMaxIdleTime: 5s - - # If non-zero, the server will allow worker threads to finish processing requests after the server - # socket has been closed for the given amount of time. - shutdownGracePeriod: 2s - - # If true, the HTTP server will prefer X-Forwarded headers over their non-forwarded equivalents. - useForwardedHeaders: true - - # If true, forces the HTTP connector to use off-heap, direct buffers. - useDirectBuffers: true - - # The hostname of the interface to which the HTTP server socket wil be found. If omitted, the - # socket will listen on all interfaces. - # bindHost: app1.example.com - -# Trust store for SSL trusted Certificate Authorities -# ssl: -# trustStore: /etc/ssl/hpcs-keystore.jks -# trustStorePassword: <%= @keystore_pass %> - - # HTTP request log settings - requestLog: - # Settings for logging to stdout. - console: - # If true, addToBatch log statements to stdout. - enabled: false - - # Settings for logging to a file. Possibly to move to this and off syslog if we end up with man http requests - file: - # If true, addToBatch log statements to a file. - enabled: false - - # The file to which statements will be logged. - currentLogFilename: ./logs/som-metrics-persister-requests.log - - # When the log file rolls over, the file will be archived to example-2012-03-15.log.gz, - # example.log will be truncated, and new statements written to it. - archivedLogFilenamePattern: ./logs/som-metrics-persister-requests-%d.log.gz - - # The maximum number of log files to archive. - archivedFileCount: 5 - - # Settings for logging to syslog. - syslog: - - # If true, addToBatch log statements to syslog. - enabled: true - - # The hostname of the syslog server to which statements will be sent. - # N.B.: If this is the local host, the local syslog instance will need to be configured to - # listen on an inet socket, not just a Unix socket. - host: localhost - - # The syslog facility to which statements will be sent. - facility: local0 - # Logging settings. logging: - # The default level of all loggers. Can be OFF, ERROR, WARN, INFO, DEBUG, TRACE, or ALL. - level: INFO + # The default level of all loggers. Can be OFF, ERROR, WARN, INFO, + # DEBUG, TRACE, or ALL. + level: DEBUG # Logger-specific levels. loggers: - com.hpcloud: INFO - com.hpcloud.som: INFO - com.yammer.metrics.reporting.JmxReporter: ERROR - -# Settings for logging to a file. - file: - # If true, addToBatch log statements to a file. - enabled: true - # Do not addToBatch log statements below this threshold to the file. - threshold: ALL - # The file to which statements will be logged. - currentLogFilename: /var/log/som/som-persister.log - # When the log file rolls over, the file will be archived to example-2012-03-15.log.gz, - # example.log will be truncated, and new statements written to it. - archivedLogFilenamePattern: /var/log/som/som-persister%d.log.gz - # The maximum number of log files to archive. - archivedFileCount: 10 + # Sets the level for 'com.example.app' to DEBUG. + com.example.app: DEBUG # Settings for logging to stdout. console: - # If true, addToBatch log statements to stdout. - enabled: false + # If true, write log statements to stdout. + enabled: true # Do not display log statements below this threshold to stdout. threshold: ALL + # The time zone in which dates should be displayed. + timeZone: UTC + + # A custom Logback format string. + logFormat: null + + # Settings for logging to a file. + file: + + # If true, write log statements to a file. + enabled: true + + # Do not write log statements below this threshold to the file. + threshold: ALL + + # The time zone in which dates should be displayed. + timeZone: UTC + + # A custom Logback format string. + logFormat: null + + # The file to which statements will be logged. + # + # If enabled is true, this must be specified. + currentLogFilename: ./logs/mon-persister.log + + # If true, log files are rotated and archived. + archive: true + + # When the log file rolls over, the file will be archived to + # app-2012-03-15.log.gz, example.log will be truncated, + # and new statements written to it. + # + # If archive is true, this must be specified. + archivedLogFilenamePattern: ./logs/mon-persister-%d.log.gz + + # The maximum number of log files to archive. + archivedFileCount: 5 + # Settings for logging to syslog. syslog: - # If true, addToBatch log statements to syslog. + # If true, write log statements to syslog. enabled: false - # The hostname of the syslog server to which statements will be sent. - # N.B.: If this is the local host, the local syslog instance will need to be configured to - # listen on an inet socket, not just a Unix socket. + # The hostname of the syslog server to which statements will be + # sent. + # + # N.B.: If this is the local host, the local syslog instance + # will need to be configured to listen on an inet socket, not just + # a Unix socket. host: localhost # The syslog facility to which statements will be sent. + # + # Can be one of: {AUTH, AUTHPRIV, DAEMON, CRON, FTP, LPR, KERN, + # MAIL, NEWS, SYSLOG, USER, UUCP, LOCAL0, LOCAL1, LOCAL2, LOCAL3, + # LOCAL4, LOCAL5, LOCAL6, LOCAL7}. facility: local0 + + # The time zone in which dates should be displayed. + timeZone: UTC + + # A custom Logback format string. + logFormat: null \ No newline at end of file diff --git a/src/deb/init/mon-persister.conf b/src/deb/init/mon-persister.conf index f25fbcd0..8a939c38 100644 --- a/src/deb/init/mon-persister.conf +++ b/src/deb/init/mon-persister.conf @@ -8,4 +8,4 @@ respawn setgid persister setuid persister -exec /usr/bin/java -Xmx8g -cp /opt/mon/mon-persister.jar com.hpcloud.mon.persister.PersisterApp --config /etc/mon/persister-config.yml +exec /usr/bin/java -Xmx8g -cp /opt/mon/mon-persister.jar com.hpcloud.MonPersisterService --config /etc/mon/persister-config.yml