From ad226c0d864ed6becfb400a08d0c13b5d70fe659 Mon Sep 17 00:00:00 2001 From: Mykyta Karpin Date: Mon, 27 Mar 2017 17:06:49 +0300 Subject: [PATCH] Provide compatibility with mos10 and ubuntu 16.04 Change-Id: Id16f9708311c7001c1fe461d6a943318774c239e --- .../puppet/manifests/zookeeper.pp | 26 +++++++-- .../patches/kafka/templates/broker.unit.erb | 22 +++++++ .../patches/zookeeper/manifests/service.pp | 57 +++++++++++++++++++ .../zookeeper/templates/conf/environment.erb | 20 +++++++ .../zookeeper/templates/zookeeper.service.erb | 27 +++++++++ pre_build_hook | 16 ++++-- 6 files changed, 157 insertions(+), 11 deletions(-) create mode 100644 deployment_scripts/puppet/patches/kafka/templates/broker.unit.erb create mode 100644 deployment_scripts/puppet/patches/zookeeper/manifests/service.pp create mode 100644 deployment_scripts/puppet/patches/zookeeper/templates/conf/environment.erb create mode 100644 deployment_scripts/puppet/patches/zookeeper/templates/zookeeper.service.erb diff --git a/deployment_scripts/puppet/manifests/zookeeper.pp b/deployment_scripts/puppet/manifests/zookeeper.pp index b2b22d8..dba0b4d 100644 --- a/deployment_scripts/puppet/manifests/zookeeper.pp +++ b/deployment_scripts/puppet/manifests/zookeeper.pp @@ -19,11 +19,27 @@ $addresses_map = hiera('kafka::addresses_map') $heap_size = hiera('zookeeper::jvm_heap_size') $datastore = hiera('kafka::data_dir') -class { 'zookeeper': - servers => $addresses_map, - id => $myid, - datastore => $datastore, - java_opts => "-Xmx${heap_size}G -Xms${heap_size}G", +if versioncmp($::operatingsystemmajrelease, '16') >= 0 { + + class { 'zookeeper': + servers => $addresses_map, + id => $myid, + datastore => $datastore, + java_opts => "-Xmx${heap_size}G -Xms${heap_size}G", + service_provider => 'systemd', + manage_service => true, + manage_service_file => true, + packages => ['zookeeper'], + rollingfile_threshold => 'DEBUG', + } +} else { + + class { 'zookeeper': + servers => $addresses_map, + id => $myid, + datastore => $datastore, + java_opts => "-Xmx${heap_size}G -Xms${heap_size}G", + } } file { '/etc/logrotate.d/zookeeper.conf': diff --git a/deployment_scripts/puppet/patches/kafka/templates/broker.unit.erb b/deployment_scripts/puppet/patches/kafka/templates/broker.unit.erb new file mode 100644 index 0000000..5f605f1 --- /dev/null +++ b/deployment_scripts/puppet/patches/kafka/templates/broker.unit.erb @@ -0,0 +1,22 @@ +[Unit] +Description=Apache Kafka server (broker) +Documentation=http://kafka.apache.org/documentation.html +Requires=network.target +After=network.target + +[Service] +Type=simple +User=kafka +Group=kafka +SyslogIdentifier=kafka +Environment='KAFKA_HEAP_OPTS=<%= @heap_opts %>' +Environment='KAFKA_LOG4J_OPTS=<%= @log4j_opts %>' +Environment='KAFKA_JMX_OPTS=<%= @jmx_opts %>' +Environment='KAFKA_OPTS=<%= @opts %>' +ExecStart=/opt/kafka/bin/kafka-server-start.sh /opt/kafka/config/server.properties +Restart=always +LimitNOFILE=65536 +LimitCORE=infinity + +[Install] +WantedBy=multi-user.target diff --git a/deployment_scripts/puppet/patches/zookeeper/manifests/service.pp b/deployment_scripts/puppet/patches/zookeeper/manifests/service.pp new file mode 100644 index 0000000..950721a --- /dev/null +++ b/deployment_scripts/puppet/patches/zookeeper/manifests/service.pp @@ -0,0 +1,57 @@ +# Class: zookeeper::service +# +# Should not be included directly +# +class zookeeper::service( + $zoo_dir = $zookeeper::cfg_dir, + $log_dir = $zookeeper::log_dir, + $pid_file = undef, + $service_provider = undef, # init mechanism + $cfg_dir = '/etc/zookeeper/conf', + $service_name = 'zookeeper', + $service_ensure = 'running', + $manage_service_file = true, + $user = 'zookeeper', + $group = 'zookeeper', + $zoo_main = 'org.apache.zookeeper.server.quorum.QuorumPeerMain', + $log4j_prop = 'INFO,ROLLINGFILE', +){ + require ::zookeeper::install + + if $manage_service_file == true { + if $service_provider == 'systemd' { + file { '/lib/systemd/system/zookeeper.service': + ensure => 'present', + content => template('zookeeper/zookeeper.service.erb'), + } ~> + exec { 'systemctl daemon-reload # for zookeeper': + refreshonly => true, + path => $::path, + notify => Service[$service_name] + } + } elsif ( $service_provider == 'init' or $service_provider == 'redhat') { + file {"/etc/init.d/${service_name}": + ensure => present, + content => template('zookeeper/zookeeper.init.erb'), + mode => '0755', + notify => Service[$service_name] + } + } + } + + service { $service_name: + ensure => $service_ensure, + hasstatus => true, + hasrestart => true, + provider => $service_provider, + enable => true, + require => [ + Class['::zookeeper::install'], + File["${cfg_dir}/zoo.cfg"] + ], + subscribe => [ + File["${cfg_dir}/myid"], File["${cfg_dir}/zoo.cfg"], + File["${cfg_dir}/environment"], File["${cfg_dir}/log4j.properties"], + ] + } +} diff --git a/deployment_scripts/puppet/patches/zookeeper/templates/conf/environment.erb b/deployment_scripts/puppet/patches/zookeeper/templates/conf/environment.erb new file mode 100644 index 0000000..7ceeb0c --- /dev/null +++ b/deployment_scripts/puppet/patches/zookeeper/templates/conf/environment.erb @@ -0,0 +1,20 @@ +NAME=zookeeper +ZOOCFGDIR=<%= @cfg_dir %> + +# TODO this is really ugly +# How to find out, which jars are needed? +# seems, that log4j requires the log4j.properties file to be in the classpath +CLASSPATH="<%= @cfg_dir %>:/usr/share/java/jline.jar:/usr/share/java/log4j-1.2.jar:/usr/share/java/xercesImpl.jar:/usr/share/java/xmlParserAPIs.jar:/usr/share/java/netty.jar:/usr/share/java/slf4j-api.jar:/usr/share/java/slf4j-log4j12.jar:/usr/share/java/zookeeper.jar" + +ZOOCFG="<%= @cfg_dir %>/zoo.cfg" +ZOO_LOG_DIR=<%= @log_dir %> +USER=<%= @user %> +GROUP=<%= @group %> +PIDDIR=<%= @pid_dir %> +PIDFILE=<%= @pid_path %> +SCRIPTNAME=/etc/init.d/$NAME +JAVA=<%= @java_bin %> +ZOOMAIN="<%= @zoo_main %>" +ZOO_LOG4J_PROP="<%= @log4j_prop %>" +JMXLOCALONLY=false +JAVA_OPTS="<%= @java_opts %>" \ No newline at end of file diff --git a/deployment_scripts/puppet/patches/zookeeper/templates/zookeeper.service.erb b/deployment_scripts/puppet/patches/zookeeper/templates/zookeeper.service.erb new file mode 100644 index 0000000..cefb562 --- /dev/null +++ b/deployment_scripts/puppet/patches/zookeeper/templates/zookeeper.service.erb @@ -0,0 +1,27 @@ +# Autogenerated by Puppet from zookeeper/templates/zookeeper.service.erb +# Do not edit + +[Unit] +Description=Apache ZooKeeper +<%if @systemd_unit_want -%> +Want=<%=@systemd_unit_want %> +<% end -%> +<%if @systemd_unit_after -%> +After=<%=@systemd_unit_after %> +<% end -%> +ConditionPathExists=/var/lib/zookeeper/myid + +[Service] +Type=simple +User=<%= @user %> +Group=<%= @group %> +SyslogIdentifier=zookeeper +EnvironmentFile=/etc/zookeeper/conf/environment +EnvironmentFile=-/etc/default/zookeeper +ExecStart=/usr/bin/java -cp ${CLASSPATH} $JAVA_OPTS -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.local.only=${JMXLOCALONLY} -Dzookeeper.log.dir=${ZOO_LOG_DIR} -Dzookeeper.root.logger=${ZOO_LOG4J_PROP} $ZOOMAIN $ZOOCFG +SuccessExitStatus=143 +# ZooKeeper is "fail-fast", see https://zookeeper.apache.org/doc/r3.4.8/zookeeperAdmin.html#sc_supervision +Restart=always + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/pre_build_hook b/pre_build_hook index 0ae37a3..236d507 100755 --- a/pre_build_hook +++ b/pre_build_hook @@ -15,16 +15,20 @@ download_puppet_module "java" "${JAVA_MODULE_URL}" download_puppet_module "stdlib" "${STDLIB_MODULE_URL}" download_puppet_module "systemd" "${SYSTEMD_MODULE_URL}" download_puppet_module "zookeeper" "${ZOOKEEPER_MODULE_URL}" -download_puppet_module "kafka" "${KAFKA_MODULE_URL}" +download_puppet_module "kafka" "${KAFKA_MODULE_URL}" # Patching modules PATCH_DIR="deployment_scripts/puppet/patches" MODULES_DIR="deployment_scripts/puppet/modules" -cp -f $PATCH_DIR/zookeeper/manifests/post_install.pp $MODULES_DIR/zookeeper/manifests -cp -f $PATCH_DIR/zookeeper/templates/conf/zoo.cfg.erb $MODULES_DIR/zookeeper/templates/conf -cp -f $PATCH_DIR/kafka/manifests/init.pp $MODULES_DIR/kafka/manifests -cp -f $PATCH_DIR/kafka/manifests/broker/service.pp $MODULES_DIR/kafka/manifests/broker -cp -f $PATCH_DIR/kafka/templates/init.erb $MODULES_DIR/kafka/templates +cp -f $PATCH_DIR/zookeeper/manifests/post_install.pp $MODULES_DIR/zookeeper/manifests +cp -f $PATCH_DIR/zookeeper/manifests/service.pp $MODULES_DIR/zookeeper/manifests +cp -f $PATCH_DIR/zookeeper/templates/conf/zoo.cfg.erb $MODULES_DIR/zookeeper/templates/conf +cp -f $PATCH_DIR/zookeeper/templates/conf/environment.erb $MODULES_DIR/zookeeper/templates/conf +cp -f $PATCH_DIR/zookeeper/templates/zookeeper.service.erb $MODULES_DIR/zookeeper/templates/ +cp -f $PATCH_DIR/kafka/manifests/init.pp $MODULES_DIR/kafka/manifests +cp -f $PATCH_DIR/kafka/manifests/broker/service.pp $MODULES_DIR/kafka/manifests/broker +cp -f $PATCH_DIR/kafka/templates/init.erb $MODULES_DIR/kafka/templates +cp -f $PATCH_DIR/kafka/templates/broker.unit.erb $MODULES_DIR/kafka/templates KAFKA_TARBALL_URL="http://mirrors.ukfast.co.uk/sites/ftp.apache.org/kafka/0.10.0.0/kafka_2.11-0.10.0.0.tgz" download_file "${KAFKA_TARBALL_URL}" kafka_2.11-0.10.0.0.tgz repositories/ubuntu/kafka/0.10.0.0