From a8682baa189e2b0afda2de47e59646ec546d75c9 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Tue, 13 Feb 2018 11:44:57 +1100 Subject: [PATCH] Enable build logs on builder, web app on launchers There is currently a confusing array of exporting options for exporting build and upload logs. The upload logs have never really worked, and the info is better given from image-list end-points. The build logs since Ia7415d2fbbb320f8eddc4e46c3a055414df5f997 are logged into separate files into /var/log/nodepool/builds. With a separate builder, it can not access the webapp ports provided by the launcher, so the redirects there are wrong -- and we don't deploy apache on the launcher to expose them. If you're using single node anyway, it's more than likely you have zuul which installs the main website so can't enable this. For these reasons deprecate and remove these old settings. Instead, add a separate flag to each of the builder and launcher. builder::enable_build_log_via_http will deploy apache and a config file that exposes /var/log/nodepool/builds launcher::enable_webapp will deploy apache and redirects to the internal webapp Note this does not handle a launcher and builder on the same host; since the webapp, and indeed refactoring this module in general for multiple daemons, is under some development, we leave this as a future task. Depends-On: https://review.openstack.org/543667 Change-Id: I447886dd32f7f3bc6758ffd7a1b725689d04ee68 --- manifests/builder.pp | 36 ++++++++- manifests/init.pp | 112 ++++---------------------- manifests/launcher.pp | 25 ++++++ templates/nodepool-builder.vhost.erb | 22 +++++ templates/nodepool-launcher.vhost.erb | 19 +++++ templates/nodepool-log.vhost.erb | 40 --------- 6 files changed, 117 insertions(+), 137 deletions(-) create mode 100644 templates/nodepool-builder.vhost.erb create mode 100644 templates/nodepool-launcher.vhost.erb delete mode 100644 templates/nodepool-log.vhost.erb diff --git a/manifests/builder.pp b/manifests/builder.pp index 1a36122..a405967 100644 --- a/manifests/builder.pp +++ b/manifests/builder.pp @@ -17,7 +17,10 @@ class nodepool::builder( $statsd_host = undef, $nodepool_ssh_public_key = undef, - $image_log_document_root = '/var/log/nodepool/image', + # If true, export build logs from $build_log_document_root via apache + $enable_build_log_via_http = false, + $build_log_document_root = '/var/log/nodepool/builds', + $vhost_name = $::fqdn, $builder_logging_conf_template = 'nodepool/nodepool-builder.logging.conf.erb', $environment = {}, $build_workers = '1', @@ -94,4 +97,35 @@ class nodepool::builder( ], } + if $enable_build_log_via_http == true { + include ::httpd + + ::httpd::vhost { $vhost_name: + port => 80, + priority => '50', + docroot => 'MEANINGLESS_ARGUMENT', + template => 'nodepool/nodepool-builder.vhost.erb', + } + if ! defined(Httpd::Mod['rewrite']) { + httpd::mod { 'rewrite': ensure => present } + } + if ! defined(Httpd::Mod['proxy']) { + httpd::mod { 'proxy': ensure => present } + } + if ! defined(Httpd::Mod['proxy_http']) { + httpd::mod { 'proxy_http': ensure => present } + } + } + + file { $build_log_document_root: + ensure => directory, + mode => '0755', + owner => 'nodepool', + group => 'nodepool', + require => [ + User['nodepool'], + File['/var/log/nodepool'], + ], + } + } diff --git a/manifests/init.pp b/manifests/init.pp index 39e5ea5..e6a0619 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -25,14 +25,23 @@ class nodepool ( $git_source_repo = 'https://git.openstack.org/openstack-infra/nodepool', $revision = 'master', $statsd_host = undef, + # The following have all been deprecated and are left only for + # argument compatability + # - To export the image logs on builders use + # builder::enable_build_log_via_http + # - To enable the webapp on launchers use launcher::enable_webapp + # - Upload logs were never really useful, use the webapp endpoint + # to see status + # - TODO: common apache layout with config merging so launcher + # and builder on same host works + $enable_image_log_via_http = undef, + $image_log_document_root = undef, $vhost_name = $::fqdn, - $image_log_document_root = '/var/log/nodepool/image', - $image_log_periodic_cleanup = false, - $enable_image_log_via_http = false, - $upload_log_document_root = '/var/log/nodepool/upload', - $upload_log_periodic_cleanup = false, - # note : not currently supported - $enable_upload_log_via_http = false, + $image_log_periodic_cleanup = undef, + $upload_log_document_root = undef, + $upload_log_periodic_cleanup = undef, + $enable_upload_log_via_http = undef, + # /end $environment = {}, # enable sudo for nodepool user. Useful for using dib with nodepool $sudo = true, @@ -333,95 +342,6 @@ class nodepool ( require => File['/etc/init.d/nodepool'], } - if $image_log_document_root == $upload_log_document_root { - # It makes no sense to ask to not export build or upload logs, but - # then have them log to the same directory that will be exported. - if (($enable_image_log_via_http and !$enable_upload_log_via_http) or - ($enable_upload_log_via_http and !$enable_image_log_via_http) - ) { - fail('Unexported logs in same directory as exported logs!') - } - } - - # we only need to create the upload log dir if it is separate to the - # image log. - $separate_upload_log_dir = - $image_log_document_root != $upload_log_document_root - - if $enable_image_log_via_http == true or - $enable_upload_log_via_http == true { - # Setup apache for log access - include ::httpd - - ::httpd::vhost { $vhost_name: - port => 80, - priority => '50', - docroot => 'MEANINGLESS_ARGUMENT', - template => 'nodepool/nodepool-log.vhost.erb', - } - if ! defined(Httpd::Mod['rewrite']) { - httpd::mod { 'rewrite': ensure => present } - } - if ! defined(Httpd::Mod['proxy']) { - httpd::mod { 'proxy': ensure => present } - } - if ! defined(Httpd::Mod['proxy_http']) { - httpd::mod { 'proxy_http': ensure => present } - } - } - - if $image_log_document_root != '/var/log/nodepool' { - file { $image_log_document_root: - ensure => directory, - mode => '0755', - owner => 'nodepool', - group => 'nodepool', - require => [ - User['nodepool'], - File['/var/log/nodepool'], - ], - } - } - - # we only need this if it is different to the image_log - if $separate_upload_log_dir - { - file { $upload_log_document_root: - ensure => directory, - mode => '0755', - owner => 'nodepool', - group => 'nodepool', - require => [ - User['nodepool'], - File['/var/log/nodepool'], - ], - } - } - - # run a cleanup on the image log directory to cleanup logs for - # images that are no longer being built - if $image_log_periodic_cleanup == true { - cron { 'image_log_cleanup': - user => 'nodepool', - hour => '1', - minute => '0', - command => "find ${image_log_document_root} \\( -name '*.log' -o -name '*.log.*' \\) -mtime +7 -execdir rm {} \\;", - environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin', - } - } - - # run a cleanup on the upload log directory to cleanup logs for - # providers that are no long uploading - if $upload_log_periodic_cleanup == true { - cron { 'upload_log_cleanup': - user => 'nodepool', - hour => '1', - minute => '0', - command => "find ${upload_log_document_root} \\( -name '*.log' -o -name '*.log.*' \\) -mtime +7 -execdir rm {} \\;", - environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin', - } - } - if $sudo == true { $sudo_file_ensure = present } diff --git a/manifests/launcher.pp b/manifests/launcher.pp index 00036be..a35b0b9 100644 --- a/manifests/launcher.pp +++ b/manifests/launcher.pp @@ -19,6 +19,10 @@ class nodepool::launcher( $statsd_prefix = undef, $nodepool_ssh_public_key = undef, $launcher_logging_conf_template = 'nodepool/nodepool-launcher.logging.conf.erb', + # If true, an apache will be setup to redirect webapp end-points to + # the local webapp instance (on port 8005) + $enable_webapp = false, + $vhost_name = $::fqdn, ) { if ! defined(File['/home/nodepool/.ssh']) { @@ -87,4 +91,25 @@ class nodepool::launcher( File['/etc/nodepool/launcher-logging.conf'], ], } + + if $enable_webapp == true { + include ::httpd + + ::httpd::vhost { $vhost_name: + port => 80, + priority => '50', + docroot => 'MEANINGLESS_ARGUMENT', + template => 'nodepool/nodepool-launcher.vhost.erb', + } + if ! defined(Httpd::Mod['rewrite']) { + httpd::mod { 'rewrite': ensure => present } + } + if ! defined(Httpd::Mod['proxy']) { + httpd::mod { 'proxy': ensure => present } + } + if ! defined(Httpd::Mod['proxy_http']) { + httpd::mod { 'proxy_http': ensure => present } + } + } + } diff --git a/templates/nodepool-builder.vhost.erb b/templates/nodepool-builder.vhost.erb new file mode 100644 index 0000000..fffc1d6 --- /dev/null +++ b/templates/nodepool-builder.vhost.erb @@ -0,0 +1,22 @@ + + ServerName <%= scope.lookupvar("nodepool::builder::vhost_name") %> + + DocumentRoot <%= scope.lookupvar("nodepool::builder::build_log_document_root") %> + > + Options <%= scope.lookupvar("httpd::params::options") %> + AllowOverride None + Require all granted + + + ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/nodepool_error.log + LogLevel warn + CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/nodepool_access.log combined + ServerSignature Off + + AddType text/plain .log + + + SetOutputFilter DEFLATE + + + diff --git a/templates/nodepool-launcher.vhost.erb b/templates/nodepool-launcher.vhost.erb new file mode 100644 index 0000000..3af503a --- /dev/null +++ b/templates/nodepool-launcher.vhost.erb @@ -0,0 +1,19 @@ + + ServerName <%= scope.lookupvar("nodepool::launcher::vhost_name") %> + + ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/nodepool_error.log + LogLevel warn + CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/nodepool_access.log combined + ServerSignature Off + + + SetOutputFilter DEFLATE + + + RewriteEngine on + RewriteRule ^/image-list$ http://127.0.0.1:8005/image-list [P] + RewriteRule ^/dib-image-list$ http://127.0.0.1:8005/dib-image-list [P] + RewriteRule ^/image-list.json$ http://127.0.0.1:8005/image-list.json [P] + RewriteRule ^/dib-image-list.json$ http://127.0.0.1:8005/dib-image-list.json [P] + + diff --git a/templates/nodepool-log.vhost.erb b/templates/nodepool-log.vhost.erb deleted file mode 100644 index 17dfb4d..0000000 --- a/templates/nodepool-log.vhost.erb +++ /dev/null @@ -1,40 +0,0 @@ - - ServerName <%= scope.lookupvar("nodepool::vhost_name") %> - - DocumentRoot <%= scope.lookupvar("nodepool::image_log_document_root") %> - - <% if scope.lookupvar("nodepool::enable_image_log_via_http") -%> - > - Options <%= scope.lookupvar("httpd::params::options") %> - AllowOverride None - Require all granted - - <% end -%> - - <% if scope.lookupvar("nodepool::enable_upload_log_via_http") and scope.lookupvar("nodepool::separate_upload_log_dir") -%> - Alias "/upload" <%= scope.lookupvar("nodepool::upload_log_document_root") %> - > - Options <%= scope.lookupvar("httpd::params::options") %> - AllowOverride None - Require all granted - - <% end -%> - - ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/nodepool_error.log - LogLevel warn - CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/nodepool_access.log combined - ServerSignature Off - - AddType text/plain .log - - RewriteEngine on - RewriteRule ^/image-list$ http://127.0.0.1:8005/image-list [P] - RewriteRule ^/dib-image-list$ http://127.0.0.1:8005/dib-image-list [P] - RewriteRule ^/image-list.json$ http://127.0.0.1:8005/image-list.json [P] - RewriteRule ^/dib-image-list.json$ http://127.0.0.1:8005/dib-image-list.json [P] - - - SetOutputFilter DEFLATE - - -