From 41fcff81b26202c354e27902eaa85212a92a334c Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 18 Jul 2018 08:10:21 -0500 Subject: [PATCH] Serve pypi proxy cache from port 80 Oh easy_install. In the configure-mirrors zuul-jobs role we write out ~/.pydistutils.cfg. One of the things we do is: allow_hosts = {{ mirror_fqdn }} which results in things like allow_hosts = mirror.dfw.rax.openstack.org True story though - that doesn't match port 8080, so it breaks easy_install if we set index_url = http://mirror.dfw.rax.openstack.org:8080/pypi/simple Is there anything easy_install does right? No? We should probably completely rethink how we're plumbing mirror info into our nodes, but it is non-trivial to get a port into that file and various OpenStack projects are currently blocked on the pypi mirror being out of date. Serve the pypi proxy content from port 80. Change-Id: I518cf55fc6efe7db9493e4f5ec3601cb67f54999 --- .../templates/mirror.vhost.erb | 72 +++++++++++++------ 1 file changed, 50 insertions(+), 22 deletions(-) diff --git a/modules/openstack_project/templates/mirror.vhost.erb b/modules/openstack_project/templates/mirror.vhost.erb index c881340fcf..041e3a21eb 100644 --- a/modules/openstack_project/templates/mirror.vhost.erb +++ b/modules/openstack_project/templates/mirror.vhost.erb @@ -38,17 +38,53 @@ LogFormat "%h %l %u %t \"%r\" %>s %b %{cache-status}e \"%{Referer}i\" \"%{User-a - RewriteEngine On + # Caching reverse proxy for things that don't make sense in AFS + # + # General cache rules + CacheRoot "/var/cache/apache2/proxy" + CacheDirLevels 5 + CacheDirLength 2 + # SSL support + SSLProxyEngine on + # Prevent thundering herds. + CacheLock on + CacheLockPath "/tmp/mod_cache-lock" + CacheLockMaxAge 5 + # 5GiB + CacheMaxFileSize 5368709120 + CacheStoreExpired On - # Pypi's bandersnatch URL's are: - # /pypi/simple/index.html - # /pypi/simple/a/a/(index.html)? - # /pypi/simple/a/a/a-etc.whl - # /pypi/simple/a/abcd/(index.html)? - # /pypi/simple/a/abcd/abcd-etc.whl - RewriteCond %{REQUEST_URI} ^/pypi/simple/([^/])([^/]*) - RewriteCond %{DOCUMENT_ROOT}/pypi/simple/$1/$1$2 -d - RewriteRule ^/pypi/simple/([^/])([^/]*)(/.*)?$ /pypi/simple/$1/$1$2$3 [L] + # Added Aug 2017 in an attempt to avoid occasional 502 errors (around + # 0.05% of requests) of the type: + # + # End of file found: ... AH01102: error reading status line from remote server ... + # + # Per [1]: + # + # This avoids the "proxy: error reading status line from remote + # server" error message caused by the race condition that the backend + # server closed the pooled connection after the connection check by the + # proxy and before data sent by the proxy reached the backend. + # + # [1] https://httpd.apache.org/docs/2.4/mod/mod_proxy_http.html + SetEnv proxy-initial-not-pooled 1 + + RewriteEngine On + # pypi + CacheEnable disk "/pypi" + ProxyPass "/pypi/" "https://pypi.org/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/pypi/" "https://pypi.org/ + + # files.pythonhosted.org + CacheEnable disk "/pypifiles" + ProxyPass "/pypifiles/" "https://files.pythonhosted.org/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/pypifiles/" "https://files.pythonhosted.org/" + + # Rewrite the locations of the actual files + + SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE + Substitute "s|https://files.pythonhosted.org/|/pypifiles/|ni" + # Wheel URL's are: # /wheel/{distro}-{distro-version}/a/a/a-etc.whl @@ -58,29 +94,21 @@ LogFormat "%h %l %u %t \"%r\" %>s %b %{cache-status}e \"%{Referer}i\" \"%{User-a RewriteCond %{DOCUMENT_ROOT}/wheel/$1/$2/$2$3 -d RewriteRule ^/wheel/([^/]+)/([^/])([^/]*)(/.*)?$ /wheel/$1/$2/$2$3$4 [L] - # TODO(jhesketh): Remove this after bandersnatch implements pep503 - # https://bitbucket.org/pypa/bandersnatch/pull-requests/20/fully-implement-pep-503-normalization/diff - # Special cases for openstack.nose_plugin & backports.* + RewriteCond %{REQUEST_URI} ^/wheel/ RewriteRule ^(.*)/openstack-nose-plugin(.*)$ $1/openstack.nose_plugin$2 + RewriteCond %{REQUEST_URI} ^/wheel/ RewriteRule ^(.*)/backports-(.*)$ $1/backports.$2 - RewriteCond %{REQUEST_URI} ^/pypi/simple/([^/])([^/]*) - RewriteCond %{DOCUMENT_ROOT}/pypi/simple/$1/$1$2 -d - RewriteRule ^/pypi/simple/([^/])([^/]*)(/.*)?$ /pypi/simple/$1/$1$2$3 [L] - # Try again but replacing -'s with .'s + RewriteCond %{REQUEST_URI} ^/wheel/ RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-f RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-d RewriteRule (.*)-(.*) $1.$2 [N] - RewriteCond %{REQUEST_URI} ^/pypi/simple/([^/])([^/]*) - RewriteCond %{DOCUMENT_ROOT}/pypi/simple/$1/$1$2 -d - RewriteRule ^/pypi/simple/([^/])([^/]*)(/.*)?$ /pypi/simple/$1/$1$2$3 [L] - ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_error.log LogLevel warn - CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_access.log combined + CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_access.log combined-cache ServerSignature Off