Stabilize E2E Tests.

Change-Id: Ic25ef2ee089410a9d4f57e80a8f5499ffc07fdfa
This commit is contained in:
Malini Kamalambal 2015-09-01 16:06:26 -04:00
parent ccc58cfd78
commit 93fecc3ec3
11 changed files with 116 additions and 167 deletions

View File

@ -20,41 +20,20 @@ from flask import Flask
from flask import make_response
from flask import render_template
from flask import request
from flask_wtf import form
app = Flask(__name__)
@app.route('/')
def index():
return 'Test Flask Site from 23.253.156.204'
return 'Test Flask Site'
@app.route('/hello/', strict_slashes=False)
def hello():
return 'halo halo from 23.253.156.204'
@app.route('/hello/<user_name>/')
def hello_user(user_name):
print(request.headers)
return render_template('hello.html', name=user_name)
@app.route('/test/host-header/')
@app.route('/test/host-header/', strict_slashes=False)
def test_host_header():
print(request.headers)
return render_template('hello.html', name=request.headers['Host'])
@app.route('/hello/<user_name>/upload/', methods=['GET', 'POST'])
def upload_file(user_name):
if request.method == 'POST':
form.photo.data.save('/tmp/{0}.jpg'.format(user_name))
else:
return render_template('gorilla.html', name=user_name)
@app.route('/test/camera.jpg', strict_slashes=False)
def test_jpg():
response = make_response(render_template('jpg.html', name='jpg'))

View File

@ -13,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import hashlib
import random
import time
@ -99,8 +100,18 @@ class TestBase(fixtures.BaseTestFixture):
:returns: True/False
"""
origin_content = self.get_content(url=origin_url)
origin_content_hash = hashlib.md5()
origin_content_hash.update(origin_content)
cdn_content = self.get_content(url=cdn_url)
self.assertEqual(origin_content, cdn_content)
cdn_content_hash = hashlib.md5()
cdn_content_hash.update(cdn_content)
self.assertEqual(
origin_content_hash.hexdigest(), cdn_content_hash.hexdigest(),
msg='Contents do not match \n \
Origin url {0} \n \
CDN enabled url {1}'.format(origin_url, cdn_url))
def setup_cname(self, name, cname):
"""Create a CNAME record and wait for propagation."""

View File

@ -98,8 +98,8 @@ class TestCaching(base.TestBase):
cdn_url = 'http://' + self.test_domain + self.cacheable_endpoint
# Verify content is not cached
self.get_from_cdn_enabled_url(cdn_url=cdn_url, count=4)
self.assertCacheStatus(cdn_url=cdn_url,
edge_server = self.get_from_cdn_enabled_url(cdn_url=cdn_url, count=1)
self.assertCacheStatus(cdn_url=cdn_url, edge_server=edge_server,
status_list=['TCP_MISS', 'TCP_REFRESH_MISS'])
def test_cache_rules(self):
@ -151,30 +151,33 @@ class TestCaching(base.TestBase):
# Verify cdn hit on rule urls
cdn_jpg_url = cdn_url + self.jpg_path
self.get_from_cdn_enabled_url(cdn_url=cdn_jpg_url, count=2)
self.assertCacheStatus(cdn_url=cdn_jpg_url,
edge_server = self.get_from_cdn_enabled_url(
cdn_url=cdn_jpg_url, count=1)
self.assertCacheStatus(cdn_url=cdn_jpg_url, edge_server=edge_server,
status_list=['TCP_HIT', 'TCP_MEM_HIT'])
cdn_txt_url = cdn_url + self.txt_path
self.get_from_cdn_enabled_url(cdn_url=cdn_txt_url, count=2)
self.assertCacheStatus(cdn_url=cdn_txt_url,
edge_server = self.get_from_cdn_enabled_url(
cdn_url=cdn_txt_url, count=1)
self.assertCacheStatus(cdn_url=cdn_txt_url, edge_server=edge_server,
status_list=['TCP_HIT', 'TCP_MEM_HIT'])
cdn_zip_url = cdn_url + self.zip_path
self.get_from_cdn_enabled_url(cdn_url=cdn_zip_url, count=2)
self.assertCacheStatus(cdn_url=cdn_zip_url,
edge_server = self.get_from_cdn_enabled_url(
cdn_url=cdn_zip_url, count=1)
self.assertCacheStatus(cdn_url=cdn_zip_url, edge_server=edge_server,
status_list=['TCP_HIT', 'TCP_MEM_HIT'])
time.sleep(max(jpg_ttl, zip_ttl, txt_ttl))
# Verify that content in cache is stale/removed after the ttl expires
self.assertCacheStatus(
cdn_url=cdn_jpg_url,
cdn_url=cdn_jpg_url, edge_server=edge_server,
status_list=['TCP_REFRESH_HIT', 'TCP_REFRESH_MISS', 'TCP_MISS'])
self.assertCacheStatus(
cdn_url=cdn_txt_url,
cdn_url=cdn_txt_url, edge_server=edge_server,
status_list=['TCP_REFRESH_HIT', 'TCP_REFRESH_MISS', 'TCP_MISS'])
self.assertCacheStatus(
cdn_url=cdn_zip_url,
cdn_url=cdn_zip_url, edge_server=edge_server,
status_list=['TCP_REFRESH_HIT', 'TCP_REFRESH_MISS', 'TCP_MISS'])
def test_update_cache_rules(self):
@ -220,14 +223,15 @@ class TestCaching(base.TestBase):
origin_url=origin_jpg, cdn_url=cdn_jpg_url)
# Verify that content is cached after two requests
self.get_from_cdn_enabled_url(cdn_url=cdn_jpg_url, count=2)
self.assertCacheStatus(cdn_url=cdn_jpg_url,
edge_server = self.get_from_cdn_enabled_url(
cdn_url=cdn_jpg_url, count=1)
self.assertCacheStatus(cdn_url=cdn_jpg_url, edge_server=edge_server,
status_list=['TCP_HIT', 'TCP_MEM_HIT'])
# Verify that content in cache is stale/removed after the ttl expires
time.sleep(jpg_ttl + 10)
self.assertCacheStatus(
cdn_url=cdn_jpg_url,
cdn_url=cdn_jpg_url, edge_server=edge_server,
status_list=['TCP_REFRESH_HIT', 'TCP_REFRESH_MISS', 'TCP_MISS'])
# Update cache rules
@ -245,14 +249,15 @@ class TestCaching(base.TestBase):
request_body=test_data)
# Verify that content is cached after two requests
self.get_from_cdn_enabled_url(cdn_url=cdn_jpg_url, count=2)
self.assertCacheStatus(cdn_url=cdn_jpg_url,
edge_server = self.get_from_cdn_enabled_url(
cdn_url=cdn_jpg_url, count=1)
self.assertCacheStatus(cdn_url=cdn_jpg_url, edge_server=edge_server,
status_list=['TCP_HIT', 'TCP_MEM_HIT'])
time.sleep(new_ttl)
# Verify that content in cache is stale/removed after the ttl expires
self.assertCacheStatus(
cdn_url=cdn_jpg_url,
cdn_url=cdn_jpg_url, edge_server=edge_server,
status_list=['TCP_REFRESH_HIT', 'TCP_REFRESH_MISS', 'TCP_MISS'])
def tearDown(self):

View File

@ -15,8 +15,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import time
from tests.endtoend import base
@ -30,9 +28,7 @@ class TestWebsiteCDN(base.TestBase):
sub_domain = base.random_string(prefix='test-cdn-')
self.test_domain = sub_domain + '.' + self.dns_config.test_domain
print('Domain Name', self.test_domain)
self.origin = self.test_config.wordpress_origin
self.origin = self.test_config.alternate_origin
self.cname_rec = []
def test_enable_cdn(self):
@ -43,7 +39,7 @@ class TestWebsiteCDN(base.TestBase):
"port": 80,
"ssl": False}]
caching_list = []
self.service_name = base.random_string(prefix='testService-')
self.service_name = base.random_string(prefix='e2e-cdn-')
resp = self.setup_service(
service_name=self.service_name,
@ -66,8 +62,7 @@ class TestWebsiteCDN(base.TestBase):
origin_url = 'http://' + self.origin
cdn_enabled_url = 'http://' + self.test_domain
self.assertSameContent(origin_url=origin_url,
cdn_url=cdn_enabled_url)
self.assertSameContent(origin_url=origin_url, cdn_url=cdn_enabled_url)
if self.test_config.webpagetest_enabled:
self.run_webpagetest(url=cdn_enabled_url)
@ -78,8 +73,6 @@ class TestWebsiteCDN(base.TestBase):
sub_domain2 = base.random_string(prefix='test-cdn-')
self.test_domain2 = sub_domain2 + '.' + self.dns_config.test_domain
print('Additional Domain Name', self.test_domain2)
# Create a Poppy Service for the test website with two domains
domain_list = [{"domain": self.test_domain},
{"domain": self.test_domain2}]
@ -87,7 +80,7 @@ class TestWebsiteCDN(base.TestBase):
"port": 80,
"ssl": False}]
caching_list = []
self.service_name = base.random_string(prefix='testService-')
self.service_name = base.random_string(prefix='e2e-cdn-')
resp = self.setup_service(
service_name=self.service_name,
@ -128,68 +121,6 @@ class TestWebsiteCDN(base.TestBase):
print(wpt_result_1)
print(wpt_result_2)
def test_purge(self):
# Create a Poppy Service for the test website
domain_list = [{"domain": self.test_domain}]
origin_list = [{"origin": self.origin,
"port": 80,
"ssl": False}]
rule1 = self.cacherules_config.cache_rule1
ttlrule1 = self.cacherules_config.ttl_rule1
caching_list = [{"name": "images", "ttl": ttlrule1, "rules":
[{"name": "image_rule", "request_url": rule1}]}]
self.service_name = base.random_string(prefix='testService-')
resp = self.setup_service(
service_name=self.service_name,
domain_list=domain_list,
origin_list=origin_list,
caching_list=caching_list,
flavor_id=self.poppy_config.flavor)
self.service_location = resp.headers['location']
resp = self.poppy_client.get_service(location=self.service_location)
links = resp.json()['links']
access_url = [link['href'] for link in links if
link['rel'] == 'access_url']
# Adds cname records corresponding to the test domains
rec = self.setup_cname(name=self.test_domain, cname=access_url[0])
if rec:
self.cname_rec.append(rec[0])
origin_url = 'http://' + self.origin
cdn_enabled_url = 'http://' + self.test_domain
self.assertSameContent(origin_url=origin_url,
cdn_url=cdn_enabled_url)
# Purge object in rule 1 and ensure it gets a TCP_MISS
self.get_from_cdn_enabled_url(cdn_url=cdn_enabled_url + rule1, count=2)
self.assertCacheStatus(cdn_url=cdn_enabled_url + rule1,
status_list=['TCP_HIT', 'TCP_MEM_HIT'])
self.poppy_client.purge_asset(location=self.service_location,
asset_url=rule1)
# Wait for purge to complete & verify that content is fetched from
# origin for subsequent call.
# @todo: Change the sleep to check the real status of purge. As is
# there is no way a poppy user can get the purge status.
time.sleep(self.purge_config.purge_wait_time)
self.assertCacheStatus(
cdn_url=cdn_enabled_url + rule1,
status_list=['TCP_REFRESH_HIT', 'TCP_REFRESH_MISS', 'TCP_MISS'])
# Currently not supported
# Purge all content and ensure rule 2 gets a TCP_MISS
# self.poppy_client.purge_asset(location=self.service_location)
# self.wait_for_CDN_status(cdn_url=cdn_enabled_url, status='TCP_MISS')
# self.assertCDNMiss(cdn_url=cdn_enabled_url + rule2)
def tearDown(self):
self.poppy_client.delete_service(location=self.service_location)
for record in self.cname_rec:

View File

@ -45,6 +45,12 @@ class TestHostHeaders(base.TestBase):
self.cname_rec = []
def test_origin_host_header(self):
'''Test for CDN Service with origin host header
In this case the CDN provider will inject 'Host: self.default_origin'
header when forwarding requests from the CDN enabled url to the
origin server.
'''
domains = [{'domain': self.test_domain}]
origins = [{
"origin": self.default_origin,
@ -82,6 +88,15 @@ class TestHostHeaders(base.TestBase):
self.assertIn(self.default_origin, resp.content)
def test_custom_host_header(self):
'''Test for CDN Service with custom host header
In this case the CDN provider will inject the header
'Host: custom_value'
when forwarding requests from the CDN enabled url to the origin server.
The custom value is whatever was specified when the service was created
For eg. in the test below the header injected will be
'Host: llama-llama-red-pajama.com'
'''
domains = [{'domain': self.test_domain}]
host_header_value = 'llama-llama-red-pajama.com'
origins = [{
@ -122,6 +137,12 @@ class TestHostHeaders(base.TestBase):
self.assertIn(host_header_value, resp.content)
def test_default_host_header(self):
'''Test for CDN Service with default host header
The default host header is the domain value.
In this case Akamai will inject the 'Host: self.test_domain' header
when forwarding requests from the CDN enable url to the origin server.
'''
domains = [{'domain': self.test_domain}]
origins = [{
"origin": self.default_origin,

View File

@ -155,7 +155,7 @@ class TestIpRestrictions(base.TestBase):
cdn_url = 'http://' + test_domain
resp = self.http_client.get(url=cdn_url)
self.assertEqual(resp.status_code, 200)
self.assertIn(self.default_origin, resp.content)
self.assertIn('Test Flask Site', resp.content)
# Verify wpt cannot fetch cdn content
wpt_result = self.run_webpagetest(url=cdn_url)

View File

@ -99,13 +99,13 @@ class TestOriginHeaders(base.TestBase):
self.cname_rec.append(rec[0])
cdn_url = 'http://' + self.test_domain + self.expires_path
self.get_from_cdn_enabled_url(cdn_url=cdn_url, count=4)
self.assertCacheStatus(
cdn_url=cdn_url, status_list=['TCP_HIT', 'TCP_MEM_HIT'])
edge_server = self.get_from_cdn_enabled_url(cdn_url=cdn_url, count=1)
self.assertCacheStatus(cdn_url=cdn_url, edge_server=edge_server,
status_list=['TCP_HIT', 'TCP_MEM_HIT'])
time.sleep(self.origin_header_config.expires_ttl + 2)
self.assertCacheStatus(
cdn_url=cdn_url, status_list=['TCP_MISS', 'TCP_REFRESH_MISS'])
self.assertCacheStatus(cdn_url=cdn_url, edge_server=edge_server,
status_list=['TCP_MISS', 'TCP_REFRESH_MISS'])
def test_expires_header_with_caching_ttl(self):
domains = [{'domain': self.test_domain}]
@ -141,17 +141,17 @@ class TestOriginHeaders(base.TestBase):
self.cname_rec.append(rec[0])
cdn_url = 'http://' + self.test_domain + self.expires_path
self.get_from_cdn_enabled_url(cdn_url=cdn_url, count=4)
self.assertCacheStatus(
cdn_url=cdn_url, status_list=['TCP_HIT', 'TCP_MEM_HIT'])
edge_server = self.get_from_cdn_enabled_url(cdn_url=cdn_url, count=1)
self.assertCacheStatus(cdn_url=cdn_url, edge_server=edge_server,
status_list=['TCP_HIT', 'TCP_MEM_HIT'])
time.sleep(self.origin_header_config.expires_ttl + 2)
self.assertCacheStatus(
cdn_url=cdn_url, status_list=['TCP_HIT', 'TCP_MEM_HIT'])
self.assertCacheStatus(cdn_url=cdn_url, edge_server=edge_server,
status_list=['TCP_HIT', 'TCP_MEM_HIT'])
time.sleep(self.origin_header_config.service_ttl + 2)
self.assertCacheStatus(
cdn_url=cdn_url, status_list=['TCP_MISS', 'TCP_REFRESH_MISS'])
self.assertCacheStatus(cdn_url=cdn_url, edge_server=edge_server,
status_list=['TCP_MISS', 'TCP_REFRESH_MISS'])
def test_cache_control_header_no_caching_ttl(self):
domains = [{'domain': self.test_domain}]
@ -185,13 +185,13 @@ class TestOriginHeaders(base.TestBase):
self.cname_rec.append(rec[0])
cdn_url = 'http://' + self.test_domain + self.cache_control_path
self.get_from_cdn_enabled_url(cdn_url=cdn_url, count=4)
self.assertCacheStatus(
cdn_url=cdn_url, status_list=['TCP_HIT', 'TCP_MEM_HIT'])
edge_server = self.get_from_cdn_enabled_url(cdn_url=cdn_url, count=1)
self.assertCacheStatus(cdn_url=cdn_url, edge_server=edge_server,
status_list=['TCP_HIT', 'TCP_MEM_HIT'])
time.sleep(self.origin_header_config.cache_control_ttl + 2)
self.assertCacheStatus(
cdn_url=cdn_url, status_list=['TCP_MISS', 'TCP_REFRESH_MISS'])
self.assertCacheStatus(cdn_url=cdn_url, edge_server=edge_server,
status_list=['TCP_MISS', 'TCP_REFRESH_MISS'])
def test_cache_control_header_with_caching_ttl(self):
domains = [{'domain': self.test_domain}]
@ -227,17 +227,17 @@ class TestOriginHeaders(base.TestBase):
self.cname_rec.append(rec[0])
cdn_url = 'http://' + self.test_domain + self.expires_path
self.get_from_cdn_enabled_url(cdn_url=cdn_url, count=4)
self.assertCacheStatus(
cdn_url=cdn_url, status_list=['TCP_HIT', 'TCP_MEM_HIT'])
edge_server = self.get_from_cdn_enabled_url(cdn_url=cdn_url, count=1)
self.assertCacheStatus(cdn_url=cdn_url, edge_server=edge_server,
status_list=['TCP_HIT', 'TCP_MEM_HIT'])
time.sleep(self.origin_header_config.cache_control_ttl + 2)
self.assertCacheStatus(
cdn_url=cdn_url, status_list=['TCP_HIT', 'TCP_MEM_HIT'])
self.assertCacheStatus(cdn_url=cdn_url, edge_server=edge_server,
status_list=['TCP_HIT', 'TCP_MEM_HIT'])
time.sleep(self.origin_header_config.service_ttl + 2)
self.assertCacheStatus(
cdn_url=cdn_url, status_list=['TCP_MISS', 'TCP_REFRESH_MISS'])
self.assertCacheStatus(cdn_url=cdn_url, edge_server=edge_server,
status_list=['TCP_MISS', 'TCP_REFRESH_MISS'])
def test_expires_and_cache_control_header_no_caching_ttl(self):
domains = [{'domain': self.test_domain}]
@ -272,13 +272,13 @@ class TestOriginHeaders(base.TestBase):
cdn_url = \
'http://' + self.test_domain + self.expires_and_cache_control_path
self.get_from_cdn_enabled_url(cdn_url=cdn_url, count=4)
self.assertCacheStatus(
cdn_url=cdn_url, status_list=['TCP_HIT', 'TCP_MEM_HIT'])
edge_server = self.get_from_cdn_enabled_url(cdn_url=cdn_url, count=4)
self.assertCacheStatus(cdn_url=cdn_url, edge_server=edge_server,
status_list=['TCP_HIT', 'TCP_MEM_HIT'])
time.sleep(self.origin_header_config.cache_control_ttl + 2)
self.assertCacheStatus(
cdn_url=cdn_url, status_list=['TCP_MISS', 'TCP_REFRESH_MISS'])
self.assertCacheStatus(cdn_url=cdn_url, edge_server=edge_server,
status_list=['TCP_MISS', 'TCP_REFRESH_MISS'])
def test_expires_and_cache_control_header_with_caching_ttl(self):
domains = [{'domain': self.test_domain}]
@ -315,17 +315,17 @@ class TestOriginHeaders(base.TestBase):
cdn_url = \
'http://' + self.test_domain + self.expires_and_cache_control_path
self.get_from_cdn_enabled_url(cdn_url=cdn_url, count=4)
self.assertCacheStatus(
cdn_url=cdn_url, status_list=['TCP_HIT', 'TCP_MEM_HIT'])
edge_server = self.get_from_cdn_enabled_url(cdn_url=cdn_url, count=1)
self.assertCacheStatus(cdn_url=cdn_url, edge_server=edge_server,
status_list=['TCP_HIT', 'TCP_MEM_HIT'])
time.sleep(self.origin_header_config.cache_control_ttl + 2)
self.assertCacheStatus(
cdn_url=cdn_url, status_list=['TCP_HIT', 'TCP_MEM_HIT'])
self.assertCacheStatus(cdn_url=cdn_url, edge_server=edge_server,
status_list=['TCP_HIT', 'TCP_MEM_HIT'])
time.sleep(self.origin_header_config.service_ttl + 2)
self.assertCacheStatus(
cdn_url=cdn_url, status_list=['TCP_MISS', 'TCP_REFRESH_MISS'])
self.assertCacheStatus(cdn_url=cdn_url, edge_server=edge_server,
status_list=['TCP_MISS', 'TCP_REFRESH_MISS'])
def tearDown(self):
self.poppy_client.delete_service(location=self.service_location)

View File

@ -88,9 +88,10 @@ class TestPurge(base.TestBase):
cdn_jpg_url = cdn_url + self.purge_path
# Purge object in rule 1 and ensure it gets a TCP_MISS
self.get_from_cdn_enabled_url(cdn_url=cdn_jpg_url, count=2)
self.assertCacheStatus(
cdn_url=cdn_jpg_url, status_list=['TCP_HIT', 'TCP_MEM_HIT'])
edge_server = self.get_from_cdn_enabled_url(
cdn_url=cdn_jpg_url, count=1)
self.assertCacheStatus(cdn_url=cdn_jpg_url, edge_server=edge_server,
status_list=['TCP_HIT', 'TCP_MEM_HIT'])
url_param = {
'url': self.purge_path,
'hard': False

View File

@ -30,7 +30,7 @@ class TestSSLCDN(base.TestBase):
if self.test_config.run_ssl_tests is False:
self.skipTest('SSL tests are currently disabled in configuration')
self.test_domain = base.random_string(prefix='TestCDN-SSL')
self.test_domain = base.random_string(prefix='e2e-ssl-')
self.origin = self.test_config.ssl_origin
def test_shared_ssl_enable_cdn(self):
@ -40,7 +40,7 @@ class TestSSLCDN(base.TestBase):
"certificate": "shared"}]
origin_list = [{"origin": self.origin, "port": 443, "ssl": True}]
caching_list = []
self.service_name = base.random_string(prefix='testService-')
self.service_name = base.random_string(prefix='e2e-service-')
resp = self.setup_service(
service_name=self.service_name,
@ -53,7 +53,7 @@ class TestSSLCDN(base.TestBase):
resp = self.poppy_client.get_service(location=self.service_location)
links = resp.json()['links']
origin_url = 'http://' + self.origin
origin_url = 'https://' + self.origin
access_url = [link['href'] for link in links if
link['rel'] == 'access_url']
cdn_url = 'https://' + access_url[0]
@ -63,11 +63,6 @@ class TestSSLCDN(base.TestBase):
self.assertSameContent(origin_url=origin_url,
cdn_url=cdn_url)
# Benchmark page load metrics for the CDN enabled website
if self.test_config.webpagetest_enabled:
wpt_test_results = self.run_webpagetest(url=cdn_url)
print(wpt_test_results)
def tearDown(self):
self.poppy_client.delete_service(location=self.service_location)
super(TestSSLCDN, self).tearDown()

View File

@ -52,13 +52,17 @@ class TestConfig(data_interfaces.ConfigSectionInterface):
@property
def default_origin(self):
"""Default origin for all tests."""
"""Default origin for all tests
Use the test site below.
https://github.com/stackforge/poppy/tree/master/docker/e2e_test.
"""
return self.get('default_origin')
@property
def wordpress_origin(self):
"""IP address for wordpress origin."""
return self.get('wordpress_origin')
def alternate_origin(self):
"""Alternate origin with any type of content."""
return self.get('alternate_origin')
@property
def ssl_origin(self):

View File

@ -38,7 +38,9 @@ test_locations=Wellington:Chrome, Indore:Firefox, Stockholm:Safari, Dulles:Firef
[test_configuration]
default_origin=127.0.0.1
wordpress_origin=1.2.3.4
# Use alternate_origin to test content heavy websites. The default origin
# while useful to validate cdn functionality, is a very light flask site.
alternate_origin=1.2.3.4
ssl_origin=1.2.3.4
run_ssl_tests=False
webpagetest_enabled=False