Merge "[sitemap] if last-modified not set in header use current date"

This commit is contained in:
Jenkins 2015-10-20 01:12:12 +00:00 committed by Gerrit Code Review
commit 47c1209f61
1 changed files with 5 additions and 2 deletions

View File

@ -72,7 +72,10 @@ class SitemapSpider(spiders.CrawlSpider):
if path.startswith("/%s" % entry):
item['changefreq'] = 'weekly'
lastmod = time.strptime(response.headers['Last-Modified'],
"%a, %d %b %Y %H:%M:%S %Z")
if 'Last-Modified' in response.headers:
timestamp = response.headers['Last-Modified']
else:
timestamp = response.headers['Date']
lastmod = time.strptime(timestamp, "%a, %d %b %Y %H:%M:%S %Z")
item['lastmod'] = time.strftime("%Y-%m-%dT%H:%M:%S%z", lastmod)
return item