diff --git a/mirror_fetchers.py b/mirror_fetchers.py index 7ed2860..0249049 100755 --- a/mirror_fetchers.py +++ b/mirror_fetchers.py @@ -4,6 +4,7 @@ import copy import datetime import json import os +import re import sys import urllib @@ -86,13 +87,20 @@ while day < datetime.datetime.now(): day += one_day -# Reprocess the world? -if False: +# Reprocess the last 90 days? +FILENAME_RE = re.compile('merged/([0-9]+)/([0-9]+)/([0-9]+)') +if True: for dirpath, subdirs, files in os.walk('merged'): for filename in files: - if filename.endswith('.json'): - continue - changed_merge_files[os.path.join(dirpath, filename)] = True + m = FILENAME_RE.match('%s/%s' % (dirpath, filename)) + if m: + dt = datetime.datetime(int(m.group(1)), + int(m.group(2)), + int(m.group(3))) + age = datetime.datetime.now() - dt + if age.days < 90: + print 'Will reprocess %s' % filename + changed_merge_files[os.path.join(dirpath, filename)] = True print 'Processing changed merge files' patches = {}