Only reprocess files 90 days old (or less).

mirror_fetchers.py

@@ -4,6 +4,7 @@ import copy
 import datetime
 import json
 import os
+import re
 import sys
 import urllib
 
@@ -86,13 +87,20 @@ while day < datetime.datetime.now():
 
     day += one_day
 
-# Reprocess the world?
-if False:
+# Reprocess the last 90 days?
+FILENAME_RE = re.compile('merged/([0-9]+)/([0-9]+)/([0-9]+)')
+if True:
     for dirpath, subdirs, files in os.walk('merged'):
         for filename in files:
-            if filename.endswith('.json'):
-                continue
-            changed_merge_files[os.path.join(dirpath, filename)] = True
+            m = FILENAME_RE.match('%s/%s' % (dirpath, filename))
+            if m:
+                dt = datetime.datetime(int(m.group(1)),
+                                       int(m.group(2)),
+                                       int(m.group(3)))
+                age = datetime.datetime.now() - dt
+                if age.days < 90:
+                    print 'Will reprocess %s' % filename
+                    changed_merge_files[os.path.join(dirpath, filename)] = True
 
 print 'Processing changed merge files'
 patches = {}