Only reprocess files 90 days old (or less).
This commit is contained in:
parent
ba73f3ba95
commit
dbd02963a1
|
@ -4,6 +4,7 @@ import copy
|
||||||
import datetime
|
import datetime
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import sys
|
import sys
|
||||||
import urllib
|
import urllib
|
||||||
|
|
||||||
|
@ -86,12 +87,19 @@ while day < datetime.datetime.now():
|
||||||
|
|
||||||
day += one_day
|
day += one_day
|
||||||
|
|
||||||
# Reprocess the world?
|
# Reprocess the last 90 days?
|
||||||
if False:
|
FILENAME_RE = re.compile('merged/([0-9]+)/([0-9]+)/([0-9]+)')
|
||||||
|
if True:
|
||||||
for dirpath, subdirs, files in os.walk('merged'):
|
for dirpath, subdirs, files in os.walk('merged'):
|
||||||
for filename in files:
|
for filename in files:
|
||||||
if filename.endswith('.json'):
|
m = FILENAME_RE.match('%s/%s' % (dirpath, filename))
|
||||||
continue
|
if m:
|
||||||
|
dt = datetime.datetime(int(m.group(1)),
|
||||||
|
int(m.group(2)),
|
||||||
|
int(m.group(3)))
|
||||||
|
age = datetime.datetime.now() - dt
|
||||||
|
if age.days < 90:
|
||||||
|
print 'Will reprocess %s' % filename
|
||||||
changed_merge_files[os.path.join(dirpath, filename)] = True
|
changed_merge_files[os.path.join(dirpath, filename)] = True
|
||||||
|
|
||||||
print 'Processing changed merge files'
|
print 'Processing changed merge files'
|
||||||
|
|
Loading…
Reference in New Issue