Use a deque to avoid keeping all the scanned files alive
Avoid keeping all the files that have been read and scanned in memory by use a deque and popping files off after they have been read (this allows the gc to clean them up).
This commit is contained in:
parent
1a6a4658b4
commit
e050803234
|
@ -31,6 +31,7 @@ What is checked:
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import collections
|
||||||
import os
|
import os
|
||||||
|
|
||||||
from six.moves import configparser
|
from six.moves import configparser
|
||||||
|
@ -140,13 +141,14 @@ def main():
|
||||||
args['ignore'].update(cfg.pop("ignore", set()))
|
args['ignore'].update(cfg.pop("ignore", set()))
|
||||||
args.update(cfg)
|
args.update(cfg)
|
||||||
|
|
||||||
files = []
|
files = collections.deque()
|
||||||
for filename in utils.find_files(args.pop('paths', []), FILE_PATTERNS):
|
for filename in utils.find_files(args.pop('paths', []), FILE_PATTERNS):
|
||||||
files.append(file_parser.parse(filename))
|
files.append(file_parser.parse(filename))
|
||||||
|
|
||||||
ignoreables = frozenset(args.pop('ignore', []))
|
ignoreables = frozenset(args.pop('ignore', []))
|
||||||
errors = 0
|
errors = 0
|
||||||
for f in files:
|
while files:
|
||||||
|
f = files.popleft()
|
||||||
for c in fetch_checks(args):
|
for c in fetch_checks(args):
|
||||||
try:
|
try:
|
||||||
reports = set(c.REPORTS)
|
reports = set(c.REPORTS)
|
||||||
|
|
Loading…
Reference in New Issue