diff --git a/aeromancer/cli/project.py b/aeromancer/cli/project.py index ecd95e8..2eeb5a3 100644 --- a/aeromancer/cli/project.py +++ b/aeromancer/cli/project.py @@ -3,6 +3,7 @@ import os from aeromancer.db.models import * from aeromancer import project +from aeromancer import utils from cliff.command import Command from cliff.lister import Lister @@ -57,6 +58,20 @@ class Rescan(Command): session.commit() +class Discover(Command): + "Find all projects in the repository root" + + log = logging.getLogger(__name__) + + def take_action(self, parsed_args): + session = self.app.get_db_session() + for project_name in project.discover(self.app.options.repo_root): + full_path = os.path.join(self.app.options.repo_root, + project_name) + project.add_or_update(session, project_name, full_path) + session.commit() + + class Remove(Command): "Remove a project from the database" diff --git a/aeromancer/project.py b/aeromancer/project.py index ad4958b..6953d36 100644 --- a/aeromancer/project.py +++ b/aeromancer/project.py @@ -1,9 +1,12 @@ +import glob import io +import itertools import logging import os import subprocess from aeromancer.db.models import Project, File +from aeromancer import utils from sqlalchemy.orm.exc import NoResultFound @@ -42,6 +45,18 @@ def update(session, proj_obj): _update_project_files(session, proj_obj) +def _find_files_in_project(path): + """Return a list of the files managed in the project. + + Uses 'git ls-files' + """ + with utils.working_dir(path): + cmd = subprocess.Popen(['git', 'ls-files', '-z'], + stdout=subprocess.PIPE) + output = cmd.communicate()[0] + return output.split('\0') + + def _update_project_files(session, proj_obj): """Update the files stored for each project""" # Delete any existing files in case the list of files being @@ -50,22 +65,31 @@ def _update_project_files(session, proj_obj): for file_obj in proj_obj.files: session.delete(file_obj) + # FIXME(dhellmann): Concurrency? + # Now load the files currently being managed by git. - before = os.getcwd() - os.chdir(proj_obj.path) - try: - cmd = subprocess.Popen(['git', 'ls-files', '-z'], - stdout=subprocess.PIPE) - output = cmd.communicate()[0] - filenames = output.split('\0') - finally: - os.chdir(before) - for filename in filenames: + for filename in _find_files_in_project(proj_obj.path): fullname = os.path.join(proj_obj.path, filename) if not os.path.isfile(fullname): continue with io.open(fullname, mode='r', encoding='utf-8') as f: - body = f.read() + try: + body = f.read() + except UnicodeDecodeError: + # FIXME(dhellmann): Be smarter about trying other + # encodings? + LOG.warn('Could not read %s as a UTF-8 encoded file, ignoring', + fullname) + continue lines = body.splitlines() LOG.info('%s/%s has %s lines', proj_obj.name, filename, len(lines)) session.add(File(project=proj_obj, name=filename, path=fullname)) + + +def discover(repo_root): + """Discover project-like directories under the repository root""" + with utils.working_dir(repo_root): + return itertools.chain( + glob.glob('openstack*/*'), + glob.glob('stackforge/*'), + ) diff --git a/aeromancer/utils.py b/aeromancer/utils.py new file mode 100644 index 0000000..8ad9c77 --- /dev/null +++ b/aeromancer/utils.py @@ -0,0 +1,10 @@ +import contextlib +import os + + +@contextlib.contextmanager +def working_dir(new_dir): + before = os.getcwd() + os.chdir(new_dir) + yield + os.chdir(before) diff --git a/setup.cfg b/setup.cfg index bccec4f..d04074f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -54,3 +54,4 @@ aeromancer.cli = list = aeromancer.cli.project:List remove = aeromancer.cli.project:Remove rescan = aeromancer.cli.project:Rescan + discover = aeromancer.cli.project:Discover