diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..53395021 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +__pycache__ +.tox +.eggs +pegleg.egg-info +/ChangeLog +/AUTHORS +*.swp diff --git a/src/bin/pegleg/.gitignore b/src/bin/pegleg/.gitignore deleted file mode 100644 index 3c822bb4..00000000 --- a/src/bin/pegleg/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -__pycache__ -/.tox -/.eggs -/pegleg.egg-info -/ChangeLog -/AUTHORS diff --git a/src/bin/pegleg/pegleg/cli.py b/src/bin/pegleg/pegleg/cli.py index d2e7ac3d..9590d262 100644 --- a/src/bin/pegleg/pegleg/cli.py +++ b/src/bin/pegleg/pegleg/cli.py @@ -1,4 +1,6 @@ from . import engine +from pegleg import config + import click import logging import sys @@ -13,14 +15,13 @@ CONTEXT_SETTINGS = { @click.group(context_settings=CONTEXT_SETTINGS) -@click.pass_context @click.option( '-v', '--verbose', is_flag=bool, default=False, help='Enable debug logging') -def main(ctx, *, verbose): +def main(*, verbose): if verbose: log_level = logging.DEBUG else: @@ -29,21 +30,36 @@ def main(ctx, *, verbose): @main.group(help='Commands related to sites') -def site(): - pass +@click.option( + '-p', + '--primary', + 'primary_repo', + required=True, + help= + 'Path to the root of the primary (containing site_definition.yaml) repo.') +@click.option( + '-a', + '--auxiliary', + 'aux_repo', + multiple=True, + help='Path to the root of an auxiliary repo.') +def site(primary_repo, aux_repo): + config.set_primary_repo(primary_repo) + config.set_auxiliary_repo_list(aux_repo or []) @site.command(help='Output complete config for one site') @click.option( - '-o', - '--output', - 'output_stream', - type=click.File(mode='w'), + '-s', + '--save-location', + 'save_location', + type=click.Path( + file_okay=False, dir_okay=True, writable=True, resolve_path=True), default=sys.stdout, help='Where to output') @click.argument('site_name') -def collect(*, output_stream, site_name): - engine.site.collect(site_name, output_stream) +def collect(*, save_location, site_name): + engine.site.collect(site_name, save_location) @site.command(help='Find sites impacted by changed files') @@ -161,5 +177,24 @@ def site_type(*, revision, site_type): @LINT_OPTION @main.command(help='Sanity checks for repository content') -def lint(*, fail_on_missing_sub_src): - engine.lint.full(fail_on_missing_sub_src) +@click.option( + '-p', + '--primary', + 'primary_repo', + required=True, + help= + 'Path to the root of the primary (containing site_definition.yaml) repo.') +@click.option( + '-a', + '--auxiliary', + 'aux_repo', + multiple=True, + help='Path to the root of a auxiliary repo.') +def lint(*, fail_on_missing_sub_src, primary_repo, aux_repo): + config.set_primary_repo(primary_repo) + config.set_auxiliary_repo_list(aux_repo or []) + warns = engine.lint.full(fail_on_missing_sub_src) + if warns: + click.echo("Linting passed, but produced some warnings.") + for w in warns: + click.echo(w) diff --git a/src/bin/pegleg/pegleg/config.py b/src/bin/pegleg/pegleg/config.py new file mode 100644 index 00000000..44bbd680 --- /dev/null +++ b/src/bin/pegleg/pegleg/config.py @@ -0,0 +1,41 @@ +from copy import copy + +try: + if GLOBAL_CONTEXT: + pass +except NameError: + GLOBAL_CONTEXT = { + 'primary_repo': './', + 'aux_repos': [], + } + + +def get_primary_repo(): + return GLOBAL_CONTEXT['primary_repo'] + + +def set_primary_repo(r): + GLOBAL_CONTEXT['primary_repo'] = r + + +def set_auxiliary_repo_list(a): + GLOBAL_CONTEXT['aux_repos'] = copy(a) + + +def add_auxiliary_repo(a): + GLOBAL_CONTEXT['aux_repos'].append(a) + + +def get_auxiliary_repo_list(): + return GLOBAL_CONTEXT['aux_repos'] + + +def each_auxiliary_repo(): + for a in GLOBAL_CONTEXT['aux_repos']: + yield a + + +def all_repos(): + repos = [get_primary_repo()] + repos.extend(get_auxiliary_repo_list()) + return repos diff --git a/src/bin/pegleg/pegleg/engine/lint.py b/src/bin/pegleg/pegleg/engine/lint.py index cfd89fe7..39b9f9f4 100644 --- a/src/bin/pegleg/pegleg/engine/lint.py +++ b/src/bin/pegleg/pegleg/engine/lint.py @@ -1,11 +1,12 @@ -from pegleg.engine import util import click -import jsonschema import logging import os import pkg_resources import yaml +from pegleg.engine import util +from pegleg import config + __all__ = ['full'] LOG = logging.getLogger(__name__) @@ -19,11 +20,14 @@ DECKHAND_SCHEMAS = { def full(fail_on_missing_sub_src=False): errors = [] - errors.extend(_verify_no_unexpected_files()) + warns = [] + warns.extend(_verify_no_unexpected_files()) errors.extend(_verify_file_contents()) errors.extend(_verify_deckhand_render(fail_on_missing_sub_src)) if errors: - raise click.ClickException('\n'.join(['Linting failed:'] + errors)) + raise click.ClickException('\n'.join( + ['Linting failed:'] + errors + ['Linting warnings:'] + warns)) + return warns def _verify_no_unexpected_files(): @@ -36,16 +40,16 @@ def _verify_no_unexpected_files(): found_directories = util.files.existing_directories() LOG.debug('found_directories: %s', found_directories) - errors = [] + msgs = [] for unused_dir in sorted(found_directories - expected_directories): - errors.append('%s exists, but is unused' % unused_dir) + msgs.append('%s exists, but is unused' % unused_dir) for missing_dir in sorted(expected_directories - found_directories): if not missing_dir.endswith('common'): - errors.append( + msgs.append( '%s was not found, but expected by manifest' % missing_dir) - return errors + return msgs def _verify_file_contents(): @@ -89,18 +93,6 @@ def _verify_document(document, schemas, filename): document.get('metadata', {}).get('name', '') ]) errors = [] - try: - jsonschema.validate(document, schemas['root']) - try: - jsonschema.validate(document['metadata'], - schemas[document['metadata']['schema']]) - except Exception as e: - errors.append('%s (document %s) failed Deckhand metadata schema ' - 'validation: %s' % (filename, name, e)) - except Exception as e: - errors.append( - '%s (document %s) failed Deckhand root schema validation: %s' % - (filename, name, e)) layer = _layer(document) if layer is not None and layer != _expected_layer(filename): @@ -147,8 +139,11 @@ def _layer(data): def _expected_layer(filename): - parts = os.path.normpath(filename).split(os.sep) - return parts[0] + for r in config.all_repos(): + if filename.startswith(r + "/"): + partial_name = filename[len(r) + 1:] + parts = os.path.normpath(partial_name).split(os.sep) + return parts[0] def _load_schemas(): diff --git a/src/bin/pegleg/pegleg/engine/site.py b/src/bin/pegleg/pegleg/engine/site.py index 50bb634e..9a12369f 100644 --- a/src/bin/pegleg/pegleg/engine/site.py +++ b/src/bin/pegleg/pegleg/engine/site.py @@ -1,16 +1,36 @@ -from pegleg.engine import util +import os +import click import collections import csv import json import yaml +import logging +from pegleg.engine import util __all__ = ['collect', 'impacted', 'list_', 'show', 'render'] +LOG = logging.getLogger(__name__) -def collect(site_name, output_stream): - for filename in util.definition.site_files(site_name): - with open(filename) as f: - output_stream.writelines(f.readlines()) + +def collect(site_name, save_location): + try: + save_files = dict() + for (repo_base, + filename) in util.definition.site_files_by_repo(site_name): + repo_name = os.path.normpath(repo_base).split(os.sep)[-1] + if repo_name not in save_files: + save_files[repo_name] = open( + os.path.join(save_location, repo_name + ".yaml"), "w") + LOG.debug("Collecting file %s to file %s" % + (filename, + os.path.join(save_location, repo_name + '.yaml'))) + with open(filename) as f: + save_files[repo_name].writelines(f.readlines()) + except Exception as ex: + raise click.ClickException("Error saving output: %s" % str(ex)) + finally: + for f in save_files.values(): + f.close() def impacted(input_stream, output_stream): @@ -35,8 +55,7 @@ def render(site_name, output_stream): rendered_documents, errors = util.deckhand.deckhand_render( documents=documents) - for d in documents: - output_stream.writelines(yaml.dump(d)) + yaml.dump_all(rendered_documents, output_stream, default_flow_style=False) def list_(output_stream): diff --git a/src/bin/pegleg/pegleg/engine/util/definition.py b/src/bin/pegleg/pegleg/engine/util/definition.py index 967590ef..807d0f8e 100644 --- a/src/bin/pegleg/pegleg/engine/util/definition.py +++ b/src/bin/pegleg/pegleg/engine/util/definition.py @@ -1,6 +1,8 @@ -from . import files import click +from pegleg import config +from . import files + __all__ = [ 'create', 'load', @@ -31,19 +33,21 @@ def create(*, site_name, site_type, revision): files.dump(path(site_name), definition) -def load(site): - return files.slurp(path(site)) +def load(site, primary_repo_base=None): + return files.slurp(path(site, primary_repo_base)) -def load_as_params(site_name): - definition = load(site_name) +def load_as_params(site_name, primary_repo_base=None): + definition = load(site_name, primary_repo_base) params = definition.get('data', {}) params['site_name'] = site_name return params -def path(site_name): - return 'site/%s/site-definition.yaml' % site_name +def path(site_name, primary_repo_base=None): + if not primary_repo_base: + primary_repo_base = config.get_primary_repo() + return '%s/site/%s/site-definition.yaml' % (primary_repo_base, site_name) def pluck(site_definition, key): @@ -61,3 +65,14 @@ def site_files(site_name): for filename in files.search(files.directories_for(**params)): yield filename yield path(site_name) + + +def site_files_by_repo(site_name): + """Yield tuples of repo_base, file_name.""" + params = load_as_params(site_name) + dir_map = files.directories_for_each_repo(**params) + for repo, dl in dir_map.items(): + for filename in files.search(dl): + yield (repo, filename) + if repo == config.get_primary_repo(): + yield (repo, path(site_name)) diff --git a/src/bin/pegleg/pegleg/engine/util/files.py b/src/bin/pegleg/pegleg/engine/util/files.py index ea09c0b9..8bf9d17a 100644 --- a/src/bin/pegleg/pegleg/engine/util/files.py +++ b/src/bin/pegleg/pegleg/engine/util/files.py @@ -1,6 +1,11 @@ import click import os import yaml +import logging + +from pegleg import config + +LOG = logging.getLogger(__name__) __all__ = [ 'all', @@ -23,7 +28,10 @@ DIR_DEPTHS = { def all(): - return search(DIR_DEPTHS.keys()) + return search([ + os.path.join(r, k) for r in config.all_repos() + for k in DIR_DEPTHS.keys() + ]) def create_global_directories(revision): @@ -83,7 +91,7 @@ def _create_tree(root_path, *, tree=FULL_STRUCTURE): def directories_for(*, site_name, revision, site_type): - return [ + library_list = [ _global_common_path(), _global_revision_path(revision), _site_type_common_path(site_type), @@ -91,6 +99,32 @@ def directories_for(*, site_name, revision, site_type): _site_path(site_name), ] + return [ + os.path.join(b, l) for b in config.all_repos() for l in library_list + ] + + +def directories_for_each_repo(*, site_name, revision, site_type): + """Provide directories for each repo. + + When producing bucketized output files, the documents collected + must be collated by repo. Provide the list of source directories + by repo. + """ + library_list = [ + _global_common_path(), + _global_revision_path(revision), + _site_type_common_path(site_type), + _site_type_revision_path(site_type, revision), + _site_path(site_name), + ] + + dir_map = dict() + for r in config.all_repos(): + dir_map[r] = [os.path.join(r, l) for l in library_list] + + return dir_map + def _global_common_path(): return 'global/common' @@ -112,24 +146,32 @@ def _site_path(site_name): return 'site/%s' % site_name -def list_sites(): - for path in os.listdir('site'): - joined_path = os.path.join('site', path) +def list_sites(primary_repo_base=None): + """Get a list of site defintion directories in the primary repo.""" + if not primary_repo_base: + primary_repo_base = config.get_primary_repo() + for path in os.listdir(os.path.join(primary_repo_base, 'site')): + joined_path = os.path.join(primary_repo_base, 'site', path) if os.path.isdir(joined_path): yield path def directory_for(*, path): - parts = os.path.normpath(path).split(os.sep) - depth = DIR_DEPTHS.get(parts[0]) - if depth is not None: - return os.path.join(*parts[:depth + 1]) + for r in config.all_repos(): + if path.startswith(r + "/"): + partial_path = path[len(r) + 1:] + parts = os.path.normpath(partial_path).split(os.sep) + depth = DIR_DEPTHS.get(parts[0]) + if depth is not None: + return os.path.join(r, *parts[:depth + 1]) def existing_directories(): directories = set() - for search_path, depth in DIR_DEPTHS.items(): - directories.update(_recurse_subdirs(search_path, depth)) + for r in config.all_repos(): + for search_path, depth in DIR_DEPTHS.items(): + directories.update( + _recurse_subdirs(os.path.join(r, search_path), depth)) return directories @@ -141,7 +183,7 @@ def slurp(path): with open(path) as f: try: - return yaml.load(f) + return yaml.safe_load(f) except Exception as e: raise click.ClickException('Failed to parse %s:\n%s' % (path, e)) @@ -158,18 +200,24 @@ def dump(path, data): def _recurse_subdirs(search_path, depth): directories = set() - for path in os.listdir(search_path): - joined_path = os.path.join(search_path, path) - if os.path.isdir(joined_path): - if depth == 1: - directories.add(joined_path) - else: - directories.update(_recurse_subdirs(joined_path, depth - 1)) + try: + for path in os.listdir(search_path): + joined_path = os.path.join(search_path, path) + if os.path.isdir(joined_path): + if depth == 1: + directories.add(joined_path) + else: + directories.update( + _recurse_subdirs(joined_path, depth - 1)) + except FileNotFoundError: + pass return directories def search(search_paths): for search_path in search_paths: + LOG.debug("Recursively collecting YAMLs from %s" % search_path) for root, _dirs, filenames in os.walk(search_path): for filename in filenames: - yield os.path.join(root, filename) + if filename.endswith(".yaml"): + yield os.path.join(root, filename) diff --git a/tools/pegleg.sh b/tools/pegleg.sh index 9c85d3f2..7c524a27 100755 --- a/tools/pegleg.sh +++ b/tools/pegleg.sh @@ -8,14 +8,16 @@ realpath() { SCRIPT_DIR=$(realpath "$(dirname "${0}")") SOURCE_DIR=${SCRIPT_DIR}/pegleg -if [ -d "$PWD/global" ]; then - WORKSPACE="$PWD" -else - WORKSPACE=$(realpath "${SCRIPT_DIR}/..") +if [ -z "${WORKSPACE}" ]; then + WORKSPACE="/" fi IMAGE=${IMAGE:-quay.io/attcomdev/pegleg:latest} +echo +echo "== NOTE: Workspace $WORKSPACE is available as /workspace in container context ==" +echo + docker run --rm -t \ --net=none \ -v "${WORKSPACE}:/workspace" \