Support multiple repo for document source
- A single primary repo must be specified which holds the site_definition.yaml file. - Zero or more auxiliary repos can be specified which have additional documents used in the site definition. - Collected documents are written to a file named after their source repo. Collection must always be given a output directory rather than a single file now. Change-Id: Iceda4da18c4df45d917d88a49144e39e3f1743ed
This commit is contained in:
Scott Hussey
parent
f042fc6a7b
commit
8224e6fc21
|
|
@ -0,0 +1,7 @@
|
|||
__pycache__
|
||||
.tox
|
||||
.eggs
|
||||
pegleg.egg-info
|
||||
/ChangeLog
|
||||
/AUTHORS
|
||||
*.swp
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
__pycache__
|
||||
/.tox
|
||||
/.eggs
|
||||
/pegleg.egg-info
|
||||
/ChangeLog
|
||||
/AUTHORS
|
||||
|
|
@ -1,4 +1,6 @@
|
|||
from . import engine
|
||||
from pegleg import config
|
||||
|
||||
import click
|
||||
import logging
|
||||
import sys
|
||||
|
|
@ -13,14 +15,13 @@ CONTEXT_SETTINGS = {
|
|||
|
||||
|
||||
@click.group(context_settings=CONTEXT_SETTINGS)
|
||||
@click.pass_context
|
||||
@click.option(
|
||||
'-v',
|
||||
'--verbose',
|
||||
is_flag=bool,
|
||||
default=False,
|
||||
help='Enable debug logging')
|
||||
def main(ctx, *, verbose):
|
||||
def main(*, verbose):
|
||||
if verbose:
|
||||
log_level = logging.DEBUG
|
||||
else:
|
||||
|
|
@ -29,21 +30,36 @@ def main(ctx, *, verbose):
|
|||
|
||||
|
||||
@main.group(help='Commands related to sites')
|
||||
def site():
|
||||
pass
|
||||
@click.option(
|
||||
'-p',
|
||||
'--primary',
|
||||
'primary_repo',
|
||||
required=True,
|
||||
help=
|
||||
'Path to the root of the primary (containing site_definition.yaml) repo.')
|
||||
@click.option(
|
||||
'-a',
|
||||
'--auxiliary',
|
||||
'aux_repo',
|
||||
multiple=True,
|
||||
help='Path to the root of an auxiliary repo.')
|
||||
def site(primary_repo, aux_repo):
|
||||
config.set_primary_repo(primary_repo)
|
||||
config.set_auxiliary_repo_list(aux_repo or [])
|
||||
|
||||
|
||||
@site.command(help='Output complete config for one site')
|
||||
@click.option(
|
||||
'-o',
|
||||
'--output',
|
||||
'output_stream',
|
||||
type=click.File(mode='w'),
|
||||
'-s',
|
||||
'--save-location',
|
||||
'save_location',
|
||||
type=click.Path(
|
||||
file_okay=False, dir_okay=True, writable=True, resolve_path=True),
|
||||
default=sys.stdout,
|
||||
help='Where to output')
|
||||
@click.argument('site_name')
|
||||
def collect(*, output_stream, site_name):
|
||||
engine.site.collect(site_name, output_stream)
|
||||
def collect(*, save_location, site_name):
|
||||
engine.site.collect(site_name, save_location)
|
||||
|
||||
|
||||
@site.command(help='Find sites impacted by changed files')
|
||||
|
|
@ -161,5 +177,24 @@ def site_type(*, revision, site_type):
|
|||
|
||||
@LINT_OPTION
|
||||
@main.command(help='Sanity checks for repository content')
|
||||
def lint(*, fail_on_missing_sub_src):
|
||||
engine.lint.full(fail_on_missing_sub_src)
|
||||
@click.option(
|
||||
'-p',
|
||||
'--primary',
|
||||
'primary_repo',
|
||||
required=True,
|
||||
help=
|
||||
'Path to the root of the primary (containing site_definition.yaml) repo.')
|
||||
@click.option(
|
||||
'-a',
|
||||
'--auxiliary',
|
||||
'aux_repo',
|
||||
multiple=True,
|
||||
help='Path to the root of a auxiliary repo.')
|
||||
def lint(*, fail_on_missing_sub_src, primary_repo, aux_repo):
|
||||
config.set_primary_repo(primary_repo)
|
||||
config.set_auxiliary_repo_list(aux_repo or [])
|
||||
warns = engine.lint.full(fail_on_missing_sub_src)
|
||||
if warns:
|
||||
click.echo("Linting passed, but produced some warnings.")
|
||||
for w in warns:
|
||||
click.echo(w)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,41 @@
|
|||
from copy import copy
|
||||
|
||||
try:
|
||||
if GLOBAL_CONTEXT:
|
||||
pass
|
||||
except NameError:
|
||||
GLOBAL_CONTEXT = {
|
||||
'primary_repo': './',
|
||||
'aux_repos': [],
|
||||
}
|
||||
|
||||
|
||||
def get_primary_repo():
|
||||
return GLOBAL_CONTEXT['primary_repo']
|
||||
|
||||
|
||||
def set_primary_repo(r):
|
||||
GLOBAL_CONTEXT['primary_repo'] = r
|
||||
|
||||
|
||||
def set_auxiliary_repo_list(a):
|
||||
GLOBAL_CONTEXT['aux_repos'] = copy(a)
|
||||
|
||||
|
||||
def add_auxiliary_repo(a):
|
||||
GLOBAL_CONTEXT['aux_repos'].append(a)
|
||||
|
||||
|
||||
def get_auxiliary_repo_list():
|
||||
return GLOBAL_CONTEXT['aux_repos']
|
||||
|
||||
|
||||
def each_auxiliary_repo():
|
||||
for a in GLOBAL_CONTEXT['aux_repos']:
|
||||
yield a
|
||||
|
||||
|
||||
def all_repos():
|
||||
repos = [get_primary_repo()]
|
||||
repos.extend(get_auxiliary_repo_list())
|
||||
return repos
|
||||
|
|
@ -1,11 +1,12 @@
|
|||
from pegleg.engine import util
|
||||
import click
|
||||
import jsonschema
|
||||
import logging
|
||||
import os
|
||||
import pkg_resources
|
||||
import yaml
|
||||
|
||||
from pegleg.engine import util
|
||||
from pegleg import config
|
||||
|
||||
__all__ = ['full']
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
|
@ -19,11 +20,14 @@ DECKHAND_SCHEMAS = {
|
|||
|
||||
def full(fail_on_missing_sub_src=False):
|
||||
errors = []
|
||||
errors.extend(_verify_no_unexpected_files())
|
||||
warns = []
|
||||
warns.extend(_verify_no_unexpected_files())
|
||||
errors.extend(_verify_file_contents())
|
||||
errors.extend(_verify_deckhand_render(fail_on_missing_sub_src))
|
||||
if errors:
|
||||
raise click.ClickException('\n'.join(['Linting failed:'] + errors))
|
||||
raise click.ClickException('\n'.join(
|
||||
['Linting failed:'] + errors + ['Linting warnings:'] + warns))
|
||||
return warns
|
||||
|
||||
|
||||
def _verify_no_unexpected_files():
|
||||
|
|
@ -36,16 +40,16 @@ def _verify_no_unexpected_files():
|
|||
found_directories = util.files.existing_directories()
|
||||
LOG.debug('found_directories: %s', found_directories)
|
||||
|
||||
errors = []
|
||||
msgs = []
|
||||
for unused_dir in sorted(found_directories - expected_directories):
|
||||
errors.append('%s exists, but is unused' % unused_dir)
|
||||
msgs.append('%s exists, but is unused' % unused_dir)
|
||||
|
||||
for missing_dir in sorted(expected_directories - found_directories):
|
||||
if not missing_dir.endswith('common'):
|
||||
errors.append(
|
||||
msgs.append(
|
||||
'%s was not found, but expected by manifest' % missing_dir)
|
||||
|
||||
return errors
|
||||
return msgs
|
||||
|
||||
|
||||
def _verify_file_contents():
|
||||
|
|
@ -89,18 +93,6 @@ def _verify_document(document, schemas, filename):
|
|||
document.get('metadata', {}).get('name', '')
|
||||
])
|
||||
errors = []
|
||||
try:
|
||||
jsonschema.validate(document, schemas['root'])
|
||||
try:
|
||||
jsonschema.validate(document['metadata'],
|
||||
schemas[document['metadata']['schema']])
|
||||
except Exception as e:
|
||||
errors.append('%s (document %s) failed Deckhand metadata schema '
|
||||
'validation: %s' % (filename, name, e))
|
||||
except Exception as e:
|
||||
errors.append(
|
||||
'%s (document %s) failed Deckhand root schema validation: %s' %
|
||||
(filename, name, e))
|
||||
|
||||
layer = _layer(document)
|
||||
if layer is not None and layer != _expected_layer(filename):
|
||||
|
|
@ -147,8 +139,11 @@ def _layer(data):
|
|||
|
||||
|
||||
def _expected_layer(filename):
|
||||
parts = os.path.normpath(filename).split(os.sep)
|
||||
return parts[0]
|
||||
for r in config.all_repos():
|
||||
if filename.startswith(r + "/"):
|
||||
partial_name = filename[len(r) + 1:]
|
||||
parts = os.path.normpath(partial_name).split(os.sep)
|
||||
return parts[0]
|
||||
|
||||
|
||||
def _load_schemas():
|
||||
|
|
|
|||
|
|
@ -1,16 +1,36 @@
|
|||
from pegleg.engine import util
|
||||
import os
|
||||
import click
|
||||
import collections
|
||||
import csv
|
||||
import json
|
||||
import yaml
|
||||
import logging
|
||||
|
||||
from pegleg.engine import util
|
||||
__all__ = ['collect', 'impacted', 'list_', 'show', 'render']
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
def collect(site_name, output_stream):
|
||||
for filename in util.definition.site_files(site_name):
|
||||
with open(filename) as f:
|
||||
output_stream.writelines(f.readlines())
|
||||
|
||||
def collect(site_name, save_location):
|
||||
try:
|
||||
save_files = dict()
|
||||
for (repo_base,
|
||||
filename) in util.definition.site_files_by_repo(site_name):
|
||||
repo_name = os.path.normpath(repo_base).split(os.sep)[-1]
|
||||
if repo_name not in save_files:
|
||||
save_files[repo_name] = open(
|
||||
os.path.join(save_location, repo_name + ".yaml"), "w")
|
||||
LOG.debug("Collecting file %s to file %s" %
|
||||
(filename,
|
||||
os.path.join(save_location, repo_name + '.yaml')))
|
||||
with open(filename) as f:
|
||||
save_files[repo_name].writelines(f.readlines())
|
||||
except Exception as ex:
|
||||
raise click.ClickException("Error saving output: %s" % str(ex))
|
||||
finally:
|
||||
for f in save_files.values():
|
||||
f.close()
|
||||
|
||||
|
||||
def impacted(input_stream, output_stream):
|
||||
|
|
@ -35,8 +55,7 @@ def render(site_name, output_stream):
|
|||
|
||||
rendered_documents, errors = util.deckhand.deckhand_render(
|
||||
documents=documents)
|
||||
for d in documents:
|
||||
output_stream.writelines(yaml.dump(d))
|
||||
yaml.dump_all(rendered_documents, output_stream, default_flow_style=False)
|
||||
|
||||
|
||||
def list_(output_stream):
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
from . import files
|
||||
import click
|
||||
|
||||
from pegleg import config
|
||||
from . import files
|
||||
|
||||
__all__ = [
|
||||
'create',
|
||||
'load',
|
||||
|
|
@ -31,19 +33,21 @@ def create(*, site_name, site_type, revision):
|
|||
files.dump(path(site_name), definition)
|
||||
|
||||
|
||||
def load(site):
|
||||
return files.slurp(path(site))
|
||||
def load(site, primary_repo_base=None):
|
||||
return files.slurp(path(site, primary_repo_base))
|
||||
|
||||
|
||||
def load_as_params(site_name):
|
||||
definition = load(site_name)
|
||||
def load_as_params(site_name, primary_repo_base=None):
|
||||
definition = load(site_name, primary_repo_base)
|
||||
params = definition.get('data', {})
|
||||
params['site_name'] = site_name
|
||||
return params
|
||||
|
||||
|
||||
def path(site_name):
|
||||
return 'site/%s/site-definition.yaml' % site_name
|
||||
def path(site_name, primary_repo_base=None):
|
||||
if not primary_repo_base:
|
||||
primary_repo_base = config.get_primary_repo()
|
||||
return '%s/site/%s/site-definition.yaml' % (primary_repo_base, site_name)
|
||||
|
||||
|
||||
def pluck(site_definition, key):
|
||||
|
|
@ -61,3 +65,14 @@ def site_files(site_name):
|
|||
for filename in files.search(files.directories_for(**params)):
|
||||
yield filename
|
||||
yield path(site_name)
|
||||
|
||||
|
||||
def site_files_by_repo(site_name):
|
||||
"""Yield tuples of repo_base, file_name."""
|
||||
params = load_as_params(site_name)
|
||||
dir_map = files.directories_for_each_repo(**params)
|
||||
for repo, dl in dir_map.items():
|
||||
for filename in files.search(dl):
|
||||
yield (repo, filename)
|
||||
if repo == config.get_primary_repo():
|
||||
yield (repo, path(site_name))
|
||||
|
|
|
|||
|
|
@ -1,6 +1,11 @@
|
|||
import click
|
||||
import os
|
||||
import yaml
|
||||
import logging
|
||||
|
||||
from pegleg import config
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
__all__ = [
|
||||
'all',
|
||||
|
|
@ -23,7 +28,10 @@ DIR_DEPTHS = {
|
|||
|
||||
|
||||
def all():
|
||||
return search(DIR_DEPTHS.keys())
|
||||
return search([
|
||||
os.path.join(r, k) for r in config.all_repos()
|
||||
for k in DIR_DEPTHS.keys()
|
||||
])
|
||||
|
||||
|
||||
def create_global_directories(revision):
|
||||
|
|
@ -83,7 +91,7 @@ def _create_tree(root_path, *, tree=FULL_STRUCTURE):
|
|||
|
||||
|
||||
def directories_for(*, site_name, revision, site_type):
|
||||
return [
|
||||
library_list = [
|
||||
_global_common_path(),
|
||||
_global_revision_path(revision),
|
||||
_site_type_common_path(site_type),
|
||||
|
|
@ -91,6 +99,32 @@ def directories_for(*, site_name, revision, site_type):
|
|||
_site_path(site_name),
|
||||
]
|
||||
|
||||
return [
|
||||
os.path.join(b, l) for b in config.all_repos() for l in library_list
|
||||
]
|
||||
|
||||
|
||||
def directories_for_each_repo(*, site_name, revision, site_type):
|
||||
"""Provide directories for each repo.
|
||||
|
||||
When producing bucketized output files, the documents collected
|
||||
must be collated by repo. Provide the list of source directories
|
||||
by repo.
|
||||
"""
|
||||
library_list = [
|
||||
_global_common_path(),
|
||||
_global_revision_path(revision),
|
||||
_site_type_common_path(site_type),
|
||||
_site_type_revision_path(site_type, revision),
|
||||
_site_path(site_name),
|
||||
]
|
||||
|
||||
dir_map = dict()
|
||||
for r in config.all_repos():
|
||||
dir_map[r] = [os.path.join(r, l) for l in library_list]
|
||||
|
||||
return dir_map
|
||||
|
||||
|
||||
def _global_common_path():
|
||||
return 'global/common'
|
||||
|
|
@ -112,24 +146,32 @@ def _site_path(site_name):
|
|||
return 'site/%s' % site_name
|
||||
|
||||
|
||||
def list_sites():
|
||||
for path in os.listdir('site'):
|
||||
joined_path = os.path.join('site', path)
|
||||
def list_sites(primary_repo_base=None):
|
||||
"""Get a list of site defintion directories in the primary repo."""
|
||||
if not primary_repo_base:
|
||||
primary_repo_base = config.get_primary_repo()
|
||||
for path in os.listdir(os.path.join(primary_repo_base, 'site')):
|
||||
joined_path = os.path.join(primary_repo_base, 'site', path)
|
||||
if os.path.isdir(joined_path):
|
||||
yield path
|
||||
|
||||
|
||||
def directory_for(*, path):
|
||||
parts = os.path.normpath(path).split(os.sep)
|
||||
depth = DIR_DEPTHS.get(parts[0])
|
||||
if depth is not None:
|
||||
return os.path.join(*parts[:depth + 1])
|
||||
for r in config.all_repos():
|
||||
if path.startswith(r + "/"):
|
||||
partial_path = path[len(r) + 1:]
|
||||
parts = os.path.normpath(partial_path).split(os.sep)
|
||||
depth = DIR_DEPTHS.get(parts[0])
|
||||
if depth is not None:
|
||||
return os.path.join(r, *parts[:depth + 1])
|
||||
|
||||
|
||||
def existing_directories():
|
||||
directories = set()
|
||||
for search_path, depth in DIR_DEPTHS.items():
|
||||
directories.update(_recurse_subdirs(search_path, depth))
|
||||
for r in config.all_repos():
|
||||
for search_path, depth in DIR_DEPTHS.items():
|
||||
directories.update(
|
||||
_recurse_subdirs(os.path.join(r, search_path), depth))
|
||||
return directories
|
||||
|
||||
|
||||
|
|
@ -141,7 +183,7 @@ def slurp(path):
|
|||
|
||||
with open(path) as f:
|
||||
try:
|
||||
return yaml.load(f)
|
||||
return yaml.safe_load(f)
|
||||
except Exception as e:
|
||||
raise click.ClickException('Failed to parse %s:\n%s' % (path, e))
|
||||
|
||||
|
|
@ -158,18 +200,24 @@ def dump(path, data):
|
|||
|
||||
def _recurse_subdirs(search_path, depth):
|
||||
directories = set()
|
||||
for path in os.listdir(search_path):
|
||||
joined_path = os.path.join(search_path, path)
|
||||
if os.path.isdir(joined_path):
|
||||
if depth == 1:
|
||||
directories.add(joined_path)
|
||||
else:
|
||||
directories.update(_recurse_subdirs(joined_path, depth - 1))
|
||||
try:
|
||||
for path in os.listdir(search_path):
|
||||
joined_path = os.path.join(search_path, path)
|
||||
if os.path.isdir(joined_path):
|
||||
if depth == 1:
|
||||
directories.add(joined_path)
|
||||
else:
|
||||
directories.update(
|
||||
_recurse_subdirs(joined_path, depth - 1))
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
return directories
|
||||
|
||||
|
||||
def search(search_paths):
|
||||
for search_path in search_paths:
|
||||
LOG.debug("Recursively collecting YAMLs from %s" % search_path)
|
||||
for root, _dirs, filenames in os.walk(search_path):
|
||||
for filename in filenames:
|
||||
yield os.path.join(root, filename)
|
||||
if filename.endswith(".yaml"):
|
||||
yield os.path.join(root, filename)
|
||||
|
|
|
|||
|
|
@ -8,14 +8,16 @@ realpath() {
|
|||
|
||||
SCRIPT_DIR=$(realpath "$(dirname "${0}")")
|
||||
SOURCE_DIR=${SCRIPT_DIR}/pegleg
|
||||
if [ -d "$PWD/global" ]; then
|
||||
WORKSPACE="$PWD"
|
||||
else
|
||||
WORKSPACE=$(realpath "${SCRIPT_DIR}/..")
|
||||
if [ -z "${WORKSPACE}" ]; then
|
||||
WORKSPACE="/"
|
||||
fi
|
||||
|
||||
IMAGE=${IMAGE:-quay.io/attcomdev/pegleg:latest}
|
||||
|
||||
echo
|
||||
echo "== NOTE: Workspace $WORKSPACE is available as /workspace in container context =="
|
||||
echo
|
||||
|
||||
docker run --rm -t \
|
||||
--net=none \
|
||||
-v "${WORKSPACE}:/workspace" \
|
||||
|
|
|
|||
Loading…
Reference in New Issue