Add site render using deckhand and use deckhand for linting

Change-Id: I8ba6abe6d93db12ded63817a1f5a79fe55d73786

tools/pegleg.sh

@@ -2,6 +2,10 @@
 
 set -e
 
+realpath() {
+    [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}"
+}
+
 SCRIPT_DIR=$(realpath "$(dirname "${0}")")
 SOURCE_DIR=${SCRIPT_DIR}/pegleg
 if [ -d "$PWD/global" ]; then

tools/pegleg/pegleg/cli.py

@@ -32,7 +32,6 @@ def main(ctx, *, verbose):
 def site():
     pass
 
-
 @site.command(help='Output complete config for one site')
 @click.option(
     '-o',
@@ -45,7 +44,6 @@ def site():
 def collect(*, output_stream, site_name):
     engine.site.collect(site_name, output_stream)
 
-
 @site.command(help='Find sites impacted by changed files')
 @click.option(
     '-i',
@@ -89,6 +87,18 @@ def show(*, output_stream, site_name):
     engine.site.show(site_name, output_stream)
 
 
+@site.command('render', help='Render a site through the deckhand engine')
+@click.option(
+    '-o',
+    '--output',
+    'output_stream',
+    type=click.File(mode='w'),
+    default=sys.stdout,
+    help='Where to output')
+@click.argument('site_name')
+def render(*, output_stream, site_name):
+    engine.site.render(site_name, output_stream)
+
 def _validate_revision_callback(_ctx, _param, value):
     if value is not None and value.startswith('v'):
         return value
@@ -114,6 +124,14 @@ SITE_TYPE_OPTION = click.option(
     required=True,
     help='Site type to use ("large", "medium", "cicd", "labs", etc.')
 
+LINT_OPTION = click.option(
+    '-f',
+    '--fail-on-missing-sub-src',
+    required=False,
+    type=click.BOOL,
+    default=True,
+    help="Raise deckhand exception on missing substition sources. Defaults to True.")
+
 
 @stub.command('global', help='Add global structure for a new revision')
 @RELEASE_OPTION
@@ -136,6 +154,7 @@ def site_type(*, revision, site_type):
     engine.stub.site_type(revision, site_type)
 
 
+@LINT_OPTION
 @main.command(help='Sanity checks for repository content')
-def lint():
-    engine.lint.full()
+def lint(*, fail_on_missing_sub_src):
+    engine.lint.full(fail_on_missing_sub_src)

tools/pegleg/pegleg/engine/lint.py

@@ -17,10 +17,11 @@ DECKHAND_SCHEMAS = {
 }
 
 
-def full():
+def full(fail_on_missing_sub_src=False):
     errors = []
     errors.extend(_verify_no_unexpected_files())
     errors.extend(_verify_file_contents())
+    errors.extend(_verify_deckhand_render(fail_on_missing_sub_src))
     if errors:
         raise click.ClickException('\n'.join(['Linting failed:'] + errors))
 
@@ -122,6 +123,20 @@ def _verify_document(document, schemas, filename):
                     % (filename, name))
     return errors
 
+def _verify_deckhand_render(fail_on_missing_sub_src=False):
+
+    documents = []
+
+    for filename in util.files.all():
+        with open(filename) as f:
+            documents.extend(list(yaml.safe_load_all(f)))
+
+    rendered_documents, errors = util.deckhand.deckhand_render(
+        documents=documents,
+        fail_on_missing_sub_src=fail_on_missing_sub_src,
+        validate=True,
+    )
+    return errors
 
 def _layer(data):
     if hasattr(data, 'get'):

tools/pegleg/pegleg/engine/site.py

@@ -2,8 +2,9 @@ from pegleg.engine import util
 import collections
 import csv
 import json
+import yaml
 
-__all__ = ['collect', 'impacted', 'list_', 'show']
+__all__ = ['collect', 'impacted', 'list_', 'show', 'render']
 
 
 def collect(site_name, output_stream):
@@ -25,6 +26,17 @@ def impacted(input_stream, output_stream):
     for site_name in sorted(impacted_sites):
         output_stream.write(site_name + '\n')
 
+def render(site_name, output_stream):
+    documents = []
+    for filename in util.definition.site_files(site_name):
+        with open(filename) as f:
+            documents.extend(list(yaml.safe_load_all(f)))
+
+    rendered_documents, errors = util.deckhand.deckhand_render(
+        documents=documents
+    )
+    for d in documents:
+        output_stream.writelines(yaml.dump(d))
 
 def list_(output_stream):
     fieldnames = ['site_name', 'site_type', 'revision']

tools/pegleg/pegleg/engine/util/__init__.py

@@ -1,3 +1,4 @@
 # flake8: noqa
 from . import definition
 from . import files
+from . import deckhand

tools/pegleg/pegleg/engine/util/deckhand.py

@@ -0,0 +1,42 @@
+from deckhand.engine import layering
+from deckhand import errors as dh_errors
+
+def load_schemas_from_docs(documents):
+    '''
+    Fills the cache of known schemas from the document set
+    '''
+
+    errors = []
+    SCHEMA_SCHEMA = "deckhand/DataSchema/v1"
+
+    schema_set = dict()
+    for document in documents:
+        if document.get('schema', '') == SCHEMA_SCHEMA:
+            name = document['metadata']['name']
+            if name in schema_set:
+                errors.append('Duplicate schema specified for: %s' % name)
+
+            schema_set[name] = document['data']
+
+    return schema_set, errors
+
+def deckhand_render(documents=[], fail_on_missing_sub_src=False, validate=False):
+
+    errors = []
+    rendered_documents = []
+
+    schemas, schema_errors = load_schemas_from_docs(documents)
+    errors.extend(schema_errors)
+
+    try:
+        deckhand_eng = layering.DocumentLayering(
+            documents,
+            substitution_sources=documents,
+            fail_on_missing_sub_src=fail_on_missing_sub_src,
+            validate=validate)
+        rendered_documents = [dict(d) for d in deckhand_eng.render()]
+    except dh_errors.DeckhandException as e:
+        errors.append('An unknown Deckhand exception occurred while trying'
+                      ' to render documents: %s' % str(e))
+
+    return rendered_documents, errors

tools/pegleg/requirements.txt

@@ -1,3 +1,4 @@
 click==6.7
 jsonschema==2.6.0
 pyyaml==3.12
+git+https://github.com/att-comdev/deckhand.git@02c6a8dc1fe1a2ff82daba18f4df573d6d20cfca#egg=deckhand