From 26e6792690581243e18248c5ac1a88481537d500 Mon Sep 17 00:00:00 2001 From: Mark Burnett Date: Mon, 5 Mar 2018 14:10:03 -0600 Subject: [PATCH] Allow adding new definitions to PKICatalog * Detect and re-use existing Certs/Keys * Negative functional test for join with missing cert * Positive functional test to generate cert after initial construction * Extract some promenade test code into tools/g2/lib/promenade.sh * Add timestamps to tar'd up files Change-Id: Ib717785fc2c8f6cd1db1970ecdf1f5184ed40e92 --- docs/source/configuration/pki-catalog.rst | 5 +- examples/basic/PKICatalog-addition.yaml | 21 +++ examples/basic/PKICatalog.yaml | 7 - promenade/config.py | 15 ++- promenade/control/join_scripts.py | 5 +- promenade/exceptions.py | 23 ++++ promenade/generator.py | 153 +++++++++++++++++----- promenade/pki.py | 29 +++- promenade/renderer.py | 17 ++- promenade/tar_bundler.py | 2 + tools/g2/lib/config.sh | 1 + tools/g2/lib/promenade.sh | 56 ++++++++ tools/g2/manifests/resiliency.json | 35 ++++- tools/g2/stages/build-scripts.sh | 3 - tools/g2/stages/fail-join-node.sh | 49 +++++++ tools/g2/stages/generate-certificates.sh | 59 ++++++++- tools/g2/stages/join-nodes.sh | 50 +------ 17 files changed, 425 insertions(+), 105 deletions(-) create mode 100644 examples/basic/PKICatalog-addition.yaml create mode 100755 tools/g2/stages/fail-join-node.sh diff --git a/docs/source/configuration/pki-catalog.rst b/docs/source/configuration/pki-catalog.rst index 2d3dc3b3..a891372f 100644 --- a/docs/source/configuration/pki-catalog.rst +++ b/docs/source/configuration/pki-catalog.rst @@ -1,7 +1,10 @@ PKI Catalog =========== -Configuration for certificate and keypair generation in the cluster. +Configuration for certificate and keypair generation in the cluster. The +``promenade generate-certs`` command will read all ``PKICatalog`` documents and +either find pre-existing certificates/keys, or generate new ones based on the +given definition. Sample Document diff --git a/examples/basic/PKICatalog-addition.yaml b/examples/basic/PKICatalog-addition.yaml new file mode 100644 index 00000000..ff90f910 --- /dev/null +++ b/examples/basic/PKICatalog-addition.yaml @@ -0,0 +1,21 @@ +--- +schema: promenade/PKICatalog/v1 +metadata: + schema: metadata/Document/v1 + name: cluster-certificates-addition + layeringDefinition: + abstract: false + layer: site +data: + certificate_authorities: + kubernetes: + description: CA for Kubernetes components + certificates: + - document_name: kubelet-n3 + common_name: system:node:n3 + hosts: + - n3 + - 192.168.77.13 + groups: + - system:nodes +... diff --git a/examples/basic/PKICatalog.yaml b/examples/basic/PKICatalog.yaml index 3e6f93da..f0efdf67 100644 --- a/examples/basic/PKICatalog.yaml +++ b/examples/basic/PKICatalog.yaml @@ -48,13 +48,6 @@ data: - 192.168.77.12 groups: - system:nodes - - document_name: kubelet-n3 - common_name: system:node:n3 - hosts: - - n3 - - 192.168.77.13 - groups: - - system:nodes - document_name: scheduler description: Service certificate for Kubernetes scheduler common_name: system:kube-scheduler diff --git a/promenade/config.py b/promenade/config.py index 44b67ccd..cc7bd52c 100644 --- a/promenade/config.py +++ b/promenade/config.py @@ -94,7 +94,7 @@ class Configuration: 'No document found matching kind=%s schema=%s name=%s' % (kind, schema, name)) - def iterate(self, *, kind=None, schema=None, labels=None): + def iterate(self, *, kind=None, schema=None, labels=None, name=None): if kind is not None: if schema is not None: raise AssertionError( @@ -102,9 +102,14 @@ class Configuration: schema = 'promenade/%s/v1' % kind for document in self.documents: - if _matches_filter(document, schema=schema, labels=labels): + if _matches_filter( + document, schema=schema, labels=labels, name=name): yield document + def find(self, *args, **kwargs): + for doc in self.iterate(*args, **kwargs): + return doc + def extract_genesis_config(self): LOG.debug('Extracting genesis config.') documents = [] @@ -179,7 +184,7 @@ class Configuration: ['/apiserver', '--apiserver-count=2', '--v=5']) -def _matches_filter(document, *, schema, labels): +def _matches_filter(document, *, schema, labels, name): matches = True if schema is not None and not document.get('schema', '').startswith(schema): @@ -194,6 +199,10 @@ def _matches_filter(document, *, schema, labels): if document_labels[key] != value: matches = False + if name is not None: + if _mg(document, 'name') != name: + matches = False + return matches diff --git a/promenade/control/join_scripts.py b/promenade/control/join_scripts.py index f852a770..313d9e68 100644 --- a/promenade/control/join_scripts.py +++ b/promenade/control/join_scripts.py @@ -50,8 +50,9 @@ class JoinScriptsResource(BaseResource): design_ref, allow_missing_substitutions=False, leave_kubectl=leave_kubectl) - except exceptions.DeckhandException as e: - raise falcon.HTTPInternalServerError(description=str(e)) + except exceptions.DeckhandException: + LOG.exception('Caught Deckhand render error for configuration') + raise if config.get_path('KubernetesNode:.', SENTINEL) != SENTINEL: raise exceptions.ExistingKubernetesNodeDocumentError( diff --git a/promenade/exceptions.py b/promenade/exceptions.py index eac443e1..8e94af2f 100644 --- a/promenade/exceptions.py +++ b/promenade/exceptions.py @@ -222,6 +222,24 @@ class PromenadeException(Exception): LOG.error(self.title + (self.description or '')) +class PKIError(PromenadeException): + """ + A parent error for PKI-related issues. + """ + title = 'PKI Error' + # NOTE(mark-burnett): The API should never see these errors. + status = falcon.HTTP_500 + + +class IncompletePKIPairError(PKIError): + """ + An incomplete pair (Certificate + Key or Pub + Priv) was found in cache. + """ + title = 'Incomplete Pair Error' + # NOTE(mark-burnett): The API should never see these errors. + status = falcon.HTTP_500 + + class ApiError(PromenadeException): """ An error to handle general api errors. @@ -294,6 +312,11 @@ class ValidationException(PromenadeException): class DeckhandException(PromenadeException): title = 'Deckhand Engine Error' + status = falcon.HTTP_400 + + +class TemplateRenderException(PromenadeException): + title = 'Template Rendering Error Error' status = falcon.HTTP_500 diff --git a/promenade/generator.py b/promenade/generator.py index 2b8bad82..cb470192 100644 --- a/promenade/generator.py +++ b/promenade/generator.py @@ -1,4 +1,6 @@ -from . import logging, pki +from . import exceptions, logging, pki +import collections +import itertools import os import yaml @@ -12,35 +14,129 @@ class Generator: self.config = config self.keys = pki.PKI() self.documents = [] + self.outputs = collections.defaultdict(dict) @property def cluster_domain(self): return self.config['KubernetesNetwork:dns.cluster_domain'] def generate(self, output_dir): - for ca_name, ca_def in self.config[ - 'PKICatalog:certificate_authorities'].items(): - self.gen('ca', ca_name) - for cert_def in ca_def.get('certificates', []): - hosts = cert_def.get('hosts', []) - hosts.extend( - get_host_list( - cert_def.get('kubernetes_service_names', []))) - self.gen( - 'certificate', - cert_def['document_name'], - ca=ca_name, - cn=cert_def['common_name'], - hosts=hosts, - groups=cert_def.get('groups', [])) - for keypair_def in self.config['PKICatalog:keypairs']: - self.gen('keypair', keypair_def['name']) - _write(output_dir, self.documents) + for catalog in self.config.iterate(kind='PKICatalog'): + for ca_name, ca_def in catalog['data'].get( + 'certificate_authorities', {}).items(): + ca_cert, ca_key = self.get_or_gen_ca(ca_name) - def gen(self, kind, *args, **kwargs): - method = getattr(self.keys, 'generate_' + kind) + for cert_def in ca_def.get('certificates', []): + document_name = cert_def['document_name'] + cert, key = self.get_or_gen_cert( + document_name, + ca_cert=ca_cert, + ca_key=ca_key, + cn=cert_def['common_name'], + hosts=_extract_hosts(cert_def), + groups=cert_def.get('groups', [])) - self.documents.extend(method(*args, **kwargs)) + for keypair_def in catalog['data'].get('keypairs', []): + document_name = keypair_def['name'] + self.get_or_gen_keypair(document_name) + + self._write(output_dir) + + def get_or_gen_ca(self, document_name): + kinds = [ + 'CertificateAuthority', + 'CertificateAuthorityKey', + ] + return self._get_or_gen(self.gen_ca, kinds, document_name) + + def get_or_gen_cert(self, document_name, **kwargs): + kinds = [ + 'Certificate', + 'CertificateKey', + ] + return self._get_or_gen(self.gen_cert, kinds, document_name, **kwargs) + + def get_or_gen_keypair(self, document_name): + kinds = [ + 'PublicKey', + 'PrivateKey', + ] + return self._get_or_gen(self.gen_keypair, kinds, document_name) + + def gen_ca(self, document_name, **kwargs): + return self.keys.generate_ca(document_name, **kwargs) + + def gen_cert(self, document_name, *, ca_cert, ca_key, **kwargs): + ca_cert_data = ca_cert['data'] + ca_key_data = ca_key['data'] + return self.keys.generate_certificate( + document_name, ca_cert=ca_cert_data, ca_key=ca_key_data, **kwargs) + + def gen_keypair(self, document_name): + return self.keys.generate_keypair(document_name) + + def _get_or_gen(self, generator, kinds, document_name, *args, **kwargs): + docs = self._find_docs(kinds, document_name) + if not docs: + docs = generator(document_name, *args, **kwargs) + + # Adding these to output should be idempotent, so we use a dict. + for doc in docs: + self.outputs[doc['schema']][doc['metadata']['name']] = doc + + return docs + + def _find_docs(self, kinds, document_name): + schemas = ['deckhand/%s/v1' % k for k in kinds] + docs = self._find_in_config(schemas, document_name) + if docs: + if len(docs) == len(kinds): + LOG.debug('Found docs in input config named %s, kinds: %s', + document_name, kinds) + return docs + else: + raise exceptions.IncompletePKIPairError( + 'Incomplete set %s ' + 'for name: %s' % (kinds, document_name)) + + else: + docs = self._find_in_outputs(schemas, document_name) + if docs: + LOG.debug('Found docs in current outputs named %s, kinds: %s', + document_name, kinds) + return docs + else: + LOG.debug('No docs existing docs named %s, kinds: %s', + document_name, kinds) + return [] + + def _find_in_config(self, schemas, document_name): + result = [] + for schema in schemas: + doc = self.config.find(schema=schema, name=document_name) + if doc: + result.append(doc) + return result + + def _find_in_outputs(self, schemas, document_name): + result = [] + for schema in schemas: + if document_name in self.outputs.get(schema, {}): + result.append(self.outputs[schema][document_name]) + return result + + def _write(self, output_dir): + docs = list( + itertools.chain.from_iterable( + v.values() for v in self.outputs.values())) + with open(os.path.join(output_dir, 'certificates.yaml'), 'w') as f: + # Don't use safe_dump_all so we can block format certificate data. + yaml.dump_all( + docs, + stream=f, + default_flow_style=False, + explicit_start=True, + indent=2) def get_host_list(service_names): @@ -52,12 +148,7 @@ def get_host_list(service_names): return service_list -def _write(output_dir, docs): - with open(os.path.join(output_dir, 'certificates.yaml'), 'w') as f: - # Don't use safe_dump_all so we can block format certificate data. - yaml.dump_all( - docs, - stream=f, - default_flow_style=False, - explicit_start=True, - indent=2) +def _extract_hosts(cert_def): + hosts = cert_def.get('hosts', []) + hosts.extend(get_host_list(cert_def.get('kubernetes_service_names', []))) + return hosts diff --git a/promenade/pki.py b/promenade/pki.py index 63866ebc..196ac628 100644 --- a/promenade/pki.py +++ b/promenade/pki.py @@ -14,7 +14,6 @@ LOG = logging.getLogger(__name__) class PKI: def __init__(self): - self.certificate_authorities = {} self._ca_config_string = None @property @@ -40,7 +39,6 @@ class PKI: files={ 'csr.json': self.csr(name=ca_name, groups=['Kubernetes']), }) - self.certificate_authorities[ca_name] = result return (self._wrap_ca(ca_name, result['cert']), self._wrap_ca_key(ca_name, result['key'])) @@ -56,7 +54,19 @@ class PKI: return (self._wrap_pub_key(name, pub_result['pub.pem']), self._wrap_priv_key(name, priv_result['priv.pem'])) - def generate_certificate(self, name, *, ca, cn, groups=[], hosts=[]): + def generate_certificate(self, + name, + *, + ca_cert, + ca_key, + cn, + groups=None, + hosts=None): + if groups is None: + groups = [] + if hosts is None: + hosts = [] + result = self._cfssl( [ 'gencert', '-ca', 'ca.pem', '-ca-key', 'ca-key.pem', '-config', @@ -64,8 +74,8 @@ class PKI: ], files={ 'ca-config.json': self.ca_config, - 'ca.pem': self.certificate_authorities[ca]['cert'], - 'ca-key.pem': self.certificate_authorities[ca]['key'], + 'ca.pem': ca_cert, + 'ca-key.pem': ca_key, 'csr.json': self.csr(name=cn, groups=groups, hosts=hosts), }) @@ -75,12 +85,17 @@ class PKI: def csr(self, *, name, - groups=[], - hosts=[], + groups=None, + hosts=None, key={ 'algo': 'rsa', 'size': 2048 }): + if groups is None: + groups = [] + if hosts is None: + hosts = [] + return json.dumps({ 'CN': name, 'key': key, diff --git a/promenade/renderer.py b/promenade/renderer.py index b002a4ea..5b4100fe 100644 --- a/promenade/renderer.py +++ b/promenade/renderer.py @@ -1,4 +1,4 @@ -from . import logging, tar_bundler +from . import exceptions, logging, tar_bundler import base64 import datetime import io @@ -75,7 +75,13 @@ def render_template_into_bundler(*, bundler, config, destination_path, with open(source_path) as f: template = env.from_string(f.read()) now = int(datetime.datetime.utcnow().timestamp()) - data = template.render(config=config, now=now) + try: + data = template.render(config=config, now=now) + except jinja2.exceptions.TemplateRuntimeError as e: + LOG.exception('Error rendering template (%s)' % source_path) + raise exceptions.TemplateRenderException( + 'Error rendering template (%s): %s' % (source_path, e)) + bundler.add(path=destination_path, data=data, mode=mode) @@ -91,7 +97,12 @@ def render_template(config, *, template, context=None): env = _build_env() template_obj = env.from_string(template_contents.decode('utf-8')) - return template_obj.render(config=config, **context) + try: + return template_obj.render(config=config, **context) + except jinja2.exceptions.TemplateRuntimeError as e: + LOG.exception('Error rendering template (%s)' % template) + raise exceptions.TemplateRenderException( + 'Error rendering template (%s): %s' % (template, e)) def _build_env(): diff --git a/promenade/tar_bundler.py b/promenade/tar_bundler.py index c697c4f9..ba0daea2 100644 --- a/promenade/tar_bundler.py +++ b/promenade/tar_bundler.py @@ -1,6 +1,7 @@ import hashlib import io import tarfile +import time from promenade import logging @@ -25,6 +26,7 @@ class TarBundler: data_bytes = data tar_info.size = len(data_bytes) tar_info.mode = mode + tar_info.mtime = int(time.time()) if tar_info.size > 0: # Ignore bandit false positive: B303:blacklist diff --git a/tools/g2/lib/config.sh b/tools/g2/lib/config.sh index ec61f804..1baed6c0 100644 --- a/tools/g2/lib/config.sh +++ b/tools/g2/lib/config.sh @@ -4,6 +4,7 @@ export BASE_IMAGE_URL=${BASE_IMAGE_URL:-https://cloud-images.ubuntu.com/releases export IMAGE_PROMENADE=${IMAGE_PROMENADE:-quay.io/attcomdev/promenade:latest} export NGINX_DIR="${TEMP_DIR}/nginx" export NGINX_URL="http://192.168.77.1:7777" +export PROMENADE_BASE_URL="http://promenade-api.ucp.svc.cluster.local" export PROMENADE_DEBUG=${PROMENADE_DEBUG:-0} export REGISTRY_DATA_DIR=${REGISTRY_DATA_DIR:-/mnt/registry} export VIRSH_POOL=${VIRSH_POOL:-promenade} diff --git a/tools/g2/lib/promenade.sh b/tools/g2/lib/promenade.sh index 9a3af8b0..ca386492 100644 --- a/tools/g2/lib/promenade.sh +++ b/tools/g2/lib/promenade.sh @@ -5,3 +5,59 @@ promenade_teardown_node() { ssh_cmd "${TARGET}" /usr/local/bin/promenade-teardown kubectl_cmd "${VIA}" delete node "${TARGET}" } + +promenade_render_curl_url() { + NAME=${1} + USE_DECKHAND=${2} + DECKHAND_REVISION=${3} + shift 3 + LABELS=(${@}) + + LABEL_PARAMS= + for label in "${LABELS[@]}"; do + LABEL_PARAMS+="&labels.dynamic=${label}" + done + + BASE_URL="${PROMENADE_BASE_URL}/api/v1.0/join-scripts" + if [[ ${USE_DECKHAND} == 1 ]]; then + DESIGN_REF="design_ref=deckhand%2Bhttp://deckhand-int.ucp.svc.cluster.local:9000/api/v1.0/revisions/${DECKHAND_REVISION}/rendered-documents" + else + DESIGN_REF="design_ref=${NGINX_URL}/promenade.yaml" + fi + HOST_PARAMS="hostname=${NAME}&ip=$(config_vm_ip "${NAME}")" + + echo "${BASE_URL}?${DESIGN_REF}&${HOST_PARAMS}&leave_kubectl=true${LABEL_PARAMS}" +} + +promenade_render_validate_url() { + echo "${PROMENADE_BASE_URL}/api/v1.0/validatedesign" +} + +promenade_render_validate_body() { + USE_DECKHAND=${1} + DECKHAND_REVISION=${2} + + if [[ ${USE_DECKHAND} == 1 ]]; then + JSON="{\"rel\":\"design\",\"href\":\"deckhand+http://deckhand-int.ucp.svc.cluster.local:9000/api/v1.0/revisions/${DECKHAND_REVISION}/rendered-documents\",\"type\":\"application/x-yaml\"}" + else + JSON="{\"rel\":\"design\",\"href\":\"${NGINX_URL}/promenade.yaml\",\"type\":\"application/x-yaml\"}" + fi + + echo ${JSON} +} + +promenade_health_check() { + VIA=${1} + log "Checking Promenade API health" + MAX_HEALTH_ATTEMPTS=6 + for attempt in $(seq ${MAX_HEALTH_ATTEMPTS}); do + if ssh_cmd "${VIA}" curl -v --fail "${PROMENADE_BASE_URL}/api/v1.0/health"; then + log "Promenade API healthy" + break + elif [[ $attempt == "${MAX_HEALTH_ATTEMPTS}" ]]; then + log "Promenade health check failed, max retries (${MAX_HEALTH_ATTEMPTS}) exceeded." + exit 1 + fi + sleep 10 + done +} diff --git a/tools/g2/manifests/resiliency.json b/tools/g2/manifests/resiliency.json index d30d9d73..40bbedcb 100644 --- a/tools/g2/manifests/resiliency.json +++ b/tools/g2/manifests/resiliency.json @@ -18,7 +18,10 @@ }, { "name": "Generate Certificates", - "script": "generate-certificates.sh" + "script": "generate-certificates.sh", + "arguments": [ + "-x", "PKICatalog-addition.yaml" + ] }, { "name": "Build Scripts", @@ -40,6 +43,36 @@ "-v", "n0", "-n", "n1", "-n", "n2", + "-l", "calico-etcd=enabled", + "-l", "kubernetes-apiserver=enabled", + "-l", "kubernetes-controller-manager=enabled", + "-l", "kubernetes-etcd=enabled", + "-l", "kubernetes-scheduler=enabled", + "-l", "ucp-control-plane=enabled", + "-e", "kubernetes n0 n0 n1 n2", + "-e", "calico n0 n0 n1 n2" + ] + }, + { + "name": "Verify Join Failure", + "script": "fail-join-node.sh", + "arguments": [ + "-v", "n0", + "-n", "n3" + ] + }, + { + "name": "Update Generated Certs", + "script": "generate-certificates.sh", + "arguments": [ + "-u" + ] + }, + { + "name": "Join Final Master", + "script": "join-nodes.sh", + "arguments": [ + "-v", "n0", "-n", "n3", "-l", "calico-etcd=enabled", "-l", "coredns=enabled", diff --git a/tools/g2/stages/build-scripts.sh b/tools/g2/stages/build-scripts.sh index 5b2ff9c2..77796940 100755 --- a/tools/g2/stages/build-scripts.sh +++ b/tools/g2/stages/build-scripts.sh @@ -19,6 +19,3 @@ docker run --rm -t \ --validators \ -o scripts \ config/*.yaml - -mkdir -p "${NGINX_DIR}" -cat "${TEMP_DIR}"/config/*.yaml > "${TEMP_DIR}/nginx/promenade.yaml" diff --git a/tools/g2/stages/fail-join-node.sh b/tools/g2/stages/fail-join-node.sh new file mode 100755 index 00000000..64bd4107 --- /dev/null +++ b/tools/g2/stages/fail-join-node.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +set -e + +source "${GATE_UTILS}" + +while getopts "n:v:" opt; do + case "${opt}" in + n) + NODE="${OPTARG}" + ;; + v) + VIA=${OPTARG} + ;; + *) + echo "Unknown option" + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +if [ $# -gt 0 ]; then + echo "Unknown arguments specified: ${*}" + exit 1 +fi + +SCRIPT_DIR="${TEMP_DIR}/join-fail-curled-scripts" + +mkdir -p "${SCRIPT_DIR}" + +CURL_ARGS=("-v" "--fail" "--max-time" "300") + +promenade_health_check "${VIA}" + +LABELS=( + "foo=bar" +) + +USE_DECKHAND=0 +JOIN_CURL_URL="$(promenade_render_curl_url "${NODE}" "${USE_DECKHAND}" "" "${LABELS[@]}")" +log "Attempting to get join script (should fail) via: ${JOIN_CURL_URL}" +if ! ssh_cmd "${VIA}" curl "${CURL_ARGS[@]}" \ + "${JOIN_CURL_URL}" > "${SCRIPT_DIR}/join-${NODE}.sh"; then + log "Failed to get join script" +else + log "No failure when fetching join script" + exit 1 +fi diff --git a/tools/g2/stages/generate-certificates.sh b/tools/g2/stages/generate-certificates.sh index ae237cc4..920c9eee 100755 --- a/tools/g2/stages/generate-certificates.sh +++ b/tools/g2/stages/generate-certificates.sh @@ -9,11 +9,61 @@ mkdir -p "${OUTPUT_DIR}" chmod 777 "${OUTPUT_DIR}" OUTPUT_FILE="${OUTPUT_DIR}/combined.yaml" +CERTIFICATES_FILE="${OUTPUT_DIR}/certificates.yaml" +OLD_CERTIFICATES_FILE="${OUTPUT_DIR}/certificates-old.yaml" + +IS_UPDATE=0 +DO_EXCLUDE=0 +EXCLUDE_PATTERNS=() + +while getopts "ux:" opt; do + case "${opt}" in + u) + IS_UPDATE=1 + ;; + x) + DO_EXCLUDE=1 + EXCLUDE_PATTERNS+=("${OPTARG}") + ;; + *) + echo "Unknown option" + exit 1 + ;; + esac +done +shift $((OPTIND-1)) + +function should_include_filename() { + FILENAME="${1}" + if [[ ${DO_EXCLUDE} == 1 ]]; then + for pattern in "${EXCLUDE_PATTERNS[@]}"; do + if echo "${FILENAME}" | grep "${pattern}" > /dev/null; then + return 1 + fi + done + fi + return 0 +} + +# Ensure we do not duplicate configuration on update. +rm -f "${OUTPUT_FILE}" + for source_dir in $(config_configuration); do log Copying configuration from "${source_dir}" - cat "${WORKSPACE}/${source_dir}"/*.yaml >> "${OUTPUT_FILE}" + for filename in ${WORKSPACE}/${source_dir}/*.yaml; do + if should_include_filename "${filename}"; then + log Including config from "$filename" + cat "${filename}" >> "${OUTPUT_FILE}" + else + log Excluding config from "$filename" + fi + done done +if [[ ${IS_UPDATE} == "1" && -e ${CERTIFICATES_FILE} ]]; then + mv "${CERTIFICATES_FILE}" "${OLD_CERTIFICATES_FILE}" +fi + log "Setting up local caches.." nginx_cache_and_replace_tar_urls "${OUTPUT_DIR}"/*.yaml registry_replace_references "${OUTPUT_DIR}"/*.yaml @@ -30,3 +80,10 @@ docker run --rm -t \ generate-certs \ -o /target \ "${FILES[@]}" + +if [[ -e "${OLD_CERTIFICATES_FILE}" ]]; then + rm -f "${OLD_CERTIFICATES_FILE}" +fi + +mkdir -p "${NGINX_DIR}" +cat "${TEMP_DIR}"/config/*.yaml > "${TEMP_DIR}/nginx/promenade.yaml" diff --git a/tools/g2/stages/join-nodes.sh b/tools/g2/stages/join-nodes.sh index 5a21316b..e6adeaa4 100755 --- a/tools/g2/stages/join-nodes.sh +++ b/tools/g2/stages/join-nodes.sh @@ -10,6 +10,7 @@ declare -a NODES GET_KEYSTONE_TOKEN=0 USE_DECKHAND=0 +DECKHAND_REVISION='' while getopts "d:e:l:n:tv:" opt; do case "${opt}" in @@ -46,43 +47,11 @@ if [ $# -gt 0 ]; then fi SCRIPT_DIR="${TEMP_DIR}/curled-scripts" -BASE_PROM_URL="http://promenade-api.ucp.svc.cluster.local" echo Etcd Clusters: "${ETCD_CLUSTERS[@]}" echo Labels: "${LABELS[@]}" echo Nodes: "${NODES[@]}" -render_curl_url() { - NAME=${1} - shift - LABELS=(${@}) - - LABEL_PARAMS= - for label in "${LABELS[@]}"; do - LABEL_PARAMS+="&labels.dynamic=${label}" - done - - BASE_URL="${BASE_PROM_URL}/api/v1.0/join-scripts" - if [[ ${USE_DECKHAND} == 1 ]]; then - DESIGN_REF="design_ref=deckhand%2Bhttp://deckhand-int.ucp.svc.cluster.local:9000/api/v1.0/revisions/${DECKHAND_REVISION}/rendered-documents" - else - DESIGN_REF="design_ref=${NGINX_URL}/promenade.yaml" - fi - HOST_PARAMS="hostname=${NAME}&ip=$(config_vm_ip "${NAME}")" - - echo "${BASE_URL}?${DESIGN_REF}&${HOST_PARAMS}&leave_kubectl=true${LABEL_PARAMS}" -} - -render_validate_body() { - if [[ ${USE_DECKHAND} == 1 ]]; then - JSON="{\"rel\":\"design\",\"href\":\"deckhand+http://deckhand-int.ucp.svc.cluster.local:9000/api/v1.0/revisions/${DECKHAND_REVISION}/rendered-documents\",\"type\":\"application/x-yaml\"}" - else - JSON="{\"rel\":\"design\",\"href\":\"${NGINX_URL}/promenade.yaml\",\"type\":\"application/x-yaml\"}" - fi - - echo ${JSON} -} - mkdir -p "${SCRIPT_DIR}" for NAME in "${NODES[@]}"; do @@ -100,23 +69,12 @@ for NAME in "${NODES[@]}"; do CURL_ARGS+=("-H" "X-Auth-Token: ${TOKEN}") fi - log "Checking Promenade API health" - MAX_HEALTH_ATTEMPTS=6 - for attempt in $(seq ${MAX_HEALTH_ATTEMPTS}); do - if ssh_cmd "${VIA}" curl -v "${CURL_ARGS[@]}" "${BASE_PROM_URL}/api/v1.0/health"; then - log "Promenade API healthy" - break - elif [[ $attempt == "${MAX_HEALTH_ATTEMPTS}" ]]; then - log "Promenade health check failed, max retries (${MAX_HEALTH_ATTEMPTS}) exceeded." - exit 1 - fi - sleep 10 - done + promenade_health_check "${VIA}" log "Validating documents" - ssh_cmd "${VIA}" curl -v "${CURL_ARGS[@]}" -X POST -H "Content-Type: application/json" -d $(render_validate_body) "${BASE_PROM_URL}/api/v1.0/validatedesign" + ssh_cmd "${VIA}" curl -v "${CURL_ARGS[@]}" -X POST -H "Content-Type: application/json" -d "$(promenade_render_validate_body "${USE_DECKHAND}" "${DECKHAND_REVISION}")" "$(promenade_render_validate_url)" - JOIN_CURL_URL="$(render_curl_url "${NAME}" "${LABELS[@]}")" + JOIN_CURL_URL="$(promenade_render_curl_url "${NAME}" "${USE_DECKHAND}" "${DECKHAND_REVISION}" "${LABELS[@]}")" log "Fetching join script via: ${JOIN_CURL_URL}" ssh_cmd "${VIA}" curl "${CURL_ARGS[@]}" \ "${JOIN_CURL_URL}" > "${SCRIPT_DIR}/join-${NAME}.sh"