363 lines
16 KiB
Python
363 lines
16 KiB
Python
# vim: tabstop=4 shiftwidth=4 softtabstop=4
|
|
|
|
# Copyright (C) 2012 Yahoo! Inc. All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License..
|
|
|
|
import abc
|
|
import copy
|
|
|
|
from anvil import cfg
|
|
from anvil import colorizer
|
|
from anvil import env
|
|
from anvil import exceptions as excp
|
|
from anvil import importer
|
|
from anvil import log as logging
|
|
from anvil import persona as _persona
|
|
from anvil import phase
|
|
from anvil import shell as sh
|
|
from anvil import utils
|
|
|
|
import six
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
BASE_ENTRYPOINTS = {
|
|
'build': 'anvil.components.base_build:BuildComponent',
|
|
}
|
|
BASE_PYTHON_ENTRYPOINTS = dict(BASE_ENTRYPOINTS)
|
|
BASE_PYTHON_ENTRYPOINTS.update({
|
|
'build': 'anvil.components.base_build:PythonBuildComponent',
|
|
})
|
|
SPECIAL_GROUPS = _persona.SPECIAL_GROUPS
|
|
|
|
|
|
class PhaseFunctors(object):
|
|
def __init__(self, start, run, end):
|
|
self.start = start
|
|
self.run = run
|
|
self.end = end
|
|
|
|
|
|
class Action(object):
|
|
__meta__ = abc.ABCMeta
|
|
needs_sudo = True
|
|
|
|
def __init__(self, name, distro, root_dir, cli_opts):
|
|
self.distro = distro
|
|
self.name = name
|
|
# Root directory where all files/downloads will be based at
|
|
self.root_dir = root_dir
|
|
# Action phases are tracked in this directory
|
|
self.phase_dir = sh.joinpths(root_dir, 'phases')
|
|
|
|
# Yamls are loaded (with its reference links) using this instance at the
|
|
# given component directory where component configuration will be found.
|
|
self.config_loader = cfg.YamlMergeLoader(root_dir,
|
|
origins_path=cli_opts['origins_fn'])
|
|
|
|
# Stored for components to get any options
|
|
self.cli_opts = cli_opts
|
|
|
|
@abc.abstractproperty
|
|
@property
|
|
def lookup_name(self):
|
|
# Name that will be used to lookup this module
|
|
# in any configuration (may or may not be the same as the name
|
|
# of this action)....
|
|
raise NotImplementedError()
|
|
|
|
@abc.abstractmethod
|
|
def _run(self, persona, groups):
|
|
"""Run the phases of processing for this action.
|
|
|
|
Subclasses are expected to override this method to
|
|
do something useful.
|
|
"""
|
|
raise NotImplementedError()
|
|
|
|
def _make_default_entry_points(self, component_name, component_options):
|
|
if component_options.get('python_entrypoints'):
|
|
return BASE_PYTHON_ENTRYPOINTS.copy()
|
|
return BASE_ENTRYPOINTS.copy()
|
|
|
|
def _merge_subsystems(self, distro_subsystems, desired_subsystems):
|
|
subsystems = {}
|
|
for subsystem_name in desired_subsystems:
|
|
# Return a deep copy so that later instances can not modify
|
|
# other instances subsystem accidentally...
|
|
subsystems[subsystem_name] = copy.deepcopy(distro_subsystems.get(subsystem_name, {}))
|
|
return subsystems
|
|
|
|
def _construct_siblings(self, name, siblings, base_params, sibling_instances):
|
|
# First setup the sibling instance action references
|
|
for (action, _entry_point) in siblings.items():
|
|
if action not in sibling_instances:
|
|
sibling_instances[action] = {}
|
|
there_siblings = {}
|
|
for (action, entry_point) in siblings.items():
|
|
sibling_params = utils.merge_dicts(base_params, self.cli_opts, preserve=True)
|
|
# Give the sibling the reference to all other siblings being created
|
|
# which will be populated when they are created (now or later) for
|
|
# the same action
|
|
sibling_params['instances'] = sibling_instances[action]
|
|
a_sibling = importer.construct_entry_point(entry_point, **sibling_params)
|
|
# Update the sibling we are returning and the corresponding
|
|
# siblings for that action (so that the sibling can have the
|
|
# correct 'sibling' instances associated with it, if it needs those...)
|
|
there_siblings[action] = a_sibling
|
|
# Update all siblings being constructed so that there siblings will
|
|
# be correct when fetched...
|
|
sibling_instances[action][name] = a_sibling
|
|
return there_siblings
|
|
|
|
def _construct_instances(self, persona):
|
|
"""Create component objects for each component in the persona."""
|
|
# Keeps track of all sibling instances across all components + actions
|
|
# so that each instance or sibling instance will be connected to the
|
|
# right set of siblings....
|
|
sibling_instances = {}
|
|
components_created = set()
|
|
groups = []
|
|
for group in persona.matched_components:
|
|
instances = utils.OrderedDict()
|
|
for c in group:
|
|
if c in components_created:
|
|
raise RuntimeError("Can not duplicate component %s in a"
|
|
" later group %s" % (c, group.id))
|
|
d_component = self.distro.extract_component(
|
|
c, self.lookup_name, default_entry_point_creator=self._make_default_entry_points)
|
|
LOG.debug("Constructing component %r (%s)", c, d_component.entry_point)
|
|
d_subsystems = d_component.options.pop('subsystems', {})
|
|
sibling_params = {}
|
|
sibling_params['name'] = c
|
|
# First create its siblings with a 'minimal' set of options
|
|
# This is done, so that they will work in a minimal state, they do not
|
|
# get access to the persona options since those are action specific (or could be),
|
|
# if this is not useful, we can give them full access, unsure if its worse or better...
|
|
active_subsystems = self._merge_subsystems(distro_subsystems=d_subsystems,
|
|
desired_subsystems=persona.wanted_subsystems.get(c, []))
|
|
sibling_params['subsystems'] = active_subsystems
|
|
sibling_params['siblings'] = {} # This gets adjusted during construction
|
|
sibling_params['distro'] = self.distro
|
|
sibling_params['options'] = self.config_loader.load(
|
|
distro=d_component, component=c,
|
|
origins_patch=self.cli_opts.get('origins_patch'))
|
|
LOG.debug("Constructing %r %s siblings...", c, len(d_component.siblings))
|
|
my_siblings = self._construct_siblings(c, d_component.siblings, sibling_params, sibling_instances)
|
|
# Now inject the full options and create the target instance
|
|
# with the full set of options and not the restricted set that
|
|
# siblings get...
|
|
instance_params = dict(sibling_params)
|
|
instance_params['instances'] = instances
|
|
instance_params['options'] = self.config_loader.load(
|
|
distro=d_component, component=c, persona=persona,
|
|
origins_patch=self.cli_opts.get('origins_patch'))
|
|
instance_params['siblings'] = my_siblings
|
|
instance_params = utils.merge_dicts(instance_params, self.cli_opts, preserve=True)
|
|
instances[c] = importer.construct_entry_point(d_component.entry_point, **instance_params)
|
|
if c not in SPECIAL_GROUPS:
|
|
components_created.add(c)
|
|
groups.append((group.id, instances))
|
|
return groups
|
|
|
|
def _verify_components(self, groups):
|
|
for group, instances in groups:
|
|
LOG.info("Verifying that the components of group %s are ready"
|
|
" to rock-n-roll.", colorizer.quote(group))
|
|
for _c, instance in six.iteritems(instances):
|
|
instance.verify()
|
|
|
|
def _warm_components(self, groups):
|
|
for group, instances in groups:
|
|
LOG.info("Warming up component configurations of group %s.",
|
|
colorizer.quote(group))
|
|
for _c, instance in six.iteritems(instances):
|
|
instance.warm_configs()
|
|
|
|
def _on_start(self, persona, groups):
|
|
LOG.info("Booting up your components.")
|
|
LOG.debug("Starting environment settings:")
|
|
utils.log_object(env.get(), logger=LOG, level=logging.DEBUG, item_max_len=64)
|
|
sh.mkdirslist(self.phase_dir)
|
|
self._verify_components(groups)
|
|
self._warm_components(groups)
|
|
|
|
def _on_finish(self, persona, groups):
|
|
LOG.info("Tearing down your components.")
|
|
LOG.debug("Final environment settings:")
|
|
utils.log_object(env.get(), logger=LOG, level=logging.DEBUG, item_max_len=64)
|
|
|
|
def _get_phase_filename(self, phase_name):
|
|
# Do some canonicalization of the phase name so its in a semi-standard format...
|
|
phase_name = phase_name.lower().strip()
|
|
phase_name = phase_name.replace("-", '_')
|
|
phase_name = phase_name.replace(" ", "_")
|
|
if not phase_name:
|
|
raise ValueError("Phase name must not be empty")
|
|
return sh.joinpths(self.phase_dir, "%s.phases" % (phase_name))
|
|
|
|
def _run_many_phase(self, functors, group, instances, phase_name, *inv_phase_names):
|
|
"""Run a given 'functor' across all of the components, passing *all* instances to run."""
|
|
|
|
# This phase recorder will be used to check if a given component
|
|
# and action has ran in the past, if so that components action
|
|
# will not be ran again. It will also be used to mark that a given
|
|
# component has completed a phase (if that phase runs).
|
|
if not phase_name:
|
|
phase_recorder = phase.NullPhaseRecorder()
|
|
else:
|
|
phase_recorder = phase.PhaseRecorder(self._get_phase_filename(phase_name))
|
|
|
|
# These phase recorders will be used to undo other actions activities
|
|
# ie, when an install completes you want the uninstall phase to be
|
|
# removed from that actions phase file (and so on). This list will be
|
|
# used to accomplish that.
|
|
neg_phase_recs = []
|
|
if inv_phase_names:
|
|
for n in inv_phase_names:
|
|
if not n:
|
|
neg_phase_recs.append(phase.NullPhaseRecorder())
|
|
else:
|
|
neg_phase_recs.append(phase.PhaseRecorder(self._get_phase_filename(n)))
|
|
|
|
def change_activate(instance, on_off):
|
|
# Activate/deactivate a component instance and there siblings (if any)
|
|
#
|
|
# This is used when you say are looking at components
|
|
# that have been activated before your component has been.
|
|
#
|
|
# Typically this is useful for checking if a previous component
|
|
# has a shared dependency with your component and if so then there
|
|
# is no need to reinstall said dependency...
|
|
instance.activated = on_off
|
|
for (_name, sibling_instance) in instance.siblings.items():
|
|
sibling_instance.activated = on_off
|
|
|
|
def run_inverse_recorders(c_name):
|
|
for n in neg_phase_recs:
|
|
n.unmark(c_name)
|
|
|
|
# Reset all activations
|
|
for c, instance in six.iteritems(instances):
|
|
change_activate(instance, False)
|
|
|
|
# Run all components which have not been ran previously (due to phase tracking)
|
|
instances_started = utils.OrderedDict()
|
|
for c, instance in six.iteritems(instances):
|
|
if c in SPECIAL_GROUPS:
|
|
c = "%s_%s" % (c, group)
|
|
if c in phase_recorder:
|
|
LOG.debug("Skipping phase named %r for component %r since it already happened.", phase_name, c)
|
|
else:
|
|
try:
|
|
with phase_recorder.mark(c):
|
|
if functors.start:
|
|
functors.start(instance)
|
|
instances_started[c] = instance
|
|
except excp.NoTraceException:
|
|
pass
|
|
if functors.run:
|
|
results = functors.run(list(six.itervalues(instances_started)))
|
|
else:
|
|
results = [None] * len(instances_started)
|
|
instances_ran = instances_started
|
|
for i, (c, instance) in enumerate(six.iteritems(instances_ran)):
|
|
result = results[i]
|
|
try:
|
|
with phase_recorder.mark(c):
|
|
if functors.end:
|
|
functors.end(instance, result)
|
|
except excp.NoTraceException:
|
|
pass
|
|
for c, instance in six.iteritems(instances_ran):
|
|
change_activate(instance, True)
|
|
run_inverse_recorders(c)
|
|
|
|
def _run_phase(self, functors, group, instances, phase_name, *inv_phase_names):
|
|
"""Run a given 'functor' across all of the components, in order individually."""
|
|
|
|
# This phase recorder will be used to check if a given component
|
|
# and action has ran in the past, if so that components action
|
|
# will not be ran again. It will also be used to mark that a given
|
|
# component has completed a phase (if that phase runs).
|
|
if not phase_name:
|
|
phase_recorder = phase.NullPhaseRecorder()
|
|
else:
|
|
phase_recorder = phase.PhaseRecorder(self._get_phase_filename(phase_name))
|
|
|
|
# These phase recorders will be used to undo other actions activities
|
|
# ie, when an install completes you want the uninstall phase to be
|
|
# removed from that actions phase file (and so on). This list will be
|
|
# used to accomplish that.
|
|
neg_phase_recs = []
|
|
if inv_phase_names:
|
|
for n in inv_phase_names:
|
|
if not n:
|
|
neg_phase_recs.append(phase.NullPhaseRecorder())
|
|
else:
|
|
neg_phase_recs.append(phase.PhaseRecorder(self._get_phase_filename(n)))
|
|
|
|
def change_activate(instance, on_off):
|
|
# Activate/deactivate a component instance and there siblings (if any)
|
|
#
|
|
# This is used when you say are looking at components
|
|
# that have been activated before your component has been.
|
|
#
|
|
# Typically this is useful for checking if a previous component
|
|
# has a shared dependency with your component and if so then there
|
|
# is no need to reinstall said dependency...
|
|
instance.activated = on_off
|
|
for (_name, sibling_instance) in instance.siblings.items():
|
|
sibling_instance.activated = on_off
|
|
|
|
def run_inverse_recorders(c_name):
|
|
for n in neg_phase_recs:
|
|
n.unmark(c_name)
|
|
|
|
# Reset all activations
|
|
for c, instance in six.iteritems(instances):
|
|
change_activate(instance, False)
|
|
|
|
# Run all components which have not been ran previously (due to phase tracking)
|
|
for c, instance in six.iteritems(instances):
|
|
if c in SPECIAL_GROUPS:
|
|
c = "%s_%s" % (c, group)
|
|
if c in phase_recorder:
|
|
LOG.debug("Skipping phase named %r for component %r since it already happened.", phase_name, c)
|
|
else:
|
|
try:
|
|
with phase_recorder.mark(c):
|
|
if functors.start:
|
|
functors.start(instance)
|
|
if functors.run:
|
|
result = functors.run(instance)
|
|
else:
|
|
result = None
|
|
if functors.end:
|
|
functors.end(instance, result)
|
|
except excp.NoTraceException:
|
|
pass
|
|
change_activate(instance, True)
|
|
run_inverse_recorders(c)
|
|
|
|
def run(self, persona):
|
|
groups = self._construct_instances(persona)
|
|
LOG.info("Processing components for action %s.", colorizer.quote(self.name))
|
|
for group in persona.matched_components:
|
|
utils.log_iterable(group,
|
|
header="Activating group %s in the following order" % colorizer.quote(group.id),
|
|
logger=LOG)
|
|
self._on_start(persona, groups)
|
|
self._run(persona, groups)
|
|
self._on_finish(persona, groups)
|