diff --git a/src/bin/shipyard_airflow/shipyard_airflow/dags/armada_deploy_site.py b/src/bin/shipyard_airflow/shipyard_airflow/dags/armada_deploy_site.py index 47440160..6c39a983 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/dags/armada_deploy_site.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/dags/armada_deploy_site.py @@ -13,11 +13,20 @@ # limitations under the License. from airflow.models import DAG -from airflow.operators import ArmadaGetReleasesOperator -from airflow.operators import ArmadaGetStatusOperator -from airflow.operators import ArmadaPostApplyOperator -from config_path import config_path +try: + from airflow.operators import ArmadaGetReleasesOperator + from airflow.operators import ArmadaGetStatusOperator + from airflow.operators import ArmadaPostApplyOperator + from config_path import config_path +except ImportError: + from shipyard_airflow.plugins.armada_get_releases import \ + ArmadaGetReleasesOperator + from shipyard_airflow.plugins.armada_get_status import \ + ArmadaGetStatusOperator + from shipyard_airflow.plugins.armada_post_apply import \ + ArmadaPostApplyOperator + from shipyard_airflow.dags.config_path import config_path def deploy_site_armada(parent_dag_name, child_dag_name, args): diff --git a/src/bin/shipyard_airflow/shipyard_airflow/dags/common_step_factory.py b/src/bin/shipyard_airflow/shipyard_airflow/dags/common_step_factory.py index fb155a42..e6e54aa4 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/dags/common_step_factory.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/dags/common_step_factory.py @@ -11,24 +11,49 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from airflow.operators import ConcurrencyCheckOperator -from airflow.operators import DeckhandRetrieveRenderedDocOperator -from airflow.operators import DeploymentConfigurationOperator -from airflow.operators import DeckhandCreateSiteActionTagOperator - from airflow.operators.bash_operator import BashOperator from airflow.operators.python_operator import BranchPythonOperator from airflow.operators.python_operator import PythonOperator from airflow.operators.subdag_operator import SubDagOperator -from armada_deploy_site import deploy_site_armada -from config_path import config_path -from destroy_node import destroy_server -from drydock_deploy_site import deploy_site_drydock -from failure_handlers import step_failure_handler -from preflight_checks import all_preflight_checks -from validate_site_design import validate_site_design -import dag_names as dn +try: + # Operators are loaded from being registered to airflow.operators + # in a deployed fashion + from airflow.operators import ConcurrencyCheckOperator + from airflow.operators import DeckhandRetrieveRenderedDocOperator + from airflow.operators import DeploymentConfigurationOperator + from airflow.operators import DeckhandCreateSiteActionTagOperator +except ImportError: + # for local testing, they are loaded from their source directory + from shipyard_airflow.plugins.concurrency_check_operator import \ + ConcurrencyCheckOperator + from shipyard_airflow.plugins.deckhand_retrieve_rendered_doc import \ + DeckhandRetrieveRenderedDocOperator + from shipyard_airflow.plugins.deployment_configuration_operator import \ + DeploymentConfigurationOperator + from shipyard_airflow.plugins.deckhand_create_site_action_tag import \ + DeckhandCreateSiteActionTagOperator + +try: + # modules reside in a flat directory when deployed with dags + from armada_deploy_site import deploy_site_armada + from config_path import config_path + from destroy_node import destroy_server + from drydock_deploy_site import deploy_site_drydock + from failure_handlers import step_failure_handler + from preflight_checks import all_preflight_checks + from validate_site_design import validate_site_design + import dag_names as dn +except ImportError: + # for testing, specify the qualified source directory + from shipyard_airflow.dags.armada_deploy_site import deploy_site_armada + from shipyard_airflow.dags.config_path import config_path + from shipyard_airflow.dags.destroy_node import destroy_server + from shipyard_airflow.dags.drydock_deploy_site import deploy_site_drydock + from shipyard_airflow.dags.failure_handlers import step_failure_handler + from shipyard_airflow.dags.preflight_checks import all_preflight_checks + from shipyard_airflow.dags.validate_site_design import validate_site_design + import shipyard_airflow.dags.dag_names as dn class CommonStepFactory(object): diff --git a/src/bin/shipyard_airflow/shipyard_airflow/dags/deploy_site.py b/src/bin/shipyard_airflow/shipyard_airflow/dags/deploy_site.py index 5881e6f6..ecb8a2d8 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/dags/deploy_site.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/dags/deploy_site.py @@ -16,7 +16,11 @@ from datetime import timedelta import airflow from airflow import DAG -from common_step_factory import CommonStepFactory +try: + from common_step_factory import CommonStepFactory +except ImportError: + from shipyard_airflow.dags.common_step_factory import CommonStepFactory + """deploy_site the top-level orchestration DAG for deploying a site using the Undercloud diff --git a/src/bin/shipyard_airflow/shipyard_airflow/dags/destroy_node.py b/src/bin/shipyard_airflow/shipyard_airflow/dags/destroy_node.py index c8f3ab2f..3926fbcb 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/dags/destroy_node.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/dags/destroy_node.py @@ -13,14 +13,29 @@ # limitations under the License. from airflow.models import DAG -from airflow.operators import DrydockDestroyNodeOperator -from airflow.operators import PromenadeCheckEtcdOperator -from airflow.operators import PromenadeClearLabelsOperator -from airflow.operators import PromenadeDecommissionNodeOperator -from airflow.operators import PromenadeDrainNodeOperator -from airflow.operators import PromenadeShutdownKubeletOperator -from config_path import config_path +try: + from airflow.operators import DrydockDestroyNodeOperator + from airflow.operators import PromenadeCheckEtcdOperator + from airflow.operators import PromenadeClearLabelsOperator + from airflow.operators import PromenadeDecommissionNodeOperator + from airflow.operators import PromenadeDrainNodeOperator + from airflow.operators import PromenadeShutdownKubeletOperator + from config_path import config_path +except ImportError: + from shipyard_airflow.plugins.drydock_destroy_nodes import \ + DrydockDestroyNodeOperator + from shipyard_airflow.plugins.promenade_check_etcd import \ + PromenadeCheckEtcdOperator + from shipyard_airflow.plugins.promenade_clear_labels import \ + PromenadeClearLabelsOperator + from shipyard_airflow.plugins.promenade_decommission_node import \ + PromenadeDecommissionNodeOperator + from shipyard_airflow.plugins.promenade_drain_node import \ + PromenadeDrainNodeOperator + from shipyard_airflow.plugins.promenade_shutdown_kubelet import \ + PromenadeShutdownKubeletOperator + from shipyard_airflow.dags.config_path import config_path def destroy_server(parent_dag_name, child_dag_name, args): diff --git a/src/bin/shipyard_airflow/shipyard_airflow/dags/drydock_deploy_site.py b/src/bin/shipyard_airflow/shipyard_airflow/dags/drydock_deploy_site.py index f635a382..f2f32eb1 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/dags/drydock_deploy_site.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/dags/drydock_deploy_site.py @@ -13,11 +13,20 @@ # limitations under the License. from airflow.models import DAG -from airflow.operators import DrydockNodesOperator -from airflow.operators import DrydockPrepareSiteOperator -from airflow.operators import DrydockVerifySiteOperator -from config_path import config_path +try: + from airflow.operators import DrydockNodesOperator + from airflow.operators import DrydockPrepareSiteOperator + from airflow.operators import DrydockVerifySiteOperator + from config_path import config_path +except ImportError: + from shipyard_airflow.plugins.drydock_nodes import \ + DrydockNodesOperator + from shipyard_airflow.plugins.drydock_prepare_site import \ + DrydockPrepareSiteOperator + from shipyard_airflow.plugins.drydock_verify_site import \ + DrydockVerifySiteOperator + from shipyard_airflow.dags.config_path import config_path def deploy_site_drydock(parent_dag_name, child_dag_name, args): diff --git a/src/bin/shipyard_airflow/shipyard_airflow/dags/preflight_checks.py b/src/bin/shipyard_airflow/shipyard_airflow/dags/preflight_checks.py index ebd4add1..ebafc26f 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/dags/preflight_checks.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/dags/preflight_checks.py @@ -13,9 +13,14 @@ # limitations under the License. from airflow.models import DAG -from airflow.operators import UcpHealthCheckOperator -from config_path import config_path +try: + from airflow.operators import UcpHealthCheckOperator + from config_path import config_path +except ImportError: + from shipyard_airflow.plugins.ucp_preflight_check_operator import \ + UcpHealthCheckOperator + from shipyard_airflow.dags.config_path import config_path def all_preflight_checks(parent_dag_name, child_dag_name, args): diff --git a/src/bin/shipyard_airflow/shipyard_airflow/dags/redeploy_server.py b/src/bin/shipyard_airflow/shipyard_airflow/dags/redeploy_server.py index 37efa287..8d247d5f 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/dags/redeploy_server.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/dags/redeploy_server.py @@ -16,7 +16,11 @@ from datetime import timedelta import airflow from airflow import DAG -from common_step_factory import CommonStepFactory +try: + from common_step_factory import CommonStepFactory +except ImportError: + from shipyard_airflow.dags.common_step_factory import CommonStepFactory + """redeploy_server The top-level orchestration DAG for redeploying a server using the Undercloud diff --git a/src/bin/shipyard_airflow/shipyard_airflow/dags/update_site.py b/src/bin/shipyard_airflow/shipyard_airflow/dags/update_site.py index af8bc22a..624411aa 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/dags/update_site.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/dags/update_site.py @@ -16,7 +16,10 @@ from datetime import timedelta import airflow from airflow import DAG -from common_step_factory import CommonStepFactory +try: + from common_step_factory import CommonStepFactory +except ImportError: + from shipyard_airflow.dags.common_step_factory import CommonStepFactory """update_site diff --git a/src/bin/shipyard_airflow/shipyard_airflow/dags/update_software.py b/src/bin/shipyard_airflow/shipyard_airflow/dags/update_software.py index 4d2bd7ea..c2933f33 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/dags/update_software.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/dags/update_software.py @@ -16,8 +16,12 @@ from datetime import timedelta import airflow from airflow import DAG -from common_step_factory import CommonStepFactory -from validate_site_design import SOFTWARE +try: + from common_step_factory import CommonStepFactory + from validate_site_design import SOFTWARE +except ImportError: + from shipyard_airflow.dags.common_step_factory import CommonStepFactory + from shipyard_airflow.dags.validate_site_design import SOFTWARE """update_software diff --git a/src/bin/shipyard_airflow/shipyard_airflow/dags/validate_site_design.py b/src/bin/shipyard_airflow/shipyard_airflow/dags/validate_site_design.py index 9dbdaf4d..dc1ec798 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/dags/validate_site_design.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/dags/validate_site_design.py @@ -13,13 +13,23 @@ # limitations under the License. from airflow.models import DAG -from airflow.operators import ArmadaValidateDesignOperator -from airflow.operators import DeckhandValidateSiteDesignOperator -from airflow.operators import DrydockValidateDesignOperator -from airflow.operators import PromenadeValidateSiteDesignOperator - -from config_path import config_path +try: + from airflow.operators import ArmadaValidateDesignOperator + from airflow.operators import DeckhandValidateSiteDesignOperator + from airflow.operators import DrydockValidateDesignOperator + from airflow.operators import PromenadeValidateSiteDesignOperator + from config_path import config_path +except ImportError: + from shipyard_airflow.plugins.armada_validate_design import \ + ArmadaValidateDesignOperator + from shipyard_airflow.plugins.deckhand_validate_site import \ + DeckhandValidateSiteDesignOperator + from shipyard_airflow.plugins.drydock_validate_design import \ + DrydockValidateDesignOperator + from shipyard_airflow.plugins.promenade_validate_site_design import \ + PromenadeValidateSiteDesignOperator + from shipyard_airflow.dags.config_path import config_path BAREMETAL = 'baremetal' SOFTWARE = 'software' diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_base_operator.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_base_operator.py index 1be7d3ba..3e46113a 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_base_operator.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_base_operator.py @@ -20,11 +20,20 @@ from airflow.utils.decorators import apply_defaults import armada.common.client as client import armada.common.session as session -from get_k8s_pod_port_ip import get_pod_port_ip -from service_token import shipyard_service_token -from ucp_base_operator import UcpBaseOperator -import service_endpoint -from xcom_pusher import XcomPusher + +try: + from get_k8s_pod_port_ip import get_pod_port_ip + import service_endpoint + from service_token import shipyard_service_token + from ucp_base_operator import UcpBaseOperator + from xcom_pusher import XcomPusher +except ImportError: + from shipyard_airflow.plugins.get_k8s_pod_port_ip import get_pod_port_ip + from shipyard_airflow.plugins import service_endpoint + from shipyard_airflow.plugins.service_token import shipyard_service_token + from shipyard_airflow.plugins.ucp_base_operator import UcpBaseOperator + from shipyard_airflow.plugins.xcom_pusher import XcomPusher + LOG = logging.getLogger(__name__) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_get_releases.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_get_releases.py index b805c0b2..390e41cd 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_get_releases.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_get_releases.py @@ -16,7 +16,11 @@ import logging from airflow.exceptions import AirflowException from airflow.plugins_manager import AirflowPlugin -from armada_base_operator import ArmadaBaseOperator +try: + from armada_base_operator import ArmadaBaseOperator +except ImportError: + from shipyard_airflow.plugins.armada_base_operator import \ + ArmadaBaseOperator from armada.exceptions import api_exceptions as errors LOG = logging.getLogger(__name__) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_get_status.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_get_status.py index 1acba694..2fece597 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_get_status.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_get_status.py @@ -16,7 +16,11 @@ import logging from airflow.exceptions import AirflowException from airflow.plugins_manager import AirflowPlugin -from armada_base_operator import ArmadaBaseOperator +try: + from armada_base_operator import ArmadaBaseOperator +except ImportError: + from shipyard_airflow.plugins.armada_base_operator import \ + ArmadaBaseOperator from armada.exceptions import api_exceptions as errors LOG = logging.getLogger(__name__) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_post_apply.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_post_apply.py index 7afd6108..0f73f490 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_post_apply.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_post_apply.py @@ -16,7 +16,11 @@ import logging from airflow.exceptions import AirflowException from airflow.plugins_manager import AirflowPlugin -from armada_base_operator import ArmadaBaseOperator +try: + from armada_base_operator import ArmadaBaseOperator +except ImportError: + from shipyard_airflow.plugins.armada_base_operator import \ + ArmadaBaseOperator from armada.exceptions import api_exceptions as errors LOG = logging.getLogger(__name__) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_validate_design.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_validate_design.py index 5dec64e9..6a3eebe4 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_validate_design.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/armada_validate_design.py @@ -16,7 +16,12 @@ import logging from airflow.plugins_manager import AirflowPlugin from airflow.exceptions import AirflowException -from armada_base_operator import ArmadaBaseOperator +try: + from armada_base_operator import ArmadaBaseOperator +except ImportError: + from shipyard_airflow.plugins.armada_base_operator import \ + ArmadaBaseOperator + from armada.exceptions import api_exceptions as errors LOG = logging.getLogger(__name__) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_base_operator.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_base_operator.py index 909af305..28693618 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_base_operator.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_base_operator.py @@ -19,9 +19,15 @@ from airflow.plugins_manager import AirflowPlugin from airflow.exceptions import AirflowException from deckhand.client import client as deckhand_client -import service_endpoint -from service_token import shipyard_service_token -from ucp_base_operator import UcpBaseOperator + +try: + import service_endpoint + from service_token import shipyard_service_token + from ucp_base_operator import UcpBaseOperator +except ImportError: + from shipyard_airflow.plugins import service_endpoint + from shipyard_airflow.plugins.service_token import shipyard_service_token + from shipyard_airflow.plugins.ucp_base_operator import UcpBaseOperator LOG = logging.getLogger(__name__) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_create_site_action_tag.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_create_site_action_tag.py index 15728a9a..fe84ef2e 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_create_site_action_tag.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_create_site_action_tag.py @@ -20,7 +20,11 @@ import subprocess # nosec from airflow.plugins_manager import AirflowPlugin from airflow.exceptions import AirflowException -from deckhand_base_operator import DeckhandBaseOperator +try: + from deckhand_base_operator import DeckhandBaseOperator +except ImportError: + from shipyard_airflow.plugins.deckhand_base_operator import \ + DeckhandBaseOperator FAILED_STATUSES = ('failed', 'upstream_failed') LOG = logging.getLogger(__name__) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_retrieve_rendered_doc.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_retrieve_rendered_doc.py index cffe3a37..76f8e0c6 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_retrieve_rendered_doc.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_retrieve_rendered_doc.py @@ -16,7 +16,12 @@ import logging from airflow.plugins_manager import AirflowPlugin from airflow.exceptions import AirflowException -from deckhand_base_operator import DeckhandBaseOperator +try: + from deckhand_base_operator import DeckhandBaseOperator +except ImportError: + from shipyard_airflow.plugins.deckhand_base_operator import \ + DeckhandBaseOperator + LOG = logging.getLogger(__name__) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_validate_site.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_validate_site.py index cc77c731..aeb2a3ae 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_validate_site.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/deckhand_validate_site.py @@ -19,7 +19,11 @@ import yaml from airflow.plugins_manager import AirflowPlugin from airflow.exceptions import AirflowException -from deckhand_base_operator import DeckhandBaseOperator +try: + from deckhand_base_operator import DeckhandBaseOperator +except ImportError: + from shipyard_airflow.plugins.deckhand_base_operator import \ + DeckhandBaseOperator LOG = logging.getLogger(__name__) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_base_operator.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_base_operator.py index baf1d7d4..9473065a 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_base_operator.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_base_operator.py @@ -25,20 +25,6 @@ import drydock_provisioner.drydock_client.client as client import drydock_provisioner.drydock_client.session as session from drydock_provisioner import error as errors -try: - import service_endpoint -except ImportError: - from shipyard_airflow.plugins import service_endpoint -try: - from service_token import shipyard_service_token -except ImportError: - from shipyard_airflow.plugins.service_token import shipyard_service_token - -try: - from ucp_base_operator import UcpBaseOperator -except ImportError: - from shipyard_airflow.plugins.ucp_base_operator import UcpBaseOperator - try: from drydock_errors import ( DrydockClientUseFailureException, @@ -46,6 +32,10 @@ try: DrydockTaskNotCreatedException, DrydockTaskTimeoutException ) + import service_endpoint + from service_token import shipyard_service_token + from ucp_base_operator import UcpBaseOperator + except ImportError: from shipyard_airflow.plugins.drydock_errors import ( DrydockClientUseFailureException, @@ -53,6 +43,9 @@ except ImportError: DrydockTaskNotCreatedException, DrydockTaskTimeoutException ) + from shipyard_airflow.plugins import service_endpoint + from shipyard_airflow.plugins.service_token import shipyard_service_token + from shipyard_airflow.plugins.ucp_base_operator import UcpBaseOperator LOG = logging.getLogger(__name__) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_destroy_nodes.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_destroy_nodes.py index 33693abb..ce03a988 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_destroy_nodes.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_destroy_nodes.py @@ -16,7 +16,11 @@ import time from airflow.plugins_manager import AirflowPlugin -from drydock_base_operator import DrydockBaseOperator +try: + from drydock_base_operator import DrydockBaseOperator +except ImportError: + from shipyard_airflow.plugins.drydock_base_operator import \ + DrydockBaseOperator LOG = logging.getLogger(__name__) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_nodes.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_nodes.py index cba97c0e..f9ebbeef 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_nodes.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_nodes.py @@ -35,21 +35,15 @@ from shipyard_airflow.common.deployment_group.node_lookup import NodeLookup try: import check_k8s_node_status -except ImportError: - from shipyard_airflow.plugins import check_k8s_node_status - -try: from drydock_base_operator import DrydockBaseOperator -except ImportError: - from shipyard_airflow.plugins.drydock_base_operator import \ - DrydockBaseOperator - -try: from drydock_errors import ( DrydockTaskFailedException, DrydockTaskTimeoutException ) except ImportError: + from shipyard_airflow.plugins import check_k8s_node_status + from shipyard_airflow.plugins.drydock_base_operator import \ + DrydockBaseOperator from shipyard_airflow.plugins.drydock_errors import ( DrydockTaskFailedException, DrydockTaskTimeoutException diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_prepare_site.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_prepare_site.py index 04264a31..475e6071 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_prepare_site.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_prepare_site.py @@ -13,7 +13,11 @@ # limitations under the License. from airflow.plugins_manager import AirflowPlugin -from drydock_base_operator import DrydockBaseOperator +try: + from drydock_base_operator import DrydockBaseOperator +except ImportError: + from shipyard_airflow.plugins.drydock_base_operator import \ + DrydockBaseOperator class DrydockPrepareSiteOperator(DrydockBaseOperator): diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_validate_design.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_validate_design.py index 30c374e8..2bde3d2f 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_validate_design.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_validate_design.py @@ -19,7 +19,11 @@ import requests from airflow.plugins_manager import AirflowPlugin from airflow.exceptions import AirflowException -from drydock_base_operator import DrydockBaseOperator +try: + from drydock_base_operator import DrydockBaseOperator +except ImportError: + from shipyard_airflow.plugins.drydock_base_operator import \ + DrydockBaseOperator LOG = logging.getLogger(__name__) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_verify_site.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_verify_site.py index adbfc0e8..de55d2f2 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_verify_site.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/drydock_verify_site.py @@ -13,7 +13,11 @@ # limitations under the License. from airflow.plugins_manager import AirflowPlugin -from drydock_base_operator import DrydockBaseOperator +try: + from drydock_base_operator import DrydockBaseOperator +except ImportError: + from shipyard_airflow.plugins.drydock_base_operator import \ + DrydockBaseOperator class DrydockVerifySiteOperator(DrydockBaseOperator): diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_base_operator.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_base_operator.py index e40d39af..c4ce4859 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_base_operator.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_base_operator.py @@ -19,17 +19,11 @@ from airflow.exceptions import AirflowException try: import service_endpoint -except ImportError: - from shipyard_airflow.plugins import service_endpoint - -try: from service_token import shipyard_service_token -except ImportError: - from shipyard_airflow.plugins.service_token import shipyard_service_token - -try: from ucp_base_operator import UcpBaseOperator except ImportError: + from shipyard_airflow.plugins import service_endpoint + from shipyard_airflow.plugins.service_token import shipyard_service_token from shipyard_airflow.plugins.ucp_base_operator import UcpBaseOperator LOG = logging.getLogger(__name__) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_check_etcd.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_check_etcd.py index 944e7f2b..761c457a 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_check_etcd.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_check_etcd.py @@ -17,7 +17,11 @@ import time from airflow.plugins_manager import AirflowPlugin from airflow.exceptions import AirflowException -from promenade_base_operator import PromenadeBaseOperator +try: + from promenade_base_operator import PromenadeBaseOperator +except ImportError: + from shipyard_airflow.plugins.promenade_base_operator import \ + PromenadeBaseOperator LOG = logging.getLogger(__name__) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_clear_labels.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_clear_labels.py index 1aee2669..50b7d2a5 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_clear_labels.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_clear_labels.py @@ -17,7 +17,11 @@ import time from airflow.plugins_manager import AirflowPlugin from airflow.exceptions import AirflowException -from promenade_base_operator import PromenadeBaseOperator +try: + from promenade_base_operator import PromenadeBaseOperator +except ImportError: + from shipyard_airflow.plugins.promenade_base_operator import \ + PromenadeBaseOperator LOG = logging.getLogger(__name__) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_decommission_node.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_decommission_node.py index 8dc83172..f5360ca7 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_decommission_node.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_decommission_node.py @@ -17,7 +17,11 @@ import time from airflow.plugins_manager import AirflowPlugin from airflow.exceptions import AirflowException -from promenade_base_operator import PromenadeBaseOperator +try: + from promenade_base_operator import PromenadeBaseOperator +except ImportError: + from shipyard_airflow.plugins.promenade_base_operator import \ + PromenadeBaseOperator LOG = logging.getLogger(__name__) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_drain_node.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_drain_node.py index 47ced5ee..352111cd 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_drain_node.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_drain_node.py @@ -17,7 +17,11 @@ import time from airflow.plugins_manager import AirflowPlugin from airflow.exceptions import AirflowException -from promenade_base_operator import PromenadeBaseOperator +try: + from promenade_base_operator import PromenadeBaseOperator +except ImportError: + from shipyard_airflow.plugins.promenade_base_operator import \ + PromenadeBaseOperator LOG = logging.getLogger(__name__) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_shutdown_kubelet.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_shutdown_kubelet.py index 0a880f59..b7b92b00 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_shutdown_kubelet.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_shutdown_kubelet.py @@ -17,7 +17,11 @@ import time from airflow.plugins_manager import AirflowPlugin from airflow.exceptions import AirflowException -from promenade_base_operator import PromenadeBaseOperator +try: + from promenade_base_operator import PromenadeBaseOperator +except ImportError: + from shipyard_airflow.plugins.promenade_base_operator import \ + PromenadeBaseOperator LOG = logging.getLogger(__name__) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py index b4d5ff01..07ff1222 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py @@ -22,9 +22,8 @@ from airflow.plugins_manager import AirflowPlugin try: from promenade_base_operator import PromenadeBaseOperator except ImportError: - from shipyard_airflow.plugins.promenade_base_operator import ( + from shipyard_airflow.plugins.promenade_base_operator import \ PromenadeBaseOperator - ) LOG = logging.getLogger(__name__) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_base_operator.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_base_operator.py index d26243e3..4c804d5e 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_base_operator.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_base_operator.py @@ -23,35 +23,22 @@ from airflow.plugins_manager import AirflowPlugin from airflow.utils.decorators import apply_defaults try: + from deckhand_client_factory import DeckhandClientFactory import service_endpoint -except ImportError: - from shipyard_airflow.plugins import service_endpoint - -try: from get_k8s_logs import get_pod_logs -except ImportError: - from shipyard_airflow.plugins.get_k8s_logs import get_pod_logs - -try: from get_k8s_logs import K8sLoggingException -except ImportError: - from shipyard_airflow.plugins.get_k8s_logs import K8sLoggingException - -try: from xcom_puller import XcomPuller except ImportError: + from shipyard_airflow.plugins.deckhand_client_factory import \ + DeckhandClientFactory + from shipyard_airflow.plugins import service_endpoint + from shipyard_airflow.plugins.get_k8s_logs import get_pod_logs + from shipyard_airflow.plugins.get_k8s_logs import K8sLoggingException from shipyard_airflow.plugins.xcom_puller import XcomPuller from shipyard_airflow.common.document_validators.document_validation_utils \ import DocumentValidationUtils -try: - from deckhand_client_factory import DeckhandClientFactory -except ImportError: - from shipyard_airflow.plugins.deckhand_client_factory import ( - DeckhandClientFactory - ) - LOG = logging.getLogger(__name__) @@ -102,10 +89,11 @@ class UcpBaseOperator(BaseOperator): self.shipyard_conf = shipyard_conf self.start_time = datetime.now() self.xcom_push_flag = xcom_push - self.doc_utils = _get_document_util(self.shipyard_conf) - self.endpoints = service_endpoint.ServiceEndpoints(self.shipyard_conf) def execute(self, context): + # Setup values that depend on the shipyard configuration + self.doc_utils = _get_document_util(self.shipyard_conf) + self.endpoints = service_endpoint.ServiceEndpoints(self.shipyard_conf) # Execute UCP base function self.ucp_base(context) diff --git a/src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_preflight_check_operator.py b/src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_preflight_check_operator.py index 0393703b..3bf888a8 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_preflight_check_operator.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_preflight_check_operator.py @@ -22,17 +22,11 @@ from airflow.utils.decorators import apply_defaults try: import service_endpoint -except ImportError: - from shipyard_airflow.plugins import service_endpoint - -try: from xcom_puller import XcomPuller -except ImportError: - from shipyard_airflow.plugins.xcom_puller import XcomPuller - -try: from xcom_pusher import XcomPusher except ImportError: + from shipyard_airflow.plugins import service_endpoint + from shipyard_airflow.plugins.xcom_puller import XcomPuller from shipyard_airflow.plugins.xcom_pusher import XcomPusher LOG = logging.getLogger(__name__) diff --git a/src/bin/shipyard_airflow/tests/unit/dags/test_dags.py b/src/bin/shipyard_airflow/tests/unit/dags/test_dags.py new file mode 100644 index 00000000..36bb3702 --- /dev/null +++ b/src/bin/shipyard_airflow/tests/unit/dags/test_dags.py @@ -0,0 +1,33 @@ +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from airflow import DAG + +from shipyard_airflow.dags import deploy_site +from shipyard_airflow.dags import update_site +from shipyard_airflow.dags import update_software +from shipyard_airflow.dags import redeploy_server + + +def test_dags_load(): + """assert that each dag is a DAG object after importing the modules + + As these are the top level dags, the subdags and many operators + will be loaded as a result. This ensures that basic construction + is working for most of the workflow logic, however it tests nearly + none of the decision making. + """ + for d in [deploy_site.dag, update_site.dag, + update_software.dag, redeploy_server.dag]: + assert isinstance(d, DAG) + assert d.task_count > 0