/* * Copyright (c) 2017 Wind River Systems, Inc. * * SPDX-License-Identifier: Apache-2.0 * * * * @file * Wind River Titanium Cloud Maintenance BMC Utilities */ #include #include #include using namespace std; #include "nodeBase.h" /* for ... mtce common definitions */ #include "nodeClass.h" /* for ... */ #include "bmcUtil.h" /* for ... mtce-common bmc utility header */ /***************************************************************************** * * Name : bmc_command_send * * Description: This utility starts the bmc command handling thread * with the specified command. * * Returns : PASS if all the pre-start semantic checks pass and the * thread was started. * * Otherwise the thread was not started and some non zero * FAIL_xxxx code is returned after a representative design * log is generated. * *****************************************************************************/ int nodeLinkClass::bmc_command_send ( struct nodeLinkClass::node * node_ptr, int command ) { int rc = PASS ; /* handle 'kill of in-progress' thread or 'done but not consumed' thread */ if ( ! thread_idle ( node_ptr->bmc_thread_ctrl )) { if ( ! thread_done ( node_ptr->bmc_thread_ctrl )) { thread_kill ( node_ptr->bmc_thread_ctrl, node_ptr->bmc_thread_info ); return (RETRY); } else { mtcTimer_reset ( node_ptr->bmc_thread_ctrl.timer ); if ( thread_done_consume ( node_ptr->bmc_thread_ctrl, node_ptr->bmc_thread_info ) != PASS ) { return (RETRY); } } } node_ptr->bmc_thread_info.command = command ; /* Update / Setup the BMC access credentials */ node_ptr->thread_extra_info.bm_ip = node_ptr->bm_ip ; node_ptr->thread_extra_info.bm_un = node_ptr->bm_un ; node_ptr->thread_extra_info.bm_pw = node_ptr->bm_pw ; /* Special case handling for Redfish Root (BMC) Query command. * Current protocol override for this command that only applies * to redfish and used for the bmc protocol learning process. */ if ( command == BMC_THREAD_CMD__BMC_QUERY ) node_ptr->bmc_thread_info.proto = BMC_PROTOCOL__REDFISHTOOL ; else node_ptr->bmc_thread_info.proto = node_ptr->bmc_protocol ; if ( node_ptr->bmc_thread_info.proto == BMC_PROTOCOL__REDFISHTOOL ) { /* set the command specific redfishtool base command string */ if ( command == BMC_THREAD_CMD__RAW_GET ) node_ptr->bm_cmd = REDFISHTOOL_RAW_GET_CMD ; else node_ptr->bm_cmd = REDFISHTOOL_POWER_RESET_CMD ; /* append to the reset/power control or raw get command string */ switch (command) { case BMC_THREAD_CMD__RAW_GET: { if ( ! node_ptr->bmc_info.power_ctrl.raw_target_path.empty() ) { node_ptr->bm_cmd.append(node_ptr->bmc_info.power_ctrl.raw_target_path); } else { elog("%s is missing the raw get target", node_ptr->hostname.c_str()); return(FAIL_NOT_SUPPORTED); } break ; } case BMC_THREAD_CMD__POWER_RESET: { /* Use graceful for the first half of the retry countdown * and immediate for the remaining retries. */ if ((!node_ptr->bmc_info.power_ctrl.reset.immediate.empty()) && ( node_ptr->power_action_retries < MTC_RESET_ACTION_SWITCH_THRESHOLD)) node_ptr->bm_cmd.append(node_ptr->bmc_info.power_ctrl.reset.immediate); /* Unfaulted graceful if it exists */ else if (!node_ptr->bmc_info.power_ctrl.reset.graceful.empty()) node_ptr->bm_cmd.append(node_ptr->bmc_info.power_ctrl.reset.graceful); /* Unfaulted immediate if graceful does not exist */ else if (!node_ptr->bmc_info.power_ctrl.reset.immediate.empty()) node_ptr->bm_cmd.append(node_ptr->bmc_info.power_ctrl.reset.immediate); else { elog("%s offers no supported reset commands", node_ptr->hostname.c_str()); return(FAIL_NOT_SUPPORTED); } break ; } case BMC_THREAD_CMD__POWER_ON: { /* Use graceful for the first half of the retry countdown * and immediate for the remaining retries. */ if ((!node_ptr->bmc_info.power_ctrl.poweron.immediate.empty()) && ( node_ptr->power_action_retries < MTC_POWER_ACTION_SWITCH_THRESHOLD)) node_ptr->bm_cmd.append(node_ptr->bmc_info.power_ctrl.poweron.immediate); /* Unfaulted graceful if it exists */ else if (!node_ptr->bmc_info.power_ctrl.poweron.graceful.empty()) node_ptr->bm_cmd.append(node_ptr->bmc_info.power_ctrl.poweron.graceful); /* Unfaulted immediate if graceful does not exist */ else if (!node_ptr->bmc_info.power_ctrl.poweron.immediate.empty()) node_ptr->bm_cmd.append(node_ptr->bmc_info.power_ctrl.poweron.immediate); else { elog("%s offers no supported poweron commands", node_ptr->hostname.c_str()); return(FAIL_NOT_SUPPORTED); } break ; } case BMC_THREAD_CMD__POWER_OFF: { /* Use graceful for the first half of the retry countdown * and immediate for the remaining retries. */ if ((!node_ptr->bmc_info.power_ctrl.poweroff.immediate.empty() ) && ( node_ptr->power_action_retries < MTC_POWER_ACTION_SWITCH_THRESHOLD)) node_ptr->bm_cmd.append(node_ptr->bmc_info.power_ctrl.poweroff.immediate); /* Unfaulted graceful if it exists */ else if (!node_ptr->bmc_info.power_ctrl.poweroff.graceful.empty() ) node_ptr->bm_cmd.append(node_ptr->bmc_info.power_ctrl.poweroff.graceful); /* Unfaulted immediate if graceful does not exist */ else if (!node_ptr->bmc_info.power_ctrl.poweroff.immediate.empty()) node_ptr->bm_cmd.append(node_ptr->bmc_info.power_ctrl.poweroff.immediate); else { elog("%s offers no supported poweroff commands", node_ptr->hostname.c_str()); return(FAIL_NOT_SUPPORTED); } break ; } } node_ptr->thread_extra_info.bm_cmd = node_ptr->bm_cmd ; } #ifdef WANT_FIT_TESTING { bool want_fit = false ; int fit = FIT_CODE__BMC_COMMAND_SEND ; if ( daemon_want_fit ( fit, node_ptr->hostname, "root_query" ) == true ) { want_fit = true ; } else if ( daemon_want_fit ( fit, node_ptr->hostname, "bmc_info" ) == true ) { want_fit = true ; } else if (( command == BMC_THREAD_CMD__POWER_STATUS ) && ( daemon_want_fit ( fit, node_ptr->hostname, "power_status" ) == true )) { want_fit = true ; } else if ( daemon_want_fit ( fit, node_ptr->hostname, "reset_cause" ) == true ) { want_fit = true ; } else if (( command == BMC_THREAD_CMD__POWER_RESET ) && ( daemon_want_fit ( fit, node_ptr->hostname, "reset" ) == true )) { want_fit = true ; } else if (( command == BMC_THREAD_CMD__POWER_ON ) && ( daemon_want_fit ( fit, node_ptr->hostname, "power_on" ) == true )) { want_fit = true ; } else if (( command == BMC_THREAD_CMD__POWER_ON ) && ( daemon_want_fit ( fit, node_ptr->hostname, "power_none" ) == true )) { /* Just change the command to query status */ command = BMC_THREAD_CMD__POWER_STATUS ; } else if (( command == BMC_THREAD_CMD__POWER_OFF ) && ( daemon_want_fit ( fit, node_ptr->hostname, "power_off" ) == true )) { want_fit = true ; } else if (( command == BMC_THREAD_CMD__POWER_OFF ) && ( daemon_want_fit ( fit, node_ptr->hostname, "power_none" ) == true )) { /* Just change the command to query status */ command = BMC_THREAD_CMD__POWER_STATUS ; } else if (( command == BMC_THREAD_CMD__POWER_CYCLE ) && ( daemon_want_fit ( fit, node_ptr->hostname, "power_cycle" ) == true )) { want_fit = true ; } else if (( command == BMC_THREAD_CMD__BOOTDEV_PXE ) && ( daemon_want_fit ( fit, node_ptr->hostname, "netboot_pxe" ) == true )) { want_fit = true ; } if ( want_fit == true ) { slog ("%s FIT %s\n", node_ptr->hostname.c_str(), bmcUtil_getCmd_str(command).c_str() ); node_ptr->bmc_thread_info.status = node_ptr->bmc_thread_ctrl.status = rc = FAIL_FIT ; node_ptr->bmc_thread_info.status_string = "bmc_command_send fault insertion failure" ; return ( rc ); } } #endif if (( hostUtil_is_valid_ip_addr ( node_ptr->thread_extra_info.bm_ip ) == true ) && ( !node_ptr->thread_extra_info.bm_un.empty() ) && ( !node_ptr->thread_extra_info.bm_pw.empty ())) { node_ptr->bmc_thread_ctrl.status = rc = thread_launch ( node_ptr->bmc_thread_ctrl, node_ptr->bmc_thread_info ) ; if ( rc != PASS ) { elog ("%s failed to launch power control thread (rc:%d)\n", node_ptr->hostname.c_str(), rc ); } else { ilog ("%s %s send '%s' command (%s)", node_ptr->hostname.c_str(), node_ptr->bmc_thread_ctrl.name.c_str(), bmcUtil_getCmd_str(node_ptr->bmc_thread_info.command).c_str(), bmcUtil_getProtocol_str(node_ptr->bmc_protocol).c_str()); } node_ptr->bmc_thread_ctrl.retries = 0 ; } else { node_ptr->bmc_thread_ctrl.status = rc = node_ptr->bmc_thread_info.status = FAIL_INVALID_DATA ; node_ptr->bmc_thread_info.status_string = "one or more bmc credentials are invalid" ; wlog ("%s %s %s %s\n", node_ptr->hostname.c_str(), hostUtil_is_valid_ip_addr ( node_ptr->thread_extra_info.bm_ip ) ? "" : "bm_ip:invalid", node_ptr->thread_extra_info.bm_un.empty() ? "bm_un:empty" : "", node_ptr->thread_extra_info.bm_pw.empty() ? "bm_pw:empty" : ""); } return (rc); } /***************************************************************************** * * Name : bmc_command_recv * * Description: This utility will check for bmc command thread completion. * * Returns : PASS is returned if the thread reports done. * RETRY is returned if the thread has not completed. * FAIL_RETRY is returned after 10 back-to-back calls return RETRY. * *****************************************************************************/ int nodeLinkClass::bmc_command_recv ( struct nodeLinkClass::node * node_ptr ) { int rc = RETRY ; /* check for 'thread done' completion */ if ( thread_done( node_ptr->bmc_thread_ctrl ) == true ) { if ( node_ptr->bmc_protocol == BMC_PROTOCOL__REDFISHTOOL ) { if (( rc = node_ptr->bmc_thread_info.status ) != PASS ) { /* handle the redfishtool root query as a special case because * it is likely to fail and we don't want un-necessary error logs */ if ((( node_ptr->bmc_thread_info.command == BMC_THREAD_CMD__BMC_QUERY ) || ( node_ptr->bmc_thread_info.command == BMC_THREAD_CMD__BMC_INFO )) && (( rc == FAIL_SYSTEM_CALL ) || ( rc == FAIL_NOT_ACTIVE ))) { blog ("%s bmc redfish %s failed", node_ptr->hostname.c_str(), bmcUtil_getCmd_str( node_ptr->bmc_thread_info.command).c_str()); } else { elog ("%s bmc redfish %s command failed (%s) (data:%s) (rc:%d:%d:%s)\n", node_ptr->hostname.c_str(), bmcUtil_getCmd_str(node_ptr->bmc_thread_info.command).c_str(), bmcUtil_getProtocol_str(node_ptr->bmc_protocol).c_str(), node_ptr->bmc_thread_info.data.c_str(), rc, node_ptr->bmc_thread_info.status, node_ptr->bmc_thread_info.status_string.c_str()); } goto bmc_command_recv_cleanup; } else { rc = PASS ; } } else /* default is ipmi */ { if (( rc = node_ptr->bmc_thread_info.status ) != PASS ) { /* Don't log an error if this is just the BMC Query failure * used for protocol learning */ if ( node_ptr->bmc_thread_info.command != BMC_THREAD_CMD__BMC_QUERY ) { elog ("%s %s command failed (%s) (data:%s) (rc:%d:%d:%s)\n", node_ptr->hostname.c_str(), bmcUtil_getCmd_str(node_ptr->bmc_thread_info.command).c_str(), bmcUtil_getProtocol_str(node_ptr->bmc_protocol).c_str(), node_ptr->bmc_thread_info.data.c_str(), rc, node_ptr->bmc_thread_info.status, node_ptr->bmc_thread_info.status_string.c_str()); } } else { if ( node_ptr->bmc_thread_info.command == BMC_THREAD_CMD__POWER_RESET ) { if ( node_ptr->bmc_thread_info.data.find(IPMITOOL_POWER_RESET_RESP) == std::string::npos ) rc = FAIL_RESET_CONTROL ; } else if ( node_ptr->bmc_thread_info.command == BMC_THREAD_CMD__POWER_OFF ) { if ( node_ptr->bmc_thread_info.data.find(IPMITOOL_POWER_OFF_RESP) == std::string::npos ) rc = FAIL_POWER_CONTROL ; } else if ( node_ptr->bmc_thread_info.command == BMC_THREAD_CMD__POWER_ON ) { if ( node_ptr->bmc_thread_info.data.find(IPMITOOL_POWER_ON_RESP) == std::string::npos ) rc = FAIL_POWER_CONTROL ; } else if ( node_ptr->bmc_thread_info.command == BMC_THREAD_CMD__POWER_CYCLE ) { if ( node_ptr->bmc_thread_info.data.find(IPMITOOL_POWER_CYCLE_RESP) == std::string::npos ) rc = FAIL_POWER_CONTROL ; } if ( rc ) { node_ptr->bmc_thread_info.status = rc ; node_ptr->bmc_thread_info.status_string = ("power command failed"); wlog ("%s %s Response: %s\n", node_ptr->hostname.c_str(), bmcUtil_getCmd_str( node_ptr->bmc_thread_info.command).c_str(), node_ptr->bmc_thread_info.data.c_str()); } else { blog1 ("%s %s Response: %s\n", node_ptr->hostname.c_str(), bmcUtil_getCmd_str( node_ptr->bmc_thread_info.command).c_str(), node_ptr->bmc_thread_info.data.c_str()); } } } #ifdef WANT_FIT_TESTING if ( rc == PASS ) { bool want_fit = false ; int fit = FIT_CODE__BMC_COMMAND_RECV ; if ( daemon_want_fit ( fit, node_ptr->hostname, "root_query" ) == true ) { want_fit = true ; } if ( daemon_want_fit ( fit, node_ptr->hostname, "bmc_info" ) == true ) { want_fit = true ; } else if ( daemon_want_fit ( fit, node_ptr->hostname, "reset_cause" ) == true ) { want_fit = true ; } else if (( node_ptr->bmc_thread_info.command == BMC_THREAD_CMD__POWER_RESET ) && ( daemon_want_fit ( fit, node_ptr->hostname, "reset" ) == true )) { want_fit = true ; } else if (( node_ptr->bmc_thread_info.command == BMC_THREAD_CMD__POWER_ON ) && ( daemon_want_fit ( fit, node_ptr->hostname, "power_on" ) == true )) { want_fit = true ; } else if (( node_ptr->bmc_thread_info.command == BMC_THREAD_CMD__POWER_OFF ) && ( daemon_want_fit ( fit, node_ptr->hostname, "power_off" ) == true )) { want_fit = true ; } else if (( node_ptr->bmc_thread_info.command == BMC_THREAD_CMD__POWER_CYCLE ) && ( daemon_want_fit ( fit, node_ptr->hostname, "power_cycle" ) == true )) { want_fit = true ; } else if (( node_ptr->bmc_thread_info.command == BMC_THREAD_CMD__POWER_STATUS ) && ( daemon_want_fit ( fit, node_ptr->hostname, "power_status" ) == true )) { want_fit = true ; } else if (( node_ptr->bmc_thread_info.command == BMC_THREAD_CMD__BOOTDEV_PXE ) && ( daemon_want_fit ( fit, node_ptr->hostname, "netboot_pxe" ) == true )) { want_fit = true ; } if ( want_fit == true ) { node_ptr->bmc_thread_info.status = rc = FAIL_FIT ; node_ptr->bmc_thread_info.status_string = "bmc_command_recv fault insertion failure" ; } } #endif } /* handle max retries reached */ if ( rc == PASS ) { ; } else if ( node_ptr->bmc_thread_ctrl.retries++ >= BMC__MAX_RECV_RETRIES ) { wlog ("%s %s command timeout (%d of %d)\n", node_ptr->hostname.c_str(), bmcUtil_getCmd_str(node_ptr->bmc_thread_info.command).c_str(), node_ptr->bmc_thread_ctrl.retries, BMC__MAX_RECV_RETRIES); rc = FAIL_RETRY; } /* handle progressive retry */ else { if ( node_ptr->bmc_thread_ctrl.id == 0 ) { wlog ("%s %s command not-running\n", node_ptr->hostname.c_str(), bmcUtil_getCmd_str(node_ptr->bmc_thread_info.command).c_str()); rc = FAIL_NOT_ACTIVE ; } else { /* The BMC is sometimes slow, * No need to log till we reach half of the retry threshold */ if ( node_ptr->bmc_thread_ctrl.retries > (BMC__MAX_RECV_RETRIES/2) ) { ilog ("%s %s command in-progress (polling %d of %d)\n", node_ptr->hostname.c_str(), bmcUtil_getCmd_str(node_ptr->bmc_thread_info.command).c_str(), node_ptr->bmc_thread_ctrl.retries, BMC__MAX_RECV_RETRIES); } rc = RETRY ; } } bmc_command_recv_cleanup: if ( rc != RETRY ) { if ( rc != PASS ) { ilog ("%s %s recv '%s' command (%s) (rc:%d)", node_ptr->hostname.c_str(), node_ptr->bmc_thread_ctrl.name.c_str(), bmcUtil_getCmd_str(node_ptr->bmc_thread_info.command).c_str(), bmcUtil_getProtocol_str(node_ptr->bmc_protocol).c_str(), rc); } node_ptr->bmc_thread_ctrl.done = true ; node_ptr->bmc_thread_ctrl.retries = 0 ; node_ptr->bmc_thread_ctrl.id = 0 ; node_ptr->bmc_thread_info.id = 0 ; node_ptr->bmc_thread_info.command = 0 ; } return (rc); } /***************************************************************************** * * Name : bmc_command_done * * Description: This utility frees the bmc command thread for next execution. * *****************************************************************************/ void nodeLinkClass::bmc_command_done ( struct nodeLinkClass::node * node_ptr ) { node_ptr->bmc_thread_ctrl.done = true ; }