#!/bin/bash
#< Display active/inactive status for members of a BIGIP pool, optionally checking HTTP
# poolstats.sh
#
# version 0.1 20070222 KWaldron Base functionality
# version 0.2 20070222 KWaldron Resolution of hostnames
# Operates in quiet mode by default
# version 0.3 20070223 KWaldron -n option added
# version 0.4 20070226 KWaldron -R option added (not tested)
#
# Requires:
# valip - See http://www.zazzybob.com/bin/tars/valip.tar.gz
# common_functions.sh - See http://www.zazzybob.com/bin/tars/common_functions.sh.tar.gz
# Server is the remote server to connect to - i.e. the BIGIP
SERVER="192.168.0.1"
REMOTE_USER="root"
AWK="/usr/bin/awk"
BASENAME="/bin/basename"
BC="/usr/bin/bc"
BIGPIPE="/bin/bigpipe"
CAT="/usr/bin/cat"
CURL="/usr/bin/curl"
CUT="/usr/bin/cut"
ECHO="/bin/echo"
EGREP="/bin/egrep"
GREP="/bin/grep"
HOST="/usr/bin/host"
PRINTF="/usr/bin/printf"
RM="/usr/bin/rm"
SED="/usr/bin/sed"
SSH="/usr/bin/ssh"
WC="/usr/bin/wc"
# Dependencies
COMMON_FUNCTIONS="./common_functions.sh"
SERVICES="/etc/services"
TMP_ROOT="/tmp"
TMP_FILE="${TMP_ROOT}/poolstats.sh.$$"
TMP_FILE_RATIO="${TMP_ROOT}/poolstats.sh.ratio.$$"
THIS_PROG=$( ${BASENAME} $0 )
SUCCESS=0
ERROR=1
DEBUG=0
NAGIOS=0
URL=""
RATIO=0
RESOLVE=0
VERBOSE=0
DOWN_FAIL_RATIO=0
HTTP_FAIL_RATIO=0
function cleanup {
${RM} -f ${TMP_FILE} ${TMP_FILE_RATIO}
}
trap "cleanup; exit ${ERROR}" 1 2 3 15
function usage {
{
${ECHO} "Usage: ${THIS_PROG} [-dnrv] -p <pool> [-u <url>]"
${ECHO} " -d - Turn some debugging output on"
${ECHO} " -n - Format output for use with check_pool.sh nagios plugin"
${ECHO} " -p - Specify pool to check"
${ECHO} " -u - Check HTTP status code for <url>"
${ECHO} " -r - Resolve hostnames"
${ECHO} " -R - Perform checks on a ratio based pool (Not Tested)"
${ECHO} " -v - Print verbose output"
} >&2
}
function check_args {
ERROR_COUNT=0
if [ -z ${POOL_NAME} ]; then
print_error "No pool specified"
(( ERROR_COUNT = ERROR_COUNT + 1 ))
fi
if [ "${ERROR_COUNT}" -gt "0" ]; then
exit ${ERROR}
fi
}
function check_pool_exists {
OUTPUT=$( ${SSH} ${REMOTE_USER}@${SERVER} "${BIGPIPE} pool ${POOL_NAME} list 2>&1" )
(( DEBUG )) && {
print_debug "--[ check_pool_exists ]--"
print_debug "${OUTPUT}"
}
# save output incase we want to check ratios
RATIO_TMP="${OUTPUT}"
${ECHO} "${OUTPUT}" | ${GREP} "The requested pool.*was not found" >/dev/null 2>&1
if [ "$?" -eq "0" ]; then
print_error "pool ${POOL_NAME} does not exist"
exit ${ERROR}
fi
}
function print_header {
(( VERBOSE )) && {
if [ -n "${URL}" ]; then
${PRINTF} "%16s\t%16s\t%16s\t%16s\t%s\n\n" "POOL" "MEMBER" "PORT" "STATUS" "HTTP STATUS"
else
${PRINTF} "%16s\t%16s\t%16s\t%s\n\n" "POOL" "MEMBER" "PORT" "STATUS"
fi
}
}
function parse_pool_status {
SUMMARY=$( ${ECHO} "${POOL_STATUS}" | ${SED} -n 's/^.*POOL MEMBER \([^ ]*\)[ ][ ]*\([^ ]*\).*$/\1|\2/p' )
(( RATIO )) && {
NUM_MEMBERS=$( ${ECHO} "${RATIO_TMP}" | ${GREP} "^[ ]*member.*$" | ${WC} -l )
NUM_RATIO_MEMBERS=$( ${ECHO} "${RATIO_TMP}" | ${GREP} "^[ ]*member.*ratio [0-9]\{1,\}$" | ${WC} -l )
if [ "${NUM_MEMBERS}" -ne "${NUM_RATIO_MEMBERS}" ]; then
print_error "Cannot use -R when pool does not use ratios"
exit ${ERROR}
fi
RATIO_TOTAL=$( ${ECHO} "${RATIO_TMP}" | ${GREP} "^[ ]*member.*ratio.*$" | ${AWK} '{ ratio += $NF } END{ print ratio }' )
}
MEMBER_COUNT=0
DOWN_COUNT=0
HTTP_FAIL_COUNT=0
${ECHO} "${SUMMARY}" | while IFS="|"; read MEMBER; do
set -- ${MEMBER}
S_POOL=$( ${ECHO} $1 | ${CUT} -d'/' -f1 )
S_TMP=$( ${ECHO} $1 | ${CUT} -d'/' -f2 )
S_MEMBER_IP=$( ${ECHO} ${S_TMP} | ${CUT} -d':' -f1 )
S_PORT=$( ${ECHO} ${S_TMP} | ${CUT} -d':' -f2 )
NUM_PORT=$( ${GREP} "${S_PORT}" "${SERVICES}" | ${GREP} "/tcp" | ${AWK} '{print $2}' | ${CUT} -d'/' -f1 | ${SED} -n '1p' )
(( RATIO )) && {
S_RATIO=$( ${ECHO} "${RATIO_TMP}" | ${GREP} "^[ ]*member ${S_MEMBER_IP}:.*ratio" | ${SED} 's/^.*ratio \([0-9]\{1,\}\)$/\1/' )
}
if [ -n "${URL}" ]; then
(( DEBUG )) && {
print_debug "--[ parse_pool_status: url_checks ]--"
print_debug "${URL}"
}
${ECHO} "${S_PORT}" | ${EGREP} "^(http|webcache)$" >/dev/null 2>&1
if [ "$?" -ne "0" ]; then
print_error "Port ${S_PORT} not supported for HTTP checks"
exit ${ERROR}
fi
URL_PART_1=$( ${ECHO} "${URL}" | ${SED} -n "s=http://\([^/]*\)/.*$=\1=p" )
URL_PART_2=$( ${ECHO} "${URL}" | ${SED} -n "s=http://[^/]*\(/.*\)$=\1=p" )
S_HTTP_RESULT=$( ${CURL} -o /dev/null -w %{http_code} -s -H "Host: ${URL_PART_1}" http://${S_MEMBER_IP}:${NUM_PORT}${URL_PART_2} )
if [ ${S_HTTP_RESULT} = "000" ]; then
S_HTTP_RESULT="DOWN"
fi
if [ ${S_HTTP_RESULT} != "200" ]; then
(( HTTP_FAIL_COUNT = HTTP_FAIL_COUNT + 1 ))
(( RATIO )) && {
HTTP_FAIL_TMP=$( ${ECHO} "scale=2; ( ${S_RATIO} / ${RATIO_TOTAL} ) * 100" | ${BC} | ${SED} 's/\..*$//' )
(( HTTP_FAIL_RATIO = HTTP_FAIL_RATIO + HTTP_FAIL_TMP ))
}
fi
fi
(( DEBUG )) && {
print_debug "--[ parse_pool_status: S_* vars ]--"
for VAR in S_POOL S_TMP S_MEMBER_IP S_PORT S_RATIO; do
print_debug "Variable: ${VAR} Value: $( eval ${ECHO} \$${VAR} )"
done
}
S_STATUS=$2
${ECHO} "${S_STATUS}" | ${GREP} "DOWN" >/dev/null 2>&1
if [ "$?" -eq "0" ]; then
(( DOWN_COUNT = DOWN_COUNT + 1 ))
(( RATIO )) && {
DOWN_FAIL_TMP=$( ${ECHO} "scale=2; ( ${S_RATIO} / ${RATIO_TOTAL} ) * 100" | ${BC} | ${SED} 's/\..*$//' )
(( DOWN_FAIL_RATIO = DOWN_FAIL_RATIO + DOWN_FAIL_TMP ))
}
fi
(( RESOLVE )) && {
RESOLVED_HOSTNAME=$( ${HOST} "${S_MEMBER_IP}" 2>&1 | ${AWK} '{print $NF}' )
${ECHO} "${OUTPUT}" | ${GREP} "3(NXDOMAIN)" >/dev/null 2>&1
if [ "$?" -ne "0" ]; then
(( DEBUG )) && {
print_debug "Resolved ${S_MEMBER_IP} to ${RESOLVED_HOSTNAME}"
}
${ECHO} "${RESOLVED_HOSTNAME}" | ${GREP} "\.$" >/dev/null 2>&1
if [ "$?" -eq "0" ]; then
RESOLVED_HOSTNAME=$( ${ECHO} "${RESOLVED_HOSTNAME}" | ${SED} 's/\.$//' )
fi
S_MEMBER_IP=${RESOLVED_HOSTNAME}
else
(( DEBUG )) && {
print_debug "Could not resolve ${S_MEMBER_IP} - continuing..."
}
fi
}
(( MEMBER_COUNT = MEMBER_COUNT + 1 ))
(( VERBOSE )) && {
if [ -n "${URL}" ]; then
${PRINTF} "%16s\t%16s\t%16s\t%16s\t%s\n" "${S_POOL}" "${S_MEMBER_IP}" "${NUM_PORT}" "${S_STATUS}" "${S_HTTP_RESULT}"
else
${PRINTF} "%16s\t%16s\t%16s\t%s\n" "${S_POOL}" "${S_MEMBER_IP}" "${NUM_PORT}" "${S_STATUS}"
fi
}
if [ -n "${URL}" ]; then
${ECHO} "${DOWN_COUNT}|${MEMBER_COUNT}|${HTTP_FAIL_COUNT}" > ${TMP_FILE}
else
${ECHO} "${DOWN_COUNT}|${MEMBER_COUNT}" > ${TMP_FILE}
fi
(( RATIO )) && {
if [ -n "${URL}" ]; then
${ECHO} "${DOWN_FAIL_RATIO}|${HTTP_FAIL_RATIO}" > ${TMP_FILE_RATIO}
else
${ECHO} "${DOWN_FAIL_RATIO}" > ${TMP_FILE_RATIO}
fi
}
done
(( ! VERBOSE || NAGIOS )) && {
while IFS="|"; read LINE; do
set -- ${LINE}
DOWN_COUNT=$1
MEMBER_COUNT=$2
BIGIP_FAIL_PERCENT=$( ${ECHO} "scale=2; ( ${DOWN_COUNT} / ${MEMBER_COUNT} ) * 100" | ${BC} | ${SED} 's/\..*$//' )
(( RATIO )) && {
if [ -z "${URL}" ]; then
DOWN_FAIL_RATIO=$( ${CAT} ${TMP_FILE_RATIO} )
else
DOWN_FAIL_RATIO=$( ${CUT} -d"|" -f1 ${TMP_FILE_RATIO} )
fi
}
(( NAGIOS )) && {
(( RATIO )) && {
(( DEBUG )) && {
print_debug "==[ DOWN_FAIL_RATIO ]==[ ${DOWN_FAIL_RATIO} ]=="
}
${ECHO} "${DOWN_COUNT}:${MEMBER_COUNT}:${DOWN_FAIL_RATIO}"
} || {
${ECHO} "${DOWN_COUNT}:${MEMBER_COUNT}:${BIGIP_FAIL_PERCENT}"
}
} || {
(( RATIO )) && {
${ECHO} "${DOWN_COUNT} down out of ${MEMBER_COUNT} in BIG-IP ( ${BIGIP_FAIL_PERCENT}% )"
} || {
${ECHO} "${DOWN_COUNT} down out of ${MEMBER_COUNT} in BIG-IP ( ${DOWN_FAIL_RATIO}% )"
}
}
if [ -n "${URL}" ]; then
HTTP_FAIL_COUNT=$3
HTTP_FAIL_PERCENT=$( ${ECHO} "scale=2; ( ${HTTP_FAIL_COUNT} / ${MEMBER_COUNT} ) * 100" | ${BC} | ${SED} 's/\..*$//' )
(( RATIO )) && {
HTTP_FAIL_RATIO=$( ${CUT} -d"|" -f2 ${TMP_FILE_RATIO} )
(( DEBUG )) && {
print_debug "==[ HTTP_FAIL_RATIO ]==[ ${HTTP_FAIL_RATIO} ]=="
}
}
(( NAGIOS )) && {
(( RATIO )) && {
${ECHO} "${HTTP_FAIL_COUNT}:${MEMBER_COUNT}:${HTTP_FAIL_RATIO}"
} || {
${ECHO} "${HTTP_FAIL_COUNT}:${MEMBER_COUNT}:${HTTP_FAIL_PERCENT}"
}
} || {
(( RATIO )) && {
${ECHO} "${HTTP_FAIL_COUNT} out of ${MEMBER_COUNT} returned non-200 HTTP status ( ${HTTP_FAIL_PERCENT}% )"
} || {
${ECHO} "${HTTP_FAIL_COUNT} out of ${MEMBER_COUNT} returned non-200 HTTP status ( ${HTTP_FAIL_RATIO}% )"
}
}
fi
done < ${TMP_FILE} # will only ever be a single line
}
}
function get_pool_status {
POOL_STATUS=$( ${SSH} ${REMOTE_USER}@${SERVER} "${BIGPIPE} pool ${POOL_NAME} show 2>&1" )
(( DEBUG )) && {
print_debug "--[ get_pool_status ]--"
print_debug "${POOL_STATUS}"
}
parse_pool_status
}
if [ "$#" -lt "2" ]; then
usage
exit ${ERROR}
fi
. ${COMMON_FUNCTIONS} 2>/dev/null || {
${ECHO} "Error: Could not load common functions"
exit 255
}
while getopts ":dnp:u:rRv" OPTION; do
case ${OPTION} in
"d" ) DEBUG=1
;;
"n" ) NAGIOS=1
;;
"u" ) URL="${OPTARG}"
;;
"p" ) POOL_NAME="${OPTARG}"
;;
"r" ) RESOLVE=1
;;
"R" ) RATIO=1
;;
"v" ) VERBOSE=1
;;
* ) usage
exit ${ERROR}
;;
esac
done
shift $(( ${OPTIND} - 1 ))
if [ "$#" -ne "0" ]; then
usage
exit "${ERROR}"
fi
check_args
check_pool_exists
print_header
get_pool_status
cleanup
exit "${SUCCESS}"