#!/bin/sh
#
#   Support:      linux-ha-dev@lists.tummy.com
#   License:      GNU General Public License (GPL)
#
#	Basic tests of sanity for a newly-built version of
#	linux-HA software (heartbeat)
#
#	Conditions for running:
#
#	Heartbeat must be installed.
#
#	Must be root.
#
#	CANNOT have a real heartbeat configuration.
#
#	Must have networking configured with one working
#		network interface.
#
#	should not have $TESTIP below used for anything ;-)
#
#	should have multicast address $MCASTIP port 694
#		available
#		(you don't need a multicast capable router).
#
#
# Note: you might have to change TESTIP and MCASTIP
#
DEFAULTINTERFACE="" #  Used when all guesses fail (which they shouldnt)
TESTIP=10.253.252.251
MCASTIP=225.224.223.222
#
IFCONFIG="@IFCONFIG@ @IFCONFIG_A_OPT@"
HADIR=@sysconfdir@/ha.d
HBSCRIPT=@INITDIR@/heartbeat@INIT_EXT@
STONITH=@sbindir@/stonith
OCF_TESTER=@sbindir@/ocf-tester
LIBDIR=@libdir@/
HBLIB=@libdir@/heartbeat
SCRIPTDIR=@HA_NOARCHDATAHBDIR@
APPHBD=$HBLIB/apphbd
CLSTATUS=@bindir@/cl_status
APPHBTEST=$HBLIB/apphbtest
IPCTEST=$HBLIB/ipctest
MGMTDTEST=$SCRIPTDIR/mgmtdtest
LRMTEST=$SCRIPTDIR/lrmtest/LRMBasicSanityCheck
STONITHDTEST=$SCRIPTDIR/stonithdtest/STONITHDBasicSanityCheck
CRMTEST="@PYTHON@ $SCRIPTDIR/cts/CTSlab.py --bsc"
SNMPAGENTTEST=$SCRIPTDIR/SNMPAgentSanityCheck
BASE64_MD5_TEST=$HBLIB/base64_md5_test
MALLOC_CHECK_=2; export MALLOC_CHECK_
TESTPROG=@TEST@
#
IDENTSTRING="Linux-HA TEST configuration file - REMOVEME!!"
DUMMYNODE=ImAlwaysDead.com
LOCALNODE=`uname -n`
RSCDIR=$HADIR/resource.d
PATH=$PATH:/sbin:/usr/sbin
export PATH

# Some signals to trap and act on
SIGLIST="0 1 2 3 6 15"

errcount=0

# Make a safe place to store logs
maketempdir()
{
	i=1

	while [ $i -gt 0 ]; do
		tmp="/tmp/lha-dir-$$-$i"
		if (umask 077 && mkdir "$tmp"); then
			echo "$tmp"
			return 0
		fi
		i=$((i+1))
	done

	echo "Could not create tempoary directory to store logs" >& 2
	return 1
}

LOGDIR=`maketempdir` || exit 1
LOGFILE="$LOGDIR/log"
touch "$LOGFILE"

# CTSlab.py needs this otherwise it hangs.
# Presumably it tries to stat or open the log file after
# relinquishing root privileges. It would be nice to get to
# the bottom of this and make a cleaner fix.
chmod o+x "$LOGDIR"

cd $HADIR
ulimit -c unlimited


GetAllIFNames() {

    if [ "apple" = "@build_vendor@" ]; then
	# "flags=8863" signifies an active interface that supports tcp
	$IFCONFIG | grep '^[a-zA-Z]' | grep "flags=8863" | sed 's%:* .*%%'
    else
	$IFCONFIG | grep '^[a-zA-Z]' | sed 's%:* .*%%'
    fi
}

GuessIFname() {
  # Machines with multiple nics rarely got this right
  #  using ip and route should prove much more reliable
  #  for those that have it

  external_host=123.0.0.1
  if
    [ -x /sbin/ip ]
  then
    /sbin/ip r g $external_host | head -n 1 | awk '{print $5}'
  else
    route get $external_host | grep interface | awk '{print $2}'
  fi
}

INTERFACE=`GuessIFname`
if
  [ "x" = "x$INTERFACE" ]
then
  INTERFACE=`GetAllIFNames | grep -v '^lo' | head -n 1`
fi

if
  [ "x" = "x$INTERFACE" ]
then
  INTERFACE=$DEFAULTINTERFACE
fi

if
  [ "x" = "x$INTERFACE" ]
then
  echo "No valid interface found.  Please modify DEFAULTINTERFACE in this script and re-execute"
  exit 1
fi

echo "Using interface: $INTERFACE"
#
#	Is it safe to overwrite this config file?
#
CheckConfigFile() {
  if
    [ ! -f $1 ] 
  then
    return 0
  fi
  if
    grep "$IDENTSTRING" $1 >/dev/null 2>&1
  then
    return 0
  else
    return 1
  fi
}

GenerateAuthKeys() {
  if
    [ "$AuthTry" = "" ]
  then
    AuthTry=1
  else
    AuthTry=`expr "$AuthTry" + 1`
  fi
  cat <<-! >$1
	#	$IDENTSTRING
	#
	auth $AuthTry
	1 sha1 SuperSecretKey--SHHH!!!
	2 md5  Encript-string-for-md5!!
	3 crc
	!
  chmod 600 $1
}

GenerateHaCf() {
  cat <<-! >$1
	#	$IDENTSTRING
	logfile   /dev/null
	debugfile $LOGFILE
	keepalive 10ms
	debug 1
	deadtime 5
	initdead 5
	auto_failback on
	stonith_host * null $LOCALNODE $DUMMYNODE
	mcast $INTERFACE $MCASTIP 694 0 0
	#bcast $INTERFACE
        ping 127.0.0.1
        ping_group pingus_please 127.0.0.1
	node $LOCALNODE $DUMMYNODE
	use_logd off
	respawn @HA_CCMUSER@ $HBLIB/ccm
	respawn @HA_CCMUSER@ $HBLIB/ipfail
	apiauth stonithd uid=root
	# Eventually I'd like to be able to enable this and
	# have things work right...
	#respawn root $HBLIB/hbagent
	!
  chmod 644 $1
}

GenerateCrmdHaCf() {
  cat <<-! >$1
	#	$IDENTSTRING
	syslogmsgfmt true
	logfile   /dev/null
	debugfile $LOGFILE
	keepalive 10ms
	debug    1
	deadtime 5
	initdead 5
	mcast $INTERFACE $MCASTIP 694 0 0
	#bcast $INTERFACE
        ping 127.0.0.1
        ping_group pingus_please 127.0.0.1
	node $LOCALNODE $DUMMYNODE
	use_logd off
	crm yes
	apiauth cibmon   uid=@HA_CCMUSER@
	respawn @HA_CCMUSER@ $HBLIB/cibmon -d
	respawn root $HBLIB/pingd
	!
  chmod 644 $1
#	conn_logd_time 60
#	use_logd  true
#	respawn root $HBLIB/ha_logd -c $2
#  cat <<-! >$2
#	#	$IDENTSTRING
#	logfile   /dev/null
#	debugfile $LOGFILE
#	#logfacility
#	entity ha_logd
#	#useapphbd no
#	!
}
GenerateMgmtdCrmdHaCf() {
  cat <<-! >$1
	#	$IDENTSTRING
	logfile   /dev/null
	debugfile $LOGFILE
	keepalive 10ms
	debug    1
	deadtime 5
	initdead 5
	mcast $INTERFACE $MCASTIP 694 0 0
	#bcast $INTERFACE
        ping 127.0.0.1
        ping_group pingus_please 127.0.0.1
	node $LOCALNODE $DUMMYNODE
	use_logd off
	crm yes
	apiauth mgmtd   uid=root
	respawn root $HBLIB/mgmtd -t
	!
  chmod 644 $1
}

GenerateHaResources() {
  cat <<-! >$1
	#	$IDENTSTRING
	$DUMMYNODE IPaddr::$TESTIP/30
	!
  chmod 644 $1
}
CONFIGFILES="$HADIR/ha.cf $HADIR/authkeys $HADIR/haresources"
CONFIGFILES="$CONFIGFILES @HA_VARLIBDIR@/heartbeat/crm/cib.xml"
CONFIGFILES="$CONFIGFILES @HA_VARLIBDIR@/heartbeat/crm/cib.xml.last"
CACHEFILES="@HA_VARLIBDIR@/heartbeat/hostcache  @HA_VARLIBDIR@/heartbeat/delhostcache"
CACHEFILES="$CACHEFILES @HA_VARLIBDIR@/heartbeat/crm/cib.xml.sig"
CACHEFILES="$CACHEFILES @HA_VARLIBDIR@/heartbeat/crm/cib.xml.sig.last"

SetUpConfigFiles() {
  if
    HBStatus
  then
    echo "Should not run tests with heartbeat already running."
  fi
  SaveConfigFiles
  for j in $CONFIGFILES 
  do
    if
      CheckConfigFile $j
    then
      : OK
    else
      echo "OOPS! $j already exists!"
      echo "Real configuration already set up."
      echo "Sorry..."
      exit 1
    fi
  done

  GenerateAuthKeys $HADIR/authkeys
  GenerateHaCf $HADIR/ha.cf
  GenerateHaResources $HADIR/haresources
  rm -f $HADIR/core
}

RemoveConfigFiles() {
  for j in $CONFIGFILES
  do
    if
      CheckConfigFile $j
    then
      rm -f $j
    else
      echo "OOPS! Cannot remove real config file $j!"
    fi
  done

  for j in $CACHEFILES
  do 
	rm -f $j
  done
  RestoreConfigFiles
}

SaveLog() {
  SAVELOG=/tmp/linux-ha.testlog
  chmod a+r $LOGFILE
  mv $LOGFILE $SAVELOG
  rmdir "$LOGDIR"
  echo "$errcount errors. Log file is stored in $SAVELOG"
}

RemoveConfigFilesAndSaveLogAndExit() {
  trap "" $SIGLIST		# tidying: ignore other signals
  RemoveConfigFiles
  SaveLog
  exit
}

SaveConfigFiles() {
  cd $HADIR
  if
    [ ! -d .cfsave ]
  then
    mkdir .cfsave
  fi
  mv $CONFIGFILES .cfsave >/dev/null 2>&1
  mv $CACHEFILES .cfsave >/dev/null 2>&1
}

RestoreConfigFiles() {
 mv $HADIR/.cfsave/cib.xml*  @HA_VARLIBDIR@/heartbeat/crm/ >/dev/null 2>&1
  mv $HADIR/.cfsave/hostcache @HA_VARLIBDIR@/heartbeat/  >/dev/null 2>&1
  mv $HADIR/.cfsave/delhostcache @HA_VARLIBDIR@/heartbeat/ >/dev/null 2>&1
  mv $HADIR/.cfsave/* $HADIR >/dev/null 2>&1
}

HBStart() {
  echo "Starting heartbeat"
  $HBSCRIPT start
}

HBStop() {
  echo "Stopping heartbeat"
  $HBSCRIPT stop
}

HBReload() {
  echo "Reloading heartbeat"
  $HBSCRIPT reload >/dev/null 2>&1
  rc=$?
  sleep 5
  return $rc
}

HBStatus() {
  case `$HBSCRIPT status 2>&1` in
    *running*)	true;;
    *)		false;;
  esac
}

#
#	Search the log file for the given grep pattern
#
LookForString() {
  count=1
  while
    if
      grep -i "$1" $LOGFILE
    then
      return 0
    fi
   [ $count -lt 60 ]
  do
    count=`expr $count + 1`
    sleep 1
  done
  return 1
}

#	Check for the given count of the given string
#	Complain unless the right number are there.
CheckPat()
{
  count=`egrep -ic "$1" $LOGFILE`
  min=$2
  if
    [ $# -gt 2 ]
  then
    max=$3
  else
    max=$2
  fi
  if
    [ $count -lt $min -o $count -gt $max ]
  then
    echo "ERROR: Did not find [$2:$3] occurances of $1 in $LOGFILE `date`" 2>&1 | tee -a $LOGFILE
    echo "ERROR: Found $count instead." | tee -a $LOGFILE
    errcount=`expr $errcount + 1`
  fi
}

TestHeartbeat() {
  if
    HBStatus
  then
    echo "That's weird.  Heartbeat seems to be running..."
    HBStop
  fi
  if
    $CLSTATUS hbstatus >/dev/null 2>&1
  then
    echo "$CLSTATUS shows heartbeat running" | tee -a $LOGFILE
    errcount=`expr $errcount + 1`
  fi
   
  if
    HBStart
  then
    if
      HBStatus
    then
     : COOL!
    else
     echo "Heartbeat did not start." | tee -a $LOGFILE
     exit 1
    fi
  fi

 sleep 5

  j=0
  while
    [ "$j" -lt 30 ]
  do
    if
      $CLSTATUS hbstatus >/dev/null 2>&1
    then
       break
    else
      sleep 1
    fi
    j=`expr "$j" + 1`
  done

  if
    $CLSTATUS hbstatus >/dev/null 2>&1
  then
   : OK
  else
    echo "$CLSTATUS shows heartbeat not running ($?)" | tee -a $LOGFILE
    errcount=`expr $errcount + 1`
  fi
  for node in $LOCALNODE 127.0.0.1 pingus_please
  do
    if
      $CLSTATUS nodestatus $node >/dev/null 2>&1
    then
     : $node status is OK
    else
      echo "$CLSTATUS shows status of [$node] as dead ($?)" | tee -a $LOGFILE
      errcount=`expr $errcount + 1`
    fi
  done

  #
  # Heartbeat seems to be running...
  #

  if
    LookForString "node $DUMMYNODE.*is dead" >/dev/null
  then
    : OK
  else
    echo "Does not look like we noticed $DUMMYNODE was dead" | tee -a $LOGFILE
    errcount=`expr $errcount + 1`
  fi
  if
    $CLSTATUS nodestatus $DUMMYNODE >/dev/null 2>&1
  then
    echo "$CLSTATUS shows $DUMMYNODE status as alive(!)" | tee -a $LOGFILE
    errcount=`expr $errcount + 1`
  fi


  if
    LookForString "Resetting node $DUMMYNODE with" >/dev/null &&
    LookForString "node $DUMMYNODE now reset" >/dev/null
  then
    : OK
  else
    echo "Does not look like we STONITHed $DUMMYNODE" | tee -a $LOGFILE
    errcount=`expr $errcount + 1`
  fi
  
  if
    LookForString "IPaddr.*$TESTIP" >/dev/null
  then
    :  Looks good
  else
    echo "Does not look like we took over the IP address" | tee -a $LOGFILE
    errcount=`expr $errcount + 1`
  fi

  #	Wait until heartbeat thinks things are stable
  #	that is, not in "transition"
  for j in 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
  do
    if
      STAT=`$CLSTATUS rscstatus 2>/dev/null`
    then
      case "$STAT" in
        transition)
		sleep 1
		;;
        all)	break
		;;
        *)	echo "$CLSTATUS shows resource status as $STAT" | tee -a $LOGFILE
		break
		;;
      esac
    else
      echo "$CLSTATUS rscstatus failed [$?]" | tee -a $LOGFILE
      break
    fi
  done
  if
    LookForString ARP >/dev/null
  then
    :  Looks good
  else
    echo "Does not look like we ARPed the address" | tee -a $LOGFILE
    errcount=`expr $errcount + 1`
  fi
  if
    $RSCDIR/IPaddr $TESTIP status >/dev/null 2>&1 &&
    $RSCDIR/IPaddr $TESTIP monitor >/dev/null 2>&1
  then
    : COOL!
  else
    echo "Looks like monitor operation failed" | tee -a $LOGFILE
    errcount=`expr $errcount + 1`
  fi

  #
  #	Reload test -- ha.cf changed
  #
  echo "Performing ha.cf reload test" >> $LOGFILE
  touch $HADIR/ha.cf
  if
    HBReload
  then
    : OK! reload after touching ha.cf worked!
  else
    echo "Heartbeat reload operation returned $?" | tee -a $LOGFILE
    errcount=`expr $errcount + 1`
  fi
  if
    LookForString "restart exec" >/dev/null
  then
    :  Looks good
  else
    echo "Does not look like we did a restart exec." | tee -a $LOGFILE
    errcount=`expr $errcount + 1`
  fi
  if
    HBStatus
  then
    : "OK - reload didn't kill anything ;-)"
  fi
  #
  #	Reload test -- authkeys changed
  #
  echo "Performing authkeys reload test" >> $LOGFILE
  GenerateAuthKeys $HADIR/authkeys
  if
    HBReload
  then
    : OK! reload after changing authkeys worked!
  else
    echo "Heartbeat reload operation returned $?" | tee -a $LOGFILE
    errcount=`expr $errcount + 1`
  fi
  if
    CheckPat "Signalling.* to reread config files" 2 >/dev/null
  then
    : OK
  else
    echo "Heartbeat did not reread config files exactly twice" | tee -a $LOGFILE
    errcount=`expr $errcount + 1`
  fi
  sleep 2
  if
    CheckPat "restart exec" 1 >/dev/null
  then
    :  Looks good -- did not do another exec
  else
    echo "Looks like we did an extra exec" | tee -a $LOGFILE
    errcount=`expr $errcount + 1`
  fi
  if
    HBStatus
  then
    : "OK - reload didn't kill anything ;-)"
  fi
  
  echo "Stopping heartbeat." >> $LOGFILE
  if
    HBStop
  then
    : OK!
  else
    echo "Heartbeat stop operation returned $?" | tee -a $LOGFILE
    errcount=`expr $errcount + 1`
  fi
  if
    HBStatus
  then
    echo "Looks like heartbeat did not really stop." | tee -a $LOGFILE
    echo "You\'ll probably need to kill some processes yourself."
    errcount=`expr $errcount + 1`
  fi

  if
    $RSCDIR/IPaddr $TESTIP status >/dev/null 2>&1 &&
    $RSCDIR/IPaddr $TESTIP monitor >/dev/null 2>&1
  then
    echo "Looks like the test IP address is still live..."
    errcount=`expr $errcount + 1`
  fi

}

StonithCheck() {

  serrcount=0

  echo "Checking STONITH basic sanity." | tee -a $LOGFILE
  if
    $STONITH -h >/dev/null
  then
    : OK
  else
    echo "$STONITH -h failed" | tee -a $LOGFILE
    serrcount=`expr $serrcount + 1`
  fi
  wc=`$STONITH -h | wc -l`
  if
    [ $wc -lt 100 ]
  then
    echo "$STONITH -h help message is too short ($wc lines)" | tee -a $LOGFILE
    serrcount=`expr $serrcount + 1`
  fi
  if
    FOOBARLIST=`$STONITH -t null -p "foo bar" -l`
  then
    : FOOBARLIST OK
  else
    echo "$STONITH -t null list option failed" | tee -a $LOGFILE
    serrcount=`expr $serrcount + 1`
  fi
  if
    echo $FOOBARLIST | grep foo >/dev/null &&
    echo $FOOBARLIST | grep bar >/dev/null
  then
    : OK null list
  else
    echo "$STONITH -t null list option incorrect" | tee -a $LOGFILE
    serrcount=`expr $serrcount + 1`
  fi

  if
    RESETOUT=`$STONITH -t null -p "foo bar" foo 2>&1`
  then
    case $RESETOUT in
     *"Host null-reset: foo"*)	: NULL Stonith output OK;;
     *)				echo "NULL reset failed."
  				  serrcount=`expr $serrcount + 1`;;
    esac
  else
    echo "$STONITH -t null reset failed." | tee -a $LOGFILE
  fi
  errcount=`expr $errcount + $serrcount`
}


AppHBCheck() {
	CFFILE=/var/run/$$.apphbd.cf
	clientcount=5
	cat <<-! >$CFFILE
		realtime yes
		debug_level	1
		debugfile	$LOGFILE
		logfile		/dev/null
	!
	echo "Performing apphbd success case tests" | tee -a $LOGFILE
	if
          $APPHBD -s >/dev/null 2>&1
        then
          echo "That's odd, $APPHBD is already running."
	  killcount=2
	  $APPHBD -k >/dev/null 2>&1
        else
          killcount=1
        fi
	$APPHBD -c $CFFILE
	sleep 5
	$APPHBTEST -i 1000 -p $clientcount -n 5 >> $LOGFILE 2>&1 
	for pat in "apphb_client_register:" "type=setint" "info:.*apphb_client_remove:"
	do
	  CheckPat "$pat" $clientcount
	done
	CheckPat "failed to heartbeat|resumed heartbeats" 0
	
	echo "Performing apphbd failure case tests" | tee -a $LOGFILE
	$APPHBTEST -F -i 1000 -p 1 -n 5 >>$CFFILE 2>&1
	for pat in "'failtest'.* failed to heartbeat" "'failtest'.* resumed heartbeats"
	do
          CheckPat "$pat" 1 2
	done
	sleep 5
	CheckPat "WARN:.*hangup" 1
	$APPHBD -k $CFFILE
	CheckPat "info:.*apphbd.*stopped" $killcount
	if
          $APPHBD -s >/dev/null 2>&1
        then
          echo "ERROR: $APPHBD is still running!" | tee -a $LOGFILE
        fi
}

IPCtest() {
	echo "Starting IPC tests" | tee -a $LOGFILE
	$IPCTEST >>$LOGFILE 2>&1
	ret=$?
	if [ $ret -ne 0 ]; then
		errcount=`expr $errcount + $ret`
		return $ret
	fi
	### Might here try "transient-test.sh" test.
	errcount=`expr $errcount + $ret`
	return $ret
}

LRMTest() {
  	if
	  [ ! -f $LRMTEST ] 
	then
	  return 0
	fi
	echo "Starting LRM tests" | tee -a $LOGFILE
	$LRMTEST $HBLIB >> $LOGFILE 2>&1
	ret=$?
	errcount=`expr $errcount + $ret`
	if
	  [ $ret != 0 ] 
	then
	  echo "LRM tests failed."
	fi
}

StonithDaemonTest() {
	if 
		[ ! -f $STONITHDTEST ]
	then 
		return 1
	fi
	if 
		HBStart
	then
		:
	else
		echo "Cannot start heartbeat, so do not start stonith daemon test." | tee -a $LOGFILE
		return 1
	fi

  	i=0
	while
		[ "$i" -lt 10 ]
	do
		if
			$CLSTATUS hbstatus >/dev/null 2>&1
		then
			break
		else
			sleep 1
		fi
		i=`expr "$i" + 1`
	done

	if
		$CLSTATUS hbstatus >/dev/null 2>&1
	then
		:
	else
		echo "There are internal errors in heartbeat, so do not start stonith daemon test." | tee -a $LOGFILE
		return 1
	fi

	echo "starting STONITH Daemon tests" | tee -a $LOGFILE
	$STONITHDTEST >> $LOGFILE 2>&1
	ret=$?
	errcount=`expr $errcount + $ret`
	if 
	        [ $ret != 0 ]
	then 
		echo "STONITH Daemon tests failed."
	else 
	  	echo "STONITH Daemon tests passed."
	fi
	HBStop
}

TestCRM() {
	GenerateCrmdHaCf $HADIR/ha.cf $HADIR/ha_logd.cf
	echo "Starting CRM tests" | tee -a $LOGFILE

	$CRMTEST -L $LOGFILE 2> /dev/null
	ret=$?
	if [ $ret != 0 ]; then
	    errcount=`expr $errcount + 1`
	    echo "CRM tests failed (rc=$ret)." | tee -a $LOGFILE
	    return 1
	fi	
	echo "CRM tests passed."  | tee -a $LOGFILE
	return 0
}

TestMgmtd() {
  	if [ ! -f $MGMTDTEST ]; then
	    return 0
	fi
	GenerateMgmtdCrmdHaCf $HADIR/ha.cf $HADIR/ha_logd.cf
	echo "Starting Mgmtd tests" | tee -a $LOGFILE

	if 
		HBStart
	then
		:
	else
		echo "cannot start heartbeat, so do not start mgmt daemon test." | tee -a $LOGFILE
		return 1
	fi

 	i=0
	while
		[ "$i" -lt 10 ]
	do
		if
			$CLSTATUS hbstatus >/dev/null 2>&1
		then
			break
		else
			sleep 1
		fi
		i=`expr "$i" + 1`
	done

	if
		$CLSTATUS hbstatus >/dev/null 2>&1
	then
		:
	else
		echo "There are internal errors in heartbeat, so do not start mgmt daemon test." | tee -a $LOGFILE
		$CLSTATUS hbstatus
		return 1
	fi

	sleep 10
	$MGMTDTEST > $LOGFILE 2>&1
	ret=$?
	if [ $ret != 0 ]; then
	    errcount=`expr $errcount + 1`
	    echo "Mgmtd test failed." | tee -a  $LOGFILE
	else 
	    echo "Mgmtd test passed."
	fi

	HBStop

	
}

SNMPAgentTest() {
	if 
	    [ ! -f $SNMPAGENTTEST ]
	then 
	    return 0
	fi
	if 
	    HBStart
	then
	    sleep 6
	    echo "starting SNMP Agent tests" | tee -a $LOGFILE
	    $SNMPAGENTTEST >> $LOGFILE 2>&1
	    ret=$?
	    errcount=`expr $errcount + $ret`
	    if 
	        [ $ret != 0 ]
	    then 
	        echo "SNMP Agent tests failed."
	    else 
	    	echo "SNMP Agent tests pass."
	    fi
	    HBStop
	fi
}

Base64MD5Test() {
  	if
	  [ ! -x $BASE64_MD5_TEST ] 
	then
	  return 0
	fi
	echo "Starting base64 and md5 algorithm tests" | tee -a $LOGFILE
	$BASE64_MD5_TEST >> $LOGFILE 2>&1
	ret=$?
	errcount=`expr $errcount + $ret`
	if
	  [ $ret != 0 ] 
	then
	  echo "base64 and md5 algorithm tests failed."
        else
          echo "base64 and md5 algorithm tests succeeded."
	fi
}

TestRA() {
	if [ ! -x $OCF_TESTER ]; then
		return 0
	fi
        OCF_ROOT=@OCF_ROOT_DIR@
        export OCF_ROOT
	RADIR=@OCF_RA_DIR@/@HB_PKG@
	echo "Starting Resource Agent tests" | tee -a $LOGFILE

	echo "Testing RA: Dummy" | tee -a $LOGFILE
	$OCF_TESTER -n DemoDummy $RADIR/Dummy >>$LOGFILE 2>&1
	rc=$?
	if [ $rc -ne 0 ]; then
		echo "ERROR: Dummy RA failed" | tee -a $LOGFILE
		return
	fi

	echo "Testing RA: IPaddr" | tee -a $LOGFILE
	$OCF_TESTER -o ip=127.0.0.20 -o netmask=32 -n DemoIPaddr \
		$RADIR/IPaddr >>$LOGFILE 2>&1
	rc=$?
	if [ $rc -ne 0 ]; then
		echo "ERROR: IPaddr RA failed" | tee -a $LOGFILE
		return
	fi
	
	if [ -x @IP2UTIL@ ]; then
		echo "Testing RA: IPaddr2" | tee -a $LOGFILE
		$OCF_TESTER -o ip=127.0.0.20 -o netmask=32 -n DemoIPaddr2 \
			$RADIR/IPaddr2 >>$LOGFILE 2>&1
		rc=$?
		if [ $rc -ne 0 ]; then
			echo "ERROR: IPaddr2 RA failed" | tee -a $LOGFILE
			return
		fi
	fi

	if [ `uname -s` = 'Linux' ]; then
		MNT_DIR="$LOGDIR/mnt"
		mkdir "$MNT_DIR"
		echo "Testing RA: Filesystem" | tee -a $LOGFILE
		$OCF_TESTER -o device=/dev/null -o fstype=proc -o directory=$MNT_DIR \
			-n DemoFS $RADIR/Filesystem >>$LOGFILE 2>&1
		rc=$?
		rmdir $MNT_DIR
		if [ $rc -ne 0 ]; then
			echo "ERROR: Filesystem RA failed" | tee -a $LOGFILE
			return
		fi
	fi
	echo "RA tests PASSED" | tee -a $LOGFILE
}


#
#	Check our identity.
#	Set Up Config Files.
#	Run Tests.
#	Remove Config Files.
#

ID=`@WHOAMI@`
case $ID in
  root)	: OK;;
  *)	echo "Must be root to run this.  Sorry."
	exit 1;;
esac

touch $LOGFILE
chown @HA_CCMUSER@ $LOGFILE

trap 'RemoveConfigFilesAndSaveLogAndExit' $SIGLIST

SetUpConfigFiles
> $LOGFILE

COREPID=/proc/sys/kernel/core_uses_pid
if
  [ -f "$COREPID" ]
then
  echo 1 > $COREPID
fi

# Create temp rsc dir if it doesn't yet exist, RAs depend on it
HARSCTMP=@HA_VARRUNHBRSCDIR@
if
  [ ! -d "$HARSCTMP" ]
then
  mkdir $HARSCTMP
fi

Base64MD5Test
TestRA

IPCtest
ret=$?
if [ $ret -ne 0 ]; then
	# If IPC is faulty, then it is pointless to attempt dependent checks.
	echo "ERROR: IPCtest failed; continuing is pointless" | tee -a $LOGFILE
	exit $ret
fi
# Might want to add a longer, more substantial IPC test later.

TestHeartbeat
StonithCheck
AppHBCheck
LRMTest
StonithDaemonTest
#TestCRM
SNMPAgentTest
#TestMgmtd

ls -l $HADIR/core* 2>/dev/null
if
  [ $? = 0 ]
then
  errcount=`expr $errcount + 1`
  echo "OOPS! We generated a core file!"
  file $HADIR/core*
fi

if
  egrep  'CRIT|ERROR' $LOGFILE
then
  echo "OOPS! Looks like we had some errors come up."
  errcount=`expr $errcount + 1`
fi

exit $errcount
