test/test_end_to_end.sh

   1 #!/bin/bash
   2 #Copyright (C) 2009-2010 :
   3 #    Gabes Jean, naparuba@gmail.com
   4 #    Gerhard Lausser, Gerhard.Lausser@consol.de
   5 #
   6 #This file is part of Shinken.
   7 #
   8 #Shinken is free software: you can redistribute it and/or modify
   9 #it under the terms of the GNU Affero General Public License as published by
  10 #the Free Software Foundation, either version 3 of the License, or
  11 #(at your option) any later version.
  12 #
  13 #Shinken is distributed in the hope that it will be useful,
  14 #but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 #MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 #GNU Affero General Public License for more details.
  17 #
  18 #You should have received a copy of the GNU Affero General Public License
  19 #along with Shinken.  If not, see <http://www.gnu.org/licenses/>.
  20
  21
  22 #set -x
  23
  24 echo "Begining test END TO END"
  25
  26
  27 DIR=$(cd $(dirname "$0"); pwd)
  28 echo "Going to dir $DIR/.."
  29 cd $DIR/..
  30
  31
  32
  33
  34
  35 #check for a process existance with good number
  36 function check_process_nb {
  37     NB=`ps -fu shinken | grep python | grep -v grep | grep $1 | wc -l`
  38     if [ $NB != "$2" ]
  39     then
  40         echo "Error : There is not enouth $1 launched (only $NB)."
  41         exit 2
  42     else
  43         echo "Ok, got $NB $1"
  44     fi
  45 }
  46
  47 function is_file_present {
  48     if [ -f $1 ]
  49     then
  50         echo "File $1 is present."
  51     else
  52         echo "Error : File $1 is missing!"
  53         exit 2
  54     fi
  55 }
  56
  57 function string_in_file {
  58     grep "$1" $2
  59     if [ $? != 0 ]
  60     then
  61         echo "Error : the file $2 is missing string $1 !"
  62         exit 2
  63     else
  64         echo "The string $1 is in $2"
  65     fi
  66 }
  67
  68
  69 function print_date {
  70     date +%s
  71 }
  72
  73 function check_good_run {
  74     VAR="$1"
  75     echo "Check for $NB_SCHEDULERS Scheduler"
  76     check_process_nb scheduler $NB_SCHEDULERS
  77     is_file_present $VAR/schedulerd.pid
  78
  79     echo "Check for $NB_POLLERS pollers (1 master, 1 for multiporcess module (queue manager), 4 workers)"
  80     check_process_nb poller $NB_POLLERS
  81     is_file_present $VAR/pollerd.pid
  82
  83     echo "Check for $NB_REACTIONNERS reactionners (1 master, 1 for multiporcess module (queue manager) 1 worker)"
  84     check_process_nb reactionner $NB_REACTIONNERS
  85     is_file_present $VAR/reactionnerd.pid
  86
  87     echo "Check for $NB_BROKERS brokers (one master, one for livestatus.dat, one for log)"
  88     check_process_nb broker $NB_BROKERS
  89     is_file_present $VAR/brokerd.pid
  90
  91     echo "Check for $NB_ARBITERS arbiter"
  92     check_process_nb arbiter $NB_ARBITERS
  93     is_file_present $VAR/arbiterd.pid
  94
  95     echo "Now checking for good file prensence"
  96     ls var
  97     is_file_present $VAR/nagios.log
  98     string_in_file "Waiting for initial configuration" $VAR/nagios.log
  99     string_in_file "First scheduling" $VAR/nagios.log
 100     string_in_file "OK, all schedulers configurations are dispatched :)" $VAR/nagios.log
 101     string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
 102     string_in_file "OK, no more poller sent need" $VAR/nagios.log
 103     string_in_file "OK, no more broker sent need" $VAR/nagios.log
 104 }
 105
 106
 107 #Standard launch process packets
 108 NB_SCHEDULERS=1
 109 NB_POLLERS=6
 110 NB_REACTIONNERS=3
 111 NB_BROKERS=3
 112 NB_ARBITERS=1
 113
 114
 115
 116 echo "Clean old tests and kill remaining processes"
 117 ./clean.sh
 118
 119
 120 echo "####################################################################################"
 121 echo "#                                                                                  #"
 122 echo "#                           SIMPLE START                                           #"
 123 echo "#                                                                                  #"
 124 echo "####################################################################################"
 125
 126 echo "Now we can start some launch tests"
 127 bin/launch_all_debug.sh
 128
 129
 130 echo "Now checking for existing apps"
 131
 132 echo "we can sleep 5sec for conf dispatching and so good number of process"
 133 sleep 5
 134
 135 #Now check if the run looks good with var in the direct directory
 136 check_good_run var
 137
 138 echo "First launch check OK"
 139
 140 echo "####################################################################################"
 141 echo "Now check bad start"
 142
 143 for mod in arbiter scheduler broker poller reactionner
 144 do
 145         SHINKEN_DEFAULT_FILE=test/etc/bad_start/shinken_default ./bin/init.d/shinken -d start "$mod"
 146         rc=$?
 147         if [ $rc -eq 0 ];
 148         then
 149                 echo "ERROR: $mod should not have started twice but we got successfull returncode from it !"
 150                 exit 2
 151         fi
 152 done
 153
 154 echo "####################################################################################"
 155
 156 echo "Now we clean it and test an install"
 157 ./clean.sh
 158
 159 echo "####################################################################################"
 160 echo "#                                                                                  #"
 161 echo "#                           DUMMY INSTALL                                          #"
 162 echo "#                                                                                  #"
 163 echo "####################################################################################"
 164
 165 echo "Now installing the application in DUMMY mode"
 166 python setup.py install --root=/tmp/moncul --record=INSTALLED_FILES --install-scripts=/usr/bin
 167
 168 if [ $? != '0' ]
 169 then
 170     echo "Error : the dummy install failed."
 171     exit 2
 172 fi
 173 echo "Dummy install OK"
 174
 175 echo "I reclean all for a real install"
 176 ./clean.sh
 177
 178
 179 echo "####################################################################################"
 180 echo "#                                                                                  #"
 181 echo "#                           REAL INSTALL                                           #"
 182 echo "#                                                                                  #"
 183 echo "####################################################################################"
 184
 185 echo "Now a REAL install"
 186 sudo python setup.py install --install-scripts=/usr/bin
 187 if [ $? != '0' ]
 188 then
 189     echo "Error : the real install failed."
 190     exit 2
 191 fi
 192 echo "Real install OK"
 193
 194 #Useful to take it from setup_parameter? It's just for coding here
 195 ETC=/etc/shinken
 196 is_file_present $ETC/nagios.cfg
 197 is_file_present $ETC/shinken-specific.cfg
 198 string_in_file "servicegroups.cfg" $ETC/nagios.cfg
 199 is_file_present /usr/bin/shinken-arbiter
 200
 201 ps -fu shinken
 202
 203
 204 echo "Now we can test a real run guy"
 205 /etc/init.d/shinken-scheduler -d start
 206 /etc/init.d/shinken-poller -d start
 207 /etc/init.d/shinken-reactionner -d start
 208 /etc/init.d/shinken-broker -d start
 209 /etc/init.d/shinken-arbiter -d start
 210
 211 echo "We will sleep again 5sec so every one is quite stable...."
 212 sleep 5
 213 check_good_run /var/lib/shinken
 214
 215 sudo /etc/init.d/shinken-arbiter status
 216 sudo /etc/init.d/shinken-scheduler status
 217 sudo /etc/init.d/shinken-poller status
 218 sudo /etc/init.d/shinken-reactionner status
 219 sudo /etc/init.d/shinken-broker status
 220
 221 sudo /etc/init.d/shinken-arbiter stop
 222 sudo /etc/init.d/shinken-scheduler stop
 223 sudo /etc/init.d/shinken-poller stop
 224 sudo /etc/init.d/shinken-reactionner stop
 225 sudo /etc/init.d/shinken-broker stop
 226
 227 sleep 2
 228 ps -fu shinken
 229
 230 check_process_nb arbiter 0
 231 check_process_nb scheduler 0
 232 check_process_nb broker 0
 233 check_process_nb poller 0
 234 check_process_nb reactionner 0
 235
 236
 237
 238 echo "OK Great. Even the real launch test pass. Great. I can clean after me."
 239 ./clean.sh
 240
 241
 242
 243
 244 echo "####################################################################################"
 245 echo "#                                                                                  #"
 246 echo "#                                    HA launch                                     #"
 247 echo "#                                                                                  #"
 248 echo "####################################################################################"
 249
 250 echo "Now we can start some launch tests"
 251 test/bin/launch_all_debug2.sh
 252
 253
 254 echo "Now checking for existing apps"
 255
 256 echo "we can sleep 5sec for conf dispatching and so good number of process"
 257 sleep 5
 258
 259 #The number of process changed, we mush look for it
 260
 261
 262 #Standard launch process packets
 263 NB_SCHEDULERS=2
 264 #6 for stack 1, and 2 for 2 (not active, so no worker)
 265 NB_POLLERS=8
 266 #3 for stack1, 2 for stack2 (no worker from now)
 267 NB_REACTIONNERS=5
 268 #3 for stack 1, 1 for stack2 (no livesatus.dat nor log worker launch)
 269 NB_BROKERS=4
 270 #still 1
 271 NB_ARBITERS=1
 272
 273 #Now check if the run looks good with var in the direct directory
 274 check_good_run var
 275
 276 echo "All launch of HA daemons is OK"
 277
 278 #Now we kill and see if all is OK :)
 279 #We clean the log file
 280 #$VAR/nagios.log
 281
 282
 283 #We kill the most important thing first : the scheduler-Master
 284 bin/stop_scheduler.sh
 285
 286 #We sleep to be sruethe scheduler see us
 287 sleep 2
 288 NB_SCHEDULERS=1
 289 print_date
 290
 291 #First we look is the arbiter saw the scheduler as dead
 292 string_in_file "Warning : Scheduler scheduler-Master had the configuration 0 but is dead, I am not happy." $VAR/nagios.log
 293 #Then we look if the scheduler-spare got a conf from arbiter (here, view from the arbiter)
 294 string_in_file "Dispatch OK of for conf in scheduler scheduler-Spare" $VAR/nagios.log
 295
 296 #then is the broker know it and try to connect to the new scheduler-spare
 297 string_in_file "\[broker-Master\] Connexion OK to the scheduler scheduler-Spare" $VAR/nagios.log
 298
 299
 300 echo "Now stop the poller-Master"
 301 #Now we stop the poller. We will see the sapre take the job (we hope in fact :) )
 302 bin/stop_poller.sh
 303 #check_good_run var
 304 sleep 2
 305 print_date
 306
 307 #The master should be look dead
 308 string_in_file "Warning : The poller poller-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
 309 #The spare should got the conf
 310 string_in_file "\[All\] Dispatch OK of for configuration 0 to poller poller-Slave" $VAR/nagios.log
 311 #And he should got the scheduler link (the sapre one)
 312 string_in_file "\[poller-Slave\] Connexion OK with scheduler scheduler-Spare" $VAR/nagios.log
 313
 314
 315 echo "Now stop the reactionner"
 316 bin/stop_reactionner.sh
 317 #check_good_run var
 318 sleep 2
 319 print_date
 320
 321 #The master should be look dead
 322 string_in_file "\[All\] Warning : The reactionner reactionner-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
 323 #The spare should got the conf
 324 string_in_file "\[All\] Dispatch OK of for configuration 0 to reactionner reactionner-Spare" $VAR/nagios.log
 325 #And he should got the scheduler link (the sapre one)
 326 string_in_file "\[reactionner-Spare\] Connexion OK with scheduler scheduler-Spare" $VAR/nagios.log
 327
 328
 329 echo "Now we stop... the Broker!"
 330 bin/stop_broker.sh
 331 #check_good_run var
 332 sleep 2
 333 print_date
 334
 335 #The master should be look dead
 336 string_in_file "\[All\] Warning : The broker broker-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
 337 #The spare should got the conf
 338 string_in_file "\[All\] Dispatch OK of for configuration 0 to broker broker-Slave" $VAR/nagios.log
 339 #And he should got the scheduler link (the spare one)
 340 string_in_file "\[broker-Slave\] Connexion OK to the scheduler scheduler-Spare" $VAR/nagios.log
 341 #And to other satellites
 342 string_in_file "\[broker-Slave\] Connexion OK to the reactionner reactionner-Spare" $VAR/nagios.log
 343 string_in_file "\[broker-Slave\] Connexion problem to the poller poller-Master : connection failed" $VAR/nagios.log
 344 #And should have load the modules
 345 string_in_file "\[broker-Slave\] I correctly load the modules : \['Simple-log', 'Livestatus'\]" $VAR/nagios.log
 346
 347
 348 echo "Now we clean it"
 349 ./clean.sh
 350
 351
 352
 353 echo "####################################################################################"
 354 echo "#                                                                                  #"
 355 echo "#                          Load balancing launch                                   #"
 356 echo "#                                                                                  #"
 357 echo "####################################################################################"
 358
 359 echo "Now we can start some launch tests"
 360 test/bin/launch_all_debug3.sh
 361
 362
 363 echo "Now checking for existing apps"
 364
 365 echo "we can sleep 5sec for conf dispatching and so good number of process"
 366 sleep 5
 367
 368 #The number of process changed, we mush look for it
 369
 370
 371 #Standard launch process packets
 372 NB_SCHEDULERS=2
 373 #6 for stack 1, and 6 for stack 2
 374 NB_POLLERS=12
 375 #3 for stack1, same for stack 2
 376 NB_REACTIONNERS=6
 377 #3 for stack 1, 1 for stack2 (no livestatus nor log worker launch)
 378 NB_BROKERS=4
 379 #still 1
 380 NB_ARBITERS=1
 381
 382 #Now check if the run looks good with var in the direct directory
 383 check_good_run var
 384
 385 echo "All launch of LB daemons is OK"
 386
 387
 388 #Now look if it's also good in the log file too
 389 string_in_file "Dispatch OK of for conf in scheduler scheduler-Master-2" $VAR/nagios.log
 390 string_in_file "Dispatch OK of for conf in scheduler scheduler-Master-1" $VAR/nagios.log
 391 string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
 392 string_in_file "OK, no more poller sent need" $VAR/nagios.log
 393 string_in_file "OK, no more broker sent need" $VAR/nagios.log
 394
 395 echo "Now we clean it"
 396 ./clean.sh
 397
 398
 399 echo ""
 400 echo ""
 401 echo "All check are OK. Congrats! You can go take a Beer ;)"