test/test_end_to_end.sh

   1 #!/bin/bash
   2 #Copyright (C) 2009-2010 :
   3 #    Gabes Jean, naparuba@gmail.com
   4 #    Gerhard Lausser, Gerhard.Lausser@consol.de
   5 #
   6 #This file is part of Shinken.
   7 #
   8 #Shinken is free software: you can redistribute it and/or modify
   9 #it under the terms of the GNU Affero General Public License as published by
  10 #the Free Software Foundation, either version 3 of the License, or
  11 #(at your option) any later version.
  12 #
  13 #Shinken is distributed in the hope that it will be useful,
  14 #but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 #MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 #GNU Affero General Public License for more details.
  17 #
  18 #You should have received a copy of the GNU Affero General Public License
  19 #along with Shinken.  If not, see <http://www.gnu.org/licenses/>.
  20
  21
  22 #set -x
  23
  24 echo "Begining test END TO END"
  25
  26
  27 DIR=$(cd $(dirname "$0"); pwd)
  28 echo "Going to dir $DIR/.."
  29 cd $DIR/..
  30
  31
  32
  33
  34
  35 #check for a process existance with good number
  36 function check_process_nb {
  37     NB=`ps -fu shinken | grep python | grep -v grep | grep "shinken-"$1 | wc -l`
  38     if [ $NB != "$2" ]
  39     then
  40         echo "Error : There is not enouth $1 launched (only $NB)."
  41         exit 2
  42     else
  43         echo "Ok, got $NB $1"
  44     fi
  45 }
  46
  47 function is_file_present {
  48     if [ -f $1 ]
  49     then
  50         echo "File $1 is present."
  51     else
  52         echo "Error : File $1 is missing!"
  53         exit 2
  54     fi
  55 }
  56
  57 function string_in_file {
  58     grep "$1" $2
  59     if [ $? != 0 ]
  60     then
  61         echo "Error : the file $2 is missing string $1 !"
  62         exit 2
  63     else
  64         echo "The string $1 is in $2"
  65     fi
  66 }
  67
  68
  69 function print_date {
  70     date +%s
  71 }
  72
  73 function check_good_run {
  74     VAR="$1"
  75     echo "Check for $NB_SCHEDULERS Scheduler"
  76     check_process_nb scheduler $NB_SCHEDULERS
  77     is_file_present $VAR/schedulerd.pid
  78
  79     echo "Check for $NB_POLLERS pollers (1 master, 1 for multiporcess module (queue manager), 4 workers)"
  80     check_process_nb poller $NB_POLLERS
  81     is_file_present $VAR/pollerd.pid
  82
  83     echo "Check for $NB_REACTIONNERS reactionners (1 master, 1 for multiporcess module (queue manager) 1 worker)"
  84     check_process_nb reactionner $NB_REACTIONNERS
  85     is_file_present $VAR/reactionnerd.pid
  86
  87     echo "Check for $NB_BROKERS brokers (one master, one for livestatus.dat, one for log)"
  88     check_process_nb broker $NB_BROKERS
  89     is_file_present $VAR/brokerd.pid
  90
  91     echo "Check for $NB_RECEIVERS receivers (one master)"
  92     check_process_nb receiver $NB_RECEIVERS
  93     is_file_present $VAR/receiverd.pid
  94
  95
  96     echo "Check for $NB_ARBITERS arbiter"
  97     check_process_nb arbiter $NB_ARBITERS
  98     is_file_present $VAR/arbiterd.pid
  99
 100     echo "Now checking for good file prensence"
 101     ls var
 102     is_file_present $VAR/nagios.log
 103     string_in_file "Waiting for initial configuration" $VAR/nagios.log
 104     string_in_file "First scheduling" $VAR/nagios.log
 105     string_in_file "OK, all schedulers configurations are dispatched :)" $VAR/nagios.log
 106     string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
 107     string_in_file "OK, no more poller sent need" $VAR/nagios.log
 108     string_in_file "OK, no more broker sent need" $VAR/nagios.log
 109 }
 110
 111 function localize_config {
 112     # change paths in config files (/usr/local/shinken/*) to
 113     # relative paths, so this test runs only in the current directory.
 114     # takes nagios.cfg and shinken-specific.cfg
 115     cp $1 /tmp/nagios.cfg.save
 116     cp $2 /tmp/shinken-specific.cfg.save
 117     sed -e 's/\/usr\/local\/shinken\///g' < /tmp/nagios.cfg.save > $1
 118     sed -e 's/\/usr\/local\/shinken\/var\///g' < /tmp/shinken-specific.cfg.save > $2
 119 }
 120
 121 function globalize_config {
 122     mv /tmp/nagios.cfg.save $1
 123     mv /tmp/shinken-specific.cfg.save $2
 124 }
 125
 126
 127
 128 #Standard launch process packets
 129 NB_SCHEDULERS=1
 130 NB_POLLERS=6
 131 NB_REACTIONNERS=3
 132 NB_BROKERS=3
 133 NB_RECEIVERS=1
 134 NB_ARBITERS=1
 135
 136
 137
 138 echo "Clean old tests and kill remaining processes"
 139 ./clean.sh
 140
 141
 142 echo "####################################################################################"
 143 echo "#                                                                                  #"
 144 echo "#                           SIMPLE START                                           #"
 145 echo "#                                                                                  #"
 146 echo "####################################################################################"
 147
 148 echo "Now we can start some launch tests"
 149 localize_config etc/nagios.cfg etc/shinken-specific.cfg
 150 bin/launch_all_debug.sh
 151 globalize_config etc/nagios.cfg etc/shinken-specific.cfg
 152
 153
 154 echo "Now checking for existing apps"
 155
 156 echo "we can sleep 5sec for conf dispatching and so good number of process"
 157 sleep 5
 158
 159 #Now check if the run looks good with var in the direct directory
 160 check_good_run var
 161
 162 echo "First launch check OK"
 163
 164
 165 echo "####################################################################################"
 166
 167 echo "Now we clean it and test an install"
 168 ./clean.sh
 169
 170 echo "####################################################################################"
 171 echo "#                                                                                  #"
 172 echo "#                           DUMMY INSTALL                                          #"
 173 echo "#                                                                                  #"
 174 echo "####################################################################################"
 175
 176 echo "Now installing the application in DUMMY mode"
 177 python setup.py install --root=/tmp/moncul --record=INSTALLED_FILES --install-scripts=/usr/bin
 178
 179 if [ $? != '0' ]
 180 then
 181     echo "Error : the dummy install failed."
 182     exit 2
 183 fi
 184 echo "Dummy install OK"
 185
 186 echo "I reclean all for a real install"
 187 ./clean.sh
 188
 189
 190 echo "####################################################################################"
 191 echo "#                                                                                  #"
 192 echo "#                           REAL INSTALL                                           #"
 193 echo "#                                                                                  #"
 194 echo "####################################################################################"
 195
 196 echo "Now a REAL install"
 197 sudo python setup.py install --install-scripts=/usr/bin
 198 if [ $? != '0' ]
 199 then
 200     echo "Error : the real install failed."
 201     exit 2
 202 fi
 203 echo "Real install OK"
 204
 205 #Useful to take it from setup_parameter? It's just for coding here
 206 ETC=/etc/shinken
 207 is_file_present $ETC/nagios.cfg
 208 is_file_present $ETC/shinken-specific.cfg
 209 string_in_file "servicegroups.cfg" $ETC/nagios.cfg
 210 is_file_present /usr/bin/shinken-arbiter
 211
 212 ps -fu shinken
 213
 214
 215 echo "Now we can test a real run guy"
 216 /etc/init.d/shinken-scheduler -d start
 217 /etc/init.d/shinken-poller -d start
 218 /etc/init.d/shinken-reactionner -d start
 219 /etc/init.d/shinken-broker -d start
 220 /etc/init.d/shinken-receiver -d start
 221 /etc/init.d/shinken-arbiter -d start
 222
 223 echo "We will sleep again 5sec so every one is quite stable...."
 224 sleep 5
 225 check_good_run /var/lib/shinken
 226
 227 sudo /etc/init.d/shinken-arbiter status
 228 sudo /etc/init.d/shinken-scheduler status
 229 sudo /etc/init.d/shinken-poller status
 230 sudo /etc/init.d/shinken-reactionner status
 231 sudo /etc/init.d/shinken-broker status
 232 sudo /etc/init.d/shinken-receiver status
 233
 234 sudo /etc/init.d/shinken-arbiter stop
 235 sudo /etc/init.d/shinken-scheduler stop
 236 sudo /etc/init.d/shinken-poller stop
 237 sudo /etc/init.d/shinken-reactionner stop
 238 sudo /etc/init.d/shinken-broker stop
 239 sudo /etc/init.d/shinken-receiver stop
 240
 241 sleep 2
 242 ps -fu shinken
 243
 244 check_process_nb arbiter 0
 245 check_process_nb scheduler 0
 246 check_process_nb broker 0
 247 check_process_nb receiver 0
 248 check_process_nb poller 0
 249 check_process_nb reactionner 0
 250
 251
 252
 253 echo "OK Great. Even the real launch test pass. Great. I can clean after me."
 254 ./clean.sh
 255
 256
 257
 258
 259 echo "####################################################################################"
 260 echo "#                                                                                  #"
 261 echo "#                                    HA launch                                     #"
 262 echo "#                                                                                  #"
 263 echo "####################################################################################"
 264
 265 echo "Now we can start some launch tests"
 266 localize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-ha-only.cfg
 267 test/bin/launch_all_debug2.sh
 268 globalize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-ha-only.cfg
 269
 270
 271 echo "Now checking for existing apps"
 272
 273 echo "we can sleep 5sec for conf dispatching and so good number of process"
 274 sleep 5
 275
 276 #The number of process changed, we mush look for it
 277
 278
 279 #Standard launch process packets
 280 NB_SCHEDULERS=2
 281 #6 for stack 1, and 2 for 2 (not active, so no worker)
 282 NB_POLLERS=8
 283 #3 for stack1, 2 for stack2 (no worker from now)
 284 NB_REACTIONNERS=5
 285 #3 for stack 1, 1 for stack2 (no livesatus.dat nor log worker launch)
 286 NB_BROKERS=4
 287 #Still 1 receiver
 288 NB_RECEIVERS=1
 289 #still 1
 290 NB_ARBITERS=1
 291
 292 #Now check if the run looks good with var in the direct directory
 293 check_good_run var
 294
 295 echo "All launch of HA daemons is OK"
 296
 297 #Now we kill and see if all is OK :)
 298 #We clean the log file
 299 #$VAR/nagios.log
 300
 301
 302 #We kill the most important thing first : the scheduler-Master
 303 bin/stop_scheduler.sh
 304
 305 #We sleep to be sruethe scheduler see us
 306 sleep 2
 307 NB_SCHEDULERS=1
 308 print_date
 309
 310 #First we look is the arbiter saw the scheduler as dead
 311 string_in_file "Warning : Scheduler scheduler-Master had the configuration 0 but is dead, I am not happy." $VAR/nagios.log
 312 #Then we look if the scheduler-spare got a conf from arbiter (here, view from the arbiter)
 313 string_in_file "Dispatch OK of for conf in scheduler scheduler-Spare" $VAR/nagios.log
 314
 315 #then is the broker know it and try to connect to the new scheduler-spare
 316 string_in_file "\[broker-Master\] Connexion OK to the scheduler scheduler-Spare" $VAR/nagios.log
 317
 318
 319 echo "Now stop the poller-Master"
 320 #Now we stop the poller. We will see the sapre take the job (we hope in fact :) )
 321 bin/stop_poller.sh
 322 #check_good_run var
 323 sleep 2
 324 print_date
 325
 326 #The master should be look dead
 327 string_in_file "Warning : The poller poller-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
 328 #The spare should got the conf
 329 string_in_file "\[All\] Dispatch OK of for configuration 0 to poller poller-Slave" $VAR/nagios.log
 330 #And he should got the scheduler link (the sapre one)
 331 string_in_file "\[poller-Slave\] Connexion OK with scheduler scheduler-Spare" $VAR/nagios.log
 332
 333
 334 echo "Now stop the reactionner"
 335 bin/stop_reactionner.sh
 336 #check_good_run var
 337 sleep 2
 338 print_date
 339
 340 #The master should be look dead
 341 string_in_file "\[All\] Warning : The reactionner reactionner-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
 342 #The spare should got the conf
 343 string_in_file "\[All\] Dispatch OK of for configuration 0 to reactionner reactionner-Spare" $VAR/nagios.log
 344 #And he should got the scheduler link (the sapre one)
 345 string_in_file "\[reactionner-Spare\] Connexion OK with scheduler scheduler-Spare" $VAR/nagios.log
 346
 347
 348 echo "Now we stop... the Broker!"
 349 bin/stop_broker.sh
 350 #check_good_run var
 351 sleep 2
 352 print_date
 353
 354 #The master should be look dead
 355 string_in_file "\[All\] Warning : The broker broker-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
 356 #The spare should got the conf
 357 string_in_file "\[All\] Dispatch OK of for configuration 0 to broker broker-Slave" $VAR/nagios.log
 358 #And he should got the scheduler link (the spare one)
 359 string_in_file "\[broker-Slave\] Connexion OK to the scheduler scheduler-Spare" $VAR/nagios.log
 360 #And to other satellites
 361 string_in_file "\[broker-Slave\] Connexion OK to the reactionner reactionner-Spare" $VAR/nagios.log
 362 string_in_file "\[broker-Slave\] Connexion problem to the poller poller-Master : connection failed" $VAR/nagios.log
 363 #And should have load the modules
 364 string_in_file "\[broker-Slave\] I correctly loaded the modules : \['Simple-log', 'Livestatus'\]" $VAR/nagios.log
 365
 366
 367 echo "Now we clean it"
 368 ./clean.sh
 369
 370
 371
 372 echo "####################################################################################"
 373 echo "#                                                                                  #"
 374 echo "#                          Load balancing launch                                   #"
 375 echo "#                                                                                  #"
 376 echo "####################################################################################"
 377
 378 echo "Now we can start some launch tests"
 379 localize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-lb-only.cfg
 380 test/bin/launch_all_debug3.sh
 381 globalize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-lb-only.cfg
 382
 383
 384 echo "Now checking for existing apps"
 385
 386 echo "we can sleep 5sec for conf dispatching and so good number of process"
 387 sleep 5
 388
 389 #The number of process changed, we mush look for it
 390
 391
 392 #Standard launch process packets
 393 NB_SCHEDULERS=2
 394 #6 for stack 1, and 6 for stack 2
 395 NB_POLLERS=12
 396 #3 for stack1, same for stack 2
 397 NB_REACTIONNERS=6
 398 #3 for stack 1, 1 for stack2 (no livestatus nor log worker launch)
 399 NB_BROKERS=4
 400 #STill one receivers
 401 NB_RECEIVERS=1
 402 #still 1
 403 NB_ARBITERS=1
 404
 405 #Now check if the run looks good with var in the direct directory
 406 check_good_run var
 407
 408 echo "All launch of LB daemons is OK"
 409
 410
 411 #Now look if it's also good in the log file too
 412 string_in_file "Dispatch OK of for conf in scheduler scheduler-Master-2" $VAR/nagios.log
 413 string_in_file "Dispatch OK of for conf in scheduler scheduler-Master-1" $VAR/nagios.log
 414 string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
 415 string_in_file "OK, no more poller sent need" $VAR/nagios.log
 416 string_in_file "OK, no more broker sent need" $VAR/nagios.log
 417
 418 echo "Now we clean it"
 419 ./clean.sh
 420
 421
 422
 423 echo "####################################################################################"
 424 echo "#                                                                                  #"
 425 echo "#                              Passive Poller                                      #"
 426 echo "#                                                                                  #"
 427 echo "####################################################################################"
 428
 429 echo "Now we can start some launch tests"
 430 localize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-passive-poller.cfg
 431 test/bin/launch_all_debug4.sh
 432 globalize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-passive-poller.cfg
 433
 434
 435 echo "Now checking for existing apps"
 436
 437 echo "we can sleep 5sec for conf dispatching and so good number of process"
 438 sleep 5
 439
 440 #The number of process changed, we mush look for it
 441
 442
 443 #Standard launch process packets
 444 NB_SCHEDULERS=2
 445 #6 for stack 1, and 6 for stack 2
 446 NB_POLLERS=12
 447 #3 for stack1, Only 2 for stack 2 because it is not active
 448 NB_REACTIONNERS=5
 449 #3 for stack 1, 1 for stack2 (no livestatus nor log worker launch)
 450 NB_BROKERS=4
 451 #Still oen receiver
 452 NB_RECEIVERS=1
 453 #still 1
 454 NB_ARBITERS=1
 455
 456 #Now check if the run looks good with var in the direct directory
 457 check_good_run var
 458
 459 echo "All launch of LB daemons is OK"
 460
 461
 462 #Now look if it's also good in the log file too
 463 string_in_file "Dispatch OK of for conf in scheduler scheduler-Master-2" $VAR/nagios.log
 464 string_in_file "Dispatch OK of for conf in scheduler scheduler-Master-1" $VAR/nagios.log
 465 string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
 466 string_in_file "OK, no more poller sent need" $VAR/nagios.log
 467 string_in_file "OK, no more broker sent need" $VAR/nagios.log
 468 # We should see the poller 2 say it is passive
 469 string_in_file "\[poller-Master-2\] Passive mode enabled." $VAR/nagios.log
 470 # and the schedulers should connect to it too
 471 string_in_file "Connexion OK to the poller poller-Master-2" $VAR/nagios.log
 472
 473
 474 echo "Now we clean it"
 475 #./clean.sh
 476
 477
 478 echo ""
 479 echo ""
 480 echo "All check are OK. Congrats! You can go take a Beer ;)"