test/test_end_to_end.sh

   1 #!/bin/bash
   2 #Copyright (C) 2009-2010 :
   3 #    Gabes Jean, naparuba@gmail.com
   4 #    Gerhard Lausser, Gerhard.Lausser@consol.de
   5 #
   6 #This file is part of Shinken.
   7 #
   8 #Shinken is free software: you can redistribute it and/or modify
   9 #it under the terms of the GNU Affero General Public License as published by
  10 #the Free Software Foundation, either version 3 of the License, or
  11 #(at your option) any later version.
  12 #
  13 #Shinken is distributed in the hope that it will be useful,
  14 #but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 #MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 #GNU Affero General Public License for more details.
  17 #
  18 #You should have received a copy of the GNU Affero General Public License
  19 #along with Shinken.  If not, see <http://www.gnu.org/licenses/>.
  20
  21
  22 #set -x
  23
  24 echo "Begining test END TO END"
  25
  26
  27 DIR=$(cd $(dirname "$0"); pwd)
  28 echo "Going to dir $DIR/.."
  29 cd $DIR/..
  30
  31
  32
  33
  34
  35 #check for a process existance with good number
  36 function check_process_nb {
  37     NB=`ps -fu shinken | grep python | grep -v grep | grep "shinken-"$1 | wc -l`
  38     if [ $NB != "$2" ]
  39     then
  40         echo "Error : There is not enouth $1 launched (only $NB)."
  41         exit 2
  42     else
  43         echo "Ok, got $NB $1"
  44     fi
  45 }
  46
  47 function is_file_present {
  48     if [ -f $1 ]
  49     then
  50         echo "File $1 is present."
  51     else
  52         echo "Error : File $1 is missing!"
  53         exit 2
  54     fi
  55 }
  56
  57 function string_in_file {
  58     grep "$1" $2
  59     if [ $? != 0 ]
  60     then
  61         echo "Error : the file $2 is missing string $1 !"
  62         exit 2
  63     else
  64         echo "The string $1 is in $2"
  65     fi
  66 }
  67
  68
  69 function print_date {
  70     date +%s
  71 }
  72
  73 function check_good_run {
  74     VAR="$1"
  75     echo "Check for $NB_SCHEDULERS Scheduler"
  76     check_process_nb scheduler $NB_SCHEDULERS
  77     is_file_present $VAR/schedulerd.pid
  78
  79     echo "Check for $NB_POLLERS pollers (1 master, 1 for multiporcess module (queue manager), 4 workers)"
  80     check_process_nb poller $NB_POLLERS
  81     is_file_present $VAR/pollerd.pid
  82
  83     echo "Check for $NB_REACTIONNERS reactionners (1 master, 1 for multiporcess module (queue manager) 1 worker)"
  84     check_process_nb reactionner $NB_REACTIONNERS
  85     is_file_present $VAR/reactionnerd.pid
  86
  87     echo "Check for $NB_BROKERS brokers (one master, one for livestatus.dat, one for log)"
  88     check_process_nb broker $NB_BROKERS
  89     is_file_present $VAR/brokerd.pid
  90
  91     echo "Check for $NB_ARBITERS arbiter"
  92     check_process_nb arbiter $NB_ARBITERS
  93     is_file_present $VAR/arbiterd.pid
  94
  95     echo "Now checking for good file prensence"
  96     ls var
  97     is_file_present $VAR/nagios.log
  98     string_in_file "Waiting for initial configuration" $VAR/nagios.log
  99     string_in_file "First scheduling" $VAR/nagios.log
 100     string_in_file "OK, all schedulers configurations are dispatched :)" $VAR/nagios.log
 101     string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
 102     string_in_file "OK, no more poller sent need" $VAR/nagios.log
 103     string_in_file "OK, no more broker sent need" $VAR/nagios.log
 104 }
 105
 106 function localize_config {
 107     # change paths in config files (/usr/local/shinken/*) to
 108     # relative paths, so this test runs only in the current directory.
 109     # takes nagios.cfg and shinken-specific.cfg
 110     cp $1 /tmp/nagios.cfg.save
 111     cp $2 /tmp/shinken-specific.cfg.save
 112     sed -e 's/\/usr\/local\/shinken\///g' < /tmp/nagios.cfg.save > $1
 113     sed -e 's/\/usr\/local\/shinken\/var\///g' < /tmp/shinken-specific.cfg.save > $2
 114 }
 115
 116 function globalize_config {
 117     mv /tmp/nagios.cfg.save $1
 118     mv /tmp/shinken-specific.cfg.save $2
 119 }
 120
 121
 122
 123 #Standard launch process packets
 124 NB_SCHEDULERS=1
 125 NB_POLLERS=6
 126 NB_REACTIONNERS=3
 127 NB_BROKERS=3
 128 NB_ARBITERS=1
 129
 130
 131
 132 echo "Clean old tests and kill remaining processes"
 133 ./clean.sh
 134
 135
 136 echo "####################################################################################"
 137 echo "#                                                                                  #"
 138 echo "#                           SIMPLE START                                           #"
 139 echo "#                                                                                  #"
 140 echo "####################################################################################"
 141
 142 echo "Now we can start some launch tests"
 143 localize_config etc/nagios.cfg etc/shinken-specific.cfg
 144 bin/launch_all_debug.sh
 145 globalize_config etc/nagios.cfg etc/shinken-specific.cfg
 146
 147
 148 echo "Now checking for existing apps"
 149
 150 echo "we can sleep 5sec for conf dispatching and so good number of process"
 151 sleep 5
 152
 153 #Now check if the run looks good with var in the direct directory
 154 check_good_run var
 155
 156 echo "First launch check OK"
 157
 158
 159 echo "####################################################################################"
 160
 161 echo "Now we clean it and test an install"
 162 ./clean.sh
 163
 164 echo "####################################################################################"
 165 echo "#                                                                                  #"
 166 echo "#                           DUMMY INSTALL                                          #"
 167 echo "#                                                                                  #"
 168 echo "####################################################################################"
 169
 170 echo "Now installing the application in DUMMY mode"
 171 python setup.py install --root=/tmp/moncul --record=INSTALLED_FILES --install-scripts=/usr/bin
 172
 173 if [ $? != '0' ]
 174 then
 175     echo "Error : the dummy install failed."
 176     exit 2
 177 fi
 178 echo "Dummy install OK"
 179
 180 echo "I reclean all for a real install"
 181 ./clean.sh
 182
 183
 184 echo "####################################################################################"
 185 echo "#                                                                                  #"
 186 echo "#                           REAL INSTALL                                           #"
 187 echo "#                                                                                  #"
 188 echo "####################################################################################"
 189
 190 echo "Now a REAL install"
 191 sudo python setup.py install --install-scripts=/usr/bin
 192 if [ $? != '0' ]
 193 then
 194     echo "Error : the real install failed."
 195     exit 2
 196 fi
 197 echo "Real install OK"
 198
 199 #Useful to take it from setup_parameter? It's just for coding here
 200 ETC=/etc/shinken
 201 is_file_present $ETC/nagios.cfg
 202 is_file_present $ETC/shinken-specific.cfg
 203 string_in_file "servicegroups.cfg" $ETC/nagios.cfg
 204 is_file_present /usr/bin/shinken-arbiter
 205
 206 ps -fu shinken
 207
 208
 209 echo "Now we can test a real run guy"
 210 /etc/init.d/shinken-scheduler -d start
 211 /etc/init.d/shinken-poller -d start
 212 /etc/init.d/shinken-reactionner -d start
 213 /etc/init.d/shinken-broker -d start
 214 /etc/init.d/shinken-arbiter -d start
 215
 216 echo "We will sleep again 5sec so every one is quite stable...."
 217 sleep 5
 218 check_good_run /var/lib/shinken
 219
 220 sudo /etc/init.d/shinken-arbiter status
 221 sudo /etc/init.d/shinken-scheduler status
 222 sudo /etc/init.d/shinken-poller status
 223 sudo /etc/init.d/shinken-reactionner status
 224 sudo /etc/init.d/shinken-broker status
 225
 226 sudo /etc/init.d/shinken-arbiter stop
 227 sudo /etc/init.d/shinken-scheduler stop
 228 sudo /etc/init.d/shinken-poller stop
 229 sudo /etc/init.d/shinken-reactionner stop
 230 sudo /etc/init.d/shinken-broker stop
 231
 232 sleep 2
 233 ps -fu shinken
 234
 235 check_process_nb arbiter 0
 236 check_process_nb scheduler 0
 237 check_process_nb broker 0
 238 check_process_nb poller 0
 239 check_process_nb reactionner 0
 240
 241
 242
 243 echo "OK Great. Even the real launch test pass. Great. I can clean after me."
 244 ./clean.sh
 245
 246
 247
 248
 249 echo "####################################################################################"
 250 echo "#                                                                                  #"
 251 echo "#                                    HA launch                                     #"
 252 echo "#                                                                                  #"
 253 echo "####################################################################################"
 254
 255 echo "Now we can start some launch tests"
 256 localize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-ha-only.cfg
 257 test/bin/launch_all_debug2.sh
 258 globalize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-ha-only.cfg
 259
 260
 261 echo "Now checking for existing apps"
 262
 263 echo "we can sleep 5sec for conf dispatching and so good number of process"
 264 sleep 5
 265
 266 #The number of process changed, we mush look for it
 267
 268
 269 #Standard launch process packets
 270 NB_SCHEDULERS=2
 271 #6 for stack 1, and 2 for 2 (not active, so no worker)
 272 NB_POLLERS=8
 273 #3 for stack1, 2 for stack2 (no worker from now)
 274 NB_REACTIONNERS=5
 275 #3 for stack 1, 1 for stack2 (no livesatus.dat nor log worker launch)
 276 NB_BROKERS=4
 277 #still 1
 278 NB_ARBITERS=1
 279
 280 #Now check if the run looks good with var in the direct directory
 281 check_good_run var
 282
 283 echo "All launch of HA daemons is OK"
 284
 285 #Now we kill and see if all is OK :)
 286 #We clean the log file
 287 #$VAR/nagios.log
 288
 289
 290 #We kill the most important thing first : the scheduler-Master
 291 bin/stop_scheduler.sh
 292
 293 #We sleep to be sruethe scheduler see us
 294 sleep 2
 295 NB_SCHEDULERS=1
 296 print_date
 297
 298 #First we look is the arbiter saw the scheduler as dead
 299 string_in_file "Warning : Scheduler scheduler-Master had the configuration 0 but is dead, I am not happy." $VAR/nagios.log
 300 #Then we look if the scheduler-spare got a conf from arbiter (here, view from the arbiter)
 301 string_in_file "Dispatch OK of for conf in scheduler scheduler-Spare" $VAR/nagios.log
 302
 303 #then is the broker know it and try to connect to the new scheduler-spare
 304 string_in_file "\[broker-Master\] Connexion OK to the scheduler scheduler-Spare" $VAR/nagios.log
 305
 306
 307 echo "Now stop the poller-Master"
 308 #Now we stop the poller. We will see the sapre take the job (we hope in fact :) )
 309 bin/stop_poller.sh
 310 #check_good_run var
 311 sleep 2
 312 print_date
 313
 314 #The master should be look dead
 315 string_in_file "Warning : The poller poller-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
 316 #The spare should got the conf
 317 string_in_file "\[All\] Dispatch OK of for configuration 0 to poller poller-Slave" $VAR/nagios.log
 318 #And he should got the scheduler link (the sapre one)
 319 string_in_file "\[poller-Slave\] Connexion OK with scheduler scheduler-Spare" $VAR/nagios.log
 320
 321
 322 echo "Now stop the reactionner"
 323 bin/stop_reactionner.sh
 324 #check_good_run var
 325 sleep 2
 326 print_date
 327
 328 #The master should be look dead
 329 string_in_file "\[All\] Warning : The reactionner reactionner-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
 330 #The spare should got the conf
 331 string_in_file "\[All\] Dispatch OK of for configuration 0 to reactionner reactionner-Spare" $VAR/nagios.log
 332 #And he should got the scheduler link (the sapre one)
 333 string_in_file "\[reactionner-Spare\] Connexion OK with scheduler scheduler-Spare" $VAR/nagios.log
 334
 335
 336 echo "Now we stop... the Broker!"
 337 bin/stop_broker.sh
 338 #check_good_run var
 339 sleep 2
 340 print_date
 341
 342 #The master should be look dead
 343 string_in_file "\[All\] Warning : The broker broker-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
 344 #The spare should got the conf
 345 string_in_file "\[All\] Dispatch OK of for configuration 0 to broker broker-Slave" $VAR/nagios.log
 346 #And he should got the scheduler link (the spare one)
 347 string_in_file "\[broker-Slave\] Connexion OK to the scheduler scheduler-Spare" $VAR/nagios.log
 348 #And to other satellites
 349 string_in_file "\[broker-Slave\] Connexion OK to the reactionner reactionner-Spare" $VAR/nagios.log
 350 string_in_file "\[broker-Slave\] Connexion problem to the poller poller-Master : connection failed" $VAR/nagios.log
 351 #And should have load the modules
 352 string_in_file "\[broker-Slave\] I correctly loaded the modules : \['Simple-log', 'Livestatus'\]" $VAR/nagios.log
 353
 354
 355 echo "Now we clean it"
 356 ./clean.sh
 357
 358
 359
 360 echo "####################################################################################"
 361 echo "#                                                                                  #"
 362 echo "#                          Load balancing launch                                   #"
 363 echo "#                                                                                  #"
 364 echo "####################################################################################"
 365
 366 echo "Now we can start some launch tests"
 367 localize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-lb-only.cfg
 368 test/bin/launch_all_debug3.sh
 369 globalize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-lb-only.cfg
 370
 371
 372 echo "Now checking for existing apps"
 373
 374 echo "we can sleep 5sec for conf dispatching and so good number of process"
 375 sleep 5
 376
 377 #The number of process changed, we mush look for it
 378
 379
 380 #Standard launch process packets
 381 NB_SCHEDULERS=2
 382 #6 for stack 1, and 6 for stack 2
 383 NB_POLLERS=12
 384 #3 for stack1, same for stack 2
 385 NB_REACTIONNERS=6
 386 #3 for stack 1, 1 for stack2 (no livestatus nor log worker launch)
 387 NB_BROKERS=4
 388 #still 1
 389 NB_ARBITERS=1
 390
 391 #Now check if the run looks good with var in the direct directory
 392 check_good_run var
 393
 394 echo "All launch of LB daemons is OK"
 395
 396
 397 #Now look if it's also good in the log file too
 398 string_in_file "Dispatch OK of for conf in scheduler scheduler-Master-2" $VAR/nagios.log
 399 string_in_file "Dispatch OK of for conf in scheduler scheduler-Master-1" $VAR/nagios.log
 400 string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
 401 string_in_file "OK, no more poller sent need" $VAR/nagios.log
 402 string_in_file "OK, no more broker sent need" $VAR/nagios.log
 403
 404 echo "Now we clean it"
 405 ./clean.sh
 406
 407
 408
 409 echo "####################################################################################"
 410 echo "#                                                                                  #"
 411 echo "#                              Passive Poller                                      #"
 412 echo "#                                                                                  #"
 413 echo "####################################################################################"
 414
 415 echo "Now we can start some launch tests"
 416 localize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-passive-poller.cfg
 417 test/bin/launch_all_debug4.sh
 418 globalize_config etc/nagios.cfg test/etc/test_stack2/shinken-specific-passive-poller.cfg
 419
 420
 421 echo "Now checking for existing apps"
 422
 423 echo "we can sleep 5sec for conf dispatching and so good number of process"
 424 sleep 5
 425
 426 #The number of process changed, we mush look for it
 427
 428
 429 #Standard launch process packets
 430 NB_SCHEDULERS=2
 431 #6 for stack 1, and 6 for stack 2
 432 NB_POLLERS=12
 433 #3 for stack1, Only 2 for stack 2 because it is not active
 434 NB_REACTIONNERS=5
 435 #3 for stack 1, 1 for stack2 (no livestatus nor log worker launch)
 436 NB_BROKERS=4
 437 #still 1
 438 NB_ARBITERS=1
 439
 440 #Now check if the run looks good with var in the direct directory
 441 check_good_run var
 442
 443 echo "All launch of LB daemons is OK"
 444
 445
 446 #Now look if it's also good in the log file too
 447 string_in_file "Dispatch OK of for conf in scheduler scheduler-Master-2" $VAR/nagios.log
 448 string_in_file "Dispatch OK of for conf in scheduler scheduler-Master-1" $VAR/nagios.log
 449 string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
 450 string_in_file "OK, no more poller sent need" $VAR/nagios.log
 451 string_in_file "OK, no more broker sent need" $VAR/nagios.log
 452 # We should see the poller 2 say it is passive
 453 string_in_file "\[poller-Master-2\] Passive mode enabled." $VAR/nagios.log
 454 # and the schedulers should connect to it too
 455 string_in_file "Connexion OK to the poller poller-Master-2" $VAR/nagios.log
 456
 457
 458 echo "Now we clean it"
 459 #./clean.sh
 460
 461
 462 echo ""
 463 echo ""
 464 echo "All check are OK. Congrats! You can go take a Beer ;)"