test/test_end_to_end.sh

   1 #!/bin/bash
   2 #Copyright (C) 2009-2010 :
   3 #    Gabes Jean, naparuba@gmail.com
   4 #    Gerhard Lausser, Gerhard.Lausser@consol.de
   5 #
   6 #This file is part of Shinken.
   7 #
   8 #Shinken is free software: you can redistribute it and/or modify
   9 #it under the terms of the GNU Affero General Public License as published by
  10 #the Free Software Foundation, either version 3 of the License, or
  11 #(at your option) any later version.
  12 #
  13 #Shinken is distributed in the hope that it will be useful,
  14 #but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 #MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 #GNU Affero General Public License for more details.
  17 #
  18 #You should have received a copy of the GNU Affero General Public License
  19 #along with Shinken.  If not, see <http://www.gnu.org/licenses/>.
  20
  21
  22 #set -x
  23
  24 echo "Begining test END TO END"
  25
  26
  27 DIR=$(cd $(dirname "$0"); pwd)
  28 echo "Going to dir $DIR/.."
  29 cd $DIR/..
  30
  31
  32
  33
  34
  35 #check for a process existance with good number
  36 function check_process_nb {
  37     NB=`ps -fu shinken | grep python | grep -v grep | grep $1 | wc -l`
  38     if [ $NB != "$2" ]
  39     then
  40         echo "Error : There is not enouth $1 launched (only $NB)."
  41         exit 2
  42     else
  43         echo "Ok, got $NB $1"
  44     fi
  45 }
  46
  47 function is_file_present {
  48     if [ -f $1 ]
  49     then
  50         echo "File $1 is present."
  51     else
  52         echo "Error : File $1 is missing!"
  53         exit 2
  54     fi
  55 }
  56
  57 function string_in_file {
  58     grep "$1" $2
  59     if [ $? != 0 ]
  60     then
  61         echo "Error : the file $2 is missing string $1 !"
  62         exit 2
  63     else
  64         echo "The string $1 is in $2"
  65     fi
  66 }
  67
  68
  69 function print_date {
  70     date +%s
  71 }
  72
  73 function check_good_run {
  74     VAR="$1"
  75     echo "Check for $NB_SCHEDULERS Scheduler"
  76     check_process_nb scheduler $NB_SCHEDULERS
  77     is_file_present $VAR/schedulerd.pid
  78
  79     echo "Check for $NB_POLLERS pollers (1 master, 1 for multiporcess module (queue manager), 4 workers)"
  80     check_process_nb poller $NB_POLLERS
  81     is_file_present $VAR/pollerd.pid
  82
  83     echo "Check for $NB_REACTIONNERS reactionners (1 master, 1 for multiporcess module (queue manager) 1 worker)"
  84     check_process_nb reactionner $NB_REACTIONNERS
  85     is_file_present $VAR/reactionnerd.pid
  86
  87     echo "Check for $NB_BROKERS brokers (one master, one for livestatus.dat, one for log)"
  88     check_process_nb broker $NB_BROKERS
  89     is_file_present $VAR/brokerd.pid
  90
  91     echo "Check for $NB_ARBITERS arbiter"
  92     check_process_nb arbiter $NB_ARBITERS
  93     is_file_present $VAR/arbiterd.pid
  94
  95     echo "Now checking for good file prensence"
  96     ls var
  97     is_file_present $VAR/nagios.log
  98     string_in_file "Waiting for initial configuration" $VAR/nagios.log
  99     string_in_file "First scheduling" $VAR/nagios.log
 100     string_in_file "OK, all schedulers configurations are dispatched :)" $VAR/nagios.log
 101     string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
 102     string_in_file "OK, no more poller sent need" $VAR/nagios.log
 103     string_in_file "OK, no more broker sent need" $VAR/nagios.log
 104 }
 105
 106
 107 #Standard launch process packets
 108 NB_SCHEDULERS=1
 109 NB_POLLERS=6
 110 NB_REACTIONNERS=3
 111 NB_BROKERS=3
 112 NB_ARBITERS=1
 113
 114
 115
 116 echo "Clean old tests and kill remaining processes"
 117 ./clean.sh
 118
 119
 120 echo "####################################################################################"
 121 echo "#                                                                                  #"
 122 echo "#                           SIMPLE START                                           #"
 123 echo "#                                                                                  #"
 124 echo "####################################################################################"
 125
 126 echo "Now we can start some launch tests"
 127 bin/launch_all_debug.sh
 128
 129
 130 echo "Now checking for existing apps"
 131
 132 echo "we can sleep 5sec for conf dispatching and so good number of process"
 133 sleep 5
 134
 135 #Now check if the run looks good with var in the direct directory
 136 check_good_run var
 137
 138 echo "First launch check OK"
 139
 140
 141 echo "####################################################################################"
 142
 143 echo "Now we clean it and test an install"
 144 ./clean.sh
 145
 146 echo "####################################################################################"
 147 echo "#                                                                                  #"
 148 echo "#                           DUMMY INSTALL                                          #"
 149 echo "#                                                                                  #"
 150 echo "####################################################################################"
 151
 152 echo "Now installing the application in DUMMY mode"
 153 python setup.py install --root=/tmp/moncul --record=INSTALLED_FILES --install-scripts=/usr/bin
 154
 155 if [ $? != '0' ]
 156 then
 157     echo "Error : the dummy install failed."
 158     exit 2
 159 fi
 160 echo "Dummy install OK"
 161
 162 echo "I reclean all for a real install"
 163 ./clean.sh
 164
 165
 166 echo "####################################################################################"
 167 echo "#                                                                                  #"
 168 echo "#                           REAL INSTALL                                           #"
 169 echo "#                                                                                  #"
 170 echo "####################################################################################"
 171
 172 echo "Now a REAL install"
 173 sudo python setup.py install --install-scripts=/usr/bin
 174 if [ $? != '0' ]
 175 then
 176     echo "Error : the real install failed."
 177     exit 2
 178 fi
 179 echo "Real install OK"
 180
 181 #Useful to take it from setup_parameter? It's just for coding here
 182 ETC=/etc/shinken
 183 is_file_present $ETC/nagios.cfg
 184 is_file_present $ETC/shinken-specific.cfg
 185 string_in_file "servicegroups.cfg" $ETC/nagios.cfg
 186 is_file_present /usr/bin/shinken-arbiter
 187
 188 ps -fu shinken
 189
 190
 191 echo "Now we can test a real run guy"
 192 /etc/init.d/shinken-scheduler -d start
 193 /etc/init.d/shinken-poller -d start
 194 /etc/init.d/shinken-reactionner -d start
 195 /etc/init.d/shinken-broker -d start
 196 /etc/init.d/shinken-arbiter -d start
 197
 198 echo "We will sleep again 5sec so every one is quite stable...."
 199 sleep 5
 200 check_good_run /var/lib/shinken
 201
 202 sudo /etc/init.d/shinken-arbiter status
 203 sudo /etc/init.d/shinken-scheduler status
 204 sudo /etc/init.d/shinken-poller status
 205 sudo /etc/init.d/shinken-reactionner status
 206 sudo /etc/init.d/shinken-broker status
 207
 208 sudo /etc/init.d/shinken-arbiter stop
 209 sudo /etc/init.d/shinken-scheduler stop
 210 sudo /etc/init.d/shinken-poller stop
 211 sudo /etc/init.d/shinken-reactionner stop
 212 sudo /etc/init.d/shinken-broker stop
 213
 214 sleep 2
 215 ps -fu shinken
 216
 217 check_process_nb arbiter 0
 218 check_process_nb scheduler 0
 219 check_process_nb broker 0
 220 check_process_nb poller 0
 221 check_process_nb reactionner 0
 222
 223
 224
 225 echo "OK Great. Even the real launch test pass. Great. I can clean after me."
 226 ./clean.sh
 227
 228
 229
 230
 231 echo "####################################################################################"
 232 echo "#                                                                                  #"
 233 echo "#                                    HA launch                                     #"
 234 echo "#                                                                                  #"
 235 echo "####################################################################################"
 236
 237 echo "Now we can start some launch tests"
 238 test/bin/launch_all_debug2.sh
 239
 240
 241 echo "Now checking for existing apps"
 242
 243 echo "we can sleep 5sec for conf dispatching and so good number of process"
 244 sleep 5
 245
 246 #The number of process changed, we mush look for it
 247
 248
 249 #Standard launch process packets
 250 NB_SCHEDULERS=2
 251 #6 for stack 1, and 2 for 2 (not active, so no worker)
 252 NB_POLLERS=8
 253 #3 for stack1, 2 for stack2 (no worker from now)
 254 NB_REACTIONNERS=5
 255 #3 for stack 1, 1 for stack2 (no livesatus.dat nor log worker launch)
 256 NB_BROKERS=4
 257 #still 1
 258 NB_ARBITERS=1
 259
 260 #Now check if the run looks good with var in the direct directory
 261 check_good_run var
 262
 263 echo "All launch of HA daemons is OK"
 264
 265 #Now we kill and see if all is OK :)
 266 #We clean the log file
 267 #$VAR/nagios.log
 268
 269
 270 #We kill the most important thing first : the scheduler-Master
 271 bin/stop_scheduler.sh
 272
 273 #We sleep to be sruethe scheduler see us
 274 sleep 2
 275 NB_SCHEDULERS=1
 276 print_date
 277
 278 #First we look is the arbiter saw the scheduler as dead
 279 string_in_file "Warning : Scheduler scheduler-Master had the configuration 0 but is dead, I am not happy." $VAR/nagios.log
 280 #Then we look if the scheduler-spare got a conf from arbiter (here, view from the arbiter)
 281 string_in_file "Dispatch OK of for conf in scheduler scheduler-Spare" $VAR/nagios.log
 282
 283 #then is the broker know it and try to connect to the new scheduler-spare
 284 string_in_file "\[broker-Master\] Connexion OK to the scheduler scheduler-Spare" $VAR/nagios.log
 285
 286
 287 echo "Now stop the poller-Master"
 288 #Now we stop the poller. We will see the sapre take the job (we hope in fact :) )
 289 bin/stop_poller.sh
 290 #check_good_run var
 291 sleep 2
 292 print_date
 293
 294 #The master should be look dead
 295 string_in_file "Warning : The poller poller-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
 296 #The spare should got the conf
 297 string_in_file "\[All\] Dispatch OK of for configuration 0 to poller poller-Slave" $VAR/nagios.log
 298 #And he should got the scheduler link (the sapre one)
 299 string_in_file "\[poller-Slave\] Connexion OK with scheduler scheduler-Spare" $VAR/nagios.log
 300
 301
 302 echo "Now stop the reactionner"
 303 bin/stop_reactionner.sh
 304 #check_good_run var
 305 sleep 2
 306 print_date
 307
 308 #The master should be look dead
 309 string_in_file "\[All\] Warning : The reactionner reactionner-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
 310 #The spare should got the conf
 311 string_in_file "\[All\] Dispatch OK of for configuration 0 to reactionner reactionner-Spare" $VAR/nagios.log
 312 #And he should got the scheduler link (the sapre one)
 313 string_in_file "\[reactionner-Spare\] Connexion OK with scheduler scheduler-Spare" $VAR/nagios.log
 314
 315
 316 echo "Now we stop... the Broker!"
 317 bin/stop_broker.sh
 318 #check_good_run var
 319 sleep 2
 320 print_date
 321
 322 #The master should be look dead
 323 string_in_file "\[All\] Warning : The broker broker-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
 324 #The spare should got the conf
 325 string_in_file "\[All\] Dispatch OK of for configuration 0 to broker broker-Slave" $VAR/nagios.log
 326 #And he should got the scheduler link (the spare one)
 327 string_in_file "\[broker-Slave\] Connexion OK to the scheduler scheduler-Spare" $VAR/nagios.log
 328 #And to other satellites
 329 string_in_file "\[broker-Slave\] Connexion OK to the reactionner reactionner-Spare" $VAR/nagios.log
 330 string_in_file "\[broker-Slave\] Connexion problem to the poller poller-Master : connection failed" $VAR/nagios.log
 331 #And should have load the modules
 332 string_in_file "\[broker-Slave\] I correctly load the modules : \['Simple-log', 'Livestatus'\]" $VAR/nagios.log
 333
 334
 335 echo "Now we clean it"
 336 ./clean.sh
 337
 338
 339
 340 echo "####################################################################################"
 341 echo "#                                                                                  #"
 342 echo "#                          Load balancing launch                                   #"
 343 echo "#                                                                                  #"
 344 echo "####################################################################################"
 345
 346 echo "Now we can start some launch tests"
 347 test/bin/launch_all_debug3.sh
 348
 349
 350 echo "Now checking for existing apps"
 351
 352 echo "we can sleep 5sec for conf dispatching and so good number of process"
 353 sleep 5
 354
 355 #The number of process changed, we mush look for it
 356
 357
 358 #Standard launch process packets
 359 NB_SCHEDULERS=2
 360 #6 for stack 1, and 6 for stack 2
 361 NB_POLLERS=12
 362 #3 for stack1, same for stack 2
 363 NB_REACTIONNERS=6
 364 #3 for stack 1, 1 for stack2 (no livestatus nor log worker launch)
 365 NB_BROKERS=4
 366 #still 1
 367 NB_ARBITERS=1
 368
 369 #Now check if the run looks good with var in the direct directory
 370 check_good_run var
 371
 372 echo "All launch of LB daemons is OK"
 373
 374
 375 #Now look if it's also good in the log file too
 376 string_in_file "Dispatch OK of for conf in scheduler scheduler-Master-2" $VAR/nagios.log
 377 string_in_file "Dispatch OK of for conf in scheduler scheduler-Master-1" $VAR/nagios.log
 378 string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
 379 string_in_file "OK, no more poller sent need" $VAR/nagios.log
 380 string_in_file "OK, no more broker sent need" $VAR/nagios.log
 381
 382 echo "Now we clean it"
 383 ./clean.sh
 384
 385
 386 echo ""
 387 echo ""
 388 echo "All check are OK. Congrats! You can go take a Beer ;)"