test/test_end_to_end.sh

   1 #!/bin/bash
   2 #Copyright (C) 2009-2010 :
   3 #    Gabes Jean, naparuba@gmail.com
   4 #    Gerhard Lausser, Gerhard.Lausser@consol.de
   5 #
   6 #This file is part of Shinken.
   7 #
   8 #Shinken is free software: you can redistribute it and/or modify
   9 #it under the terms of the GNU Affero General Public License as published by
  10 #the Free Software Foundation, either version 3 of the License, or
  11 #(at your option) any later version.
  12 #
  13 #Shinken is distributed in the hope that it will be useful,
  14 #but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 #MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 #GNU Affero General Public License for more details.
  17 #
  18 #You should have received a copy of the GNU Affero General Public License
  19 #along with Shinken.  If not, see <http://www.gnu.org/licenses/>.
  20
  21
  22 #set -x
  23
  24 echo "Begining test END TO END"
  25
  26
  27 DIR=$(cd $(dirname "$0"); pwd)
  28 echo "Going to dir $DIR/.."
  29 cd $DIR/..
  30
  31
  32
  33
  34
  35 #check for a process existance with good number
  36 function check_process_nb {
  37     NB=`ps -fu shinken | grep python | grep -v grep | grep $1 | wc -l`
  38     if [ $NB != "$2" ]
  39     then
  40         echo "Error : There is not enouth $1 launched (only $NB)."
  41         exit 2
  42     else
  43         echo "Ok, got $NB $1"
  44     fi
  45 }
  46
  47 function is_file_present {
  48     if [ -f $1 ]
  49     then
  50         echo "File $1 is present."
  51     else
  52         echo "Error : File $1 is missing!"
  53         exit 2
  54     fi
  55 }
  56
  57 function string_in_file {
  58     grep "$1" $2
  59     if [ $? != 0 ]
  60     then
  61         echo "Error : the file $2 is missing string $1 !"
  62         exit 2
  63     else
  64         echo "The string $1 is in $2"
  65     fi
  66 }
  67
  68
  69 function print_date {
  70     date +%s
  71 }
  72
  73 function check_good_run {
  74     VAR="$1"
  75     echo "Check for $NB_SCHEDULERS Scheduler"
  76     check_process_nb scheduler $NB_SCHEDULERS
  77     is_file_present $VAR/schedulerd.pid
  78
  79     echo "Check for $NB_POLLERS pollers (1 master, 1 for multiporcess module (queue manager), 4 workers)"
  80     check_process_nb poller $NB_POLLERS
  81     is_file_present $VAR/pollerd.pid
  82
  83     echo "Check for $NB_REACTIONNERS reactionners (1 master, 1 for multiporcess module (queue manager) 1 worker)"
  84     check_process_nb reactionner $NB_REACTIONNERS
  85     is_file_present $VAR/reactionnerd.pid
  86
  87     echo "Check for $NB_BROKERS brokers (one master, one for livestatus.dat, one for log)"
  88     check_process_nb broker $NB_BROKERS
  89     is_file_present $VAR/brokerd.pid
  90
  91     echo "Check for $NB_ARBITERS arbiter"
  92     check_process_nb arbiter $NB_ARBITERS
  93     is_file_present $VAR/arbiterd.pid
  94
  95     echo "Now checking for good file prensence"
  96     ls var
  97     is_file_present $VAR/nagios.log
  98     string_in_file "Waiting for initial configuration" $VAR/nagios.log
  99     string_in_file "First scheduling" $VAR/nagios.log
 100     string_in_file "OK, all schedulers configurations are dispatched :)" $VAR/nagios.log
 101     string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
 102     string_in_file "OK, no more poller sent need" $VAR/nagios.log
 103     string_in_file "OK, no more broker sent need" $VAR/nagios.log
 104 }
 105
 106
 107 #Standard launch process packets
 108 NB_SCHEDULERS=1
 109 NB_POLLERS=6
 110 NB_REACTIONNERS=3
 111 NB_BROKERS=3
 112 NB_ARBITERS=1
 113
 114
 115
 116 echo "Clean old tests and kill remaining processes"
 117 ./clean.sh
 118
 119
 120 echo "####################################################################################"
 121 echo "#                                                                                  #"
 122 echo "#                           SIMPLE START                                           #"
 123 echo "#                                                                                  #"
 124 echo "####################################################################################"
 125
 126 echo "Now we can start some launch tests"
 127 bin/launch_all_debug.sh
 128
 129
 130 echo "Now checking for existing apps"
 131
 132 echo "we can sleep 5sec for conf dispatching and so good number of process"
 133 sleep 5
 134
 135 #Now check if the run looks good with var in the direct directory
 136 check_good_run var
 137
 138 echo "First launch check OK"
 139
 140 echo "Now we clean it and test an install"
 141 ./clean.sh
 142
 143 echo "####################################################################################"
 144 echo "#                                                                                  #"
 145 echo "#                           DUMMY INSTALL                                          #"
 146 echo "#                                                                                  #"
 147 echo "####################################################################################"
 148
 149 echo "Now installing the application in DUMMY mode"
 150 python setup.py install --root=/tmp/moncul --record=INSTALLED_FILES --install-scripts=/usr/bin
 151
 152 if [ $? != '0' ]
 153 then
 154     echo "Error : the dummy install failed."
 155     exit 2
 156 fi
 157 echo "Dummy install OK"
 158
 159 echo "I reclean all for a real install"
 160 ./clean.sh
 161
 162
 163 echo "####################################################################################"
 164 echo "#                                                                                  #"
 165 echo "#                           REAL INSTALL                                           #"
 166 echo "#                                                                                  #"
 167 echo "####################################################################################"
 168
 169 echo "Now a REAL install"
 170 sudo python setup.py install --install-scripts=/usr/bin
 171 if [ $? != '0' ]
 172 then
 173     echo "Error : the real install failed."
 174     exit 2
 175 fi
 176 echo "Real install OK"
 177
 178 #Useful to take it from setup_parameter? It's just for coding here
 179 ETC=/etc/shinken
 180 is_file_present $ETC/nagios.cfg
 181 is_file_present $ETC/shinken-specific.cfg
 182 string_in_file "servicegroups.cfg" $ETC/nagios.cfg
 183 is_file_present /usr/bin/shinken-arbiter
 184
 185 ps -fu shinken
 186
 187
 188 echo "Now we can test a real run guy"
 189 /etc/init.d/shinken-scheduler start
 190 /etc/init.d/shinken-poller start
 191 /etc/init.d/shinken-reactionner start
 192 /etc/init.d/shinken-broker start
 193 /etc/init.d/shinken-arbiter start
 194
 195 echo "We will sleep again 5sec so every one is quite stable...."
 196 sleep 5
 197 check_good_run /var/lib/shinken
 198
 199 sudo /etc/init.d/shinken-arbiter status
 200 sudo /etc/init.d/shinken-scheduler status
 201 sudo /etc/init.d/shinken-poller status
 202 sudo /etc/init.d/shinken-reactionner status
 203 sudo /etc/init.d/shinken-broker status
 204
 205 sudo /etc/init.d/shinken-arbiter stop
 206 sudo /etc/init.d/shinken-scheduler stop
 207 sudo /etc/init.d/shinken-poller stop
 208 sudo /etc/init.d/shinken-reactionner stop
 209 sudo /etc/init.d/shinken-broker stop
 210
 211 sleep 2
 212 ps -fu shinken
 213
 214 check_process_nb arbiter 0
 215 check_process_nb scheduler 0
 216 check_process_nb broker 0
 217 check_process_nb poller 0
 218 check_process_nb reactionner 0
 219
 220
 221
 222 echo "OK Great. Even the real launch test pass. Great. I can clean after me."
 223 ./clean.sh
 224
 225
 226
 227
 228 echo "####################################################################################"
 229 echo "#                                                                                  #"
 230 echo "#                                    HA launch                                     #"
 231 echo "#                                                                                  #"
 232 echo "####################################################################################"
 233
 234 echo "Now we can start some launch tests"
 235 test/bin/launch_all_debug2.sh
 236
 237
 238 echo "Now checking for existing apps"
 239
 240 echo "we can sleep 5sec for conf dispatching and so good number of process"
 241 sleep 5
 242
 243 #The number of process changed, we mush look for it
 244
 245
 246 #Standard launch process packets
 247 NB_SCHEDULERS=2
 248 #6 for stack 1, and 2 for 2 (not active, so no worker)
 249 NB_POLLERS=8
 250 #3 for stack1, 2 for stack2 (no worker from now)
 251 NB_REACTIONNERS=5
 252 #3 for stack 1, 1 for stack2 (no livesatus.dat nor log worker launch)
 253 NB_BROKERS=4
 254 #still 1
 255 NB_ARBITERS=1
 256
 257 #Now check if the run looks good with var in the direct directory
 258 check_good_run var
 259
 260 echo "All launch of HA daemons is OK"
 261
 262 #Now we kill and see if all is OK :)
 263 #We clean the log file
 264 #$VAR/nagios.log
 265
 266
 267 #We kill the most important thing first : the scheduler-Master
 268 bin/stop_scheduler.sh
 269
 270 #We sleep to be sruethe scheduler see us
 271 sleep 2
 272 NB_SCHEDULERS=1
 273 print_date
 274
 275 #First we look is the arbiter saw the scheduler as dead
 276 string_in_file "Warning : Scheduler scheduler-Master had the configuration 0 but is dead, I am not happy." $VAR/nagios.log
 277 #Then we look if the scheduler-spare got a conf from arbiter (here, view from the arbiter)
 278 string_in_file "Dispatch OK of for conf in scheduler scheduler-Spare" $VAR/nagios.log
 279
 280 #then is the broker know it and try to connect to the new scheduler-spare
 281 string_in_file "\[broker-Master\] Connexion OK to the scheduler scheduler-Spare" $VAR/nagios.log
 282
 283
 284 echo "Now stop the poller-Master"
 285 #Now we stop the poller. We will see the sapre take the job (we hope in fact :) )
 286 bin/stop_poller.sh
 287 #check_good_run var
 288 sleep 2
 289 print_date
 290
 291 #The master should be look dead
 292 string_in_file "Warning : The poller poller-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
 293 #The spare should got the conf
 294 string_in_file "\[All\] Dispatch OK of for configuration 0 to poller poller-Slave" $VAR/nagios.log
 295 #And he should got the scheduler link (the sapre one)
 296 string_in_file "\[poller-Slave\] Connexion OK with scheduler scheduler-Spare" $VAR/nagios.log
 297
 298
 299 echo "Now stop the reactionner"
 300 bin/stop_reactionner.sh
 301 #check_good_run var
 302 sleep 2
 303 print_date
 304
 305 #The master should be look dead
 306 string_in_file "\[All\] Warning : The reactionner reactionner-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
 307 #The spare should got the conf
 308 string_in_file "\[All\] Dispatch OK of for configuration 0 to reactionner reactionner-Spare" $VAR/nagios.log
 309 #And he should got the scheduler link (the sapre one)
 310 string_in_file "\[reactionner-Spare\] Connexion OK with scheduler scheduler-Spare" $VAR/nagios.log
 311
 312
 313 echo "Now we stop... the Broker!"
 314 bin/stop_broker.sh
 315 #check_good_run var
 316 sleep 2
 317 print_date
 318
 319 #The master should be look dead
 320 string_in_file "\[All\] Warning : The broker broker-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
 321 #The spare should got the conf
 322 string_in_file "\[All\] Dispatch OK of for configuration 0 to broker broker-Slave" $VAR/nagios.log
 323 #And he should got the scheduler link (the spare one)
 324 string_in_file "\[broker-Slave\] Connexion OK to the scheduler scheduler-Spare" $VAR/nagios.log
 325 #And to other satellites
 326 string_in_file "\[broker-Slave\] Connexion OK to the reactionner reactionner-Spare" $VAR/nagios.log
 327 string_in_file "\[broker-Slave\] Connexion problem to the poller poller-Master : connection failed" $VAR/nagios.log
 328 #And should have load the modules
 329 string_in_file "\[broker-Slave\] I correctly load the modules : \['Simple-log', 'Livestatus'\]" $VAR/nagios.log
 330
 331
 332 echo "Now we clean it"
 333 ./clean.sh
 334
 335
 336
 337 echo "####################################################################################"
 338 echo "#                                                                                  #"
 339 echo "#                          Load balancing launch                                   #"
 340 echo "#                                                                                  #"
 341 echo "####################################################################################"
 342
 343 echo "Now we can start some launch tests"
 344 test/bin/launch_all_debug3.sh
 345
 346
 347 echo "Now checking for existing apps"
 348
 349 echo "we can sleep 5sec for conf dispatching and so good number of process"
 350 sleep 5
 351
 352 #The number of process changed, we mush look for it
 353
 354
 355 #Standard launch process packets
 356 NB_SCHEDULERS=2
 357 #6 for stack 1, and 6 for stack 2
 358 NB_POLLERS=12
 359 #3 for stack1, same for stack 2
 360 NB_REACTIONNERS=6
 361 #3 for stack 1, 1 for stack2 (no livestatus nor log worker launch)
 362 NB_BROKERS=4
 363 #still 1
 364 NB_ARBITERS=1
 365
 366 #Now check if the run looks good with var in the direct directory
 367 check_good_run var
 368
 369 echo "All launch of LB daemons is OK"
 370
 371
 372 #Now look if it's also good in the log file too
 373 string_in_file "Dispatch OK of for conf in scheduler scheduler-Master-2" $VAR/nagios.log
 374 string_in_file "Dispatch OK of for conf in scheduler scheduler-Master-1" $VAR/nagios.log
 375 string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
 376 string_in_file "OK, no more poller sent need" $VAR/nagios.log
 377 string_in_file "OK, no more broker sent need" $VAR/nagios.log
 378
 379 echo "Now we clean it"
 380 ./clean.sh
 381
 382
 383 echo ""
 384 echo ""
 385 echo "All check are OK. Congrats! You can go take a Beer ;)"