test/test_end_to_end.sh

   1 #!/bin/bash
   2 #Copyright (C) 2009-2010 :
   3 #    Gabes Jean, naparuba@gmail.com
   4 #    Gerhard Lausser, Gerhard.Lausser@consol.de
   5 #
   6 #This file is part of Shinken.
   7 #
   8 #Shinken is free software: you can redistribute it and/or modify
   9 #it under the terms of the GNU Affero General Public License as published by
  10 #the Free Software Foundation, either version 3 of the License, or
  11 #(at your option) any later version.
  12 #
  13 #Shinken is distributed in the hope that it will be useful,
  14 #but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 #MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 #GNU Affero General Public License for more details.
  17 #
  18 #You should have received a copy of the GNU Affero General Public License
  19 #along with Shinken.  If not, see <http://www.gnu.org/licenses/>.
  20
  21
  22 #set -x
  23
  24 echo "Begining test END TO END"
  25
  26
  27 DIR=$(cd $(dirname "$0"); pwd)
  28 echo "Going to dir $DIR/.."
  29 cd $DIR/..
  30
  31
  32
  33
  34
  35 #check for a process existance with good number
  36 function check_process_nb {
  37     NB=`ps -fu shinken | grep python | grep -v grep | grep $1 | wc -l`
  38     if [ $NB != "$2" ]
  39     then
  40         echo "Error : There is not enouth $1 launched (only $NB)."
  41         exit 2
  42     else
  43         echo "Ok, got $NB $1"
  44     fi
  45 }
  46
  47 function is_file_present {
  48     if [ -f $1 ]
  49     then
  50         echo "File $1 is present."
  51     else
  52         echo "Error : File $1 is missing!"
  53         exit 2
  54     fi
  55 }
  56
  57 function string_in_file {
  58     grep "$1" $2
  59     if [ $? != 0 ]
  60     then
  61         echo "Error : the file $2 is missing string $1 !"
  62         exit 2
  63     else
  64         echo "The string $1 is in $2"
  65     fi
  66 }
  67
  68
  69 function print_date {
  70     date +%s
  71 }
  72
  73 function check_good_run {
  74     VAR="$1"
  75     echo "Check for $NB_SCHEDULERS Scheduler"
  76     check_process_nb scheduler $NB_SCHEDULERS
  77     is_file_present $VAR/schedulerd.pid
  78
  79     echo "Check for $NB_POLLERS pollers (1 master, 1 for multiporcess module (queue manager), 4 workers)"
  80     check_process_nb poller $NB_POLLERS
  81     is_file_present $VAR/pollerd.pid
  82
  83     echo "Check for $NB_REACTIONNERS reactionners (1 master, 1 for multiporcess module (queue manager) 1 worker)"
  84     check_process_nb reactionner $NB_REACTIONNERS
  85     is_file_present $VAR/reactionnerd.pid
  86
  87     echo "Check for $NB_BROKERS brokers (one master, one for livestatus.dat, one for log)"
  88     check_process_nb broker $NB_BROKERS
  89     is_file_present $VAR/brokerd.pid
  90
  91     echo "Check for $NB_ARBITERS arbiter"
  92     check_process_nb arbiter $NB_ARBITERS
  93     is_file_present $VAR/arbiterd.pid
  94
  95     echo "Now checking for good file prensence"
  96     ls var
  97     is_file_present $VAR/nagios.log
  98     string_in_file "Waiting for initial configuration" $VAR/nagios.log
  99     string_in_file "First scheduling" $VAR/nagios.log
 100     string_in_file "OK, all schedulers configurations are dispatched :)" $VAR/nagios.log
 101     string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
 102     string_in_file "OK, no more poller sent need" $VAR/nagios.log
 103     string_in_file "OK, no more broker sent need" $VAR/nagios.log
 104 }
 105
 106
 107 #Standard launch process packets
 108 NB_SCHEDULERS=1
 109 NB_POLLERS=6
 110 NB_REACTIONNERS=3
 111 NB_BROKERS=3
 112 NB_ARBITERS=1
 113
 114
 115
 116 echo "Clean old tests and kill remaining processes"
 117 ./clean.sh
 118
 119
 120 echo "####################################################################################"
 121 echo "#                                                                                  #"
 122 echo "#                           SIMPLE START                                           #"
 123 echo "#                                                                                  #"
 124 echo "####################################################################################"
 125
 126 echo "Now we can start some launch tests"
 127 bin/launch_all_debug.sh
 128
 129
 130 echo "Now checking for existing apps"
 131
 132 echo "we can sleep 5sec for conf dispatching and so good number of process"
 133 sleep 5
 134
 135 #Now check if the run looks good with var in the direct directory
 136 check_good_run var
 137
 138 echo "First launch check OK"
 139
 140 echo "Now we clean it and test an install"
 141 ./clean.sh
 142
 143 echo "####################################################################################"
 144 echo "#                                                                                  #"
 145 echo "#                           DUMMY INSTALL                                          #"
 146 echo "#                                                                                  #"
 147 echo "####################################################################################"
 148
 149 echo "Now installing the application in DUMMY mode"
 150 python setup.py install --root=/tmp/moncul --record=INSTALLED_FILES --install-scripts=/usr/bin
 151
 152 if [ $? != '0' ]
 153 then
 154     echo "Error : the dummy install failed."
 155     exit 2
 156 fi
 157 echo "Dummy install OK"
 158
 159 echo "I reclean all for a real install"
 160 ./clean.sh
 161
 162 echo "####################################################################################"
 163 echo "#                                                                                  #"
 164 echo "#                           REAL INSTALL                                           #"
 165 echo "#                                                                                  #"
 166 echo "####################################################################################"
 167
 168 echo "Now a REAL install"
 169 sudo python setup.py install --install-scripts=/usr/bin
 170 if [ $? != '0' ]
 171 then
 172     echo "Error : the real install failed."
 173     exit 2
 174 fi
 175 echo "Real install OK"
 176
 177 #Useful to take it from setup_parameter? It's just for coding here
 178 ETC=/etc/shinken
 179 is_file_present $ETC/nagios.cfg
 180 is_file_present $ETC/shinken-specific.cfg
 181 string_in_file "servicegroups.cfg" $ETC/nagios.cfg
 182 is_file_present /usr/bin/shinken-arbiter
 183
 184 echo "Now we can test a real run guy"
 185 /etc/init.d/shinken-scheduler start
 186 /etc/init.d/shinken-poller start
 187 /etc/init.d/shinken-reactionner start
 188 /etc/init.d/shinken-broker start
 189 /etc/init.d/shinken-arbiter start
 190
 191 echo "We will sleep again 5sec so every one is quite stable...."
 192 sleep 5
 193 check_good_run /var/lib/shinken
 194
 195 sudo /etc/init.d/shinken-arbiter status
 196 sudo /etc/init.d/shinken-scheduler status
 197 sudo /etc/init.d/shinken-poller status
 198 sudo /etc/init.d/shinken-reactionner status
 199 sudo /etc/init.d/shinken-broker status
 200
 201 sudo /etc/init.d/shinken-arbiter stop
 202 sudo /etc/init.d/shinken-scheduler stop
 203 sudo /etc/init.d/shinken-poller stop
 204 sudo /etc/init.d/shinken-reactionner stop
 205 sudo /etc/init.d/shinken-broker stop
 206
 207 sleep 2
 208 ps -fu shinken
 209
 210 check_process_nb arbiter 0
 211 check_process_nb scheduler 0
 212 check_process_nb broker 0
 213 check_process_nb poller 0
 214 check_process_nb reactionner 0
 215
 216
 217
 218 echo "OK Great. Even the real launch test pass. Great. I can clean after me."
 219 ./clean.sh
 220
 221
 222
 223
 224 echo "####################################################################################"
 225 echo "#                                                                                  #"
 226 echo "#                                    HA launch                                     #"
 227 echo "#                                                                                  #"
 228 echo "####################################################################################"
 229
 230 echo "Now we can start some launch tests"
 231 test/bin/launch_all_debug2.sh
 232
 233
 234 echo "Now checking for existing apps"
 235
 236 echo "we can sleep 5sec for conf dispatching and so good number of process"
 237 sleep 5
 238
 239 #The number of process changed, we mush look for it
 240
 241
 242 #Standard launch process packets
 243 NB_SCHEDULERS=2
 244 #6 for stack 1, and 2 for 2 (not active, so no worker)
 245 NB_POLLERS=8
 246 #3 for stack1, 2 for stack2 (no worker from now)
 247 NB_REACTIONNERS=5
 248 #3 for stack 1, 1 for stack2 (no livesatus.dat nor log worker launch)
 249 NB_BROKERS=4
 250 #still 1
 251 NB_ARBITERS=1
 252
 253 #Now check if the run looks good with var in the direct directory
 254 check_good_run var
 255
 256 echo "All launch of HA daemons is OK"
 257
 258 #Now we kill and see if all is OK :)
 259 #We clean the log file
 260 #$VAR/nagios.log
 261
 262
 263 #We kill the most important thing first : the scheduler-Master
 264 bin/stop_scheduler.sh
 265
 266 #We sleep to be sruethe scheduler see us
 267 sleep 2
 268 NB_SCHEDULERS=1
 269 print_date
 270
 271 #First we look is the arbiter saw the scheduler as dead
 272 string_in_file "Warning : Scheduler scheduler-Master had the configuration 0 but is dead, I am not happy." $VAR/nagios.log
 273 #Then we look if the scheduler-spare got a conf from arbiter (here, view from the arbiter)
 274 string_in_file "Dispatch OK of for conf in scheduler scheduler-Spare" $VAR/nagios.log
 275
 276 #then is the broker know it and try to connect to the new scheduler-spare
 277 string_in_file "\[broker-Master\] Connexion OK to the scheduler scheduler-Spare" $VAR/nagios.log
 278
 279
 280 echo "Now stop the poller-Master"
 281 #Now we stop the poller. We will see the sapre take the job (we hope in fact :) )
 282 bin/stop_poller.sh
 283 #check_good_run var
 284 sleep 2
 285 print_date
 286
 287 #The master should be look dead
 288 string_in_file "Warning : The poller poller-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
 289 #The spare should got the conf
 290 string_in_file "\[All\] Dispatch OK of for configuration 0 to poller poller-Slave" $VAR/nagios.log
 291 #And he should got the scheduler link (the sapre one)
 292 string_in_file "\[poller-Slave\] Connexion OK with scheduler scheduler-Spare" $VAR/nagios.log
 293
 294
 295 echo "Now stop the reactionner"
 296 bin/stop_reactionner.sh
 297 #check_good_run var
 298 sleep 2
 299 print_date
 300
 301 #The master should be look dead
 302 string_in_file "\[All\] Warning : The reactionner reactionner-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
 303 #The spare should got the conf
 304 string_in_file "\[All\] Dispatch OK of for configuration 0 to reactionner reactionner-Spare" $VAR/nagios.log
 305 #And he should got the scheduler link (the sapre one)
 306 string_in_file "\[reactionner-Spare\] Connexion OK with scheduler scheduler-Spare" $VAR/nagios.log
 307
 308
 309 echo "Now we stop... the Broker!"
 310 bin/stop_broker.sh
 311 #check_good_run var
 312 sleep 2
 313 print_date
 314
 315 #The master should be look dead
 316 string_in_file "\[All\] Warning : The broker broker-Master seems to be down, I must re-dispatch its role to someone else." $VAR/nagios.log
 317 #The spare should got the conf
 318 string_in_file "\[All\] Dispatch OK of for configuration 0 to broker broker-Slave" $VAR/nagios.log
 319 #And he should got the scheduler link (the spare one)
 320 string_in_file "\[broker-Slave\] Connexion OK to the scheduler scheduler-Spare" $VAR/nagios.log
 321 #And to other satellites
 322 string_in_file "\[broker-Slave\] Connexion OK to the reactionner reactionner-Spare" $VAR/nagios.log
 323 string_in_file "\[broker-Slave\] Connexion problem to the poller poller-Master : connection failed" $VAR/nagios.log
 324 #And should have load the modules
 325 string_in_file "\[broker-Slave\] I correctly load the modules : \['Simple-log', 'Livestatus'\]" $VAR/nagios.log
 326
 327
 328 echo "Now we clean it"
 329 ./clean.sh
 330
 331
 332
 333 echo "####################################################################################"
 334 echo "#                                                                                  #"
 335 echo "#                          Load balancing launch                                   #"
 336 echo "#                                                                                  #"
 337 echo "####################################################################################"
 338
 339 echo "Now we can start some launch tests"
 340 test/bin/launch_all_debug3.sh
 341
 342
 343 echo "Now checking for existing apps"
 344
 345 echo "we can sleep 5sec for conf dispatching and so good number of process"
 346 sleep 5
 347
 348 #The number of process changed, we mush look for it
 349
 350
 351 #Standard launch process packets
 352 NB_SCHEDULERS=2
 353 #6 for stack 1, and 6 for stack 2
 354 NB_POLLERS=12
 355 #3 for stack1, same for stack 2
 356 NB_REACTIONNERS=6
 357 #3 for stack 1, 1 for stack2 (no livestatus nor log worker launch)
 358 NB_BROKERS=4
 359 #still 1
 360 NB_ARBITERS=1
 361
 362 #Now check if the run looks good with var in the direct directory
 363 check_good_run var
 364
 365 echo "All launch of LB daemons is OK"
 366
 367
 368 #Now look if it's also good in the log file too
 369 string_in_file "Dispatch OK of for conf in scheduler scheduler-Master-2" $VAR/nagios.log
 370 string_in_file "Dispatch OK of for conf in scheduler scheduler-Master-1" $VAR/nagios.log
 371 string_in_file "OK, no more reactionner sent need" $VAR/nagios.log
 372 string_in_file "OK, no more poller sent need" $VAR/nagios.log
 373 string_in_file "OK, no more broker sent need" $VAR/nagios.log
 374
 375 echo "Now we clean it"
 376 ./clean.sh
 377
 378
 379 echo ""
 380 echo ""
 381 echo "All check are OK. Congrats! You can go take a Beer ;)"