python/samba/tests/prefork_restart.py

   1 # Tests for process restarting in the pre-fork process model
   2 #
   3 # Copyright (C) Andrew Bartlett <abartlet@samba.org> 2018
   4 #
   5 # This program is free software; you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation; either version 3 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17 #
  18 """Tests process restarting in the pre-fork process model.
  19    NOTE: As this test kills samba processes it won't play nicely with other
  20          tests, so needs to be run in it's own environment.
  21 """
  22
  23
  24 import os
  25 import signal
  26 import time
  27
  28 import samba
  29 from samba.tests import TestCase, delete_force
  30 from samba.dcerpc import echo, netlogon
  31 from samba.messaging import Messaging
  32 from samba.samdb import SamDB
  33 from samba.credentials import Credentials, DONT_USE_KERBEROS
  34 from samba.common import get_string
  35 from samba.dsdb import (
  36     UF_WORKSTATION_TRUST_ACCOUNT,
  37     UF_PASSWD_NOTREQD)
  38 from samba.dcerpc.misc import SEC_CHAN_WKSTA
  39 from samba.auth import system_session
  40
  41 NUM_WORKERS = 4
  42 MACHINE_NAME = "PFRS"
  43
  44
  45 class PreforkProcessRestartTests(TestCase):
  46
  47     def setUp(self):
  48         super().setUp()
  49         lp_ctx = self.get_loadparm()
  50         self.msg_ctx = Messaging(lp_ctx=lp_ctx)
  51
  52     def get_process_data(self):
  53         services = self.msg_ctx.irpc_all_servers()
  54
  55         processes = []
  56         for service in services:
  57             for id in service.ids:
  58                 processes.append((service.name, id.pid))
  59         return processes
  60
  61     def get_process(self, name):
  62         processes = self.get_process_data()
  63         for pname, pid in processes:
  64             if name == pname:
  65                 return pid
  66         return None
  67
  68     def get_worker_pids(self, name, workers):
  69         pids = []
  70         for x in range(workers):
  71             process_name = "prefork-worker-{0}-{1}".format(name, x)
  72             pids.append(self.get_process(process_name))
  73             self.assertIsNotNone(pids[x])
  74         return pids
  75
  76     def wait_for_workers(self, name, workers):
  77         num_workers = len(workers)
  78         for x in range(num_workers):
  79             process_name = "prefork-worker-{0}-{1}".format(name, x)
  80             self.wait_for_process(process_name, workers[x], 0, 1, 30)
  81
  82     def wait_for_process(self, name, pid, initial_delay, wait, timeout):
  83         time.sleep(initial_delay)
  84         delay = initial_delay
  85         while delay < timeout:
  86             p = self.get_process(name)
  87             if p is not None and p != pid:
  88                 # process has restarted
  89                 return
  90             time.sleep(wait)
  91             delay += wait
  92         self.fail("Times out after {0} seconds waiting for {1} to restart".
  93                   format(delay, name))
  94
  95     def check_for_duplicate_processes(self):
  96             processes = self.get_process_data()
  97             process_map = {}
  98             for name, p in processes:
  99                 if (name.startswith("prefork-") or
 100                     name.endswith("_server") or
 101                     name.endswith("srv")):
 102
 103                     if name in process_map:
 104                         if p != process_map[name]:
 105                             self.fail(
 106                                 "Duplicate process for {0}, pids {1} and {2}".
 107                                 format(name, p, process_map[name]))
 108
 109     def simple_bind(self):
 110         creds = self.insta_creds(template=self.get_credentials())
 111         creds.set_bind_dn("%s\\%s" % (creds.get_domain(),
 112                                       creds.get_username()))
 113
 114         self.samdb = SamDB(url="ldaps://%s" % os.environ["SERVER"],
 115                            lp=self.get_loadparm(),
 116                            credentials=creds)
 117
 118     def rpc_echo(self):
 119         conn = echo.rpcecho("ncalrpc:", self.get_loadparm())
 120         self.assertEqual([1, 2, 3], conn.EchoData([1, 2, 3]))
 121
 122     def netlogon(self):
 123         server = os.environ["SERVER"]
 124         host = os.environ["SERVER_IP"]
 125         lp = self.get_loadparm()
 126
 127         credentials = self.get_credentials()
 128
 129         session = system_session()
 130         ldb = SamDB(url="ldap://%s" % host,
 131                     session_info=session,
 132                     credentials=credentials,
 133                     lp=lp)
 134         machine_pass = samba.generate_random_password(32, 32)
 135         machine_name = MACHINE_NAME
 136         machine_dn = "cn=%s,%s" % (machine_name, ldb.domain_dn())
 137
 138         delete_force(ldb, machine_dn)
 139
 140         utf16pw = ('"%s"' % get_string(machine_pass)).encode('utf-16-le')
 141         ldb.add({
 142             "dn": machine_dn,
 143             "objectclass": "computer",
 144             "sAMAccountName": "%s$" % machine_name,
 145             "userAccountControl":
 146                 str(UF_WORKSTATION_TRUST_ACCOUNT | UF_PASSWD_NOTREQD),
 147             "unicodePwd": utf16pw})
 148
 149         machine_creds = Credentials()
 150         machine_creds.guess(lp)
 151         machine_creds.set_secure_channel_type(SEC_CHAN_WKSTA)
 152         machine_creds.set_kerberos_state(DONT_USE_KERBEROS)
 153         machine_creds.set_password(machine_pass)
 154         machine_creds.set_username(machine_name + "$")
 155         machine_creds.set_workstation(machine_name)
 156
 157         netlogon.netlogon(
 158             "ncacn_ip_tcp:%s[schannel,seal]" % server,
 159             lp,
 160             machine_creds)
 161
 162         delete_force(ldb, machine_dn)
 163
 164     def test_ldap_master_restart(self):
 165         # check ldap connection, do a simple bind
 166         self.simple_bind()
 167
 168         # get ldap master process
 169         pid = self.get_process("prefork-master-ldap")
 170         self.assertIsNotNone(pid)
 171
 172         # Get the worker processes
 173         workers = self.get_worker_pids("ldap", NUM_WORKERS)
 174
 175         # kill it
 176         os.kill(pid, signal.SIGTERM)
 177
 178         # wait for the process to restart
 179         self.wait_for_process("prefork-master-ldap", pid, 1, 1, 30)
 180
 181         # restarting the master restarts the workers as well, so make sure
 182         # they have finished restarting
 183         self.wait_for_workers("ldap", workers)
 184
 185         # get ldap master process
 186         new_pid = self.get_process("prefork-master-ldap")
 187         self.assertIsNotNone(new_pid)
 188
 189         # check that the pid has changed
 190         self.assertNotEqual(pid, new_pid)
 191
 192         # check that the worker processes have restarted
 193         new_workers = self.get_worker_pids("ldap", NUM_WORKERS)
 194         for x in range(NUM_WORKERS):
 195             self.assertNotEqual(workers[x], new_workers[x])
 196
 197         # check that the previous server entries have been removed.
 198         self.check_for_duplicate_processes()
 199
 200         # check ldap connection, another simple bind
 201         self.simple_bind()
 202
 203     def test_ldap_worker_restart(self):
 204         # check ldap connection, do a simple bind
 205         self.simple_bind()
 206
 207         # get ldap master process
 208         pid = self.get_process("prefork-master-ldap")
 209         self.assertIsNotNone(pid)
 210
 211         # Get the worker processes
 212         workers = self.get_worker_pids("ldap", NUM_WORKERS)
 213
 214         # kill worker 0
 215         os.kill(workers[0], signal.SIGTERM)
 216
 217         # wait for the process to restart
 218         self.wait_for_process("prefork-worker-ldap-0", pid, 1, 1, 30)
 219
 220         # get ldap master process
 221         new_pid = self.get_process("prefork-master-ldap")
 222         self.assertIsNotNone(new_pid)
 223
 224         # check that the pid has not changed
 225         self.assertEqual(pid, new_pid)
 226
 227         # check that the worker processes have restarted
 228         new_workers = self.get_worker_pids("ldap", NUM_WORKERS)
 229         # process 0 should have a new pid the others should be unchanged
 230         self.assertNotEqual(workers[0], new_workers[0])
 231         self.assertEqual(workers[1], new_workers[1])
 232         self.assertEqual(workers[2], new_workers[2])
 233         self.assertEqual(workers[3], new_workers[3])
 234
 235         # check that the previous server entries have been removed.
 236         self.check_for_duplicate_processes()
 237
 238         # check ldap connection, another simple bind
 239         self.simple_bind()
 240
 241     #
 242     # Kill all the ldap worker processes and ensure that they are restarted
 243     # correctly
 244     #
 245     def test_ldap_all_workers_restart(self):
 246         # check ldap connection, do a simple bind
 247         self.simple_bind()
 248
 249         # get ldap master process
 250         pid = self.get_process("prefork-master-ldap")
 251         self.assertIsNotNone(pid)
 252
 253         # Get the worker processes
 254         workers = self.get_worker_pids("ldap", NUM_WORKERS)
 255
 256         # kill all the worker processes
 257         for x in workers:
 258             os.kill(x, signal.SIGTERM)
 259
 260         # wait for the worker processes to restart
 261         self.wait_for_workers("ldap", workers)
 262
 263         # get ldap master process
 264         new_pid = self.get_process("prefork-master-ldap")
 265         self.assertIsNotNone(new_pid)
 266
 267         # check that the pid has not changed
 268         self.assertEqual(pid, new_pid)
 269
 270         # check that the worker processes have restarted
 271         new_workers = self.get_worker_pids("ldap", NUM_WORKERS)
 272         for x in range(NUM_WORKERS):
 273             self.assertNotEqual(workers[x], new_workers[x])
 274
 275         # check that the previous server entries have been removed.
 276         self.check_for_duplicate_processes()
 277
 278         # check ldap connection, another simple bind
 279         self.simple_bind()
 280
 281     def test_rpc_master_restart(self):
 282         # check rpc connection, make a rpc echo request
 283         self.rpc_echo()
 284
 285         # get rpc master process
 286         pid = self.get_process("prefork-master-rpc")
 287         self.assertIsNotNone(pid)
 288
 289         # Get the worker processes
 290         workers = self.get_worker_pids("rpc", NUM_WORKERS)
 291
 292         # kill it
 293         os.kill(pid, signal.SIGTERM)
 294
 295         # wait for the process to restart
 296         self.wait_for_process("prefork-master-rpc", pid, 1, 1, 30)
 297
 298         # wait for workers to restart as well
 299         self.wait_for_workers("rpc", workers)
 300
 301         # get ldap master process
 302         new_pid = self.get_process("prefork-master-rpc")
 303         self.assertIsNotNone(new_pid)
 304
 305         # check that the pid has changed
 306         self.assertNotEqual(pid, new_pid)
 307
 308         # check that the worker processes have restarted
 309         new_workers = self.get_worker_pids("rpc", NUM_WORKERS)
 310         for x in range(NUM_WORKERS):
 311             self.assertNotEqual(workers[x], new_workers[x])
 312
 313         # check that the previous server entries have been removed.
 314         self.check_for_duplicate_processes()
 315
 316         # check rpc connection, another rpc echo request
 317         self.rpc_echo()
 318
 319     def test_rpc_worker_zero_restart(self):
 320         # check rpc connection, make a rpc echo request and a netlogon request
 321         self.rpc_echo()
 322         self.netlogon()
 323
 324         # get rpc master process
 325         pid = self.get_process("prefork-master-rpc")
 326         self.assertIsNotNone(pid)
 327
 328         # Get the worker processes
 329         workers = self.get_worker_pids("rpc", NUM_WORKERS)
 330
 331         # kill worker 0
 332         os.kill(workers[0], signal.SIGTERM)
 333
 334         # wait for the process to restart
 335         self.wait_for_process("prefork-worker-rpc-0", workers[0], 1, 1, 30)
 336
 337         # get rpc master process
 338         new_pid = self.get_process("prefork-master-rpc")
 339         self.assertIsNotNone(new_pid)
 340
 341         # check that the pid has not changed
 342         self.assertEqual(pid, new_pid)
 343
 344         # check that the worker processes have restarted
 345         new_workers = self.get_worker_pids("rpc", NUM_WORKERS)
 346         # process 0 should have a new pid the others should be unchanged
 347         self.assertNotEqual(workers[0], new_workers[0])
 348         self.assertEqual(workers[1], new_workers[1])
 349         self.assertEqual(workers[2], new_workers[2])
 350         self.assertEqual(workers[3], new_workers[3])
 351
 352         # check that the previous server entries have been removed.
 353         self.check_for_duplicate_processes()
 354
 355         # check rpc connection, another rpc echo request, and netlogon request
 356         self.rpc_echo()
 357         self.netlogon()
 358
 359     def test_rpc_all_workers_restart(self):
 360         # check rpc connection, make a rpc echo request, and a netlogon request
 361         self.rpc_echo()
 362         self.netlogon()
 363
 364         # get rpc master process
 365         pid = self.get_process("prefork-master-rpc")
 366         self.assertIsNotNone(pid)
 367
 368         # Get the worker processes
 369         workers = self.get_worker_pids("rpc", NUM_WORKERS)
 370
 371         # kill all the worker processes
 372         for x in workers:
 373             os.kill(x, signal.SIGTERM)
 374
 375         # wait for the worker processes to restart
 376         for x in range(NUM_WORKERS):
 377             self.wait_for_process(
 378                 "prefork-worker-rpc-{0}".format(x), workers[x], 0, 1, 30)
 379
 380         # get rpc master process
 381         new_pid = self.get_process("prefork-master-rpc")
 382         self.assertIsNotNone(new_pid)
 383
 384         # check that the pid has not changed
 385         self.assertEqual(pid, new_pid)
 386
 387         # check that the worker processes have restarted
 388         new_workers = self.get_worker_pids("rpc", NUM_WORKERS)
 389         for x in range(NUM_WORKERS):
 390             self.assertNotEqual(workers[x], new_workers[x])
 391
 392         # check that the previous server entries have been removed.
 393         self.check_for_duplicate_processes()
 394
 395         # check rpc connection, another rpc echo request and netlogon
 396         self.rpc_echo()
 397         self.netlogon()
 398
 399     def test_master_restart_backoff(self):
 400
 401         # get kdc master process
 402         pid = self.get_process("prefork-master-echo")
 403         self.assertIsNotNone(pid)
 404
 405         #
 406         # Check that the processes get backed off as expected
 407         #
 408         # have prefork backoff increment = 5
 409         #      prefork maximum backoff   = 10
 410         backoff_increment = 5
 411         for expected in [0, 5, 10, 10]:
 412             # Get the worker processes
 413             workers = self.get_worker_pids("kdc", NUM_WORKERS)
 414
 415             process = self.get_process("prefork-master-echo")
 416             os.kill(process, signal.SIGTERM)
 417             # wait for the process to restart
 418             start = time.time()
 419             self.wait_for_process("prefork-master-echo", process, 0, 1, 30)
 420             # wait for the workers to restart as well
 421             self.wait_for_workers("echo", workers)
 422             end = time.time()
 423             duration = end - start
 424
 425             # process restart will take some time. Check that the elapsed
 426             # duration falls somewhere in the expected range, i.e. we haven't
 427             # taken longer than the backoff increment
 428             self.assertLess(duration, expected + backoff_increment)
 429             self.assertGreaterEqual(duration, expected)
 430
 431         # check that the worker processes have restarted
 432         new_workers = self.get_worker_pids("echo", NUM_WORKERS)
 433         for x in range(NUM_WORKERS):
 434             self.assertNotEqual(workers[x], new_workers[x])
 435
 436         # check that the previous server entries have been removed.
 437         self.check_for_duplicate_processes()
 438
 439     def test_worker_restart_backoff(self):
 440         #
 441         # Check that the processes get backed off as expected
 442         #
 443         # have prefork backoff increment = 5
 444         #      prefork maximum backoff   = 10
 445         backoff_increment = 5
 446         for expected in [0, 5, 10, 10]:
 447             process = self.get_process("prefork-worker-echo-2")
 448             self.assertIsNotNone(process)
 449             os.kill(process, signal.SIGTERM)
 450             # wait for the process to restart
 451             start = time.time()
 452             self.wait_for_process("prefork-worker-echo-2", process, 0, 1, 30)
 453             end = time.time()
 454             duration = end - start
 455
 456             # process restart will take some time. Check that the elapsed
 457             # duration falls somewhere in the expected range, i.e. we haven't
 458             # taken longer than the backoff increment
 459             self.assertLess(duration, expected + backoff_increment)
 460             self.assertGreaterEqual(duration, expected)
 461
 462         self.check_for_duplicate_processes()