src/applications/daemon/management/PhabricatorDaemonManagementWorkflow.php

   1 <?php
   2
   3 abstract class PhabricatorDaemonManagementWorkflow
   4   extends PhabricatorManagementWorkflow {
   5
   6   private $runDaemonsAsUser = null;
   7
   8   final protected function loadAvailableDaemonClasses() {
   9     return id(new PhutilSymbolLoader())
  10       ->setAncestorClass('PhutilDaemon')
  11       ->setConcreteOnly(true)
  12       ->selectSymbolsWithoutLoading();
  13   }
  14
  15   final protected function getLogDirectory() {
  16     $path = PhabricatorEnv::getEnvConfig('phd.log-directory');
  17     return $this->getControlDirectory($path);
  18   }
  19
  20   private function getControlDirectory($path) {
  21     if (!Filesystem::pathExists($path)) {
  22       list($err) = exec_manual('mkdir -p %s', $path);
  23       if ($err) {
  24         throw new Exception(
  25           pht(
  26             "%s requires the directory '%s' to exist, but it does not exist ".
  27             "and could not be created. Create this directory or update ".
  28             "'%s' in your configuration to point to an existing ".
  29             "directory.",
  30             'phd',
  31             $path,
  32             'phd.log-directory'));
  33       }
  34     }
  35     return $path;
  36   }
  37
  38   private function findDaemonClass($substring) {
  39     $symbols = $this->loadAvailableDaemonClasses();
  40
  41     $symbols = ipull($symbols, 'name');
  42     $match = array();
  43     foreach ($symbols as $symbol) {
  44       if (stripos($symbol, $substring) !== false) {
  45         if (strtolower($symbol) == strtolower($substring)) {
  46           $match = array($symbol);
  47           break;
  48         } else {
  49           $match[] = $symbol;
  50         }
  51       }
  52     }
  53
  54     if (count($match) == 0) {
  55       throw new PhutilArgumentUsageException(
  56         pht(
  57           "No daemons match '%s'! Use '%s' for a list of available daemons.",
  58           $substring,
  59           'phd list'));
  60     } else if (count($match) > 1) {
  61       throw new PhutilArgumentUsageException(
  62         pht(
  63           "Specify a daemon unambiguously. Multiple daemons match '%s': %s.",
  64           $substring,
  65           implode(', ', $match)));
  66     }
  67
  68     return head($match);
  69   }
  70
  71   final protected function launchDaemons(
  72     array $daemons,
  73     $debug,
  74     $run_as_current_user = false) {
  75
  76     // Convert any shorthand classnames like "taskmaster" into proper class
  77     // names.
  78     foreach ($daemons as $key => $daemon) {
  79       $class = $this->findDaemonClass($daemon['class']);
  80       $daemons[$key]['class'] = $class;
  81     }
  82
  83     $console = PhutilConsole::getConsole();
  84
  85     if (!$run_as_current_user) {
  86       // Check if the script is started as the correct user
  87       $phd_user = PhabricatorEnv::getEnvConfig('phd.user');
  88       $current_user = posix_getpwuid(posix_geteuid());
  89       $current_user = $current_user['name'];
  90       if ($phd_user && $phd_user != $current_user) {
  91         if ($debug) {
  92           throw new PhutilArgumentUsageException(
  93             pht(
  94               "You are trying to run a daemon as a nonstandard user, ".
  95               "and `%s` was not able to `%s` to the correct user. \n".
  96               'The daemons are configured to run as "%s", '.
  97               'but the current user is "%s". '."\n".
  98               'Use `%s` to run as a different user, pass `%s` to ignore this '.
  99               'warning, or edit `%s` to change the configuration.',
 100               'phd',
 101               'sudo',
 102               $phd_user,
 103               $current_user,
 104               'sudo',
 105               '--as-current-user',
 106               'phd.user'));
 107         } else {
 108           $this->runDaemonsAsUser = $phd_user;
 109           $console->writeOut(pht('Starting daemons as %s', $phd_user)."\n");
 110         }
 111       }
 112     }
 113
 114     $this->printLaunchingDaemons($daemons, $debug);
 115
 116     $trace = PhutilArgumentParser::isTraceModeEnabled();
 117
 118     $flags = array();
 119     if ($trace) {
 120       $flags[] = '--trace';
 121     }
 122
 123     if ($debug) {
 124       $flags[] = '--verbose';
 125     }
 126
 127     $instance = $this->getInstance();
 128     if ($instance) {
 129       $flags[] = '-l';
 130       $flags[] = $instance;
 131     }
 132
 133     $config = array();
 134
 135     if (!$debug) {
 136       $config['daemonize'] = true;
 137     }
 138
 139     if (!$debug) {
 140       $config['log'] = $this->getLogDirectory().'/daemons.log';
 141     }
 142
 143     $config['daemons'] = $daemons;
 144
 145     $command = csprintf('./phd-daemon %Ls', $flags);
 146
 147     $phabricator_root = dirname(phutil_get_library_root('phabricator'));
 148     $daemon_script_dir = $phabricator_root.'/scripts/daemon/';
 149
 150     if ($debug) {
 151       // Don't terminate when the user sends ^C; it will be sent to the
 152       // subprocess which will terminate normally.
 153       pcntl_signal(
 154         SIGINT,
 155         array(__CLASS__, 'ignoreSignal'));
 156
 157       echo "\n    scripts/daemon/ \$ {$command}\n\n";
 158
 159       $tempfile = new TempFile('daemon.config');
 160       Filesystem::writeFile($tempfile, json_encode($config));
 161
 162       phutil_passthru(
 163         '(cd %s && exec %C < %s)',
 164         $daemon_script_dir,
 165         $command,
 166         $tempfile);
 167     } else {
 168       try {
 169         $this->executeDaemonLaunchCommand(
 170           $command,
 171           $daemon_script_dir,
 172           $config,
 173           $this->runDaemonsAsUser);
 174       } catch (Exception $ex) {
 175         throw new PhutilArgumentUsageException(
 176           pht(
 177             'Daemons are configured to run as user "%s" in configuration '.
 178             'option `%s`, but the current user is "%s" and `phd` was unable '.
 179             'to switch to the correct user with `sudo`. Command output:'.
 180             "\n\n".
 181             '%s',
 182             $phd_user,
 183             'phd.user',
 184             $current_user,
 185             $ex->getMessage()));
 186       }
 187     }
 188   }
 189
 190   private function executeDaemonLaunchCommand(
 191     $command,
 192     $daemon_script_dir,
 193     array $config,
 194     $run_as_user = null) {
 195
 196     $is_sudo = false;
 197     if ($run_as_user) {
 198       // If anything else besides sudo should be
 199       // supported then insert it here (runuser, su, ...)
 200       $command = csprintf(
 201         'sudo -En -u %s -- %C',
 202         $run_as_user,
 203         $command);
 204       $is_sudo = true;
 205     }
 206     $future = new ExecFuture('exec %C', $command);
 207     // Play games to keep 'ps' looking reasonable.
 208     $future->setCWD($daemon_script_dir);
 209     $future->write(json_encode($config));
 210     list($stdout, $stderr) = $future->resolvex();
 211
 212     if ($is_sudo) {
 213       // On OSX, `sudo -n` exits 0 when the user does not have permission to
 214       // switch accounts without a password. This is not consistent with
 215       // sudo on Linux, and seems buggy/broken. Check for this by string
 216       // matching the output.
 217       if (preg_match('/sudo: a password is required/', $stderr)) {
 218         throw new Exception(
 219           pht(
 220             '%s exited with a zero exit code, but emitted output '.
 221             'consistent with failure under OSX.',
 222             'sudo'));
 223       }
 224     }
 225   }
 226
 227   public static function ignoreSignal($signo) {
 228     return;
 229   }
 230
 231   public static function requireExtensions() {
 232     self::mustHaveExtension('pcntl');
 233     self::mustHaveExtension('posix');
 234   }
 235
 236   private static function mustHaveExtension($ext) {
 237     if (!extension_loaded($ext)) {
 238       echo pht(
 239         "ERROR: The PHP extension '%s' is not installed. You must ".
 240         "install it to run daemons on this machine.\n",
 241         $ext);
 242       exit(1);
 243     }
 244
 245     $extension = new ReflectionExtension($ext);
 246     foreach ($extension->getFunctions() as $function) {
 247       $function = $function->name;
 248       if (!function_exists($function)) {
 249         echo pht(
 250           "ERROR: The PHP function %s is disabled. You must ".
 251           "enable it to run daemons on this machine.\n",
 252           $function.'()');
 253         exit(1);
 254       }
 255     }
 256   }
 257
 258
 259 /* -(  Commands  )----------------------------------------------------------- */
 260
 261
 262   final protected function executeStartCommand(array $options) {
 263     PhutilTypeSpec::checkMap(
 264       $options,
 265       array(
 266         'keep-leases' => 'optional bool',
 267         'force' => 'optional bool',
 268         'reserve' => 'optional float',
 269       ));
 270
 271     $console = PhutilConsole::getConsole();
 272
 273     if (!idx($options, 'force')) {
 274       $process_refs = $this->getOverseerProcessRefs();
 275       if ($process_refs) {
 276         $this->logWarn(
 277           pht('RUNNING DAEMONS'),
 278           pht('Daemons are already running:'));
 279
 280         fprintf(STDERR, '%s', "\n");
 281         foreach ($process_refs as $process_ref) {
 282           fprintf(
 283             STDERR,
 284             '%s',
 285             tsprintf(
 286               "        %s %s\n",
 287               $process_ref->getPID(),
 288               $process_ref->getCommand()));
 289         }
 290         fprintf(STDERR, '%s', "\n");
 291
 292         $this->logFail(
 293           pht('RUNNING DAEMONS'),
 294           pht(
 295             'Use "phd stop" to stop daemons, "phd restart" to restart '.
 296             'daemons, or "phd start --force" to ignore running processes.'));
 297
 298         exit(1);
 299       }
 300     }
 301
 302     if (idx($options, 'keep-leases')) {
 303       $console->writeErr("%s\n", pht('Not touching active task queue leases.'));
 304     } else {
 305       $console->writeErr("%s\n", pht('Freeing active task leases...'));
 306       $count = $this->freeActiveLeases();
 307       $console->writeErr(
 308         "%s\n",
 309         pht('Freed %s task lease(s).', new PhutilNumber($count)));
 310     }
 311
 312     $daemons = array(
 313       array(
 314         'class' => 'PhabricatorRepositoryPullLocalDaemon',
 315         'label' => 'pull',
 316       ),
 317       array(
 318         'class' => 'PhabricatorTriggerDaemon',
 319         'label' => 'trigger',
 320       ),
 321       array(
 322         'class' => 'PhabricatorFactDaemon',
 323         'label' => 'fact',
 324       ),
 325       array(
 326         'class' => 'PhabricatorTaskmasterDaemon',
 327         'label' => 'task',
 328         'pool' => PhabricatorEnv::getEnvConfig('phd.taskmasters'),
 329         'reserve' => idx($options, 'reserve', 0),
 330       ),
 331     );
 332
 333     $this->launchDaemons($daemons, $is_debug = false);
 334
 335     $console->writeErr("%s\n", pht('Done.'));
 336     return 0;
 337   }
 338
 339   final protected function executeStopCommand(array $options) {
 340     $grace_period = idx($options, 'graceful', 15);
 341     $force = idx($options, 'force');
 342
 343     $query = id(new PhutilProcessQuery())
 344       ->withIsOverseer(true);
 345
 346     $instance = $this->getInstance();
 347     if ($instance !== null && !$force) {
 348       $query->withInstances(array($instance));
 349     }
 350
 351     try {
 352       $process_refs = $query->execute();
 353     } catch (Exception $ex) {
 354       // See T13321. If this fails for some reason, just continue for now so
 355       // that daemon management still works. In the long run, we don't expect
 356       // this to fail, but I don't want to break this workflow while we iron
 357       // bugs out.
 358
 359       // See T12827. Particularly, this is likely to fail on Solaris.
 360
 361       phlog($ex);
 362
 363       $process_refs = array();
 364     }
 365
 366     if (!$process_refs) {
 367       if ($instance !== null && !$force) {
 368         $this->logInfo(
 369           pht('NO DAEMONS'),
 370           pht(
 371             'There are no running daemons for the current instance ("%s"). '.
 372             'Use "--force" to stop daemons for all instances.',
 373             $instance));
 374       } else {
 375         $this->logInfo(
 376           pht('NO DAEMONS'),
 377           pht('There are no running daemons.'));
 378       }
 379
 380       return 0;
 381     }
 382
 383     $process_refs = mpull($process_refs, null, 'getPID');
 384
 385     $stop_pids = array_keys($process_refs);
 386     $live_pids = $this->sendStopSignals($stop_pids, $grace_period);
 387
 388     $stop_pids = array_fuse($stop_pids);
 389     $live_pids = array_fuse($live_pids);
 390
 391     $dead_pids = array_diff_key($stop_pids, $live_pids);
 392
 393     foreach ($dead_pids as $dead_pid) {
 394       $dead_ref = $process_refs[$dead_pid];
 395       $this->logOkay(
 396         pht('STOP'),
 397         pht(
 398           'Stopped PID %d ("%s")',
 399           $dead_pid,
 400           $dead_ref->getCommand()));
 401     }
 402
 403     foreach ($live_pids as $live_pid) {
 404       $live_ref = $process_refs[$live_pid];
 405       $this->logFail(
 406         pht('SURVIVED'),
 407         pht(
 408           'Unable to stop PID %d ("%s").',
 409           $live_pid,
 410           $live_ref->getCommand()));
 411     }
 412
 413     if ($live_pids) {
 414       $this->logWarn(
 415         pht('SURVIVORS'),
 416         pht(
 417           'Unable to stop all daemon processes. You may need to run this '.
 418           'command as root with "sudo".'));
 419     }
 420
 421     return 0;
 422   }
 423
 424   final protected function executeReloadCommand(array $pids) {
 425     $process_refs = $this->getOverseerProcessRefs();
 426
 427     if (!$process_refs) {
 428       $this->logInfo(
 429         pht('NO DAEMONS'),
 430         pht('There are no running daemon processes to reload.'));
 431
 432       return 0;
 433     }
 434
 435     foreach ($process_refs as $process_ref) {
 436       $pid = $process_ref->getPID();
 437
 438       $this->logInfo(
 439         pht('RELOAD'),
 440         pht('Reloading process %d...', $pid));
 441
 442       posix_kill($pid, SIGHUP);
 443     }
 444
 445     return 0;
 446   }
 447
 448   private function sendStopSignals($pids, $grace_period) {
 449     // If we're doing a graceful shutdown, try SIGINT first.
 450     if ($grace_period) {
 451       $pids = $this->sendSignal($pids, SIGINT, $grace_period);
 452     }
 453
 454     // If we still have daemons, SIGTERM them.
 455     if ($pids) {
 456       $pids = $this->sendSignal($pids, SIGTERM, 15);
 457     }
 458
 459     // If the overseer is still alive, SIGKILL it.
 460     if ($pids) {
 461       $pids = $this->sendSignal($pids, SIGKILL, 0);
 462     }
 463
 464     return $pids;
 465   }
 466
 467   private function sendSignal(array $pids, $signo, $wait) {
 468     $console = PhutilConsole::getConsole();
 469
 470     $pids = array_fuse($pids);
 471
 472     foreach ($pids as $key => $pid) {
 473       if (!$pid) {
 474         // NOTE: We must have a PID to signal a daemon, since sending a signal
 475         // to PID 0 kills this process.
 476         unset($pids[$key]);
 477         continue;
 478       }
 479
 480       switch ($signo) {
 481         case SIGINT:
 482           $message = pht('Interrupting process %d...', $pid);
 483           break;
 484         case SIGTERM:
 485           $message = pht('Terminating process %d...', $pid);
 486           break;
 487         case SIGKILL:
 488           $message = pht('Killing process %d...', $pid);
 489           break;
 490       }
 491
 492       $console->writeOut("%s\n", $message);
 493       posix_kill($pid, $signo);
 494     }
 495
 496     if ($wait) {
 497       $start = PhabricatorTime::getNow();
 498       do {
 499         foreach ($pids as $key => $pid) {
 500           if (!PhabricatorDaemonReference::isProcessRunning($pid)) {
 501             $console->writeOut(pht('Process %d exited.', $pid)."\n");
 502             unset($pids[$key]);
 503           }
 504         }
 505         if (empty($pids)) {
 506           break;
 507         }
 508         usleep(100000);
 509       } while (PhabricatorTime::getNow() < $start + $wait);
 510     }
 511
 512     return $pids;
 513   }
 514
 515   private function freeActiveLeases() {
 516     $task_table = id(new PhabricatorWorkerActiveTask());
 517     $conn_w = $task_table->establishConnection('w');
 518     queryfx(
 519       $conn_w,
 520       'UPDATE %T SET leaseExpires = UNIX_TIMESTAMP()
 521         WHERE leaseExpires > UNIX_TIMESTAMP()',
 522       $task_table->getTableName());
 523     return $conn_w->getAffectedRows();
 524   }
 525
 526
 527   private function printLaunchingDaemons(array $daemons, $debug) {
 528     $console = PhutilConsole::getConsole();
 529
 530     if ($debug) {
 531       $console->writeOut(pht('Launching daemons (in debug mode):'));
 532     } else {
 533       $console->writeOut(pht('Launching daemons:'));
 534     }
 535
 536     $log_dir = $this->getLogDirectory().'/daemons.log';
 537     $console->writeOut(
 538       "\n%s\n\n",
 539       pht('(Logs will appear in "%s".)', $log_dir));
 540
 541     foreach ($daemons as $daemon) {
 542       $pool_size = pht('(Pool: %s)', idx($daemon, 'pool', 1));
 543
 544       $console->writeOut(
 545         "    %s %s\n",
 546         $pool_size,
 547         $daemon['class'],
 548         implode(' ', idx($daemon, 'argv', array())));
 549     }
 550     $console->writeOut("\n");
 551   }
 552
 553   protected function getAutoscaleReserveArgument() {
 554     return array(
 555       'name' => 'autoscale-reserve',
 556       'param' => 'ratio',
 557       'help' => pht(
 558         'Specify a proportion of machine memory which must be free '.
 559         'before autoscale pools will grow. For example, a value of 0.25 '.
 560         'means that pools will not grow unless the machine has at least '.
 561         '25%%%% of its RAM free.'),
 562     );
 563   }
 564
 565   private function selectDaemonPIDs(array $daemons, array $pids) {
 566     $console = PhutilConsole::getConsole();
 567
 568     $running_pids = array_fuse(mpull($daemons, 'getPID'));
 569     if (!$pids) {
 570       $select_pids = $running_pids;
 571     } else {
 572       // We were given a PID or set of PIDs to kill.
 573       $select_pids = array();
 574       foreach ($pids as $key => $pid) {
 575         if (!preg_match('/^\d+$/', $pid)) {
 576           $console->writeErr(pht("PID '%s' is not a valid PID.", $pid)."\n");
 577           continue;
 578         } else if (empty($running_pids[$pid])) {
 579           $console->writeErr(
 580             "%s\n",
 581             pht(
 582               'PID "%d" is not a known daemon PID.',
 583               $pid));
 584           continue;
 585         } else {
 586           $select_pids[$pid] = $pid;
 587         }
 588       }
 589     }
 590
 591     return $select_pids;
 592   }
 593
 594   protected function getOverseerProcessRefs() {
 595     $query = id(new PhutilProcessQuery())
 596       ->withIsOverseer(true);
 597
 598     $instance = PhabricatorEnv::getEnvConfig('cluster.instance');
 599     if ($instance !== null) {
 600       $query->withInstances(array($instance));
 601     }
 602
 603     return $query->execute();
 604   }
 605
 606   protected function getInstance() {
 607     return PhabricatorEnv::getEnvConfig('cluster.instance');
 608   }
 609
 610
 611 }