src/infrastructure/cluster/PhabricatorDatabaseRef.php

   1 <?php
   2
   3 final class PhabricatorDatabaseRef
   4   extends Phobject {
   5
   6   const STATUS_OKAY = 'okay';
   7   const STATUS_FAIL = 'fail';
   8   const STATUS_AUTH = 'auth';
   9   const STATUS_REPLICATION_CLIENT = 'replication-client';
  10
  11   const REPLICATION_OKAY = 'okay';
  12   const REPLICATION_MASTER_REPLICA = 'master-replica';
  13   const REPLICATION_REPLICA_NONE = 'replica-none';
  14   const REPLICATION_SLOW = 'replica-slow';
  15   const REPLICATION_NOT_REPLICATING = 'not-replicating';
  16
  17   const KEY_HEALTH = 'cluster.db.health';
  18   const KEY_REFS = 'cluster.db.refs';
  19   const KEY_INDIVIDUAL = 'cluster.db.individual';
  20
  21   private $host;
  22   private $port;
  23   private $user;
  24   private $pass;
  25   private $disabled;
  26   private $isMaster;
  27   private $isIndividual;
  28
  29   private $connectionLatency;
  30   private $connectionStatus;
  31   private $connectionMessage;
  32   private $connectionException;
  33
  34   private $replicaStatus;
  35   private $replicaMessage;
  36   private $replicaDelay;
  37
  38   private $healthRecord;
  39   private $didFailToConnect;
  40
  41   private $isDefaultPartition;
  42   private $applicationMap = array();
  43   private $masterRef;
  44   private $replicaRefs = array();
  45   private $usePersistentConnections;
  46
  47   public function setHost($host) {
  48     $this->host = $host;
  49     return $this;
  50   }
  51
  52   public function getHost() {
  53     return $this->host;
  54   }
  55
  56   public function setPort($port) {
  57     $this->port = $port;
  58     return $this;
  59   }
  60
  61   public function getPort() {
  62     return $this->port;
  63   }
  64
  65   public function setUser($user) {
  66     $this->user = $user;
  67     return $this;
  68   }
  69
  70   public function getUser() {
  71     return $this->user;
  72   }
  73
  74   public function setPass(PhutilOpaqueEnvelope $pass) {
  75     $this->pass = $pass;
  76     return $this;
  77   }
  78
  79   public function getPass() {
  80     return $this->pass;
  81   }
  82
  83   public function setIsMaster($is_master) {
  84     $this->isMaster = $is_master;
  85     return $this;
  86   }
  87
  88   public function getIsMaster() {
  89     return $this->isMaster;
  90   }
  91
  92   public function setDisabled($disabled) {
  93     $this->disabled = $disabled;
  94     return $this;
  95   }
  96
  97   public function getDisabled() {
  98     return $this->disabled;
  99   }
 100
 101   public function setConnectionLatency($connection_latency) {
 102     $this->connectionLatency = $connection_latency;
 103     return $this;
 104   }
 105
 106   public function getConnectionLatency() {
 107     return $this->connectionLatency;
 108   }
 109
 110   public function setConnectionStatus($connection_status) {
 111     $this->connectionStatus = $connection_status;
 112     return $this;
 113   }
 114
 115   public function getConnectionStatus() {
 116     if ($this->connectionStatus === null) {
 117       throw new PhutilInvalidStateException('queryAll');
 118     }
 119
 120     return $this->connectionStatus;
 121   }
 122
 123   public function setConnectionMessage($connection_message) {
 124     $this->connectionMessage = $connection_message;
 125     return $this;
 126   }
 127
 128   public function getConnectionMessage() {
 129     return $this->connectionMessage;
 130   }
 131
 132   public function setReplicaStatus($replica_status) {
 133     $this->replicaStatus = $replica_status;
 134     return $this;
 135   }
 136
 137   public function getReplicaStatus() {
 138     return $this->replicaStatus;
 139   }
 140
 141   public function setReplicaMessage($replica_message) {
 142     $this->replicaMessage = $replica_message;
 143     return $this;
 144   }
 145
 146   public function getReplicaMessage() {
 147     return $this->replicaMessage;
 148   }
 149
 150   public function setReplicaDelay($replica_delay) {
 151     $this->replicaDelay = $replica_delay;
 152     return $this;
 153   }
 154
 155   public function getReplicaDelay() {
 156     return $this->replicaDelay;
 157   }
 158
 159   public function setIsIndividual($is_individual) {
 160     $this->isIndividual = $is_individual;
 161     return $this;
 162   }
 163
 164   public function getIsIndividual() {
 165     return $this->isIndividual;
 166   }
 167
 168   public function setIsDefaultPartition($is_default_partition) {
 169     $this->isDefaultPartition = $is_default_partition;
 170     return $this;
 171   }
 172
 173   public function getIsDefaultPartition() {
 174     return $this->isDefaultPartition;
 175   }
 176
 177   public function setUsePersistentConnections($use_persistent_connections) {
 178     $this->usePersistentConnections = $use_persistent_connections;
 179     return $this;
 180   }
 181
 182   public function getUsePersistentConnections() {
 183     return $this->usePersistentConnections;
 184   }
 185
 186   public function setApplicationMap(array $application_map) {
 187     $this->applicationMap = $application_map;
 188     return $this;
 189   }
 190
 191   public function getApplicationMap() {
 192     return $this->applicationMap;
 193   }
 194
 195   public function getPartitionStateForCommit() {
 196     $state = PhabricatorEnv::getEnvConfig('cluster.databases');
 197     foreach ($state as $key => $value) {
 198       // Don't store passwords, since we don't care if they differ and
 199       // users may find it surprising.
 200       unset($state[$key]['pass']);
 201     }
 202
 203     return phutil_json_encode($state);
 204   }
 205
 206   public function setMasterRef(PhabricatorDatabaseRef $master_ref) {
 207     $this->masterRef = $master_ref;
 208     return $this;
 209   }
 210
 211   public function getMasterRef() {
 212     return $this->masterRef;
 213   }
 214
 215   public function addReplicaRef(PhabricatorDatabaseRef $replica_ref) {
 216     $this->replicaRefs[] = $replica_ref;
 217     return $this;
 218   }
 219
 220   public function getReplicaRefs() {
 221     return $this->replicaRefs;
 222   }
 223
 224   public function getDisplayName() {
 225     return $this->getRefKey();
 226   }
 227
 228   public function getRefKey() {
 229     $host = $this->getHost();
 230
 231     $port = $this->getPort();
 232     if (strlen($port)) {
 233       return "{$host}:{$port}";
 234     }
 235
 236     return $host;
 237   }
 238
 239   public static function getConnectionStatusMap() {
 240     return array(
 241       self::STATUS_OKAY => array(
 242         'icon' => 'fa-exchange',
 243         'color' => 'green',
 244         'label' => pht('Okay'),
 245       ),
 246       self::STATUS_FAIL => array(
 247         'icon' => 'fa-times',
 248         'color' => 'red',
 249         'label' => pht('Failed'),
 250       ),
 251       self::STATUS_AUTH => array(
 252         'icon' => 'fa-key',
 253         'color' => 'red',
 254         'label' => pht('Invalid Credentials'),
 255       ),
 256       self::STATUS_REPLICATION_CLIENT => array(
 257         'icon' => 'fa-eye-slash',
 258         'color' => 'yellow',
 259         'label' => pht('Missing Permission'),
 260       ),
 261     );
 262   }
 263
 264   public static function getReplicaStatusMap() {
 265     return array(
 266       self::REPLICATION_OKAY => array(
 267         'icon' => 'fa-download',
 268         'color' => 'green',
 269         'label' => pht('Okay'),
 270       ),
 271       self::REPLICATION_MASTER_REPLICA => array(
 272         'icon' => 'fa-database',
 273         'color' => 'red',
 274         'label' => pht('Replicating Master'),
 275       ),
 276       self::REPLICATION_REPLICA_NONE => array(
 277         'icon' => 'fa-download',
 278         'color' => 'red',
 279         'label' => pht('Not A Replica'),
 280       ),
 281       self::REPLICATION_SLOW => array(
 282         'icon' => 'fa-hourglass',
 283         'color' => 'red',
 284         'label' => pht('Slow Replication'),
 285       ),
 286       self::REPLICATION_NOT_REPLICATING => array(
 287         'icon' => 'fa-exclamation-triangle',
 288         'color' => 'red',
 289         'label' => pht('Not Replicating'),
 290       ),
 291     );
 292   }
 293
 294   public static function getClusterRefs() {
 295     $cache = PhabricatorCaches::getRequestCache();
 296
 297     $refs = $cache->getKey(self::KEY_REFS);
 298     if (!$refs) {
 299       $refs = self::newRefs();
 300       $cache->setKey(self::KEY_REFS, $refs);
 301     }
 302
 303     return $refs;
 304   }
 305
 306   public static function getLiveIndividualRef() {
 307     $cache = PhabricatorCaches::getRequestCache();
 308
 309     $ref = $cache->getKey(self::KEY_INDIVIDUAL);
 310     if (!$ref) {
 311       $ref = self::newIndividualRef();
 312       $cache->setKey(self::KEY_INDIVIDUAL, $ref);
 313     }
 314
 315     return $ref;
 316   }
 317
 318   public static function newRefs() {
 319     $default_port = PhabricatorEnv::getEnvConfig('mysql.port');
 320     $default_port = nonempty($default_port, 3306);
 321
 322     $default_user = PhabricatorEnv::getEnvConfig('mysql.user');
 323
 324     $default_pass = PhabricatorEnv::getEnvConfig('mysql.pass');
 325     $default_pass = phutil_string_cast($default_pass);
 326     $default_pass = new PhutilOpaqueEnvelope($default_pass);
 327
 328     $config = PhabricatorEnv::getEnvConfig('cluster.databases');
 329
 330     return id(new PhabricatorDatabaseRefParser())
 331       ->setDefaultPort($default_port)
 332       ->setDefaultUser($default_user)
 333       ->setDefaultPass($default_pass)
 334       ->newRefs($config);
 335   }
 336
 337   public static function queryAll() {
 338     $refs = self::getActiveDatabaseRefs();
 339     return self::queryRefs($refs);
 340   }
 341
 342   private static function queryRefs(array $refs) {
 343     foreach ($refs as $ref) {
 344       $conn = $ref->newManagementConnection();
 345
 346       $t_start = microtime(true);
 347       $replica_status = false;
 348       try {
 349         $replica_status = queryfx_one($conn, 'SHOW SLAVE STATUS');
 350         $ref->setConnectionStatus(self::STATUS_OKAY);
 351       } catch (AphrontAccessDeniedQueryException $ex) {
 352         $ref->setConnectionStatus(self::STATUS_REPLICATION_CLIENT);
 353         $ref->setConnectionMessage(
 354           pht(
 355             'No permission to run "SHOW SLAVE STATUS". Grant this user '.
 356             '"REPLICATION CLIENT" permission to allow Phabricator to '.
 357             'monitor replica health.'));
 358       } catch (AphrontInvalidCredentialsQueryException $ex) {
 359         $ref->setConnectionStatus(self::STATUS_AUTH);
 360         $ref->setConnectionMessage($ex->getMessage());
 361       } catch (AphrontQueryException $ex) {
 362         $ref->setConnectionStatus(self::STATUS_FAIL);
 363
 364         $class = get_class($ex);
 365         $message = $ex->getMessage();
 366         $ref->setConnectionMessage(
 367           pht(
 368             '%s: %s',
 369             get_class($ex),
 370             $ex->getMessage()));
 371       }
 372       $t_end = microtime(true);
 373       $ref->setConnectionLatency($t_end - $t_start);
 374
 375       if ($replica_status !== false) {
 376         $is_replica = (bool)$replica_status;
 377         if ($ref->getIsMaster() && $is_replica) {
 378           $ref->setReplicaStatus(self::REPLICATION_MASTER_REPLICA);
 379           $ref->setReplicaMessage(
 380             pht(
 381               'This host has a "master" role, but is replicating data from '.
 382               'another host ("%s")!',
 383               idx($replica_status, 'Master_Host')));
 384         } else if (!$ref->getIsMaster() && !$is_replica) {
 385           $ref->setReplicaStatus(self::REPLICATION_REPLICA_NONE);
 386           $ref->setReplicaMessage(
 387             pht(
 388               'This host has a "replica" role, but is not replicating data '.
 389               'from a master (no output from "SHOW SLAVE STATUS").'));
 390         } else {
 391           $ref->setReplicaStatus(self::REPLICATION_OKAY);
 392         }
 393
 394         if ($is_replica) {
 395           $latency = idx($replica_status, 'Seconds_Behind_Master');
 396           if (!strlen($latency)) {
 397             $ref->setReplicaStatus(self::REPLICATION_NOT_REPLICATING);
 398           } else {
 399             $latency = (int)$latency;
 400             $ref->setReplicaDelay($latency);
 401             if ($latency > 30) {
 402               $ref->setReplicaStatus(self::REPLICATION_SLOW);
 403               $ref->setReplicaMessage(
 404                 pht(
 405                   'This replica is lagging far behind the master. Data is at '.
 406                   'risk!'));
 407             }
 408           }
 409         }
 410       }
 411     }
 412
 413     return $refs;
 414   }
 415
 416   public function newManagementConnection() {
 417     return $this->newConnection(
 418       array(
 419         'retries' => 0,
 420         'timeout' => 2,
 421       ));
 422   }
 423
 424   public function newApplicationConnection($database) {
 425     return $this->newConnection(
 426       array(
 427         'database' => $database,
 428       ));
 429   }
 430
 431   public function isSevered() {
 432     // If we only have an individual database, never sever our connection to
 433     // it, at least for now. It's possible that using the same severing rules
 434     // might eventually make sense to help alleviate load-related failures,
 435     // but we should wait for all the cluster stuff to stabilize first.
 436     if ($this->getIsIndividual()) {
 437       return false;
 438     }
 439
 440     if ($this->didFailToConnect) {
 441       return true;
 442     }
 443
 444     $record = $this->getHealthRecord();
 445     $is_healthy = $record->getIsHealthy();
 446     if (!$is_healthy) {
 447       return true;
 448     }
 449
 450     return false;
 451   }
 452
 453   public function isReachable(AphrontDatabaseConnection $connection) {
 454     $record = $this->getHealthRecord();
 455     $should_check = $record->getShouldCheck();
 456
 457     if ($this->isSevered() && !$should_check) {
 458       return false;
 459     }
 460
 461     $this->connectionException = null;
 462     try {
 463       $connection->openConnection();
 464       $reachable = true;
 465     } catch (AphrontSchemaQueryException $ex) {
 466       // We get one of these if the database we're trying to select does not
 467       // exist. In this case, just re-throw the exception. This is expected
 468       // during first-time setup, when databases like "config" will not exist
 469       // yet.
 470       throw $ex;
 471     } catch (Exception $ex) {
 472       $this->connectionException = $ex;
 473       $reachable = false;
 474     }
 475
 476     if ($should_check) {
 477       $record->didHealthCheck($reachable);
 478     }
 479
 480     if (!$reachable) {
 481       $this->didFailToConnect = true;
 482     }
 483
 484     return $reachable;
 485   }
 486
 487   public function checkHealth() {
 488     $health = $this->getHealthRecord();
 489
 490     $should_check = $health->getShouldCheck();
 491     if ($should_check) {
 492       // This does an implicit health update.
 493       $connection = $this->newManagementConnection();
 494       $this->isReachable($connection);
 495     }
 496
 497     return $this;
 498   }
 499
 500   private function getHealthRecordCacheKey() {
 501     $host = $this->getHost();
 502     $port = $this->getPort();
 503     $key = self::KEY_HEALTH;
 504
 505     return "{$key}({$host}, {$port})";
 506   }
 507
 508   public function getHealthRecord() {
 509     if (!$this->healthRecord) {
 510       $this->healthRecord = new PhabricatorClusterServiceHealthRecord(
 511         $this->getHealthRecordCacheKey());
 512     }
 513     return $this->healthRecord;
 514   }
 515
 516   public function getConnectionException() {
 517     return $this->connectionException;
 518   }
 519
 520   public static function getActiveDatabaseRefs() {
 521     $refs = array();
 522
 523     foreach (self::getMasterDatabaseRefs() as $ref) {
 524       $refs[] = $ref;
 525     }
 526
 527     foreach (self::getReplicaDatabaseRefs() as $ref) {
 528       $refs[] = $ref;
 529     }
 530
 531     return $refs;
 532   }
 533
 534   public static function getAllMasterDatabaseRefs() {
 535     $refs = self::getClusterRefs();
 536
 537     if (!$refs) {
 538       return array(self::getLiveIndividualRef());
 539     }
 540
 541     $masters = array();
 542     foreach ($refs as $ref) {
 543       if ($ref->getIsMaster()) {
 544         $masters[] = $ref;
 545       }
 546     }
 547
 548     return $masters;
 549   }
 550
 551   public static function getMasterDatabaseRefs() {
 552     $refs = self::getAllMasterDatabaseRefs();
 553     return self::getEnabledRefs($refs);
 554   }
 555
 556   public function isApplicationHost($database) {
 557     return isset($this->applicationMap[$database]);
 558   }
 559
 560   public function loadRawMySQLConfigValue($key) {
 561     $conn = $this->newManagementConnection();
 562
 563     try {
 564       $value = queryfx_one($conn, 'SELECT @@%C', $key);
 565
 566       // NOTE: Although MySQL allows us to escape configuration values as if
 567       // they are column names, the escaping is included in the column name
 568       // of the return value: if we select "@@`x`", we get back a column named
 569       // "@@`x`", not "@@x" as we might expect.
 570       $value = head($value);
 571
 572     } catch (AphrontQueryException $ex) {
 573       $value = null;
 574     }
 575
 576     return $value;
 577   }
 578
 579   public static function getMasterDatabaseRefForApplication($application) {
 580     $masters = self::getMasterDatabaseRefs();
 581
 582     $application_master = null;
 583     $default_master = null;
 584     foreach ($masters as $master) {
 585       if ($master->isApplicationHost($application)) {
 586         $application_master = $master;
 587         break;
 588       }
 589       if ($master->getIsDefaultPartition()) {
 590         $default_master = $master;
 591       }
 592     }
 593
 594     if ($application_master) {
 595       $masters = array($application_master);
 596     } else if ($default_master) {
 597       $masters = array($default_master);
 598     } else {
 599       $masters = array();
 600     }
 601
 602     $masters = self::getEnabledRefs($masters);
 603     $master = head($masters);
 604
 605     return $master;
 606   }
 607
 608   public static function newIndividualRef() {
 609     $default_user = PhabricatorEnv::getEnvConfig('mysql.user');
 610     $default_pass = new PhutilOpaqueEnvelope(
 611       PhabricatorEnv::getEnvConfig('mysql.pass'));
 612     $default_host = PhabricatorEnv::getEnvConfig('mysql.host');
 613     $default_port = PhabricatorEnv::getEnvConfig('mysql.port');
 614
 615     return id(new self())
 616       ->setUser($default_user)
 617       ->setPass($default_pass)
 618       ->setHost($default_host)
 619       ->setPort($default_port)
 620       ->setIsIndividual(true)
 621       ->setIsMaster(true)
 622       ->setIsDefaultPartition(true)
 623       ->setUsePersistentConnections(false);
 624   }
 625
 626   public static function getAllReplicaDatabaseRefs() {
 627     $refs = self::getClusterRefs();
 628
 629     if (!$refs) {
 630       return array();
 631     }
 632
 633     $replicas = array();
 634     foreach ($refs as $ref) {
 635       if ($ref->getIsMaster()) {
 636         continue;
 637       }
 638
 639       $replicas[] = $ref;
 640     }
 641
 642     return $replicas;
 643   }
 644
 645   public static function getReplicaDatabaseRefs() {
 646     $refs = self::getAllReplicaDatabaseRefs();
 647     return self::getEnabledRefs($refs);
 648   }
 649
 650   private static function getEnabledRefs(array $refs) {
 651     foreach ($refs as $key => $ref) {
 652       if ($ref->getDisabled()) {
 653         unset($refs[$key]);
 654       }
 655     }
 656     return $refs;
 657   }
 658
 659   public static function getReplicaDatabaseRefForApplication($application) {
 660     $replicas = self::getReplicaDatabaseRefs();
 661
 662     $application_replicas = array();
 663     $default_replicas = array();
 664     foreach ($replicas as $replica) {
 665       $master = $replica->getMasterRef();
 666
 667       if ($master->isApplicationHost($application)) {
 668         $application_replicas[] = $replica;
 669       }
 670
 671       if ($master->getIsDefaultPartition()) {
 672         $default_replicas[] = $replica;
 673       }
 674     }
 675
 676     if ($application_replicas) {
 677       $replicas = $application_replicas;
 678     } else {
 679       $replicas = $default_replicas;
 680     }
 681
 682     $replicas = self::getEnabledRefs($replicas);
 683
 684     // TODO: We may have multiple replicas to choose from, and could make
 685     // more of an effort to pick the "best" one here instead of always
 686     // picking the first one. Once we've picked one, we should try to use
 687     // the same replica for the rest of the request, though.
 688
 689     return head($replicas);
 690   }
 691
 692   private function newConnection(array $options) {
 693     // If we believe the database is unhealthy, don't spend as much time
 694     // trying to connect to it, since it's likely to continue to fail and
 695     // hammering it can only make the problem worse.
 696     $record = $this->getHealthRecord();
 697     if ($record->getIsHealthy()) {
 698       $default_retries = 3;
 699       $default_timeout = 10;
 700     } else {
 701       $default_retries = 0;
 702       $default_timeout = 2;
 703     }
 704
 705     $spec = $options + array(
 706       'user' => $this->getUser(),
 707       'pass' => $this->getPass(),
 708       'host' => $this->getHost(),
 709       'port' => $this->getPort(),
 710       'database' => null,
 711       'retries' => $default_retries,
 712       'timeout' => $default_timeout,
 713       'persistent' => $this->getUsePersistentConnections(),
 714     );
 715
 716     $is_cli = (php_sapi_name() == 'cli');
 717
 718     $use_persistent = false;
 719     if (!empty($spec['persistent']) && !$is_cli) {
 720       $use_persistent = true;
 721     }
 722     unset($spec['persistent']);
 723
 724     $connection = self::newRawConnection($spec);
 725
 726     // If configured, use persistent connections. See T11672 for details.
 727     if ($use_persistent) {
 728       $connection->setPersistent($use_persistent);
 729     }
 730
 731     // Unless this is a script running from the CLI, prevent any query from
 732     // running for more than 30 seconds. See T10849 for details.
 733     if (!$is_cli) {
 734       $connection->setQueryTimeout(30);
 735     }
 736
 737     return $connection;
 738   }
 739
 740   public static function newRawConnection(array $options) {
 741     if (extension_loaded('mysqli')) {
 742       return new AphrontMySQLiDatabaseConnection($options);
 743     } else {
 744       return new AphrontMySQLDatabaseConnection($options);
 745     }
 746   }
 747
 748 }