src/infrastructure/cluster/PhabricatorClusterServiceHealthRecord.php

   1 <?php
   2
   3 class PhabricatorClusterServiceHealthRecord
   4   extends Phobject {
   5
   6   private $cacheKey;
   7   private $shouldCheck;
   8   private $isHealthy;
   9   private $upEventCount;
  10   private $downEventCount;
  11
  12   public function __construct($cache_key) {
  13     $this->cacheKey = $cache_key;
  14     $this->readState();
  15   }
  16
  17   /**
  18    * Is the database currently healthy?
  19    */
  20   public function getIsHealthy() {
  21     return $this->isHealthy;
  22   }
  23
  24
  25   /**
  26    * Should this request check database health?
  27    */
  28   public function getShouldCheck() {
  29     return $this->shouldCheck;
  30   }
  31
  32
  33   /**
  34    * How many recent health checks were successful?
  35    */
  36   public function getUpEventCount() {
  37     return $this->upEventCount;
  38   }
  39
  40
  41   /**
  42    * How many recent health checks failed?
  43    */
  44   public function getDownEventCount() {
  45     return $this->downEventCount;
  46   }
  47
  48
  49   /**
  50    * Number of failures or successes we need to see in a row before we change
  51    * the state.
  52    */
  53   public function getRequiredEventCount() {
  54     // NOTE: If you change this value, update the "Cluster: Databases" docs.
  55     return 5;
  56   }
  57
  58
  59   /**
  60    * Seconds to wait between health checks.
  61    */
  62   public function getHealthCheckFrequency() {
  63     // NOTE: If you change this value, update the "Cluster: Databases" docs.
  64     return 3;
  65   }
  66
  67
  68   public function didHealthCheck($result) {
  69     $now = microtime(true);
  70     $check_frequency = $this->getHealthCheckFrequency();
  71     $event_count = $this->getRequiredEventCount();
  72
  73     $record = $this->readHealthRecord();
  74
  75     $log = $record['log'];
  76     foreach ($log as $key => $event) {
  77       $when = idx($event, 'timestamp');
  78
  79       // If the log already has another nearby event, just ignore this one.
  80       // We raced with another process and our result can just be thrown away.
  81       if (($now - $when) <= $check_frequency) {
  82         return $this;
  83       }
  84     }
  85
  86     $log[] = array(
  87       'timestamp' => $now,
  88       'up' => $result,
  89     );
  90
  91     // Throw away older events which are now obsolete.
  92     $log = array_slice($log, -$event_count);
  93
  94     $count_up = 0;
  95     $count_down = 0;
  96     foreach ($log as $event) {
  97       if ($event['up']) {
  98         $count_up++;
  99       } else {
 100         $count_down++;
 101       }
 102     }
 103
 104     // If all of the events are the same, change the state.
 105     if ($count_up == $event_count) {
 106       $record['up'] = true;
 107     } else if ($count_down == $event_count) {
 108       $record['up'] = false;
 109     }
 110
 111     $record['log'] = $log;
 112
 113     $this->writeHealthRecord($record);
 114
 115     $this->isHealthy = $record['up'];
 116     $this->shouldCheck = false;
 117     $this->updateStatistics($record);
 118
 119     return $this;
 120   }
 121
 122
 123   private function readState() {
 124     $now = microtime(true);
 125     $check_frequency = $this->getHealthCheckFrequency();
 126
 127     $record = $this->readHealthRecord();
 128
 129     $last_check = $record['lastCheck'];
 130
 131     if (($now - $last_check) >= $check_frequency) {
 132       $record['lastCheck'] = $now;
 133       $this->writeHealthRecord($record);
 134       $this->shouldCheck = true;
 135     } else {
 136       $this->shouldCheck = false;
 137     }
 138
 139     $this->isHealthy = $record['up'];
 140     $this->updateStatistics($record);
 141   }
 142
 143   private function updateStatistics(array $record) {
 144     $this->upEventCount = 0;
 145     $this->downEventCount = 0;
 146     foreach ($record['log'] as $event) {
 147       if ($event['up']) {
 148         $this->upEventCount++;
 149       } else {
 150         $this->downEventCount++;
 151       }
 152     }
 153   }
 154
 155   public function getCacheKey() {
 156     return $this->cacheKey;
 157   }
 158
 159   private function readHealthRecord() {
 160     $cache = PhabricatorCaches::getSetupCache();
 161     $cache_key = $this->getCacheKey();
 162     $health_record = $cache->getKey($cache_key);
 163
 164     if (!is_array($health_record)) {
 165       $health_record = array(
 166         'up' => true,
 167         'lastCheck' => 0,
 168         'log' => array(),
 169       );
 170     }
 171
 172     return $health_record;
 173   }
 174
 175   private function writeHealthRecord(array $record) {
 176     $cache = PhabricatorCaches::getSetupCache();
 177     $cache_key = $this->getCacheKey();
 178     $cache->setKey($cache_key, $record);
 179   }
 180
 181 }