includes/db/loadbalancer/LBFactoryMulti.php

   1 <?php
   2 /**
   3  * Advanced generator of database load balancing objects for wiki farms.
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation; either version 2 of the License, or
   8  * (at your option) any later version.
   9  *
  10  * This program is distributed in the hope that it will be useful,
  11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  13  * GNU General Public License for more details.
  14  *
  15  * You should have received a copy of the GNU General Public License along
  16  * with this program; if not, write to the Free Software Foundation, Inc.,
  17  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  18  * http://www.gnu.org/copyleft/gpl.html
  19  *
  20  * @file
  21  * @ingroup Database
  22  */
  23
  24 /**
  25  * A multi-wiki, multi-master factory for Wikimedia and similar installations.
  26  * Ignores the old configuration globals.
  27  *
  28  * Template override precedence (highest => lowest):
  29  *   - templateOverridesByServer
  30  *   - masterTemplateOverrides
  31  *   - templateOverridesBySection/templateOverridesByCluster
  32  *   - externalTemplateOverrides
  33  *   - serverTemplate
  34  * Overrides only work on top level keys (so nested values will not be merged).
  35  *
  36  * Configuration:
  37  *     sectionsByDB                A map of database names to section names.
  38  *
  39  *     sectionLoads                A 2-d map. For each section, gives a map of server names to
  40  *                                 load ratios. For example:
  41  *                                 array(
  42  *                                     'section1' => array(
  43  *                                         'db1' => 100,
  44  *                                         'db2' => 100
  45  *                                     )
  46  *                                 )
  47  *
  48  *     serverTemplate              A server info associative array as documented for $wgDBservers.
  49  *                                 The host, hostName and load entries will be overridden.
  50  *
  51  *     groupLoadsBySection         A 3-d map giving server load ratios for each section and group.
  52  *                                 For example:
  53  *                                 array(
  54  *                                     'section1' => array(
  55  *                                         'group1' => array(
  56  *                                             'db1' => 100,
  57  *                                             'db2' => 100
  58  *                                         )
  59  *                                     )
  60  *                                 )
  61  *
  62  *     groupLoadsByDB              A 3-d map giving server load ratios by DB name.
  63  *
  64  *     hostsByName                 A map of hostname to IP address.
  65  *
  66  *     externalLoads               A map of external storage cluster name to server load map.
  67  *
  68  *     externalTemplateOverrides   A set of server info keys overriding serverTemplate for external
  69  *                                 storage.
  70  *
  71  *     templateOverridesByServer   A 2-d map overriding serverTemplate and
  72  *                                 externalTemplateOverrides on a server-by-server basis. Applies
  73  *                                 to both core and external storage.
  74  *     templateOverridesBySection  A 2-d map overriding the server info by section.
  75  *     templateOverridesByCluster  A 2-d map overriding the server info by external storage cluster.
  76  *
  77  *     masterTemplateOverrides     An override array for all master servers.
  78  *
  79  *     loadMonitorClass            Name of the LoadMonitor class to always use.
  80  *
  81  *     readOnlyBySection           A map of section name to read-only message.
  82  *                                 Missing or false for read/write.
  83  *
  84  * @ingroup Database
  85  */
  86 class LBFactoryMulti extends LBFactory {
  87         /** @var array A map of database names to section names */
  88         private $sectionsByDB;
  89
  90         /**
  91          * @var array A 2-d map. For each section, gives a map of server names to
  92          * load ratios
  93          */
  94         private $sectionLoads;
  95
  96         /**
  97          * @var array A server info associative array as documented for
  98          * $wgDBservers. The host, hostName and load entries will be
  99          * overridden
 100          */
 101         private $serverTemplate;
 102
 103         // Optional settings
 104
 105         /** @var array A 3-d map giving server load ratios for each section and group */
 106         private $groupLoadsBySection = [];
 107
 108         /** @var array A 3-d map giving server load ratios by DB name */
 109         private $groupLoadsByDB = [];
 110
 111         /** @var array A map of hostname to IP address */
 112         private $hostsByName = [];
 113
 114         /** @var array A map of external storage cluster name to server load map */
 115         private $externalLoads = [];
 116
 117         /**
 118          * @var array A set of server info keys overriding serverTemplate for
 119          * external storage
 120          */
 121         private $externalTemplateOverrides;
 122
 123         /**
 124          * @var array A 2-d map overriding serverTemplate and
 125          * externalTemplateOverrides on a server-by-server basis. Applies to both
 126          * core and external storage
 127          */
 128         private $templateOverridesByServer;
 129
 130         /** @var array A 2-d map overriding the server info by section */
 131         private $templateOverridesBySection;
 132
 133         /** @var array A 2-d map overriding the server info by external storage cluster */
 134         private $templateOverridesByCluster;
 135
 136         /** @var array An override array for all master servers */
 137         private $masterTemplateOverrides;
 138
 139         /**
 140          * @var array|bool A map of section name to read-only message. Missing or
 141          * false for read/write
 142          */
 143         private $readOnlyBySection = [];
 144
 145         // Other stuff
 146
 147         /** @var array Load balancer factory configuration */
 148         private $conf;
 149
 150         /** @var LoadBalancer[] */
 151         private $mainLBs = [];
 152
 153         /** @var LoadBalancer[] */
 154         private $extLBs = [];
 155
 156         /** @var string */
 157         private $loadMonitorClass;
 158
 159         /** @var string */
 160         private $lastWiki;
 161
 162         /** @var string */
 163         private $lastSection;
 164
 165         /**
 166          * @param array $conf
 167          * @throws MWException
 168          */
 169         public function __construct( array $conf ) {
 170                 parent::__construct( $conf );
 171
 172                 $this->conf = $conf;
 173                 $required = [ 'sectionsByDB', 'sectionLoads', 'serverTemplate' ];
 174                 $optional = [ 'groupLoadsBySection', 'groupLoadsByDB', 'hostsByName',
 175                         'externalLoads', 'externalTemplateOverrides', 'templateOverridesByServer',
 176                         'templateOverridesByCluster', 'templateOverridesBySection', 'masterTemplateOverrides',
 177                         'readOnlyBySection', 'loadMonitorClass' ];
 178
 179                 foreach ( $required as $key ) {
 180                         if ( !isset( $conf[$key] ) ) {
 181                                 throw new MWException( __CLASS__ . ": $key is required in configuration" );
 182                         }
 183                         $this->$key = $conf[$key];
 184                 }
 185
 186                 foreach ( $optional as $key ) {
 187                         if ( isset( $conf[$key] ) ) {
 188                                 $this->$key = $conf[$key];
 189                         }
 190                 }
 191         }
 192
 193         /**
 194          * @param bool|string $wiki
 195          * @return string
 196          */
 197         private function getSectionForWiki( $wiki = false ) {
 198                 if ( $this->lastWiki === $wiki ) {
 199                         return $this->lastSection;
 200                 }
 201                 list( $dbName, ) = $this->getDBNameAndPrefix( $wiki );
 202                 if ( isset( $this->sectionsByDB[$dbName] ) ) {
 203                         $section = $this->sectionsByDB[$dbName];
 204                 } else {
 205                         $section = 'DEFAULT';
 206                 }
 207                 $this->lastSection = $section;
 208                 $this->lastWiki = $wiki;
 209
 210                 return $section;
 211         }
 212
 213         /**
 214          * @param bool|string $wiki
 215          * @return LoadBalancer
 216          */
 217         public function newMainLB( $wiki = false ) {
 218                 list( $dbName, ) = $this->getDBNameAndPrefix( $wiki );
 219                 $section = $this->getSectionForWiki( $wiki );
 220                 if ( isset( $this->groupLoadsByDB[$dbName] ) ) {
 221                         $groupLoads = $this->groupLoadsByDB[$dbName];
 222                 } else {
 223                         $groupLoads = [];
 224                 }
 225
 226                 if ( isset( $this->groupLoadsBySection[$section] ) ) {
 227                         $groupLoads = array_merge_recursive( $groupLoads, $this->groupLoadsBySection[$section] );
 228                 }
 229
 230                 $readOnlyReason = $this->readOnlyReason;
 231                 // Use the LB-specific read-only reason if everything isn't already read-only
 232                 if ( $readOnlyReason === false && isset( $this->readOnlyBySection[$section] ) ) {
 233                         $readOnlyReason = $this->readOnlyBySection[$section];
 234                 }
 235
 236                 $template = $this->serverTemplate;
 237                 if ( isset( $this->templateOverridesBySection[$section] ) ) {
 238                         $template = $this->templateOverridesBySection[$section] + $template;
 239                 }
 240
 241                 return $this->newLoadBalancer(
 242                         $template,
 243                         $this->sectionLoads[$section],
 244                         $groupLoads,
 245                         $readOnlyReason
 246                 );
 247         }
 248
 249         /**
 250          * @param bool|string $wiki
 251          * @return LoadBalancer
 252          */
 253         public function getMainLB( $wiki = false ) {
 254                 $section = $this->getSectionForWiki( $wiki );
 255                 if ( !isset( $this->mainLBs[$section] ) ) {
 256                         $lb = $this->newMainLB( $wiki );
 257                         $lb->parentInfo( [ 'id' => "main-$section" ] );
 258                         $this->chronProt->initLB( $lb );
 259                         $this->mainLBs[$section] = $lb;
 260                 }
 261
 262                 return $this->mainLBs[$section];
 263         }
 264
 265         /**
 266          * @param string $cluster
 267          * @param bool|string $wiki
 268          * @throws MWException
 269          * @return LoadBalancer
 270          */
 271         protected function newExternalLB( $cluster, $wiki = false ) {
 272                 if ( !isset( $this->externalLoads[$cluster] ) ) {
 273                         throw new MWException( __METHOD__ . ": Unknown cluster \"$cluster\"" );
 274                 }
 275                 $template = $this->serverTemplate;
 276                 if ( isset( $this->externalTemplateOverrides ) ) {
 277                         $template = $this->externalTemplateOverrides + $template;
 278                 }
 279                 if ( isset( $this->templateOverridesByCluster[$cluster] ) ) {
 280                         $template = $this->templateOverridesByCluster[$cluster] + $template;
 281                 }
 282
 283                 return $this->newLoadBalancer(
 284                         $template,
 285                         $this->externalLoads[$cluster],
 286                         [],
 287                         $this->readOnlyReason
 288                 );
 289         }
 290
 291         /**
 292          * @param string $cluster External storage cluster, or false for core
 293          * @param bool|string $wiki Wiki ID, or false for the current wiki
 294          * @return LoadBalancer
 295          */
 296         public function &getExternalLB( $cluster, $wiki = false ) {
 297                 if ( !isset( $this->extLBs[$cluster] ) ) {
 298                         $this->extLBs[$cluster] = $this->newExternalLB( $cluster, $wiki );
 299                         $this->extLBs[$cluster]->parentInfo( [ 'id' => "ext-$cluster" ] );
 300                         $this->chronProt->initLB( $this->extLBs[$cluster] );
 301                 }
 302
 303                 return $this->extLBs[$cluster];
 304         }
 305
 306         /**
 307          * Make a new load balancer object based on template and load array
 308          *
 309          * @param array $template
 310          * @param array $loads
 311          * @param array $groupLoads
 312          * @param string|bool $readOnlyReason
 313          * @return LoadBalancer
 314          */
 315         private function newLoadBalancer( $template, $loads, $groupLoads, $readOnlyReason ) {
 316                 return new LoadBalancer( [
 317                         'servers' => $this->makeServerArray( $template, $loads, $groupLoads ),
 318                         'loadMonitor' => $this->loadMonitorClass,
 319                         'readOnlyReason' => $readOnlyReason,
 320                         'trxProfiler' => $this->trxProfiler
 321                 ] );
 322         }
 323
 324         /**
 325          * Make a server array as expected by LoadBalancer::__construct, using a template and load array
 326          *
 327          * @param array $template
 328          * @param array $loads
 329          * @param array $groupLoads
 330          * @return array
 331          */
 332         private function makeServerArray( $template, $loads, $groupLoads ) {
 333                 $servers = [];
 334                 $master = true;
 335                 $groupLoadsByServer = $this->reindexGroupLoads( $groupLoads );
 336                 foreach ( $groupLoadsByServer as $server => $stuff ) {
 337                         if ( !isset( $loads[$server] ) ) {
 338                                 $loads[$server] = 0;
 339                         }
 340                 }
 341                 foreach ( $loads as $serverName => $load ) {
 342                         $serverInfo = $template;
 343                         if ( $master ) {
 344                                 $serverInfo['master'] = true;
 345                                 if ( isset( $this->masterTemplateOverrides ) ) {
 346                                         $serverInfo = $this->masterTemplateOverrides + $serverInfo;
 347                                 }
 348                                 $master = false;
 349                         } else {
 350                                 $serverInfo['slave'] = true;
 351                         }
 352                         if ( isset( $this->templateOverridesByServer[$serverName] ) ) {
 353                                 $serverInfo = $this->templateOverridesByServer[$serverName] + $serverInfo;
 354                         }
 355                         if ( isset( $groupLoadsByServer[$serverName] ) ) {
 356                                 $serverInfo['groupLoads'] = $groupLoadsByServer[$serverName];
 357                         }
 358                         if ( isset( $this->hostsByName[$serverName] ) ) {
 359                                 $serverInfo['host'] = $this->hostsByName[$serverName];
 360                         } else {
 361                                 $serverInfo['host'] = $serverName;
 362                         }
 363                         $serverInfo['hostName'] = $serverName;
 364                         $serverInfo['load'] = $load;
 365                         $servers[] = $serverInfo;
 366                 }
 367
 368                 return $servers;
 369         }
 370
 371         /**
 372          * Take a group load array indexed by group then server, and reindex it by server then group
 373          * @param array $groupLoads
 374          * @return array
 375          */
 376         private function reindexGroupLoads( $groupLoads ) {
 377                 $reindexed = [];
 378                 foreach ( $groupLoads as $group => $loads ) {
 379                         foreach ( $loads as $server => $load ) {
 380                                 $reindexed[$server][$group] = $load;
 381                         }
 382                 }
 383
 384                 return $reindexed;
 385         }
 386
 387         /**
 388          * Get the database name and prefix based on the wiki ID
 389          * @param bool|string $wiki
 390          * @return array
 391          */
 392         private function getDBNameAndPrefix( $wiki = false ) {
 393                 if ( $wiki === false ) {
 394                         global $wgDBname, $wgDBprefix;
 395
 396                         return [ $wgDBname, $wgDBprefix ];
 397                 } else {
 398                         return wfSplitWikiID( $wiki );
 399                 }
 400         }
 401
 402         /**
 403          * Execute a function for each tracked load balancer
 404          * The callback is called with the load balancer as the first parameter,
 405          * and $params passed as the subsequent parameters.
 406          * @param callable $callback
 407          * @param array $params
 408          */
 409         public function forEachLB( $callback, array $params = [] ) {
 410                 foreach ( $this->mainLBs as $lb ) {
 411                         call_user_func_array( $callback, array_merge( [ $lb ], $params ) );
 412                 }
 413                 foreach ( $this->extLBs as $lb ) {
 414                         call_user_func_array( $callback, array_merge( [ $lb ], $params ) );
 415                 }
 416         }
 417
 418         public function shutdown( $flags = 0 ) {
 419                 if ( !( $flags & self::SHUTDOWN_NO_CHRONPROT ) ) {
 420                         $this->shutdownChronologyProtector( $this->chronProt );
 421                 }
 422                 $this->commitMasterChanges( __METHOD__ ); // sanity
 423         }
 424 }