includes/externalstore/ExternalStoreAccess.php

   1 <?php
   2 /**
   3  * This program is free software; you can redistribute it and/or modify
   4  * it under the terms of the GNU General Public License as published by
   5  * the Free Software Foundation; either version 2 of the License, or
   6  * (at your option) any later version.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  11  * GNU General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along
  14  * with this program; if not, write to the Free Software Foundation, Inc.,
  15  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
  16  * http://www.gnu.org/copyleft/gpl.html
  17  *
  18  * @file
  19  */
  20
  21 use Psr\Log\LoggerAwareInterface;
  22 use Psr\Log\LoggerInterface;
  23 use Psr\Log\NullLogger;
  24 use Wikimedia\RequestTimeout\TimeoutException;
  25
  26 /**
  27  * @defgroup ExternalStorage ExternalStorage
  28  *
  29  * Object storage outside the main database, see also [ExternalStore Architecture](@ref externalstorearch).
  30  */
  31
  32 /**
  33  * This is the main interface for fetching or inserting objects with [ExternalStore](@ref externalstorearch).
  34  *
  35  * This interface is meant to mimic the ExternalStoreMedium base class (which
  36  * represents a single external store protocol), and transparently uses the
  37  * right instance of that class when fetching by URL.
  38  *
  39  * @see [ExternalStore Architecture](@ref externalstorearch).
  40  * @ingroup ExternalStorage
  41  * @since 1.34
  42  */
  43 class ExternalStoreAccess implements LoggerAwareInterface {
  44         /** @var ExternalStoreFactory */
  45         private $storeFactory;
  46         /** @var LoggerInterface */
  47         private $logger;
  48
  49         /**
  50          * @param ExternalStoreFactory $factory
  51          * @param LoggerInterface|null $logger
  52          */
  53         public function __construct( ExternalStoreFactory $factory, ?LoggerInterface $logger = null ) {
  54                 $this->storeFactory = $factory;
  55                 $this->logger = $logger ?: new NullLogger();
  56         }
  57
  58         public function setLogger( LoggerInterface $logger ) {
  59                 $this->logger = $logger;
  60         }
  61
  62         /**
  63          * Fetch data from given URL
  64          *
  65          * @see ExternalStoreFactory::getStore()
  66          *
  67          * @param string $url The URL of the text to get
  68          * @param array $params Map of context parameters; same as ExternalStoreFactory::getStore()
  69          * @return string|false The text stored or false on error
  70          * @throws ExternalStoreException
  71          */
  72         public function fetchFromURL( $url, array $params = [] ) {
  73                 return $this->storeFactory->getStoreForUrl( $url, $params )->fetchFromURL( $url );
  74         }
  75
  76         /**
  77          * Fetch data from multiple URLs with a minimum of round trips
  78          *
  79          * @see ExternalStoreFactory::getStore()
  80          *
  81          * @param array $urls The URLs of the text to get
  82          * @param array $params Map of context parameters; same as ExternalStoreFactory::getStore()
  83          * @return array Map of (url => string or false if not found)
  84          * @throws ExternalStoreException
  85          */
  86         public function fetchFromURLs( array $urls, array $params = [] ) {
  87                 $batches = $this->storeFactory->getUrlsByProtocol( $urls );
  88                 $retval = [];
  89                 foreach ( $batches as $proto => $batchedUrls ) {
  90                         $store = $this->storeFactory->getStore( $proto, $params );
  91                         $retval += $store->batchFetchFromURLs( $batchedUrls );
  92                 }
  93                 // invalid, not found, db dead, etc.
  94                 $missing = array_diff( $urls, array_keys( $retval ) );
  95                 foreach ( $missing as $url ) {
  96                         $retval[$url] = false;
  97                 }
  98
  99                 return $retval;
 100         }
 101
 102         /**
 103          * Insert data into storage and return the assigned URL
 104          *
 105          * This will randomly pick one of the available write storage locations to put the data.
 106          * It will keep failing-over to any untried storage locations whenever one location is
 107          * not usable.
 108          *
 109          * @see ExternalStoreFactory::getStore()
 110          *
 111          * @param string $data
 112          * @param array $params Map of context parameters; same as ExternalStoreFactory::getStore()
 113          * @param string[]|null $tryStores Base URLs to try, e.g. [ "DB://cluster1" ]
 114          * @return string|false The URL of the stored data item, or false on error
 115          * @throws ExternalStoreException
 116          */
 117         public function insert( $data, array $params = [], ?array $tryStores = null ) {
 118                 $tryStores ??= $this->storeFactory->getWriteBaseUrls();
 119                 if ( !$tryStores ) {
 120                         throw new ExternalStoreException( "List of external stores provided is empty." );
 121                 }
 122
 123                 $error = false;      // track the last exception thrown
 124                 $readOnlyCount = 0;  // track if a store was read-only
 125                 while ( count( $tryStores ) > 0 ) {
 126                         $index = mt_rand( 0, count( $tryStores ) - 1 );
 127                         $storeUrl = $tryStores[$index];
 128
 129                         $this->logger->debug( __METHOD__ . ": trying $storeUrl" );
 130
 131                         $store = $this->storeFactory->getStoreForUrl( $storeUrl, $params );
 132                         if ( $store === false ) {
 133                                 throw new ExternalStoreException( "Invalid external storage protocol - $storeUrl" );
 134                         }
 135
 136                         $location = $this->storeFactory->getStoreLocationFromUrl( $storeUrl );
 137                         try {
 138                                 if ( $store->isReadOnly( $location ) ) {
 139                                         $readOnlyCount++;
 140                                         $msg = 'read only';
 141                                 } else {
 142                                         $url = $store->store( $location, $data );
 143                                         if ( strlen( $url ) ) {
 144                                                 // A store accepted the write; done!
 145                                                 return $url;
 146                                         }
 147                                         throw new ExternalStoreException(
 148                                                 "No URL returned by storage medium ($storeUrl)"
 149                                         );
 150                                 }
 151                         } catch ( TimeoutException $e ) {
 152                                 throw $e;
 153                         } catch ( Exception $ex ) {
 154                                 $error = $ex;
 155                                 $msg = 'caught ' . get_class( $error ) . ' exception: ' . $error->getMessage();
 156                         }
 157
 158                         unset( $tryStores[$index] ); // Don't try this one again!
 159                         $tryStores = array_values( $tryStores ); // Must have consecutive keys
 160                         $this->logger->error(
 161                                 "Unable to store text to external storage {store_path} ({failure})",
 162                                 [ 'store_path' => $storeUrl, 'failure' => $msg ]
 163                         );
 164                 }
 165
 166                 // We only get here when all stores failed.
 167                 if ( $error ) {
 168                         // At least one store threw an exception. Re-throw the most recent one.
 169                         throw $error;
 170                 } elseif ( $readOnlyCount ) {
 171                         // If no exceptions where thrown and we get here,
 172                         // this should mean that all stores were in read-only mode.
 173                         throw new ReadOnlyError();
 174                 } else {
 175                         // We shouldn't get here. If there were no failures, this method should have returned
 176                         // from inside the body of the loop.
 177                         throw new LogicException( "Unexpected failure to store text to external store" );
 178                 }
 179         }
 180
 181         /**
 182          * @param string[]|string|null $storeUrls Base URL(s) to check, e.g. [ "DB://cluster1" ]
 183          * @return bool Whether all the default insertion stores are marked as read-only
 184          * @throws ExternalStoreException
 185          */
 186         public function isReadOnly( $storeUrls = null ) {
 187                 if ( $storeUrls === null ) {
 188                         $storeUrls = $this->storeFactory->getWriteBaseUrls();
 189                 } else {
 190                         $storeUrls = is_array( $storeUrls ) ? $storeUrls : [ $storeUrls ];
 191                 }
 192
 193                 if ( !$storeUrls ) {
 194                         return false; // no stores exists which can be "read only"
 195                 }
 196
 197                 foreach ( $storeUrls as $storeUrl ) {
 198                         $store = $this->storeFactory->getStoreForUrl( $storeUrl );
 199                         $location = $this->storeFactory->getStoreLocationFromUrl( $storeUrl );
 200                         if ( $store !== false && !$store->isReadOnly( $location ) ) {
 201                                 return false; // at least one store is not read-only
 202                         }
 203                 }
 204
 205                 return true; // all stores are read-only
 206         }
 207 }