hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.regionserver;
  19
  20 import java.io.IOException;
  21 import java.util.ArrayList;
  22 import java.util.Collection;
  23 import java.util.List;
  24 import java.util.Map;
  25 import java.util.OptionalDouble;
  26 import java.util.OptionalLong;
  27 import java.util.concurrent.ConcurrentHashMap;
  28 import java.util.concurrent.ScheduledExecutorService;
  29 import java.util.concurrent.TimeUnit;
  30
  31 import org.apache.commons.lang3.StringUtils;
  32 import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
  33 import org.apache.hadoop.hbase.HConstants;
  34 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
  35 import org.apache.hadoop.hbase.HRegionInfo;
  36 import org.apache.hadoop.hbase.ServerName;
  37 import org.apache.hadoop.hbase.io.ByteBuffAllocator;
  38 import org.apache.hadoop.hbase.io.hfile.BlockCache;
  39 import org.apache.hadoop.hbase.io.hfile.CacheStats;
  40 import org.apache.hadoop.hbase.io.hfile.CombinedBlockCache;
  41 import org.apache.hadoop.hbase.mob.MobFileCache;
  42 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
  43 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
  44 import org.apache.hadoop.hbase.util.FSUtils;
  45 import org.apache.hadoop.hbase.wal.WALProvider;
  46 import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
  47 import org.apache.hadoop.hdfs.DFSHedgedReadMetrics;
  48 import org.apache.hadoop.metrics2.MetricsExecutor;
  49 import org.apache.yetus.audience.InterfaceAudience;
  50 import org.slf4j.Logger;
  51 import org.slf4j.LoggerFactory;
  52
  53 /**
  54  * Impl for exposing HRegionServer Information through Hadoop's metrics 2 system.
  55  */
  56 @InterfaceAudience.Private
  57 class MetricsRegionServerWrapperImpl
  58     implements MetricsRegionServerWrapper {
  59
  60   private static final Logger LOG = LoggerFactory.getLogger(MetricsRegionServerWrapperImpl.class);
  61
  62   private final HRegionServer regionServer;
  63   private final MetricsWALSource metricsWALSource;
  64   private final ByteBuffAllocator allocator;
  65
  66   private BlockCache blockCache;
  67   private MobFileCache mobFileCache;
  68   private CacheStats cacheStats;
  69   private CacheStats l1Stats = null;
  70   private CacheStats l2Stats = null;
  71
  72   private volatile long numStores = 0;
  73   private volatile long numWALFiles = 0;
  74   private volatile long walFileSize = 0;
  75   private volatile long numStoreFiles = 0;
  76   private volatile long memstoreSize = 0;
  77   private volatile long storeFileSize = 0;
  78   private volatile double storeFileSizeGrowthRate = 0;
  79   private volatile long maxStoreFileAge = 0;
  80   private volatile long minStoreFileAge = 0;
  81   private volatile long avgStoreFileAge = 0;
  82   private volatile long numReferenceFiles = 0;
  83   private volatile double requestsPerSecond = 0.0;
  84   private volatile long readRequestsCount = 0;
  85   private volatile double readRequestsRatePerSecond = 0;
  86   private volatile long cpRequestsCount = 0;
  87   private volatile long filteredReadRequestsCount = 0;
  88   private volatile long writeRequestsCount = 0;
  89   private volatile double writeRequestsRatePerSecond = 0;
  90   private volatile long checkAndMutateChecksFailed = 0;
  91   private volatile long checkAndMutateChecksPassed = 0;
  92   private volatile long storefileIndexSize = 0;
  93   private volatile long totalStaticIndexSize = 0;
  94   private volatile long totalStaticBloomSize = 0;
  95   private volatile long numMutationsWithoutWAL = 0;
  96   private volatile long dataInMemoryWithoutWAL = 0;
  97   private volatile double percentFileLocal = 0;
  98   private volatile double percentFileLocalSecondaryRegions = 0;
  99   private volatile long flushedCellsCount = 0;
 100   private volatile long compactedCellsCount = 0;
 101   private volatile long majorCompactedCellsCount = 0;
 102   private volatile long flushedCellsSize = 0;
 103   private volatile long compactedCellsSize = 0;
 104   private volatile long majorCompactedCellsSize = 0;
 105   private volatile long cellsCountCompactedToMob = 0;
 106   private volatile long cellsCountCompactedFromMob = 0;
 107   private volatile long cellsSizeCompactedToMob = 0;
 108   private volatile long cellsSizeCompactedFromMob = 0;
 109   private volatile long mobFlushCount = 0;
 110   private volatile long mobFlushedCellsCount = 0;
 111   private volatile long mobFlushedCellsSize = 0;
 112   private volatile long mobScanCellsCount = 0;
 113   private volatile long mobScanCellsSize = 0;
 114   private volatile long mobFileCacheAccessCount = 0;
 115   private volatile long mobFileCacheMissCount = 0;
 116   private volatile double mobFileCacheHitRatio = 0;
 117   private volatile long mobFileCacheEvictedCount = 0;
 118   private volatile long mobFileCacheCount = 0;
 119   private volatile long blockedRequestsCount = 0L;
 120   private volatile long averageRegionSize = 0L;
 121   protected final Map<String, ArrayList<Long>> requestsCountCache = new
 122       ConcurrentHashMap<String, ArrayList<Long>>();
 123
 124   private ScheduledExecutorService executor;
 125   private Runnable runnable;
 126   private long period;
 127
 128   /**
 129    * Can be null if not on hdfs.
 130    */
 131   private DFSHedgedReadMetrics dfsHedgedReadMetrics;
 132
 133   public MetricsRegionServerWrapperImpl(final HRegionServer regionServer) {
 134     this.regionServer = regionServer;
 135     initBlockCache();
 136     initMobFileCache();
 137
 138     this.period = regionServer.getConfiguration().getLong(HConstants.REGIONSERVER_METRICS_PERIOD,
 139       HConstants.DEFAULT_REGIONSERVER_METRICS_PERIOD);
 140
 141     this.executor = CompatibilitySingletonFactory.getInstance(MetricsExecutor.class).getExecutor();
 142     this.runnable = new RegionServerMetricsWrapperRunnable();
 143     this.executor.scheduleWithFixedDelay(this.runnable, this.period, this.period,
 144       TimeUnit.MILLISECONDS);
 145     this.metricsWALSource = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
 146     this.allocator = regionServer.getRpcServer().getByteBuffAllocator();
 147
 148     try {
 149       this.dfsHedgedReadMetrics = FSUtils.getDFSHedgedReadMetrics(regionServer.getConfiguration());
 150     } catch (IOException e) {
 151       LOG.warn("Failed to get hedged metrics", e);
 152     }
 153     if (LOG.isInfoEnabled()) {
 154       LOG.info("Computing regionserver metrics every " + this.period + " milliseconds");
 155     }
 156   }
 157
 158   private void initBlockCache() {
 159     this.blockCache = this.regionServer.getBlockCache().orElse(null);
 160     this.cacheStats = this.blockCache != null ? this.blockCache.getStats() : null;
 161     if (this.cacheStats != null) {
 162       if (this.cacheStats instanceof CombinedBlockCache.CombinedCacheStats) {
 163         l1Stats = ((CombinedBlockCache.CombinedCacheStats) this.cacheStats)
 164           .getLruCacheStats();
 165         l2Stats = ((CombinedBlockCache.CombinedCacheStats) this.cacheStats)
 166           .getBucketCacheStats();
 167       } else {
 168         l1Stats = this.cacheStats;
 169       }
 170     }
 171   }
 172
 173   /**
 174    * Initializes the mob file cache.
 175    */
 176   private void initMobFileCache() {
 177     this.mobFileCache = this.regionServer.getMobFileCache().orElse(null);
 178   }
 179
 180   @Override
 181   public String getClusterId() {
 182     return regionServer.getClusterId();
 183   }
 184
 185   @Override
 186   public long getStartCode() {
 187     return regionServer.getStartcode();
 188   }
 189
 190   @Override
 191   public String getZookeeperQuorum() {
 192     ZKWatcher zk = regionServer.getZooKeeper();
 193     if (zk == null) {
 194       return "";
 195     }
 196     return zk.getQuorum();
 197   }
 198
 199   @Override
 200   public String getCoprocessors() {
 201     String[] coprocessors = regionServer.getRegionServerCoprocessors();
 202     if (coprocessors == null || coprocessors.length == 0) {
 203       return "";
 204     }
 205     return StringUtils.join(coprocessors, ", ");
 206   }
 207
 208   @Override
 209   public String getServerName() {
 210     ServerName serverName = regionServer.getServerName();
 211     if (serverName == null) {
 212       return "";
 213     }
 214     return serverName.getServerName();
 215   }
 216
 217   @Override
 218   public long getNumOnlineRegions() {
 219     Collection<HRegion> onlineRegionsLocalContext = regionServer.getOnlineRegionsLocalContext();
 220     if (onlineRegionsLocalContext == null) {
 221       return 0;
 222     }
 223     return onlineRegionsLocalContext.size();
 224   }
 225
 226   @Override
 227   public long getTotalRequestCount() {
 228     return regionServer.rpcServices.requestCount.sum();
 229   }
 230
 231   @Override
 232   public long getTotalRowActionRequestCount() {
 233     return readRequestsCount + writeRequestsCount;
 234   }
 235
 236   @Override
 237   public int getSplitQueueSize() {
 238     if (this.regionServer.compactSplitThread == null) {
 239       return 0;
 240     }
 241     return this.regionServer.compactSplitThread.getSplitQueueSize();
 242   }
 243
 244   @Override
 245   public int getCompactionQueueSize() {
 246     //The thread could be zero.  if so assume there is no queue.
 247     if (this.regionServer.compactSplitThread == null) {
 248       return 0;
 249     }
 250     return this.regionServer.compactSplitThread.getCompactionQueueSize();
 251   }
 252
 253   @Override
 254   public int getSmallCompactionQueueSize() {
 255     //The thread could be zero.  if so assume there is no queue.
 256     if (this.regionServer.compactSplitThread == null) {
 257       return 0;
 258     }
 259     return this.regionServer.compactSplitThread.getSmallCompactionQueueSize();
 260   }
 261
 262   @Override
 263   public int getLargeCompactionQueueSize() {
 264     //The thread could be zero.  if so assume there is no queue.
 265     if (this.regionServer.compactSplitThread == null) {
 266       return 0;
 267     }
 268     return this.regionServer.compactSplitThread.getLargeCompactionQueueSize();
 269   }
 270
 271   @Override
 272   public int getFlushQueueSize() {
 273     //If there is no flusher there should be no queue.
 274     if (this.regionServer.getMemStoreFlusher() == null) {
 275       return 0;
 276     }
 277     return this.regionServer.getMemStoreFlusher().getFlushQueueSize();
 278   }
 279
 280   @Override
 281   public long getBlockCacheCount() {
 282     return this.blockCache != null ? this.blockCache.getBlockCount() : 0L;
 283   }
 284
 285   @Override
 286   public long getMemStoreLimit() {
 287     return this.regionServer.getRegionServerAccounting().getGlobalMemStoreLimit();
 288   }
 289
 290   @Override
 291   public long getBlockCacheSize() {
 292     return this.blockCache != null ? this.blockCache.getCurrentSize() : 0L;
 293   }
 294
 295   @Override
 296   public long getBlockCacheFreeSize() {
 297     return this.blockCache != null ? this.blockCache.getFreeSize() : 0L;
 298   }
 299
 300   @Override
 301   public long getBlockCacheHitCount() {
 302     return this.cacheStats != null ? this.cacheStats.getHitCount() : 0L;
 303   }
 304
 305   @Override
 306   public long getBlockCachePrimaryHitCount() {
 307     return this.cacheStats != null ? this.cacheStats.getPrimaryHitCount() : 0L;
 308   }
 309
 310   @Override
 311   public long getBlockCacheMissCount() {
 312     return this.cacheStats != null ? this.cacheStats.getMissCount() : 0L;
 313   }
 314
 315   @Override
 316   public long getBlockCachePrimaryMissCount() {
 317     return this.cacheStats != null ? this.cacheStats.getPrimaryMissCount() : 0L;
 318   }
 319
 320   @Override
 321   public long getBlockCacheEvictedCount() {
 322     return this.cacheStats != null ? this.cacheStats.getEvictedCount() : 0L;
 323   }
 324
 325   @Override
 326   public long getBlockCachePrimaryEvictedCount() {
 327     return this.cacheStats != null ? this.cacheStats.getPrimaryEvictedCount() : 0L;
 328   }
 329
 330   @Override
 331   public double getBlockCacheHitPercent() {
 332     double ratio = this.cacheStats != null ? this.cacheStats.getHitRatio() : 0.0;
 333     if (Double.isNaN(ratio)) {
 334       ratio = 0;
 335     }
 336     return (ratio * 100);
 337   }
 338
 339   @Override
 340   public double getBlockCacheHitCachingPercent() {
 341     double ratio = this.cacheStats != null ? this.cacheStats.getHitCachingRatio() : 0.0;
 342     if (Double.isNaN(ratio)) {
 343       ratio = 0;
 344     }
 345     return (ratio * 100);
 346   }
 347
 348   @Override
 349   public long getBlockCacheFailedInsertions() {
 350     return this.cacheStats != null ? this.cacheStats.getFailedInserts() : 0L;
 351   }
 352
 353   @Override
 354   public long getL1CacheHitCount() {
 355     return this.l1Stats != null ? this.l1Stats.getHitCount() : 0L;
 356   }
 357
 358   @Override
 359   public long getL1CacheMissCount() {
 360     return this.l1Stats != null ? this.l1Stats.getMissCount() : 0L;
 361   }
 362
 363   @Override
 364   public double getL1CacheHitRatio() {
 365     return this.l1Stats != null ? this.l1Stats.getHitRatio() : 0.0;
 366   }
 367
 368   @Override
 369   public double getL1CacheMissRatio() {
 370     return this.l1Stats != null ? this.l1Stats.getMissRatio() : 0.0;
 371   }
 372
 373   @Override
 374   public long getL2CacheHitCount() {
 375     return this.l2Stats != null ? this.l2Stats.getHitCount() : 0L;
 376   }
 377
 378   @Override
 379   public long getL2CacheMissCount() {
 380     return this.l2Stats != null ? this.l2Stats.getMissCount() : 0L;
 381   }
 382
 383   @Override
 384   public double getL2CacheHitRatio() {
 385     return this.l2Stats != null ? this.l2Stats.getHitRatio() : 0.0;
 386   }
 387
 388   @Override
 389   public double getL2CacheMissRatio() {
 390     return this.l2Stats != null ? this.l2Stats.getMissRatio() : 0.0;
 391   }
 392
 393   @Override public void forceRecompute() {
 394     this.runnable.run();
 395   }
 396
 397   @Override
 398   public long getNumStores() {
 399     return numStores;
 400   }
 401
 402   @Override
 403   public long getNumWALFiles() {
 404     return numWALFiles;
 405   }
 406
 407   @Override
 408   public long getWALFileSize() {
 409     return walFileSize;
 410   }
 411
 412   @Override
 413   public long getNumWALSlowAppend() {
 414     return metricsWALSource.getSlowAppendCount();
 415   }
 416
 417   @Override
 418   public long getNumStoreFiles() {
 419     return numStoreFiles;
 420   }
 421
 422   @Override
 423   public long getMaxStoreFileAge() {
 424     return maxStoreFileAge;
 425   }
 426
 427   @Override
 428   public long getMinStoreFileAge() {
 429     return minStoreFileAge;
 430   }
 431
 432   @Override
 433   public long getAvgStoreFileAge() {
 434     return avgStoreFileAge;
 435   }
 436
 437   @Override
 438   public long getNumReferenceFiles() {
 439     return numReferenceFiles;
 440   }
 441
 442   @Override
 443   public long getMemStoreSize() {
 444     return memstoreSize;
 445   }
 446
 447   @Override
 448   public long getStoreFileSize() {
 449     return storeFileSize;
 450   }
 451
 452   @Override
 453   public double getStoreFileSizeGrowthRate() {
 454     return storeFileSizeGrowthRate;
 455   }
 456
 457   @Override public double getRequestsPerSecond() {
 458     return requestsPerSecond;
 459   }
 460
 461   @Override
 462   public long getReadRequestsCount() {
 463     return readRequestsCount;
 464   }
 465
 466   @Override
 467   public long getCpRequestsCount() {
 468     return cpRequestsCount;
 469   }
 470
 471   @Override
 472   public double getReadRequestsRatePerSecond() {
 473     return readRequestsRatePerSecond;
 474   }
 475
 476   @Override
 477   public long getFilteredReadRequestsCount() {
 478     return filteredReadRequestsCount;
 479   }
 480
 481   @Override
 482   public long getWriteRequestsCount() {
 483     return writeRequestsCount;
 484   }
 485
 486   @Override
 487   public double getWriteRequestsRatePerSecond() {
 488     return writeRequestsRatePerSecond;
 489   }
 490
 491   @Override
 492   public long getRpcGetRequestsCount() {
 493     return regionServer.rpcServices.rpcGetRequestCount.sum();
 494   }
 495
 496   @Override
 497   public long getRpcScanRequestsCount() {
 498     return regionServer.rpcServices.rpcScanRequestCount.sum();
 499   }
 500
 501   @Override
 502   public long getRpcMultiRequestsCount() {
 503     return regionServer.rpcServices.rpcMultiRequestCount.sum();
 504   }
 505
 506   @Override
 507   public long getRpcMutateRequestsCount() {
 508     return regionServer.rpcServices.rpcMutateRequestCount.sum();
 509   }
 510
 511   @Override
 512   public long getCheckAndMutateChecksFailed() {
 513     return checkAndMutateChecksFailed;
 514   }
 515
 516   @Override
 517   public long getCheckAndMutateChecksPassed() {
 518     return checkAndMutateChecksPassed;
 519   }
 520
 521   @Override
 522   public long getStoreFileIndexSize() {
 523     return storefileIndexSize;
 524   }
 525
 526   @Override
 527   public long getTotalStaticIndexSize() {
 528     return totalStaticIndexSize;
 529   }
 530
 531   @Override
 532   public long getTotalStaticBloomSize() {
 533     return totalStaticBloomSize;
 534   }
 535
 536   @Override
 537   public long getNumMutationsWithoutWAL() {
 538     return numMutationsWithoutWAL;
 539   }
 540
 541   @Override
 542   public long getDataInMemoryWithoutWAL() {
 543     return dataInMemoryWithoutWAL;
 544   }
 545
 546   @Override
 547   public double getPercentFileLocal() {
 548     return percentFileLocal;
 549   }
 550
 551   @Override
 552   public double getPercentFileLocalSecondaryRegions() {
 553     return percentFileLocalSecondaryRegions;
 554   }
 555
 556   @Override
 557   public long getUpdatesBlockedTime() {
 558     if (this.regionServer.getMemStoreFlusher() == null) {
 559       return 0;
 560     }
 561     return this.regionServer.getMemStoreFlusher().getUpdatesBlockedMsHighWater().sum();
 562   }
 563
 564   @Override
 565   public long getFlushedCellsCount() {
 566     return flushedCellsCount;
 567   }
 568
 569   @Override
 570   public long getCompactedCellsCount() {
 571     return compactedCellsCount;
 572   }
 573
 574   @Override
 575   public long getMajorCompactedCellsCount() {
 576     return majorCompactedCellsCount;
 577   }
 578
 579   @Override
 580   public long getFlushedCellsSize() {
 581     return flushedCellsSize;
 582   }
 583
 584   @Override
 585   public long getCompactedCellsSize() {
 586     return compactedCellsSize;
 587   }
 588
 589   @Override
 590   public long getMajorCompactedCellsSize() {
 591     return majorCompactedCellsSize;
 592   }
 593
 594   @Override
 595   public long getCellsCountCompactedFromMob() {
 596     return cellsCountCompactedFromMob;
 597   }
 598
 599   @Override
 600   public long getCellsCountCompactedToMob() {
 601     return cellsCountCompactedToMob;
 602   }
 603
 604   @Override
 605   public long getCellsSizeCompactedFromMob() {
 606     return cellsSizeCompactedFromMob;
 607   }
 608
 609   @Override
 610   public long getCellsSizeCompactedToMob() {
 611     return cellsSizeCompactedToMob;
 612   }
 613
 614   @Override
 615   public long getMobFlushCount() {
 616     return mobFlushCount;
 617   }
 618
 619   @Override
 620   public long getMobFlushedCellsCount() {
 621     return mobFlushedCellsCount;
 622   }
 623
 624   @Override
 625   public long getMobFlushedCellsSize() {
 626     return mobFlushedCellsSize;
 627   }
 628
 629   @Override
 630   public long getMobScanCellsCount() {
 631     return mobScanCellsCount;
 632   }
 633
 634   @Override
 635   public long getMobScanCellsSize() {
 636     return mobScanCellsSize;
 637   }
 638
 639   @Override
 640   public long getMobFileCacheAccessCount() {
 641     return mobFileCacheAccessCount;
 642   }
 643
 644   @Override
 645   public long getMobFileCacheMissCount() {
 646     return mobFileCacheMissCount;
 647   }
 648
 649   @Override
 650   public long getMobFileCacheCount() {
 651     return mobFileCacheCount;
 652   }
 653
 654   @Override
 655   public long getMobFileCacheEvictedCount() {
 656     return mobFileCacheEvictedCount;
 657   }
 658
 659   @Override
 660   public double getMobFileCacheHitPercent() {
 661     return mobFileCacheHitRatio * 100;
 662   }
 663
 664   /**
 665    * This is the runnable that will be executed on the executor every PERIOD number of seconds
 666    * It will take metrics/numbers from all of the regions and use them to compute point in
 667    * time metrics.
 668    */
 669   public class RegionServerMetricsWrapperRunnable implements Runnable {
 670
 671     private long lastRan = 0;
 672     private long lastStoreFileSize = 0;
 673
 674     @Override
 675     synchronized public void run() {
 676       try {
 677         HDFSBlocksDistribution hdfsBlocksDistribution =
 678             new HDFSBlocksDistribution();
 679         HDFSBlocksDistribution hdfsBlocksDistributionSecondaryRegions =
 680             new HDFSBlocksDistribution();
 681
 682         long tempNumStores = 0, tempNumStoreFiles = 0, tempMemstoreSize = 0, tempStoreFileSize = 0;
 683         long tempMaxStoreFileAge = 0, tempNumReferenceFiles = 0;
 684         long avgAgeNumerator = 0, numHFiles = 0;
 685         long tempMinStoreFileAge = Long.MAX_VALUE;
 686         long tempFilteredReadRequestsCount = 0, tempCpRequestsCount = 0;
 687         long tempCheckAndMutateChecksFailed = 0;
 688         long tempCheckAndMutateChecksPassed = 0;
 689         long tempStorefileIndexSize = 0;
 690         long tempTotalStaticIndexSize = 0;
 691         long tempTotalStaticBloomSize = 0;
 692         long tempNumMutationsWithoutWAL = 0;
 693         long tempDataInMemoryWithoutWAL = 0;
 694         double tempPercentFileLocal = 0;
 695         double tempPercentFileLocalSecondaryRegions = 0;
 696         long tempFlushedCellsCount = 0;
 697         long tempCompactedCellsCount = 0;
 698         long tempMajorCompactedCellsCount = 0;
 699         long tempFlushedCellsSize = 0;
 700         long tempCompactedCellsSize = 0;
 701         long tempMajorCompactedCellsSize = 0;
 702         long tempCellsCountCompactedToMob = 0;
 703         long tempCellsCountCompactedFromMob = 0;
 704         long tempCellsSizeCompactedToMob = 0;
 705         long tempCellsSizeCompactedFromMob = 0;
 706         long tempMobFlushCount = 0;
 707         long tempMobFlushedCellsCount = 0;
 708         long tempMobFlushedCellsSize = 0;
 709         long tempMobScanCellsCount = 0;
 710         long tempMobScanCellsSize = 0;
 711         long tempBlockedRequestsCount = 0;
 712         int regionCount = 0;
 713
 714         long tempReadRequestsCount = 0;
 715         long tempWriteRequestsCount = 0;
 716         long currentReadRequestsCount = 0;
 717         long currentWriteRequestsCount = 0;
 718         long lastReadRequestsCount = 0;
 719         long lastWriteRequestsCount = 0;
 720         long readRequestsDelta = 0;
 721         long writeRequestsDelta = 0;
 722         long totalReadRequestsDelta = 0;
 723         long totalWriteRequestsDelta = 0;
 724         String encodedRegionName;
 725         for (HRegion r : regionServer.getOnlineRegionsLocalContext()) {
 726           encodedRegionName = r.getRegionInfo().getEncodedName();
 727           currentReadRequestsCount = r.getReadRequestsCount();
 728           currentWriteRequestsCount = r.getWriteRequestsCount();
 729           if (requestsCountCache.containsKey(encodedRegionName)) {
 730             lastReadRequestsCount = requestsCountCache.get(encodedRegionName).get(0);
 731             lastWriteRequestsCount = requestsCountCache.get(encodedRegionName).get(1);
 732             readRequestsDelta = currentReadRequestsCount - lastReadRequestsCount;
 733             writeRequestsDelta = currentWriteRequestsCount - lastWriteRequestsCount;
 734             totalReadRequestsDelta += readRequestsDelta;
 735             totalWriteRequestsDelta += writeRequestsDelta;
 736             //Update cache for our next comparision
 737             requestsCountCache.get(encodedRegionName).set(0,currentReadRequestsCount);
 738             requestsCountCache.get(encodedRegionName).set(1,currentWriteRequestsCount);
 739           } else {
 740             // List[0] -> readRequestCount
 741             // List[1] -> writeRequestCount
 742             ArrayList<Long> requests = new ArrayList<Long>(2);
 743             requests.add(currentReadRequestsCount);
 744             requests.add(currentWriteRequestsCount);
 745             requestsCountCache.put(encodedRegionName, requests);
 746             totalReadRequestsDelta += currentReadRequestsCount;
 747             totalWriteRequestsDelta += currentWriteRequestsCount;
 748           }
 749           tempReadRequestsCount += r.getReadRequestsCount();
 750           tempWriteRequestsCount += r.getWriteRequestsCount();
 751           tempNumMutationsWithoutWAL += r.getNumMutationsWithoutWAL();
 752           tempDataInMemoryWithoutWAL += r.getDataInMemoryWithoutWAL();
 753           tempCpRequestsCount += r.getCpRequestsCount();
 754           tempFilteredReadRequestsCount += r.getFilteredReadRequestsCount();
 755           tempCheckAndMutateChecksFailed += r.getCheckAndMutateChecksFailed();
 756           tempCheckAndMutateChecksPassed += r.getCheckAndMutateChecksPassed();
 757           tempBlockedRequestsCount += r.getBlockedRequestsCount();
 758           List<? extends Store> storeList = r.getStores();
 759           tempNumStores += storeList.size();
 760           for (Store store : storeList) {
 761             tempNumStoreFiles += store.getStorefilesCount();
 762             tempMemstoreSize += store.getMemStoreSize().getDataSize();
 763             tempStoreFileSize += store.getStorefilesSize();
 764
 765             OptionalLong storeMaxStoreFileAge = store.getMaxStoreFileAge();
 766             if (storeMaxStoreFileAge.isPresent() &&
 767                 storeMaxStoreFileAge.getAsLong() > tempMaxStoreFileAge) {
 768               tempMaxStoreFileAge = storeMaxStoreFileAge.getAsLong();
 769             }
 770
 771             OptionalLong storeMinStoreFileAge = store.getMinStoreFileAge();
 772             if (storeMinStoreFileAge.isPresent() &&
 773                 storeMinStoreFileAge.getAsLong() < tempMinStoreFileAge) {
 774               tempMinStoreFileAge = storeMinStoreFileAge.getAsLong();
 775             }
 776
 777             long storeHFiles = store.getNumHFiles();
 778             numHFiles += storeHFiles;
 779             tempNumReferenceFiles += store.getNumReferenceFiles();
 780
 781             OptionalDouble storeAvgStoreFileAge = store.getAvgStoreFileAge();
 782             if (storeAvgStoreFileAge.isPresent()) {
 783               avgAgeNumerator =
 784                   (long) (avgAgeNumerator + storeAvgStoreFileAge.getAsDouble() * storeHFiles);
 785             }
 786
 787             tempStorefileIndexSize += store.getStorefilesRootLevelIndexSize();
 788             tempTotalStaticBloomSize += store.getTotalStaticBloomSize();
 789             tempTotalStaticIndexSize += store.getTotalStaticIndexSize();
 790             tempFlushedCellsCount += store.getFlushedCellsCount();
 791             tempCompactedCellsCount += store.getCompactedCellsCount();
 792             tempMajorCompactedCellsCount += store.getMajorCompactedCellsCount();
 793             tempFlushedCellsSize += store.getFlushedCellsSize();
 794             tempCompactedCellsSize += store.getCompactedCellsSize();
 795             tempMajorCompactedCellsSize += store.getMajorCompactedCellsSize();
 796             if (store instanceof HMobStore) {
 797               HMobStore mobStore = (HMobStore) store;
 798               tempCellsCountCompactedToMob += mobStore.getCellsCountCompactedToMob();
 799               tempCellsCountCompactedFromMob += mobStore.getCellsCountCompactedFromMob();
 800               tempCellsSizeCompactedToMob += mobStore.getCellsSizeCompactedToMob();
 801               tempCellsSizeCompactedFromMob += mobStore.getCellsSizeCompactedFromMob();
 802               tempMobFlushCount += mobStore.getMobFlushCount();
 803               tempMobFlushedCellsCount += mobStore.getMobFlushedCellsCount();
 804               tempMobFlushedCellsSize += mobStore.getMobFlushedCellsSize();
 805               tempMobScanCellsCount += mobStore.getMobScanCellsCount();
 806               tempMobScanCellsSize += mobStore.getMobScanCellsSize();
 807             }
 808           }
 809
 810           HDFSBlocksDistribution distro = r.getHDFSBlocksDistribution();
 811           hdfsBlocksDistribution.add(distro);
 812           if (r.getRegionInfo().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
 813             hdfsBlocksDistributionSecondaryRegions.add(distro);
 814           }
 815           regionCount++;
 816         }
 817
 818         float localityIndex = hdfsBlocksDistribution.getBlockLocalityIndex(
 819             regionServer.getServerName().getHostname());
 820         tempPercentFileLocal = Double.isNaN(tempBlockedRequestsCount) ? 0 : (localityIndex * 100);
 821
 822         float localityIndexSecondaryRegions = hdfsBlocksDistributionSecondaryRegions
 823             .getBlockLocalityIndex(regionServer.getServerName().getHostname());
 824         tempPercentFileLocalSecondaryRegions = Double.
 825             isNaN(localityIndexSecondaryRegions) ? 0 : (localityIndexSecondaryRegions * 100);
 826
 827         // Compute the number of requests per second
 828         long currentTime = EnvironmentEdgeManager.currentTime();
 829
 830         // assume that it took PERIOD seconds to start the executor.
 831         // this is a guess but it's a pretty good one.
 832         if (lastRan == 0) {
 833           lastRan = currentTime - period;
 834         }
 835         // If we've time traveled keep the last requests per second.
 836         if ((currentTime - lastRan) > 0) {
 837           requestsPerSecond = (totalReadRequestsDelta + totalWriteRequestsDelta) /
 838               ((currentTime - lastRan) / 1000.0);
 839
 840           double readRequestsRatePerMilliSecond = (double)totalReadRequestsDelta / period;
 841           double writeRequestsRatePerMilliSecond = (double)totalWriteRequestsDelta / period;
 842
 843           readRequestsRatePerSecond = readRequestsRatePerMilliSecond * 1000.0;
 844           writeRequestsRatePerSecond = writeRequestsRatePerMilliSecond * 1000.0;
 845
 846           long intervalStoreFileSize = tempStoreFileSize - lastStoreFileSize;
 847           storeFileSizeGrowthRate = (double)intervalStoreFileSize * 1000.0 / period;
 848
 849           lastStoreFileSize = tempStoreFileSize;
 850         }
 851
 852         lastRan = currentTime;
 853
 854         final WALProvider provider = regionServer.getWalFactory().getWALProvider();
 855         final WALProvider metaProvider = regionServer.getWalFactory().getMetaWALProvider();
 856         numWALFiles = (provider == null ? 0 : provider.getNumLogFiles()) +
 857             (metaProvider == null ? 0 : metaProvider.getNumLogFiles());
 858         walFileSize = (provider == null ? 0 : provider.getLogFileSize()) +
 859             (provider == null ? 0 : provider.getLogFileSize());
 860         // Copy over computed values so that no thread sees half computed values.
 861         numStores = tempNumStores;
 862         numStoreFiles = tempNumStoreFiles;
 863         memstoreSize = tempMemstoreSize;
 864         storeFileSize = tempStoreFileSize;
 865         maxStoreFileAge = tempMaxStoreFileAge;
 866         if (regionCount > 0) {
 867           averageRegionSize = (memstoreSize + storeFileSize) / regionCount;
 868         }
 869         if (tempMinStoreFileAge != Long.MAX_VALUE) {
 870           minStoreFileAge = tempMinStoreFileAge;
 871         }
 872
 873         if (numHFiles != 0) {
 874           avgStoreFileAge = avgAgeNumerator / numHFiles;
 875         }
 876
 877         numReferenceFiles= tempNumReferenceFiles;
 878         readRequestsCount = tempReadRequestsCount;
 879         cpRequestsCount = tempCpRequestsCount;
 880         filteredReadRequestsCount = tempFilteredReadRequestsCount;
 881         writeRequestsCount = tempWriteRequestsCount;
 882         checkAndMutateChecksFailed = tempCheckAndMutateChecksFailed;
 883         checkAndMutateChecksPassed = tempCheckAndMutateChecksPassed;
 884         storefileIndexSize = tempStorefileIndexSize;
 885         totalStaticIndexSize = tempTotalStaticIndexSize;
 886         totalStaticBloomSize = tempTotalStaticBloomSize;
 887         numMutationsWithoutWAL = tempNumMutationsWithoutWAL;
 888         dataInMemoryWithoutWAL = tempDataInMemoryWithoutWAL;
 889         percentFileLocal = tempPercentFileLocal;
 890         percentFileLocalSecondaryRegions = tempPercentFileLocalSecondaryRegions;
 891         flushedCellsCount = tempFlushedCellsCount;
 892         compactedCellsCount = tempCompactedCellsCount;
 893         majorCompactedCellsCount = tempMajorCompactedCellsCount;
 894         flushedCellsSize = tempFlushedCellsSize;
 895         compactedCellsSize = tempCompactedCellsSize;
 896         majorCompactedCellsSize = tempMajorCompactedCellsSize;
 897         cellsCountCompactedToMob = tempCellsCountCompactedToMob;
 898         cellsCountCompactedFromMob = tempCellsCountCompactedFromMob;
 899         cellsSizeCompactedToMob = tempCellsSizeCompactedToMob;
 900         cellsSizeCompactedFromMob = tempCellsSizeCompactedFromMob;
 901         mobFlushCount = tempMobFlushCount;
 902         mobFlushedCellsCount = tempMobFlushedCellsCount;
 903         mobFlushedCellsSize = tempMobFlushedCellsSize;
 904         mobScanCellsCount = tempMobScanCellsCount;
 905         mobScanCellsSize = tempMobScanCellsSize;
 906         mobFileCacheAccessCount = mobFileCache != null ? mobFileCache.getAccessCount() : 0L;
 907         mobFileCacheMissCount = mobFileCache != null ? mobFileCache.getMissCount() : 0L;
 908         mobFileCacheHitRatio = mobFileCache != null ? mobFileCache.getHitRatio() : 0.0;
 909         if (Double.isNaN(mobFileCacheHitRatio)) {
 910           mobFileCacheHitRatio = 0.0;
 911         }
 912         mobFileCacheEvictedCount = mobFileCache != null ? mobFileCache.getEvictedFileCount() : 0L;
 913         mobFileCacheCount = mobFileCache != null ? mobFileCache.getCacheSize() : 0;
 914         blockedRequestsCount = tempBlockedRequestsCount;
 915       } catch (Throwable e) {
 916         LOG.warn("Caught exception! Will suppress and retry.", e);
 917       }
 918     }
 919   }
 920
 921   @Override
 922   public long getHedgedReadOps() {
 923     return this.dfsHedgedReadMetrics == null? 0: this.dfsHedgedReadMetrics.getHedgedReadOps();
 924   }
 925
 926   @Override
 927   public long getHedgedReadWins() {
 928     return this.dfsHedgedReadMetrics == null? 0: this.dfsHedgedReadMetrics.getHedgedReadWins();
 929   }
 930
 931   @Override
 932   public long getBlockedRequestsCount() {
 933     return blockedRequestsCount;
 934   }
 935
 936   @Override
 937   public long getAverageRegionSize() {
 938     return averageRegionSize;
 939   }
 940
 941   @Override
 942   public long getDataMissCount() {
 943     return this.cacheStats != null ? this.cacheStats.getDataMissCount() : 0L;
 944   }
 945
 946   @Override
 947   public long getLeafIndexMissCount() {
 948     return this.cacheStats != null ? this.cacheStats.getLeafIndexMissCount() : 0L;
 949   }
 950
 951   @Override
 952   public long getBloomChunkMissCount() {
 953     return this.cacheStats != null ? this.cacheStats.getBloomChunkMissCount() : 0L;
 954   }
 955
 956   @Override
 957   public long getMetaMissCount() {
 958     return this.cacheStats != null ? this.cacheStats.getMetaMissCount() : 0L;
 959   }
 960
 961   @Override
 962   public long getRootIndexMissCount() {
 963     return this.cacheStats != null ? this.cacheStats.getRootIndexMissCount() : 0L;
 964   }
 965
 966   @Override
 967   public long getIntermediateIndexMissCount() {
 968     return this.cacheStats != null ? this.cacheStats.getIntermediateIndexMissCount() : 0L;
 969   }
 970
 971   @Override
 972   public long getFileInfoMissCount() {
 973     return this.cacheStats != null ? this.cacheStats.getFileInfoMissCount() : 0L;
 974   }
 975
 976   @Override
 977   public long getGeneralBloomMetaMissCount() {
 978     return this.cacheStats != null ? this.cacheStats.getGeneralBloomMetaMissCount() : 0L;
 979   }
 980
 981   @Override
 982   public long getDeleteFamilyBloomMissCount() {
 983     return this.cacheStats != null ? this.cacheStats.getDeleteFamilyBloomMissCount() : 0L;
 984   }
 985
 986   @Override
 987   public long getTrailerMissCount() {
 988     return this.cacheStats != null ? this.cacheStats.getTrailerMissCount() : 0L;
 989   }
 990
 991   @Override
 992   public long getDataHitCount() {
 993     return this.cacheStats != null ? this.cacheStats.getDataHitCount() : 0L;
 994   }
 995
 996   @Override
 997   public long getLeafIndexHitCount() {
 998     return this.cacheStats != null ? this.cacheStats.getLeafIndexHitCount() : 0L;
 999   }
1000
1001   @Override
1002   public long getBloomChunkHitCount() {
1003     return this.cacheStats != null ? this.cacheStats.getBloomChunkHitCount() : 0L;
1004   }
1005
1006   @Override
1007   public long getMetaHitCount() {
1008     return this.cacheStats != null ? this.cacheStats.getMetaHitCount() : 0L;
1009   }
1010
1011   @Override
1012   public long getRootIndexHitCount() {
1013     return this.cacheStats != null ? this.cacheStats.getRootIndexHitCount() : 0L;
1014   }
1015
1016   @Override
1017   public long getIntermediateIndexHitCount() {
1018     return this.cacheStats != null ? this.cacheStats.getIntermediateIndexHitCount() : 0L;
1019   }
1020
1021   @Override
1022   public long getFileInfoHitCount() {
1023     return this.cacheStats != null ? this.cacheStats.getFileInfoHitCount() : 0L;
1024   }
1025
1026   @Override
1027   public long getGeneralBloomMetaHitCount() {
1028     return this.cacheStats != null ? this.cacheStats.getGeneralBloomMetaHitCount() : 0L;
1029   }
1030
1031   @Override
1032   public long getDeleteFamilyBloomHitCount() {
1033     return this.cacheStats != null ? this.cacheStats.getDeleteFamilyBloomHitCount() : 0L;
1034   }
1035
1036   @Override
1037   public long getTrailerHitCount() {
1038     return this.cacheStats != null ? this.cacheStats.getTrailerHitCount() : 0L;
1039   }
1040
1041   @Override
1042   public long getByteBuffAllocatorHeapAllocationBytes() {
1043     return ByteBuffAllocator.getHeapAllocationBytes(allocator, ByteBuffAllocator.HEAP);
1044   }
1045
1046   @Override
1047   public long getByteBuffAllocatorPoolAllocationBytes() {
1048     return this.allocator.getPoolAllocationBytes();
1049   }
1050
1051   @Override
1052   public double getByteBuffAllocatorHeapAllocRatio() {
1053     return ByteBuffAllocator.getHeapAllocationRatio(allocator, ByteBuffAllocator.HEAP);
1054   }
1055
1056   @Override
1057   public long getByteBuffAllocatorTotalBufferCount() {
1058     return this.allocator.getTotalBufferCount();
1059   }
1060
1061   @Override
1062   public long getByteBuffAllocatorUsedBufferCount() {
1063     return this.allocator.getUsedBufferCount();
1064   }
1065 }