hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.regionserver;
  19
  20 import java.io.IOException;
  21 import java.util.ArrayList;
  22 import java.util.Collection;
  23 import java.util.List;
  24 import java.util.Map;
  25 import java.util.OptionalDouble;
  26 import java.util.OptionalLong;
  27 import java.util.concurrent.ConcurrentHashMap;
  28 import java.util.concurrent.ScheduledExecutorService;
  29 import java.util.concurrent.TimeUnit;
  30 import org.apache.commons.lang3.StringUtils;
  31 import org.apache.hadoop.hbase.CompatibilitySingletonFactory;
  32 import org.apache.hadoop.hbase.HConstants;
  33 import org.apache.hadoop.hbase.HDFSBlocksDistribution;
  34 import org.apache.hadoop.hbase.ServerName;
  35 import org.apache.hadoop.hbase.client.RegionInfo;
  36 import org.apache.hadoop.hbase.io.ByteBuffAllocator;
  37 import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper;
  38 import org.apache.hadoop.hbase.io.hfile.BlockCache;
  39 import org.apache.hadoop.hbase.io.hfile.CacheStats;
  40 import org.apache.hadoop.hbase.io.hfile.CombinedBlockCache;
  41 import org.apache.hadoop.hbase.mob.MobFileCache;
  42 import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource;
  43 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
  44 import org.apache.hadoop.hbase.util.FSUtils;
  45 import org.apache.hadoop.hbase.wal.WALProvider;
  46 import org.apache.hadoop.hbase.zookeeper.ZKWatcher;
  47 import org.apache.hadoop.hdfs.DFSHedgedReadMetrics;
  48 import org.apache.hadoop.metrics2.MetricsExecutor;
  49 import org.apache.yetus.audience.InterfaceAudience;
  50 import org.slf4j.Logger;
  51 import org.slf4j.LoggerFactory;
  52
  53 /**
  54  * Impl for exposing HRegionServer Information through Hadoop's metrics 2 system.
  55  */
  56 @InterfaceAudience.Private
  57 class MetricsRegionServerWrapperImpl
  58     implements MetricsRegionServerWrapper {
  59
  60   private static final Logger LOG = LoggerFactory.getLogger(MetricsRegionServerWrapperImpl.class);
  61
  62   private final HRegionServer regionServer;
  63   private final MetricsWALSource metricsWALSource;
  64   private final ByteBuffAllocator allocator;
  65
  66   private BlockCache blockCache;
  67   private MobFileCache mobFileCache;
  68   private CacheStats cacheStats;
  69   private CacheStats l1Stats = null;
  70   private CacheStats l2Stats = null;
  71
  72   private volatile long numStores = 0;
  73   private volatile long numWALFiles = 0;
  74   private volatile long walFileSize = 0;
  75   private volatile long numStoreFiles = 0;
  76   private volatile long memstoreSize = 0;
  77   private volatile long storeFileSize = 0;
  78   private volatile double storeFileSizeGrowthRate = 0;
  79   private volatile long maxStoreFileAge = 0;
  80   private volatile long minStoreFileAge = 0;
  81   private volatile long avgStoreFileAge = 0;
  82   private volatile long numReferenceFiles = 0;
  83   private volatile double requestsPerSecond = 0.0;
  84   private volatile long readRequestsCount = 0;
  85   private volatile double readRequestsRatePerSecond = 0;
  86   private volatile long cpRequestsCount = 0;
  87   private volatile long filteredReadRequestsCount = 0;
  88   private volatile long writeRequestsCount = 0;
  89   private volatile double writeRequestsRatePerSecond = 0;
  90   private volatile long checkAndMutateChecksFailed = 0;
  91   private volatile long checkAndMutateChecksPassed = 0;
  92   private volatile long storefileIndexSize = 0;
  93   private volatile long totalStaticIndexSize = 0;
  94   private volatile long totalStaticBloomSize = 0;
  95   private volatile long numMutationsWithoutWAL = 0;
  96   private volatile long dataInMemoryWithoutWAL = 0;
  97   private volatile double percentFileLocal = 0;
  98   private volatile double percentFileLocalSecondaryRegions = 0;
  99   private volatile long flushedCellsCount = 0;
 100   private volatile long compactedCellsCount = 0;
 101   private volatile long majorCompactedCellsCount = 0;
 102   private volatile long flushedCellsSize = 0;
 103   private volatile long compactedCellsSize = 0;
 104   private volatile long majorCompactedCellsSize = 0;
 105   private volatile long cellsCountCompactedToMob = 0;
 106   private volatile long cellsCountCompactedFromMob = 0;
 107   private volatile long cellsSizeCompactedToMob = 0;
 108   private volatile long cellsSizeCompactedFromMob = 0;
 109   private volatile long mobFlushCount = 0;
 110   private volatile long mobFlushedCellsCount = 0;
 111   private volatile long mobFlushedCellsSize = 0;
 112   private volatile long mobScanCellsCount = 0;
 113   private volatile long mobScanCellsSize = 0;
 114   private volatile long mobFileCacheAccessCount = 0;
 115   private volatile long mobFileCacheMissCount = 0;
 116   private volatile double mobFileCacheHitRatio = 0;
 117   private volatile long mobFileCacheEvictedCount = 0;
 118   private volatile long mobFileCacheCount = 0;
 119   private volatile long blockedRequestsCount = 0L;
 120   private volatile long averageRegionSize = 0L;
 121   protected final Map<String, ArrayList<Long>> requestsCountCache = new
 122       ConcurrentHashMap<String, ArrayList<Long>>();
 123
 124   private ScheduledExecutorService executor;
 125   private Runnable runnable;
 126   private long period;
 127
 128   /**
 129    * Can be null if not on hdfs.
 130    */
 131   private DFSHedgedReadMetrics dfsHedgedReadMetrics;
 132
 133   public MetricsRegionServerWrapperImpl(final HRegionServer regionServer) {
 134     this.regionServer = regionServer;
 135     initBlockCache();
 136     initMobFileCache();
 137
 138     this.period = regionServer.getConfiguration().getLong(HConstants.REGIONSERVER_METRICS_PERIOD,
 139       HConstants.DEFAULT_REGIONSERVER_METRICS_PERIOD);
 140
 141     this.executor = CompatibilitySingletonFactory.getInstance(MetricsExecutor.class).getExecutor();
 142     this.runnable = new RegionServerMetricsWrapperRunnable();
 143     this.executor.scheduleWithFixedDelay(this.runnable, this.period, this.period,
 144       TimeUnit.MILLISECONDS);
 145     this.metricsWALSource = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
 146     this.allocator = regionServer.getRpcServer().getByteBuffAllocator();
 147
 148     try {
 149       this.dfsHedgedReadMetrics = FSUtils.getDFSHedgedReadMetrics(regionServer.getConfiguration());
 150     } catch (IOException e) {
 151       LOG.warn("Failed to get hedged metrics", e);
 152     }
 153     if (LOG.isInfoEnabled()) {
 154       LOG.info("Computing regionserver metrics every " + this.period + " milliseconds");
 155     }
 156   }
 157
 158   private void initBlockCache() {
 159     this.blockCache = this.regionServer.getBlockCache().orElse(null);
 160     this.cacheStats = this.blockCache != null ? this.blockCache.getStats() : null;
 161     if (this.cacheStats != null) {
 162       if (this.cacheStats instanceof CombinedBlockCache.CombinedCacheStats) {
 163         l1Stats = ((CombinedBlockCache.CombinedCacheStats) this.cacheStats)
 164           .getLruCacheStats();
 165         l2Stats = ((CombinedBlockCache.CombinedCacheStats) this.cacheStats)
 166           .getBucketCacheStats();
 167       } else {
 168         l1Stats = this.cacheStats;
 169       }
 170     }
 171   }
 172
 173   /**
 174    * Initializes the mob file cache.
 175    */
 176   private void initMobFileCache() {
 177     this.mobFileCache = this.regionServer.getMobFileCache().orElse(null);
 178   }
 179
 180   @Override
 181   public String getClusterId() {
 182     return regionServer.getClusterId();
 183   }
 184
 185   @Override
 186   public long getStartCode() {
 187     return regionServer.getStartcode();
 188   }
 189
 190   @Override
 191   public String getZookeeperQuorum() {
 192     ZKWatcher zk = regionServer.getZooKeeper();
 193     if (zk == null) {
 194       return "";
 195     }
 196     return zk.getQuorum();
 197   }
 198
 199   @Override
 200   public String getCoprocessors() {
 201     String[] coprocessors = regionServer.getRegionServerCoprocessors();
 202     if (coprocessors == null || coprocessors.length == 0) {
 203       return "";
 204     }
 205     return StringUtils.join(coprocessors, ", ");
 206   }
 207
 208   @Override
 209   public String getServerName() {
 210     ServerName serverName = regionServer.getServerName();
 211     if (serverName == null) {
 212       return "";
 213     }
 214     return serverName.getServerName();
 215   }
 216
 217   @Override
 218   public long getNumOnlineRegions() {
 219     Collection<HRegion> onlineRegionsLocalContext = regionServer.getOnlineRegionsLocalContext();
 220     if (onlineRegionsLocalContext == null) {
 221       return 0;
 222     }
 223     return onlineRegionsLocalContext.size();
 224   }
 225
 226   @Override
 227   public long getTotalRequestCount() {
 228     return regionServer.getRpcServices().requestCount.sum();
 229   }
 230
 231   @Override
 232   public long getTotalRowActionRequestCount() {
 233     return readRequestsCount + writeRequestsCount;
 234   }
 235
 236   @Override
 237   public int getSplitQueueSize() {
 238     final CompactSplit compactSplit = regionServer.getCompactSplitThread();
 239     return compactSplit == null ? 0 : compactSplit.getSplitQueueSize();
 240   }
 241
 242   @Override
 243   public int getCompactionQueueSize() {
 244     final CompactSplit compactSplit = regionServer.getCompactSplitThread();
 245     return compactSplit == null ? 0 : compactSplit.getCompactionQueueSize();
 246   }
 247
 248   @Override
 249   public int getSmallCompactionQueueSize() {
 250     final CompactSplit compactSplit = regionServer.getCompactSplitThread();
 251     return compactSplit == null ? 0 : compactSplit.getSmallCompactionQueueSize();
 252   }
 253
 254   @Override
 255   public int getLargeCompactionQueueSize() {
 256     final CompactSplit compactSplit = regionServer.getCompactSplitThread();
 257     return compactSplit == null ? 0 : compactSplit.getLargeCompactionQueueSize();
 258   }
 259
 260   @Override
 261   public int getFlushQueueSize() {
 262     //If there is no flusher there should be no queue.
 263     if (this.regionServer.getMemStoreFlusher() == null) {
 264       return 0;
 265     }
 266     return this.regionServer.getMemStoreFlusher().getFlushQueueSize();
 267   }
 268
 269   @Override
 270   public long getBlockCacheCount() {
 271     return this.blockCache != null ? this.blockCache.getBlockCount() : 0L;
 272   }
 273
 274   @Override
 275   public long getMemStoreLimit() {
 276     return this.regionServer.getRegionServerAccounting().getGlobalMemStoreLimit();
 277   }
 278
 279   @Override
 280   public long getBlockCacheSize() {
 281     return this.blockCache != null ? this.blockCache.getCurrentSize() : 0L;
 282   }
 283
 284   @Override
 285   public long getBlockCacheFreeSize() {
 286     return this.blockCache != null ? this.blockCache.getFreeSize() : 0L;
 287   }
 288
 289   @Override
 290   public long getBlockCacheHitCount() {
 291     return this.cacheStats != null ? this.cacheStats.getHitCount() : 0L;
 292   }
 293
 294   @Override
 295   public long getBlockCachePrimaryHitCount() {
 296     return this.cacheStats != null ? this.cacheStats.getPrimaryHitCount() : 0L;
 297   }
 298
 299   @Override
 300   public long getBlockCacheMissCount() {
 301     return this.cacheStats != null ? this.cacheStats.getMissCount() : 0L;
 302   }
 303
 304   @Override
 305   public long getBlockCachePrimaryMissCount() {
 306     return this.cacheStats != null ? this.cacheStats.getPrimaryMissCount() : 0L;
 307   }
 308
 309   @Override
 310   public long getBlockCacheEvictedCount() {
 311     return this.cacheStats != null ? this.cacheStats.getEvictedCount() : 0L;
 312   }
 313
 314   @Override
 315   public long getBlockCachePrimaryEvictedCount() {
 316     return this.cacheStats != null ? this.cacheStats.getPrimaryEvictedCount() : 0L;
 317   }
 318
 319   @Override
 320   public double getBlockCacheHitPercent() {
 321     double ratio = this.cacheStats != null ? this.cacheStats.getHitRatio() : 0.0;
 322     if (Double.isNaN(ratio)) {
 323       ratio = 0;
 324     }
 325     return (ratio * 100);
 326   }
 327
 328   @Override
 329   public double getBlockCacheHitCachingPercent() {
 330     double ratio = this.cacheStats != null ? this.cacheStats.getHitCachingRatio() : 0.0;
 331     if (Double.isNaN(ratio)) {
 332       ratio = 0;
 333     }
 334     return (ratio * 100);
 335   }
 336
 337   @Override
 338   public long getBlockCacheFailedInsertions() {
 339     return this.cacheStats != null ? this.cacheStats.getFailedInserts() : 0L;
 340   }
 341
 342   @Override
 343   public long getL1CacheHitCount() {
 344     return this.l1Stats != null ? this.l1Stats.getHitCount() : 0L;
 345   }
 346
 347   @Override
 348   public long getL1CacheMissCount() {
 349     return this.l1Stats != null ? this.l1Stats.getMissCount() : 0L;
 350   }
 351
 352   @Override
 353   public double getL1CacheHitRatio() {
 354     return this.l1Stats != null ? this.l1Stats.getHitRatio() : 0.0;
 355   }
 356
 357   @Override
 358   public double getL1CacheMissRatio() {
 359     return this.l1Stats != null ? this.l1Stats.getMissRatio() : 0.0;
 360   }
 361
 362   @Override
 363   public long getL2CacheHitCount() {
 364     return this.l2Stats != null ? this.l2Stats.getHitCount() : 0L;
 365   }
 366
 367   @Override
 368   public long getL2CacheMissCount() {
 369     return this.l2Stats != null ? this.l2Stats.getMissCount() : 0L;
 370   }
 371
 372   @Override
 373   public double getL2CacheHitRatio() {
 374     return this.l2Stats != null ? this.l2Stats.getHitRatio() : 0.0;
 375   }
 376
 377   @Override
 378   public double getL2CacheMissRatio() {
 379     return this.l2Stats != null ? this.l2Stats.getMissRatio() : 0.0;
 380   }
 381
 382   @Override public void forceRecompute() {
 383     this.runnable.run();
 384   }
 385
 386   @Override
 387   public long getNumStores() {
 388     return numStores;
 389   }
 390
 391   @Override
 392   public long getNumWALFiles() {
 393     return numWALFiles;
 394   }
 395
 396   @Override
 397   public long getWALFileSize() {
 398     return walFileSize;
 399   }
 400
 401   @Override
 402   public long getNumWALSlowAppend() {
 403     return metricsWALSource.getSlowAppendCount();
 404   }
 405
 406   @Override
 407   public long getNumStoreFiles() {
 408     return numStoreFiles;
 409   }
 410
 411   @Override
 412   public long getMaxStoreFileAge() {
 413     return maxStoreFileAge;
 414   }
 415
 416   @Override
 417   public long getMinStoreFileAge() {
 418     return minStoreFileAge;
 419   }
 420
 421   @Override
 422   public long getAvgStoreFileAge() {
 423     return avgStoreFileAge;
 424   }
 425
 426   @Override
 427   public long getNumReferenceFiles() {
 428     return numReferenceFiles;
 429   }
 430
 431   @Override
 432   public long getMemStoreSize() {
 433     return memstoreSize;
 434   }
 435
 436   @Override
 437   public long getStoreFileSize() {
 438     return storeFileSize;
 439   }
 440
 441   @Override
 442   public double getStoreFileSizeGrowthRate() {
 443     return storeFileSizeGrowthRate;
 444   }
 445
 446   @Override public double getRequestsPerSecond() {
 447     return requestsPerSecond;
 448   }
 449
 450   @Override
 451   public long getReadRequestsCount() {
 452     return readRequestsCount;
 453   }
 454
 455   @Override
 456   public long getCpRequestsCount() {
 457     return cpRequestsCount;
 458   }
 459
 460   @Override
 461   public double getReadRequestsRatePerSecond() {
 462     return readRequestsRatePerSecond;
 463   }
 464
 465   @Override
 466   public long getFilteredReadRequestsCount() {
 467     return filteredReadRequestsCount;
 468   }
 469
 470   @Override
 471   public long getWriteRequestsCount() {
 472     return writeRequestsCount;
 473   }
 474
 475   @Override
 476   public double getWriteRequestsRatePerSecond() {
 477     return writeRequestsRatePerSecond;
 478   }
 479
 480   @Override
 481   public long getRpcGetRequestsCount() {
 482     return regionServer.getRpcServices().rpcGetRequestCount.sum();
 483   }
 484
 485   @Override
 486   public long getRpcScanRequestsCount() {
 487     return regionServer.getRpcServices().rpcScanRequestCount.sum();
 488   }
 489
 490   @Override
 491   public long getRpcFullScanRequestsCount() {
 492     return regionServer.getRpcServices().rpcFullScanRequestCount.sum();
 493   }
 494
 495   @Override
 496   public long getRpcMultiRequestsCount() {
 497     return regionServer.getRpcServices().rpcMultiRequestCount.sum();
 498   }
 499
 500   @Override
 501   public long getRpcMutateRequestsCount() {
 502     return regionServer.getRpcServices().rpcMutateRequestCount.sum();
 503   }
 504
 505   @Override
 506   public long getCheckAndMutateChecksFailed() {
 507     return checkAndMutateChecksFailed;
 508   }
 509
 510   @Override
 511   public long getCheckAndMutateChecksPassed() {
 512     return checkAndMutateChecksPassed;
 513   }
 514
 515   @Override
 516   public long getStoreFileIndexSize() {
 517     return storefileIndexSize;
 518   }
 519
 520   @Override
 521   public long getTotalStaticIndexSize() {
 522     return totalStaticIndexSize;
 523   }
 524
 525   @Override
 526   public long getTotalStaticBloomSize() {
 527     return totalStaticBloomSize;
 528   }
 529
 530   @Override
 531   public long getNumMutationsWithoutWAL() {
 532     return numMutationsWithoutWAL;
 533   }
 534
 535   @Override
 536   public long getDataInMemoryWithoutWAL() {
 537     return dataInMemoryWithoutWAL;
 538   }
 539
 540   @Override
 541   public double getPercentFileLocal() {
 542     return percentFileLocal;
 543   }
 544
 545   @Override
 546   public double getPercentFileLocalSecondaryRegions() {
 547     return percentFileLocalSecondaryRegions;
 548   }
 549
 550   @Override
 551   public long getUpdatesBlockedTime() {
 552     if (this.regionServer.getMemStoreFlusher() == null) {
 553       return 0;
 554     }
 555     return this.regionServer.getMemStoreFlusher().getUpdatesBlockedMsHighWater().sum();
 556   }
 557
 558   @Override
 559   public long getFlushedCellsCount() {
 560     return flushedCellsCount;
 561   }
 562
 563   @Override
 564   public long getCompactedCellsCount() {
 565     return compactedCellsCount;
 566   }
 567
 568   @Override
 569   public long getMajorCompactedCellsCount() {
 570     return majorCompactedCellsCount;
 571   }
 572
 573   @Override
 574   public long getFlushedCellsSize() {
 575     return flushedCellsSize;
 576   }
 577
 578   @Override
 579   public long getCompactedCellsSize() {
 580     return compactedCellsSize;
 581   }
 582
 583   @Override
 584   public long getMajorCompactedCellsSize() {
 585     return majorCompactedCellsSize;
 586   }
 587
 588   @Override
 589   public long getCellsCountCompactedFromMob() {
 590     return cellsCountCompactedFromMob;
 591   }
 592
 593   @Override
 594   public long getCellsCountCompactedToMob() {
 595     return cellsCountCompactedToMob;
 596   }
 597
 598   @Override
 599   public long getCellsSizeCompactedFromMob() {
 600     return cellsSizeCompactedFromMob;
 601   }
 602
 603   @Override
 604   public long getCellsSizeCompactedToMob() {
 605     return cellsSizeCompactedToMob;
 606   }
 607
 608   @Override
 609   public long getMobFlushCount() {
 610     return mobFlushCount;
 611   }
 612
 613   @Override
 614   public long getMobFlushedCellsCount() {
 615     return mobFlushedCellsCount;
 616   }
 617
 618   @Override
 619   public long getMobFlushedCellsSize() {
 620     return mobFlushedCellsSize;
 621   }
 622
 623   @Override
 624   public long getMobScanCellsCount() {
 625     return mobScanCellsCount;
 626   }
 627
 628   @Override
 629   public long getMobScanCellsSize() {
 630     return mobScanCellsSize;
 631   }
 632
 633   @Override
 634   public long getMobFileCacheAccessCount() {
 635     return mobFileCacheAccessCount;
 636   }
 637
 638   @Override
 639   public long getMobFileCacheMissCount() {
 640     return mobFileCacheMissCount;
 641   }
 642
 643   @Override
 644   public long getMobFileCacheCount() {
 645     return mobFileCacheCount;
 646   }
 647
 648   @Override
 649   public long getMobFileCacheEvictedCount() {
 650     return mobFileCacheEvictedCount;
 651   }
 652
 653   @Override
 654   public double getMobFileCacheHitPercent() {
 655     return mobFileCacheHitRatio * 100;
 656   }
 657
 658   /**
 659    * This is the runnable that will be executed on the executor every PERIOD number of seconds
 660    * It will take metrics/numbers from all of the regions and use them to compute point in
 661    * time metrics.
 662    */
 663   public class RegionServerMetricsWrapperRunnable implements Runnable {
 664
 665     private long lastRan = 0;
 666     private long lastStoreFileSize = 0;
 667
 668     @Override
 669     synchronized public void run() {
 670       try {
 671         HDFSBlocksDistribution hdfsBlocksDistribution =
 672             new HDFSBlocksDistribution();
 673         HDFSBlocksDistribution hdfsBlocksDistributionSecondaryRegions =
 674             new HDFSBlocksDistribution();
 675
 676         long tempNumStores = 0, tempNumStoreFiles = 0, tempMemstoreSize = 0, tempStoreFileSize = 0;
 677         long tempMaxStoreFileAge = 0, tempNumReferenceFiles = 0;
 678         long avgAgeNumerator = 0, numHFiles = 0;
 679         long tempMinStoreFileAge = Long.MAX_VALUE;
 680         long tempFilteredReadRequestsCount = 0, tempCpRequestsCount = 0;
 681         long tempCheckAndMutateChecksFailed = 0;
 682         long tempCheckAndMutateChecksPassed = 0;
 683         long tempStorefileIndexSize = 0;
 684         long tempTotalStaticIndexSize = 0;
 685         long tempTotalStaticBloomSize = 0;
 686         long tempNumMutationsWithoutWAL = 0;
 687         long tempDataInMemoryWithoutWAL = 0;
 688         double tempPercentFileLocal = 0;
 689         double tempPercentFileLocalSecondaryRegions = 0;
 690         long tempFlushedCellsCount = 0;
 691         long tempCompactedCellsCount = 0;
 692         long tempMajorCompactedCellsCount = 0;
 693         long tempFlushedCellsSize = 0;
 694         long tempCompactedCellsSize = 0;
 695         long tempMajorCompactedCellsSize = 0;
 696         long tempCellsCountCompactedToMob = 0;
 697         long tempCellsCountCompactedFromMob = 0;
 698         long tempCellsSizeCompactedToMob = 0;
 699         long tempCellsSizeCompactedFromMob = 0;
 700         long tempMobFlushCount = 0;
 701         long tempMobFlushedCellsCount = 0;
 702         long tempMobFlushedCellsSize = 0;
 703         long tempMobScanCellsCount = 0;
 704         long tempMobScanCellsSize = 0;
 705         long tempBlockedRequestsCount = 0;
 706         int regionCount = 0;
 707
 708         long tempReadRequestsCount = 0;
 709         long tempWriteRequestsCount = 0;
 710         long currentReadRequestsCount = 0;
 711         long currentWriteRequestsCount = 0;
 712         long lastReadRequestsCount = 0;
 713         long lastWriteRequestsCount = 0;
 714         long readRequestsDelta = 0;
 715         long writeRequestsDelta = 0;
 716         long totalReadRequestsDelta = 0;
 717         long totalWriteRequestsDelta = 0;
 718         String encodedRegionName;
 719         for (HRegion r : regionServer.getOnlineRegionsLocalContext()) {
 720           encodedRegionName = r.getRegionInfo().getEncodedName();
 721           currentReadRequestsCount = r.getReadRequestsCount();
 722           currentWriteRequestsCount = r.getWriteRequestsCount();
 723           if (requestsCountCache.containsKey(encodedRegionName)) {
 724             lastReadRequestsCount = requestsCountCache.get(encodedRegionName).get(0);
 725             lastWriteRequestsCount = requestsCountCache.get(encodedRegionName).get(1);
 726             readRequestsDelta = currentReadRequestsCount - lastReadRequestsCount;
 727             writeRequestsDelta = currentWriteRequestsCount - lastWriteRequestsCount;
 728             totalReadRequestsDelta += readRequestsDelta;
 729             totalWriteRequestsDelta += writeRequestsDelta;
 730             //Update cache for our next comparision
 731             requestsCountCache.get(encodedRegionName).set(0,currentReadRequestsCount);
 732             requestsCountCache.get(encodedRegionName).set(1,currentWriteRequestsCount);
 733           } else {
 734             // List[0] -> readRequestCount
 735             // List[1] -> writeRequestCount
 736             ArrayList<Long> requests = new ArrayList<Long>(2);
 737             requests.add(currentReadRequestsCount);
 738             requests.add(currentWriteRequestsCount);
 739             requestsCountCache.put(encodedRegionName, requests);
 740             totalReadRequestsDelta += currentReadRequestsCount;
 741             totalWriteRequestsDelta += currentWriteRequestsCount;
 742           }
 743           tempReadRequestsCount += r.getReadRequestsCount();
 744           tempWriteRequestsCount += r.getWriteRequestsCount();
 745           tempNumMutationsWithoutWAL += r.getNumMutationsWithoutWAL();
 746           tempDataInMemoryWithoutWAL += r.getDataInMemoryWithoutWAL();
 747           tempCpRequestsCount += r.getCpRequestsCount();
 748           tempFilteredReadRequestsCount += r.getFilteredReadRequestsCount();
 749           tempCheckAndMutateChecksFailed += r.getCheckAndMutateChecksFailed();
 750           tempCheckAndMutateChecksPassed += r.getCheckAndMutateChecksPassed();
 751           tempBlockedRequestsCount += r.getBlockedRequestsCount();
 752           List<? extends Store> storeList = r.getStores();
 753           tempNumStores += storeList.size();
 754           for (Store store : storeList) {
 755             tempNumStoreFiles += store.getStorefilesCount();
 756             tempMemstoreSize += store.getMemStoreSize().getDataSize();
 757             tempStoreFileSize += store.getStorefilesSize();
 758
 759             OptionalLong storeMaxStoreFileAge = store.getMaxStoreFileAge();
 760             if (storeMaxStoreFileAge.isPresent() &&
 761                 storeMaxStoreFileAge.getAsLong() > tempMaxStoreFileAge) {
 762               tempMaxStoreFileAge = storeMaxStoreFileAge.getAsLong();
 763             }
 764
 765             OptionalLong storeMinStoreFileAge = store.getMinStoreFileAge();
 766             if (storeMinStoreFileAge.isPresent() &&
 767                 storeMinStoreFileAge.getAsLong() < tempMinStoreFileAge) {
 768               tempMinStoreFileAge = storeMinStoreFileAge.getAsLong();
 769             }
 770
 771             long storeHFiles = store.getNumHFiles();
 772             numHFiles += storeHFiles;
 773             tempNumReferenceFiles += store.getNumReferenceFiles();
 774
 775             OptionalDouble storeAvgStoreFileAge = store.getAvgStoreFileAge();
 776             if (storeAvgStoreFileAge.isPresent()) {
 777               avgAgeNumerator =
 778                   (long) (avgAgeNumerator + storeAvgStoreFileAge.getAsDouble() * storeHFiles);
 779             }
 780
 781             tempStorefileIndexSize += store.getStorefilesRootLevelIndexSize();
 782             tempTotalStaticBloomSize += store.getTotalStaticBloomSize();
 783             tempTotalStaticIndexSize += store.getTotalStaticIndexSize();
 784             tempFlushedCellsCount += store.getFlushedCellsCount();
 785             tempCompactedCellsCount += store.getCompactedCellsCount();
 786             tempMajorCompactedCellsCount += store.getMajorCompactedCellsCount();
 787             tempFlushedCellsSize += store.getFlushedCellsSize();
 788             tempCompactedCellsSize += store.getCompactedCellsSize();
 789             tempMajorCompactedCellsSize += store.getMajorCompactedCellsSize();
 790             if (store instanceof HMobStore) {
 791               HMobStore mobStore = (HMobStore) store;
 792               tempCellsCountCompactedToMob += mobStore.getCellsCountCompactedToMob();
 793               tempCellsCountCompactedFromMob += mobStore.getCellsCountCompactedFromMob();
 794               tempCellsSizeCompactedToMob += mobStore.getCellsSizeCompactedToMob();
 795               tempCellsSizeCompactedFromMob += mobStore.getCellsSizeCompactedFromMob();
 796               tempMobFlushCount += mobStore.getMobFlushCount();
 797               tempMobFlushedCellsCount += mobStore.getMobFlushedCellsCount();
 798               tempMobFlushedCellsSize += mobStore.getMobFlushedCellsSize();
 799               tempMobScanCellsCount += mobStore.getMobScanCellsCount();
 800               tempMobScanCellsSize += mobStore.getMobScanCellsSize();
 801             }
 802           }
 803
 804           HDFSBlocksDistribution distro = r.getHDFSBlocksDistribution();
 805           hdfsBlocksDistribution.add(distro);
 806           if (r.getRegionInfo().getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
 807             hdfsBlocksDistributionSecondaryRegions.add(distro);
 808           }
 809           regionCount++;
 810         }
 811
 812         float localityIndex = hdfsBlocksDistribution.getBlockLocalityIndex(
 813             regionServer.getServerName().getHostname());
 814         tempPercentFileLocal = Double.isNaN(tempBlockedRequestsCount) ? 0 : (localityIndex * 100);
 815
 816         float localityIndexSecondaryRegions = hdfsBlocksDistributionSecondaryRegions
 817             .getBlockLocalityIndex(regionServer.getServerName().getHostname());
 818         tempPercentFileLocalSecondaryRegions = Double.
 819             isNaN(localityIndexSecondaryRegions) ? 0 : (localityIndexSecondaryRegions * 100);
 820
 821         // Compute the number of requests per second
 822         long currentTime = EnvironmentEdgeManager.currentTime();
 823
 824         // assume that it took PERIOD seconds to start the executor.
 825         // this is a guess but it's a pretty good one.
 826         if (lastRan == 0) {
 827           lastRan = currentTime - period;
 828         }
 829         // If we've time traveled keep the last requests per second.
 830         if ((currentTime - lastRan) > 0) {
 831           requestsPerSecond = (totalReadRequestsDelta + totalWriteRequestsDelta) /
 832               ((currentTime - lastRan) / 1000.0);
 833
 834           double readRequestsRatePerMilliSecond = (double)totalReadRequestsDelta / period;
 835           double writeRequestsRatePerMilliSecond = (double)totalWriteRequestsDelta / period;
 836
 837           readRequestsRatePerSecond = readRequestsRatePerMilliSecond * 1000.0;
 838           writeRequestsRatePerSecond = writeRequestsRatePerMilliSecond * 1000.0;
 839
 840           long intervalStoreFileSize = tempStoreFileSize - lastStoreFileSize;
 841           storeFileSizeGrowthRate = (double)intervalStoreFileSize * 1000.0 / period;
 842
 843           lastStoreFileSize = tempStoreFileSize;
 844         }
 845
 846         lastRan = currentTime;
 847
 848         final WALProvider provider = regionServer.getWalFactory().getWALProvider();
 849         final WALProvider metaProvider = regionServer.getWalFactory().getMetaWALProvider();
 850         numWALFiles = (provider == null ? 0 : provider.getNumLogFiles()) +
 851             (metaProvider == null ? 0 : metaProvider.getNumLogFiles());
 852         walFileSize = (provider == null ? 0 : provider.getLogFileSize()) +
 853           (metaProvider == null ? 0 : metaProvider.getLogFileSize());
 854         // Copy over computed values so that no thread sees half computed values.
 855         numStores = tempNumStores;
 856         numStoreFiles = tempNumStoreFiles;
 857         memstoreSize = tempMemstoreSize;
 858         storeFileSize = tempStoreFileSize;
 859         maxStoreFileAge = tempMaxStoreFileAge;
 860         if (regionCount > 0) {
 861           averageRegionSize = (memstoreSize + storeFileSize) / regionCount;
 862         }
 863         if (tempMinStoreFileAge != Long.MAX_VALUE) {
 864           minStoreFileAge = tempMinStoreFileAge;
 865         }
 866
 867         if (numHFiles != 0) {
 868           avgStoreFileAge = avgAgeNumerator / numHFiles;
 869         }
 870
 871         numReferenceFiles= tempNumReferenceFiles;
 872         readRequestsCount = tempReadRequestsCount;
 873         cpRequestsCount = tempCpRequestsCount;
 874         filteredReadRequestsCount = tempFilteredReadRequestsCount;
 875         writeRequestsCount = tempWriteRequestsCount;
 876         checkAndMutateChecksFailed = tempCheckAndMutateChecksFailed;
 877         checkAndMutateChecksPassed = tempCheckAndMutateChecksPassed;
 878         storefileIndexSize = tempStorefileIndexSize;
 879         totalStaticIndexSize = tempTotalStaticIndexSize;
 880         totalStaticBloomSize = tempTotalStaticBloomSize;
 881         numMutationsWithoutWAL = tempNumMutationsWithoutWAL;
 882         dataInMemoryWithoutWAL = tempDataInMemoryWithoutWAL;
 883         percentFileLocal = tempPercentFileLocal;
 884         percentFileLocalSecondaryRegions = tempPercentFileLocalSecondaryRegions;
 885         flushedCellsCount = tempFlushedCellsCount;
 886         compactedCellsCount = tempCompactedCellsCount;
 887         majorCompactedCellsCount = tempMajorCompactedCellsCount;
 888         flushedCellsSize = tempFlushedCellsSize;
 889         compactedCellsSize = tempCompactedCellsSize;
 890         majorCompactedCellsSize = tempMajorCompactedCellsSize;
 891         cellsCountCompactedToMob = tempCellsCountCompactedToMob;
 892         cellsCountCompactedFromMob = tempCellsCountCompactedFromMob;
 893         cellsSizeCompactedToMob = tempCellsSizeCompactedToMob;
 894         cellsSizeCompactedFromMob = tempCellsSizeCompactedFromMob;
 895         mobFlushCount = tempMobFlushCount;
 896         mobFlushedCellsCount = tempMobFlushedCellsCount;
 897         mobFlushedCellsSize = tempMobFlushedCellsSize;
 898         mobScanCellsCount = tempMobScanCellsCount;
 899         mobScanCellsSize = tempMobScanCellsSize;
 900         mobFileCacheAccessCount = mobFileCache != null ? mobFileCache.getAccessCount() : 0L;
 901         mobFileCacheMissCount = mobFileCache != null ? mobFileCache.getMissCount() : 0L;
 902         mobFileCacheHitRatio = mobFileCache != null ? mobFileCache.getHitRatio() : 0.0;
 903         if (Double.isNaN(mobFileCacheHitRatio)) {
 904           mobFileCacheHitRatio = 0.0;
 905         }
 906         mobFileCacheEvictedCount = mobFileCache != null ? mobFileCache.getEvictedFileCount() : 0L;
 907         mobFileCacheCount = mobFileCache != null ? mobFileCache.getCacheSize() : 0;
 908         blockedRequestsCount = tempBlockedRequestsCount;
 909       } catch (Throwable e) {
 910         LOG.warn("Caught exception! Will suppress and retry.", e);
 911       }
 912     }
 913   }
 914
 915   @Override
 916   public long getHedgedReadOps() {
 917     return this.dfsHedgedReadMetrics == null? 0: this.dfsHedgedReadMetrics.getHedgedReadOps();
 918   }
 919
 920   @Override
 921   public long getHedgedReadWins() {
 922     return this.dfsHedgedReadMetrics == null? 0: this.dfsHedgedReadMetrics.getHedgedReadWins();
 923   }
 924
 925   @Override
 926   public long getHedgedReadOpsInCurThread() {
 927     return this.dfsHedgedReadMetrics == null ? 0 : this.dfsHedgedReadMetrics.getHedgedReadOpsInCurThread();
 928   }
 929
 930   @Override
 931   public long getTotalBytesRead() {
 932     return FSDataInputStreamWrapper.getTotalBytesRead();
 933   }
 934
 935   @Override
 936   public long getLocalBytesRead() {
 937     return FSDataInputStreamWrapper.getLocalBytesRead();
 938   }
 939
 940   @Override
 941   public long getShortCircuitBytesRead() {
 942     return FSDataInputStreamWrapper.getShortCircuitBytesRead();
 943   }
 944
 945   @Override
 946   public long getZeroCopyBytesRead() {
 947     return FSDataInputStreamWrapper.getZeroCopyBytesRead();
 948   }
 949
 950   @Override
 951   public long getBlockedRequestsCount() {
 952     return blockedRequestsCount;
 953   }
 954
 955   @Override
 956   public long getAverageRegionSize() {
 957     return averageRegionSize;
 958   }
 959
 960   @Override
 961   public long getDataMissCount() {
 962     return this.cacheStats != null ? this.cacheStats.getDataMissCount() : 0L;
 963   }
 964
 965   @Override
 966   public long getLeafIndexMissCount() {
 967     return this.cacheStats != null ? this.cacheStats.getLeafIndexMissCount() : 0L;
 968   }
 969
 970   @Override
 971   public long getBloomChunkMissCount() {
 972     return this.cacheStats != null ? this.cacheStats.getBloomChunkMissCount() : 0L;
 973   }
 974
 975   @Override
 976   public long getMetaMissCount() {
 977     return this.cacheStats != null ? this.cacheStats.getMetaMissCount() : 0L;
 978   }
 979
 980   @Override
 981   public long getRootIndexMissCount() {
 982     return this.cacheStats != null ? this.cacheStats.getRootIndexMissCount() : 0L;
 983   }
 984
 985   @Override
 986   public long getIntermediateIndexMissCount() {
 987     return this.cacheStats != null ? this.cacheStats.getIntermediateIndexMissCount() : 0L;
 988   }
 989
 990   @Override
 991   public long getFileInfoMissCount() {
 992     return this.cacheStats != null ? this.cacheStats.getFileInfoMissCount() : 0L;
 993   }
 994
 995   @Override
 996   public long getGeneralBloomMetaMissCount() {
 997     return this.cacheStats != null ? this.cacheStats.getGeneralBloomMetaMissCount() : 0L;
 998   }
 999
1000   @Override
1001   public long getDeleteFamilyBloomMissCount() {
1002     return this.cacheStats != null ? this.cacheStats.getDeleteFamilyBloomMissCount() : 0L;
1003   }
1004
1005   @Override
1006   public long getTrailerMissCount() {
1007     return this.cacheStats != null ? this.cacheStats.getTrailerMissCount() : 0L;
1008   }
1009
1010   @Override
1011   public long getDataHitCount() {
1012     return this.cacheStats != null ? this.cacheStats.getDataHitCount() : 0L;
1013   }
1014
1015   @Override
1016   public long getLeafIndexHitCount() {
1017     return this.cacheStats != null ? this.cacheStats.getLeafIndexHitCount() : 0L;
1018   }
1019
1020   @Override
1021   public long getBloomChunkHitCount() {
1022     return this.cacheStats != null ? this.cacheStats.getBloomChunkHitCount() : 0L;
1023   }
1024
1025   @Override
1026   public long getMetaHitCount() {
1027     return this.cacheStats != null ? this.cacheStats.getMetaHitCount() : 0L;
1028   }
1029
1030   @Override
1031   public long getRootIndexHitCount() {
1032     return this.cacheStats != null ? this.cacheStats.getRootIndexHitCount() : 0L;
1033   }
1034
1035   @Override
1036   public long getIntermediateIndexHitCount() {
1037     return this.cacheStats != null ? this.cacheStats.getIntermediateIndexHitCount() : 0L;
1038   }
1039
1040   @Override
1041   public long getFileInfoHitCount() {
1042     return this.cacheStats != null ? this.cacheStats.getFileInfoHitCount() : 0L;
1043   }
1044
1045   @Override
1046   public long getGeneralBloomMetaHitCount() {
1047     return this.cacheStats != null ? this.cacheStats.getGeneralBloomMetaHitCount() : 0L;
1048   }
1049
1050   @Override
1051   public long getDeleteFamilyBloomHitCount() {
1052     return this.cacheStats != null ? this.cacheStats.getDeleteFamilyBloomHitCount() : 0L;
1053   }
1054
1055   @Override
1056   public long getTrailerHitCount() {
1057     return this.cacheStats != null ? this.cacheStats.getTrailerHitCount() : 0L;
1058   }
1059
1060   @Override
1061   public long getByteBuffAllocatorHeapAllocationBytes() {
1062     return ByteBuffAllocator.getHeapAllocationBytes(allocator, ByteBuffAllocator.HEAP);
1063   }
1064
1065   @Override
1066   public long getByteBuffAllocatorPoolAllocationBytes() {
1067     return this.allocator.getPoolAllocationBytes();
1068   }
1069
1070   @Override
1071   public double getByteBuffAllocatorHeapAllocRatio() {
1072     return ByteBuffAllocator.getHeapAllocationRatio(allocator, ByteBuffAllocator.HEAP);
1073   }
1074
1075   @Override
1076   public long getByteBuffAllocatorTotalBufferCount() {
1077     return this.allocator.getTotalBufferCount();
1078   }
1079
1080   @Override
1081   public long getByteBuffAllocatorUsedBufferCount() {
1082     return this.allocator.getUsedBufferCount();
1083   }
1084 }