hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetricsBuilder.java

   1 /**
   2  * Copyright The Apache Software Foundation
   3  * Licensed to the Apache Software Foundation (ASF) under one
   4  * or more contributor license agreements.  See the NOTICE file
   5  * distributed with this work for additional information
   6  * regarding copyright ownership.  The ASF licenses this file
   7  * to you under the Apache License, Version 2.0 (the
   8  * "License"); you may not use this file except in compliance
   9  * with the License.  You may obtain a copy of the License at
  10  * http://www.apache.org/licenses/LICENSE-2.0
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 package org.apache.hadoop.hbase;
  19
  20 import edu.umd.cs.findbugs.annotations.Nullable;
  21
  22 import java.util.ArrayList;
  23 import java.util.Collection;
  24 import java.util.Collections;
  25 import java.util.HashMap;
  26 import java.util.List;
  27 import java.util.Map;
  28 import java.util.Set;
  29 import java.util.TreeMap;
  30 import java.util.TreeSet;
  31 import java.util.stream.Collectors;
  32 import org.apache.hadoop.hbase.replication.ReplicationLoadSink;
  33 import org.apache.hadoop.hbase.replication.ReplicationLoadSource;
  34 import org.apache.hadoop.hbase.util.Bytes;
  35 import org.apache.hadoop.hbase.util.Strings;
  36 import org.apache.yetus.audience.InterfaceAudience;
  37
  38 import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
  39 import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
  40 import org.apache.hadoop.hbase.shaded.protobuf.generated.ClusterStatusProtos;
  41 import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
  42
  43 @InterfaceAudience.Private
  44 public final class ServerMetricsBuilder {
  45
  46   /**
  47    * @param sn the server name
  48    * @return a empty metrics
  49    */
  50   public static ServerMetrics of(ServerName sn) {
  51     return newBuilder(sn).build();
  52   }
  53
  54   public static ServerMetrics of(ServerName sn, int versionNumber, String version) {
  55     return newBuilder(sn).setVersionNumber(versionNumber).setVersion(version).build();
  56   }
  57
  58   public static ServerMetrics toServerMetrics(ClusterStatusProtos.LiveServerInfo serverInfo) {
  59     return toServerMetrics(ProtobufUtil.toServerName(serverInfo.getServer()), 0, "0.0.0",
  60       serverInfo.getServerLoad());
  61   }
  62
  63   public static ServerMetrics toServerMetrics(ServerName serverName,
  64       ClusterStatusProtos.ServerLoad serverLoadPB) {
  65     return toServerMetrics(serverName, 0, "0.0.0", serverLoadPB);
  66   }
  67
  68   public static ServerMetrics toServerMetrics(ServerName serverName, int versionNumber,
  69       String version, ClusterStatusProtos.ServerLoad serverLoadPB) {
  70     return ServerMetricsBuilder.newBuilder(serverName)
  71       .setRequestCountPerSecond(serverLoadPB.getNumberOfRequests())
  72       .setRequestCount(serverLoadPB.getTotalNumberOfRequests())
  73       .setInfoServerPort(serverLoadPB.getInfoServerPort())
  74       .setMaxHeapSize(new Size(serverLoadPB.getMaxHeapMB(), Size.Unit.MEGABYTE))
  75       .setUsedHeapSize(new Size(serverLoadPB.getUsedHeapMB(), Size.Unit.MEGABYTE))
  76       .setCoprocessorNames(serverLoadPB.getCoprocessorsList().stream()
  77         .map(HBaseProtos.Coprocessor::getName).collect(Collectors.toList()))
  78       .setRegionMetrics(serverLoadPB.getRegionLoadsList().stream()
  79         .map(RegionMetricsBuilder::toRegionMetrics).collect(Collectors.toList()))
  80         .setUserMetrics(serverLoadPB.getUserLoadsList().stream()
  81             .map(UserMetricsBuilder::toUserMetrics).collect(Collectors.toList()))
  82       .setReplicationLoadSources(serverLoadPB.getReplLoadSourceList().stream()
  83           .map(ProtobufUtil::toReplicationLoadSource).collect(Collectors.toList()))
  84       .setReplicationLoadSink(serverLoadPB.hasReplLoadSink()
  85         ? ProtobufUtil.toReplicationLoadSink(serverLoadPB.getReplLoadSink())
  86         : null)
  87       .setReportTimestamp(serverLoadPB.getReportEndTime())
  88       .setLastReportTimestamp(serverLoadPB.getReportStartTime()).setVersionNumber(versionNumber)
  89       .setVersion(version).build();
  90   }
  91
  92   public static List<HBaseProtos.Coprocessor> toCoprocessor(Collection<String> names) {
  93     return names.stream()
  94         .map(n -> HBaseProtos.Coprocessor.newBuilder().setName(n).build())
  95         .collect(Collectors.toList());
  96   }
  97
  98   public static ClusterStatusProtos.ServerLoad toServerLoad(ServerMetrics metrics) {
  99     ClusterStatusProtos.ServerLoad.Builder builder = ClusterStatusProtos.ServerLoad.newBuilder()
 100         .setNumberOfRequests(metrics.getRequestCountPerSecond())
 101         .setTotalNumberOfRequests(metrics.getRequestCount())
 102         .setInfoServerPort(metrics.getInfoServerPort())
 103         .setMaxHeapMB((int) metrics.getMaxHeapSize().get(Size.Unit.MEGABYTE))
 104         .setUsedHeapMB((int) metrics.getUsedHeapSize().get(Size.Unit.MEGABYTE))
 105         .addAllCoprocessors(toCoprocessor(metrics.getCoprocessorNames())).addAllRegionLoads(
 106             metrics.getRegionMetrics().values().stream().map(RegionMetricsBuilder::toRegionLoad)
 107                 .collect(Collectors.toList())).addAllUserLoads(
 108             metrics.getUserMetrics().values().stream().map(UserMetricsBuilder::toUserMetrics)
 109                 .collect(Collectors.toList())).addAllReplLoadSource(
 110             metrics.getReplicationLoadSourceList().stream()
 111                 .map(ProtobufUtil::toReplicationLoadSource).collect(Collectors.toList()))
 112         .setReportStartTime(metrics.getLastReportTimestamp())
 113         .setReportEndTime(metrics.getReportTimestamp());
 114     if (metrics.getReplicationLoadSink() != null) {
 115       builder.setReplLoadSink(ProtobufUtil.toReplicationLoadSink(metrics.getReplicationLoadSink()));
 116     }
 117
 118     return builder.build();
 119   }
 120
 121   public static ServerMetricsBuilder newBuilder(ServerName sn) {
 122     return new ServerMetricsBuilder(sn);
 123   }
 124
 125   private final ServerName serverName;
 126   private int versionNumber;
 127   private String version = "0.0.0";
 128   private long requestCountPerSecond;
 129   private long requestCount;
 130   private Size usedHeapSize = Size.ZERO;
 131   private Size maxHeapSize = Size.ZERO;
 132   private int infoServerPort;
 133   private List<ReplicationLoadSource> sources = Collections.emptyList();
 134   @Nullable
 135   private ReplicationLoadSink sink = null;
 136   private final Map<byte[], RegionMetrics> regionStatus = new TreeMap<>(Bytes.BYTES_COMPARATOR);
 137   private final Map<byte[], UserMetrics> userMetrics = new TreeMap<>(Bytes.BYTES_COMPARATOR);
 138   private final Set<String> coprocessorNames = new TreeSet<>();
 139   private long reportTimestamp = System.currentTimeMillis();
 140   private long lastReportTimestamp = 0;
 141   private ServerMetricsBuilder(ServerName serverName) {
 142     this.serverName = serverName;
 143   }
 144
 145   public ServerMetricsBuilder setVersionNumber(int versionNumber) {
 146     this.versionNumber = versionNumber;
 147     return this;
 148   }
 149
 150   public ServerMetricsBuilder setVersion(String version) {
 151     this.version = version;
 152     return this;
 153   }
 154
 155   public ServerMetricsBuilder setRequestCountPerSecond(long value) {
 156     this.requestCountPerSecond = value;
 157     return this;
 158   }
 159
 160   public ServerMetricsBuilder setRequestCount(long value) {
 161     this.requestCount = value;
 162     return this;
 163   }
 164
 165   public ServerMetricsBuilder setUsedHeapSize(Size value) {
 166     this.usedHeapSize = value;
 167     return this;
 168   }
 169
 170   public ServerMetricsBuilder setMaxHeapSize(Size value) {
 171     this.maxHeapSize = value;
 172     return this;
 173   }
 174
 175   public ServerMetricsBuilder setInfoServerPort(int value) {
 176     this.infoServerPort = value;
 177     return this;
 178   }
 179
 180   public ServerMetricsBuilder setReplicationLoadSources(List<ReplicationLoadSource> value) {
 181     this.sources = value;
 182     return this;
 183   }
 184
 185   public ServerMetricsBuilder setReplicationLoadSink(ReplicationLoadSink value) {
 186     this.sink = value;
 187     return this;
 188   }
 189
 190   public ServerMetricsBuilder setRegionMetrics(List<RegionMetrics> value) {
 191     value.forEach(v -> this.regionStatus.put(v.getRegionName(), v));
 192     return this;
 193   }
 194
 195   public ServerMetricsBuilder setUserMetrics(List<UserMetrics> value) {
 196     value.forEach(v -> this.userMetrics.put(v.getUserName(), v));
 197     return this;
 198   }
 199
 200   public ServerMetricsBuilder setCoprocessorNames(List<String> value) {
 201     coprocessorNames.addAll(value);
 202     return this;
 203   }
 204
 205   public ServerMetricsBuilder setReportTimestamp(long value) {
 206     this.reportTimestamp = value;
 207     return this;
 208   }
 209
 210   public ServerMetricsBuilder setLastReportTimestamp(long value) {
 211     this.lastReportTimestamp = value;
 212     return this;
 213   }
 214
 215   public ServerMetrics build() {
 216     return new ServerMetricsImpl(
 217         serverName,
 218         versionNumber,
 219         version,
 220         requestCountPerSecond,
 221         requestCount,
 222         usedHeapSize,
 223         maxHeapSize,
 224         infoServerPort,
 225         sources,
 226         sink,
 227         regionStatus,
 228         coprocessorNames,
 229         reportTimestamp,
 230         lastReportTimestamp,
 231         userMetrics);
 232   }
 233
 234   private static class ServerMetricsImpl implements ServerMetrics {
 235     private final ServerName serverName;
 236     private final int versionNumber;
 237     private final String version;
 238     private final long requestCountPerSecond;
 239     private final long requestCount;
 240     private final Size usedHeapSize;
 241     private final Size maxHeapSize;
 242     private final int infoServerPort;
 243     private final List<ReplicationLoadSource> sources;
 244     @Nullable
 245     private final ReplicationLoadSink sink;
 246     private final Map<byte[], RegionMetrics> regionStatus;
 247     private final Set<String> coprocessorNames;
 248     private final long reportTimestamp;
 249     private final long lastReportTimestamp;
 250     private final Map<byte[], UserMetrics> userMetrics;
 251
 252     ServerMetricsImpl(ServerName serverName, int versionNumber, String version,
 253         long requestCountPerSecond, long requestCount, Size usedHeapSize, Size maxHeapSize,
 254         int infoServerPort, List<ReplicationLoadSource> sources, ReplicationLoadSink sink,
 255         Map<byte[], RegionMetrics> regionStatus, Set<String> coprocessorNames, long reportTimestamp,
 256         long lastReportTimestamp, Map<byte[], UserMetrics> userMetrics) {
 257       this.serverName = Preconditions.checkNotNull(serverName);
 258       this.versionNumber = versionNumber;
 259       this.version = version;
 260       this.requestCountPerSecond = requestCountPerSecond;
 261       this.requestCount = requestCount;
 262       this.usedHeapSize = Preconditions.checkNotNull(usedHeapSize);
 263       this.maxHeapSize = Preconditions.checkNotNull(maxHeapSize);
 264       this.infoServerPort = infoServerPort;
 265       this.sources = Preconditions.checkNotNull(sources);
 266       this.sink = sink;
 267       this.regionStatus = Preconditions.checkNotNull(regionStatus);
 268       this.userMetrics = Preconditions.checkNotNull(userMetrics);
 269       this.coprocessorNames =Preconditions.checkNotNull(coprocessorNames);
 270       this.reportTimestamp = reportTimestamp;
 271       this.lastReportTimestamp = lastReportTimestamp;
 272     }
 273
 274     @Override
 275     public ServerName getServerName() {
 276       return serverName;
 277     }
 278
 279     @Override
 280     public int getVersionNumber() {
 281       return versionNumber;
 282     }
 283
 284     public String getVersion() {
 285       return version;
 286     }
 287
 288     @Override
 289     public long getRequestCountPerSecond() {
 290       return requestCountPerSecond;
 291     }
 292
 293     @Override
 294     public long getRequestCount() {
 295       return requestCount;
 296     }
 297
 298     @Override
 299     public Size getUsedHeapSize() {
 300       return usedHeapSize;
 301     }
 302
 303     @Override
 304     public Size getMaxHeapSize() {
 305       return maxHeapSize;
 306     }
 307
 308     @Override
 309     public int getInfoServerPort() {
 310       return infoServerPort;
 311     }
 312
 313     @Override
 314     public List<ReplicationLoadSource> getReplicationLoadSourceList() {
 315       return Collections.unmodifiableList(sources);
 316     }
 317
 318     @Override
 319     public Map<String, List<ReplicationLoadSource>> getReplicationLoadSourceMap(){
 320       Map<String,List<ReplicationLoadSource>> sourcesMap = new HashMap<>();
 321       for(ReplicationLoadSource loadSource : sources){
 322         sourcesMap.computeIfAbsent(loadSource.getPeerID(),
 323           peerId -> new ArrayList()).add(loadSource);
 324       }
 325       return sourcesMap;
 326     }
 327
 328     @Override
 329     public ReplicationLoadSink getReplicationLoadSink() {
 330       return sink;
 331     }
 332
 333     @Override
 334     public Map<byte[], RegionMetrics> getRegionMetrics() {
 335       return Collections.unmodifiableMap(regionStatus);
 336     }
 337
 338     @Override
 339     public Map<byte[], UserMetrics> getUserMetrics() {
 340       return Collections.unmodifiableMap(userMetrics);
 341     }
 342
 343     @Override
 344     public Set<String> getCoprocessorNames() {
 345       return Collections.unmodifiableSet(coprocessorNames);
 346     }
 347
 348     @Override
 349     public long getReportTimestamp() {
 350       return reportTimestamp;
 351     }
 352
 353     @Override
 354     public long getLastReportTimestamp() {
 355       return lastReportTimestamp;
 356     }
 357
 358     @Override
 359     public String toString() {
 360       int storeCount = 0;
 361       int storeFileCount = 0;
 362       int storeRefCount = 0;
 363       int maxCompactedStoreFileRefCount = 0;
 364       long uncompressedStoreFileSizeMB = 0;
 365       long storeFileSizeMB = 0;
 366       long memStoreSizeMB = 0;
 367       long storefileIndexSizeKB = 0;
 368       long rootLevelIndexSizeKB = 0;
 369       long readRequestsCount = 0;
 370       long writeRequestsCount = 0;
 371       long filteredReadRequestsCount = 0;
 372       long bloomFilterSizeMB = 0;
 373       long compactingCellCount = 0;
 374       long compactedCellCount = 0;
 375       for (RegionMetrics r : getRegionMetrics().values()) {
 376         storeCount += r.getStoreCount();
 377         storeFileCount += r.getStoreFileCount();
 378         storeRefCount += r.getStoreRefCount();
 379         int currentMaxCompactedStoreFileRefCount = r.getMaxCompactedStoreFileRefCount();
 380         maxCompactedStoreFileRefCount = Math.max(maxCompactedStoreFileRefCount,
 381           currentMaxCompactedStoreFileRefCount);
 382         uncompressedStoreFileSizeMB += r.getUncompressedStoreFileSize().get(Size.Unit.MEGABYTE);
 383         storeFileSizeMB += r.getStoreFileSize().get(Size.Unit.MEGABYTE);
 384         memStoreSizeMB += r.getMemStoreSize().get(Size.Unit.MEGABYTE);
 385         storefileIndexSizeKB += r.getStoreFileUncompressedDataIndexSize().get(Size.Unit.KILOBYTE);
 386         readRequestsCount += r.getReadRequestCount();
 387         writeRequestsCount += r.getWriteRequestCount();
 388         filteredReadRequestsCount += r.getFilteredReadRequestCount();
 389         rootLevelIndexSizeKB += r.getStoreFileRootLevelIndexSize().get(Size.Unit.KILOBYTE);
 390         bloomFilterSizeMB += r.getBloomFilterSize().get(Size.Unit.MEGABYTE);
 391         compactedCellCount += r.getCompactedCellCount();
 392         compactingCellCount += r.getCompactingCellCount();
 393       }
 394       StringBuilder sb = Strings.appendKeyValue(new StringBuilder(), "requestsPerSecond",
 395             Double.valueOf(getRequestCountPerSecond()));
 396       Strings.appendKeyValue(sb, "numberOfOnlineRegions",
 397           Integer.valueOf(getRegionMetrics().size()));
 398       Strings.appendKeyValue(sb, "usedHeapMB", getUsedHeapSize());
 399       Strings.appendKeyValue(sb, "maxHeapMB", getMaxHeapSize());
 400       Strings.appendKeyValue(sb, "numberOfStores", storeCount);
 401       Strings.appendKeyValue(sb, "numberOfStorefiles", storeFileCount);
 402       Strings.appendKeyValue(sb, "storeRefCount", storeRefCount);
 403       Strings.appendKeyValue(sb, "maxCompactedStoreFileRefCount",
 404         maxCompactedStoreFileRefCount);
 405       Strings.appendKeyValue(sb, "storefileUncompressedSizeMB", uncompressedStoreFileSizeMB);
 406       Strings.appendKeyValue(sb, "storefileSizeMB", storeFileSizeMB);
 407       if (uncompressedStoreFileSizeMB != 0) {
 408         Strings.appendKeyValue(sb, "compressionRatio", String.format("%.4f",
 409             (float) storeFileSizeMB / (float) uncompressedStoreFileSizeMB));
 410       }
 411       Strings.appendKeyValue(sb, "memstoreSizeMB", memStoreSizeMB);
 412       Strings.appendKeyValue(sb, "readRequestsCount", readRequestsCount);
 413       Strings.appendKeyValue(sb, "filteredReadRequestsCount", filteredReadRequestsCount);
 414       Strings.appendKeyValue(sb, "writeRequestsCount", writeRequestsCount);
 415       Strings.appendKeyValue(sb, "rootIndexSizeKB", rootLevelIndexSizeKB);
 416       Strings.appendKeyValue(sb, "totalStaticIndexSizeKB", storefileIndexSizeKB);
 417       Strings.appendKeyValue(sb, "totalStaticBloomSizeKB", bloomFilterSizeMB);
 418       Strings.appendKeyValue(sb, "totalCompactingKVs", compactingCellCount);
 419       Strings.appendKeyValue(sb, "currentCompactedKVs", compactedCellCount);
 420       float compactionProgressPct = Float.NaN;
 421       if (compactingCellCount > 0) {
 422         compactionProgressPct =
 423             Float.valueOf((float) compactedCellCount / compactingCellCount);
 424       }
 425       Strings.appendKeyValue(sb, "compactionProgressPct", compactionProgressPct);
 426       Strings.appendKeyValue(sb, "coprocessors", getCoprocessorNames());
 427       return sb.toString();
 428     }
 429   }
 430 }