hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java

   1 /*
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase;
  19
  20 import static org.junit.Assert.assertArrayEquals;
  21 import static org.junit.Assert.assertEquals;
  22 import static org.junit.Assert.assertFalse;
  23 import static org.junit.Assert.assertNotEquals;
  24 import static org.junit.Assert.assertNotNull;
  25 import static org.junit.Assert.assertNull;
  26 import static org.junit.Assert.assertTrue;
  27 import static org.mockito.ArgumentMatchers.anyObject;
  28 import static org.mockito.Mockito.doReturn;
  29 import static org.mockito.Mockito.mock;
  30 import static org.mockito.Mockito.reset;
  31 import static org.mockito.Mockito.times;
  32 import static org.mockito.Mockito.verify;
  33
  34 import java.io.IOException;
  35 import java.util.ArrayList;
  36 import java.util.HashMap;
  37 import java.util.List;
  38 import java.util.Map;
  39 import java.util.Random;
  40 import java.util.concurrent.TimeUnit;
  41
  42 import org.apache.hadoop.conf.Configuration;
  43 import org.apache.hadoop.hbase.client.Admin;
  44 import org.apache.hadoop.hbase.client.Connection;
  45 import org.apache.hadoop.hbase.client.ConnectionFactory;
  46 import org.apache.hadoop.hbase.client.Get;
  47 import org.apache.hadoop.hbase.client.Put;
  48 import org.apache.hadoop.hbase.client.RegionInfo;
  49 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
  50 import org.apache.hadoop.hbase.client.RegionLocator;
  51 import org.apache.hadoop.hbase.client.Result;
  52 import org.apache.hadoop.hbase.client.Table;
  53 import org.apache.hadoop.hbase.ipc.CallRunner;
  54 import org.apache.hadoop.hbase.ipc.DelegatingRpcScheduler;
  55 import org.apache.hadoop.hbase.ipc.PriorityFunction;
  56 import org.apache.hadoop.hbase.ipc.RpcScheduler;
  57 import org.apache.hadoop.hbase.master.HMaster;
  58 import org.apache.hadoop.hbase.regionserver.HRegion;
  59 import org.apache.hadoop.hbase.regionserver.HRegionServer;
  60 import org.apache.hadoop.hbase.regionserver.RSRpcServices;
  61 import org.apache.hadoop.hbase.regionserver.SimpleRpcSchedulerFactory;
  62 import org.apache.hadoop.hbase.testclassification.MediumTests;
  63 import org.apache.hadoop.hbase.testclassification.MiscTests;
  64 import org.apache.hadoop.hbase.util.Bytes;
  65 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
  66 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
  67 import org.apache.hadoop.hbase.util.Pair;
  68 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
  69 import org.junit.AfterClass;
  70 import org.junit.BeforeClass;
  71 import org.junit.ClassRule;
  72 import org.junit.Rule;
  73 import org.junit.Test;
  74 import org.junit.experimental.categories.Category;
  75 import org.junit.rules.TestName;
  76 import org.slf4j.Logger;
  77 import org.slf4j.LoggerFactory;
  78
  79 import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
  80 import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
  81
  82 /**
  83  * Test {@link org.apache.hadoop.hbase.MetaTableAccessor}.
  84  */
  85 @Category({MiscTests.class, MediumTests.class})
  86 @SuppressWarnings("deprecation")
  87 public class TestMetaTableAccessor {
  88   @ClassRule
  89   public static final HBaseClassTestRule CLASS_RULE =
  90       HBaseClassTestRule.forClass(TestMetaTableAccessor.class);
  91
  92   private static final Logger LOG = LoggerFactory.getLogger(TestMetaTableAccessor.class);
  93   private static final  HBaseTestingUtility UTIL = new HBaseTestingUtility();
  94   private static Connection connection;
  95   private Random random = new Random();
  96
  97   @Rule
  98   public TestName name = new TestName();
  99
 100   @BeforeClass public static void beforeClass() throws Exception {
 101     UTIL.startMiniCluster(3);
 102
 103     Configuration c = new Configuration(UTIL.getConfiguration());
 104     // Tests to 4 retries every 5 seconds. Make it try every 1 second so more
 105     // responsive.  1 second is default as is ten retries.
 106     c.setLong("hbase.client.pause", 1000);
 107     c.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 10);
 108     connection = ConnectionFactory.createConnection(c);
 109   }
 110
 111   @AfterClass public static void afterClass() throws Exception {
 112     connection.close();
 113     UTIL.shutdownMiniCluster();
 114   }
 115
 116   /**
 117    * Test for HBASE-23044.
 118    */
 119   @Test
 120   public void testGetMergeRegions() throws Exception {
 121     TableName tn = TableName.valueOf(this.name.getMethodName());
 122     UTIL.createMultiRegionTable(tn, Bytes.toBytes("CF"), 4);
 123     UTIL.waitTableAvailable(tn);
 124     try (Admin admin = UTIL.getAdmin()) {
 125       List<RegionInfo> regions = admin.getRegions(tn);
 126       assertEquals(4, regions.size());
 127       admin.mergeRegionsAsync(regions.get(0).getRegionName(), regions.get(1).getRegionName(), false)
 128           .get(60, TimeUnit.SECONDS);
 129       admin.mergeRegionsAsync(regions.get(2).getRegionName(), regions.get(3).getRegionName(), false)
 130           .get(60, TimeUnit.SECONDS);
 131
 132       List<RegionInfo> mergedRegions = admin.getRegions(tn);
 133       assertEquals(2, mergedRegions.size());
 134       RegionInfo mergedRegion0 = mergedRegions.get(0);
 135       RegionInfo mergedRegion1 = mergedRegions.get(1);
 136
 137       List<RegionInfo> mergeParents =
 138           MetaTableAccessor.getMergeRegions(connection, mergedRegion0.getRegionName());
 139       assertTrue(mergeParents.contains(regions.get(0)));
 140       assertTrue(mergeParents.contains(regions.get(1)));
 141       mergeParents = MetaTableAccessor.getMergeRegions(connection, mergedRegion1.getRegionName());
 142       assertTrue(mergeParents.contains(regions.get(2)));
 143       assertTrue(mergeParents.contains(regions.get(3)));
 144
 145       // Delete merge qualifiers for mergedRegion0, then cannot getMergeRegions again
 146       MetaTableAccessor.deleteMergeQualifiers(connection, mergedRegion0);
 147       mergeParents = MetaTableAccessor.getMergeRegions(connection, mergedRegion0.getRegionName());
 148       assertNull(mergeParents);
 149
 150       mergeParents = MetaTableAccessor.getMergeRegions(connection, mergedRegion1.getRegionName());
 151       assertTrue(mergeParents.contains(regions.get(2)));
 152       assertTrue(mergeParents.contains(regions.get(3)));
 153     }
 154     UTIL.deleteTable(tn);
 155   }
 156
 157   @Test
 158   public void testAddMergeRegions() throws IOException {
 159     TableName tn = TableName.valueOf(this.name.getMethodName());
 160     Put put = new Put(Bytes.toBytes(this.name.getMethodName()));
 161     List<RegionInfo> ris = new ArrayList<>();
 162     int limit = 10;
 163     byte [] previous = HConstants.EMPTY_START_ROW;
 164     for (int i = 0; i < limit; i++) {
 165       RegionInfo ri = RegionInfoBuilder.newBuilder(tn).
 166           setStartKey(previous).setEndKey(Bytes.toBytes(i)).build();
 167       ris.add(ri);
 168     }
 169     put = MetaTableAccessor.addMergeRegions(put, ris);
 170     List<Cell> cells = put.getFamilyCellMap().get(HConstants.CATALOG_FAMILY);
 171     String previousQualifier = null;
 172     assertEquals(limit, cells.size());
 173     for (Cell cell: cells) {
 174       LOG.info(cell.toString());
 175       String qualifier = Bytes.toString(cell.getQualifierArray());
 176       assertTrue(qualifier.startsWith(HConstants.MERGE_QUALIFIER_PREFIX_STR));
 177       assertNotEquals(qualifier, previousQualifier);
 178       previousQualifier = qualifier;
 179     }
 180   }
 181
 182   @Test
 183   public void testIsMetaWhenAllHealthy() throws InterruptedException {
 184     HMaster m = UTIL.getMiniHBaseCluster().getMaster();
 185     assertTrue(m.waitForMetaOnline());
 186   }
 187
 188   @Test
 189   public void testIsMetaWhenMetaGoesOffline() throws InterruptedException {
 190     HMaster m = UTIL.getMiniHBaseCluster().getMaster();
 191     int index = UTIL.getMiniHBaseCluster().getServerWithMeta();
 192     HRegionServer rsWithMeta = UTIL.getMiniHBaseCluster().getRegionServer(index);
 193     rsWithMeta.abort("TESTING");
 194     assertTrue(m.waitForMetaOnline());
 195   }
 196
 197   /**
 198    * Does {@link MetaTableAccessor#getRegion(Connection, byte[])} and a write
 199    * against hbase:meta while its hosted server is restarted to prove our retrying
 200    * works.
 201    */
 202   @Test public void testRetrying()
 203   throws IOException, InterruptedException {
 204     final TableName tableName = TableName.valueOf(name.getMethodName());
 205     LOG.info("Started " + tableName);
 206     Table t = UTIL.createMultiRegionTable(tableName, HConstants.CATALOG_FAMILY);
 207     int regionCount = -1;
 208     try (RegionLocator r = UTIL.getConnection().getRegionLocator(tableName)) {
 209       regionCount = r.getStartKeys().length;
 210     }
 211     // Test it works getting a region from just made user table.
 212     final List<RegionInfo> regions =
 213       testGettingTableRegions(connection, tableName, regionCount);
 214     MetaTask reader = new MetaTask(connection, "reader") {
 215       @Override
 216       void metaTask() throws Throwable {
 217         testGetRegion(connection, regions.get(0));
 218         LOG.info("Read " + regions.get(0).getEncodedName());
 219       }
 220     };
 221     MetaTask writer = new MetaTask(connection, "writer") {
 222       @Override
 223       void metaTask() throws Throwable {
 224         MetaTableAccessor.addRegionToMeta(connection, regions.get(0));
 225         LOG.info("Wrote " + regions.get(0).getEncodedName());
 226       }
 227     };
 228     reader.start();
 229     writer.start();
 230
 231     // We're gonna check how it takes. If it takes too long, we will consider
 232     //  it as a fail. We can't put that in the @Test tag as we want to close
 233     //  the threads nicely
 234     final long timeOut = 180000;
 235     long startTime = System.currentTimeMillis();
 236
 237     try {
 238       // Make sure reader and writer are working.
 239       assertTrue(reader.isProgressing());
 240       assertTrue(writer.isProgressing());
 241
 242       // Kill server hosting meta -- twice  . See if our reader/writer ride over the
 243       // meta moves.  They'll need to retry.
 244       for (int i = 0; i < 2; i++) {
 245         LOG.info("Restart=" + i);
 246         UTIL.ensureSomeRegionServersAvailable(2);
 247         int index = -1;
 248         do {
 249           index = UTIL.getMiniHBaseCluster().getServerWithMeta();
 250         } while (index == -1 &&
 251           startTime + timeOut < System.currentTimeMillis());
 252
 253         if (index != -1){
 254           UTIL.getMiniHBaseCluster().abortRegionServer(index);
 255           UTIL.getMiniHBaseCluster().waitOnRegionServer(index);
 256         }
 257       }
 258
 259       assertTrue("reader: " + reader.toString(), reader.isProgressing());
 260       assertTrue("writer: " + writer.toString(), writer.isProgressing());
 261     } catch (IOException e) {
 262       throw e;
 263     } finally {
 264       reader.stop = true;
 265       writer.stop = true;
 266       reader.join();
 267       writer.join();
 268       t.close();
 269     }
 270     long exeTime = System.currentTimeMillis() - startTime;
 271     assertTrue("Timeout: test took " + exeTime / 1000 + " sec", exeTime < timeOut);
 272   }
 273
 274   /**
 275    * Thread that runs a MetaTableAccessor task until asked stop.
 276    */
 277   abstract static class MetaTask extends Thread {
 278     boolean stop = false;
 279     int count = 0;
 280     Throwable t = null;
 281     final Connection connection;
 282
 283     MetaTask(final Connection connection, final String name) {
 284       super(name);
 285       this.connection = connection;
 286     }
 287
 288     @Override
 289     public void run() {
 290       try {
 291         while(!this.stop) {
 292           LOG.info("Before " + this.getName()+ ", count=" + this.count);
 293           metaTask();
 294           this.count += 1;
 295           LOG.info("After " + this.getName() + ", count=" + this.count);
 296           Thread.sleep(100);
 297         }
 298       } catch (Throwable t) {
 299         LOG.info(this.getName() + " failed", t);
 300         this.t = t;
 301       }
 302     }
 303
 304     boolean isProgressing() throws InterruptedException {
 305       int currentCount = this.count;
 306       while(currentCount == this.count) {
 307         if (!isAlive()) return false;
 308         if (this.t != null) return false;
 309         Thread.sleep(10);
 310       }
 311       return true;
 312     }
 313
 314     @Override
 315     public String toString() {
 316       return "count=" + this.count + ", t=" +
 317         (this.t == null? "null": this.t.toString());
 318     }
 319
 320     abstract void metaTask() throws Throwable;
 321   }
 322
 323   @Test
 324   public void testGetRegionsFromMetaTable() throws IOException, InterruptedException {
 325     List<RegionInfo> regions = MetaTableLocator.getMetaRegions(UTIL.getZooKeeperWatcher());
 326     assertTrue(regions.size() >= 1);
 327     assertTrue(
 328         MetaTableLocator.getMetaRegionsAndLocations(UTIL.getZooKeeperWatcher()).size() >= 1);
 329   }
 330
 331   @Test public void testTableExists() throws IOException {
 332     final TableName tableName = TableName.valueOf(name.getMethodName());
 333     assertFalse(MetaTableAccessor.tableExists(connection, tableName));
 334     UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
 335     assertTrue(MetaTableAccessor.tableExists(connection, tableName));
 336     Admin admin = UTIL.getAdmin();
 337     admin.disableTable(tableName);
 338     admin.deleteTable(tableName);
 339     assertFalse(MetaTableAccessor.tableExists(connection, tableName));
 340     assertTrue(MetaTableAccessor.tableExists(connection,
 341         TableName.META_TABLE_NAME));
 342     UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
 343     assertTrue(MetaTableAccessor.tableExists(connection, tableName));
 344     admin.disableTable(tableName);
 345     admin.deleteTable(tableName);
 346     assertFalse(MetaTableAccessor.tableExists(connection, tableName));
 347   }
 348
 349   @Test public void testGetRegion() throws IOException, InterruptedException {
 350     final String name = this.name.getMethodName();
 351     LOG.info("Started " + name);
 352     // Test get on non-existent region.
 353     Pair<RegionInfo, ServerName> pair =
 354       MetaTableAccessor.getRegion(connection, Bytes.toBytes("nonexistent-region"));
 355     assertNull(pair);
 356     LOG.info("Finished " + name);
 357   }
 358
 359   // Test for the optimization made in HBASE-3650
 360   @Test public void testScanMetaForTable()
 361   throws IOException, InterruptedException {
 362     final TableName tableName = TableName.valueOf(name.getMethodName());
 363     LOG.info("Started " + tableName);
 364
 365     /** Create 2 tables
 366      - testScanMetaForTable
 367      - testScanMetaForTablf
 368     **/
 369
 370     UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
 371     // name that is +1 greater than the first one (e+1=f)
 372     TableName greaterName =
 373         TableName.valueOf("testScanMetaForTablf");
 374     UTIL.createTable(greaterName, HConstants.CATALOG_FAMILY);
 375
 376     // Now make sure we only get the regions from 1 of the tables at a time
 377
 378     assertEquals(1, MetaTableAccessor.getTableRegions(connection, tableName).size());
 379     assertEquals(1, MetaTableAccessor.getTableRegions(connection, greaterName).size());
 380   }
 381
 382   private static List<RegionInfo> testGettingTableRegions(final Connection connection,
 383       final TableName name, final int regionCount)
 384   throws IOException, InterruptedException {
 385     List<RegionInfo> regions = MetaTableAccessor.getTableRegions(connection, name);
 386     assertEquals(regionCount, regions.size());
 387     Pair<RegionInfo, ServerName> pair =
 388       MetaTableAccessor.getRegion(connection, regions.get(0).getRegionName());
 389     assertEquals(regions.get(0).getEncodedName(),
 390       pair.getFirst().getEncodedName());
 391     return regions;
 392   }
 393
 394   private static void testGetRegion(final Connection connection,
 395       final RegionInfo region)
 396   throws IOException, InterruptedException {
 397     Pair<RegionInfo, ServerName> pair =
 398       MetaTableAccessor.getRegion(connection, region.getRegionName());
 399     assertEquals(region.getEncodedName(),
 400       pair.getFirst().getEncodedName());
 401   }
 402
 403   @Test
 404   public void testParseReplicaIdFromServerColumn() {
 405     String column1 = HConstants.SERVER_QUALIFIER_STR;
 406     assertEquals(0,
 407         MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column1)));
 408     String column2 = column1 + MetaTableAccessor.META_REPLICA_ID_DELIMITER;
 409     assertEquals(-1,
 410         MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column2)));
 411     String column3 = column2 + "00";
 412     assertEquals(-1,
 413         MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column3)));
 414     String column4 = column3 + "2A";
 415     assertEquals(42,
 416         MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column4)));
 417     String column5 = column4 + "2A";
 418     assertEquals(-1,
 419         MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column5)));
 420     String column6 = HConstants.STARTCODE_QUALIFIER_STR;
 421     assertEquals(-1,
 422         MetaTableAccessor.parseReplicaIdFromServerColumn(Bytes.toBytes(column6)));
 423   }
 424
 425   @Test
 426   public void testMetaReaderGetColumnMethods() {
 427     assertArrayEquals(HConstants.SERVER_QUALIFIER, MetaTableAccessor.getServerColumn(0));
 428     assertArrayEquals(Bytes.toBytes(HConstants.SERVER_QUALIFIER_STR
 429       + MetaTableAccessor.META_REPLICA_ID_DELIMITER + "002A"),
 430       MetaTableAccessor.getServerColumn(42));
 431
 432     assertArrayEquals(HConstants.STARTCODE_QUALIFIER,
 433       MetaTableAccessor.getStartCodeColumn(0));
 434     assertArrayEquals(Bytes.toBytes(HConstants.STARTCODE_QUALIFIER_STR
 435       + MetaTableAccessor.META_REPLICA_ID_DELIMITER + "002A"),
 436       MetaTableAccessor.getStartCodeColumn(42));
 437
 438     assertArrayEquals(HConstants.SEQNUM_QUALIFIER,
 439       MetaTableAccessor.getSeqNumColumn(0));
 440     assertArrayEquals(Bytes.toBytes(HConstants.SEQNUM_QUALIFIER_STR
 441       + MetaTableAccessor.META_REPLICA_ID_DELIMITER + "002A"),
 442       MetaTableAccessor.getSeqNumColumn(42));
 443   }
 444
 445   @Test
 446   public void testMetaLocationsForRegionReplicas() throws IOException {
 447     ServerName serverName0 = ServerName.valueOf("foo", 60010, random.nextLong());
 448     ServerName serverName1 = ServerName.valueOf("bar", 60010, random.nextLong());
 449     ServerName serverName100 = ServerName.valueOf("baz", 60010, random.nextLong());
 450
 451     long regionId = System.currentTimeMillis();
 452     RegionInfo primary = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 453         .setStartKey(HConstants.EMPTY_START_ROW)
 454         .setEndKey(HConstants.EMPTY_END_ROW)
 455         .setSplit(false)
 456         .setRegionId(regionId)
 457         .setReplicaId(0)
 458         .build();
 459     RegionInfo replica1 = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 460         .setStartKey(HConstants.EMPTY_START_ROW)
 461         .setEndKey(HConstants.EMPTY_END_ROW)
 462         .setSplit(false)
 463         .setRegionId(regionId)
 464         .setReplicaId(1)
 465         .build();
 466     RegionInfo replica100 = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 467         .setStartKey(HConstants.EMPTY_START_ROW)
 468         .setEndKey(HConstants.EMPTY_END_ROW)
 469         .setSplit(false)
 470         .setRegionId(regionId)
 471         .setReplicaId(100)
 472         .build();
 473
 474     long seqNum0 = random.nextLong();
 475     long seqNum1 = random.nextLong();
 476     long seqNum100 = random.nextLong();
 477
 478     try (Table meta = MetaTableAccessor.getMetaHTable(connection)) {
 479       MetaTableAccessor.updateRegionLocation(connection, primary, serverName0, seqNum0,
 480         EnvironmentEdgeManager.currentTime());
 481
 482       // assert that the server, startcode and seqNum columns are there for the primary region
 483       assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true);
 484
 485       // add replica = 1
 486       MetaTableAccessor.updateRegionLocation(connection, replica1, serverName1, seqNum1,
 487         EnvironmentEdgeManager.currentTime());
 488       // check whether the primary is still there
 489       assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true);
 490       // now check for replica 1
 491       assertMetaLocation(meta, primary.getRegionName(), serverName1, seqNum1, 1, true);
 492
 493       // add replica = 1
 494       MetaTableAccessor.updateRegionLocation(connection, replica100, serverName100, seqNum100,
 495         EnvironmentEdgeManager.currentTime());
 496       // check whether the primary is still there
 497       assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true);
 498       // check whether the replica 1 is still there
 499       assertMetaLocation(meta, primary.getRegionName(), serverName1, seqNum1, 1, true);
 500       // now check for replica 1
 501       assertMetaLocation(meta, primary.getRegionName(), serverName100, seqNum100, 100, true);
 502     }
 503   }
 504
 505   public static void assertMetaLocation(Table meta, byte[] row, ServerName serverName,
 506       long seqNum, int replicaId, boolean checkSeqNum) throws IOException {
 507     Get get = new Get(row);
 508     Result result = meta.get(get);
 509     assertTrue(Bytes.equals(
 510       result.getValue(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(replicaId)),
 511       Bytes.toBytes(serverName.getHostAndPort())));
 512     assertTrue(Bytes.equals(
 513       result.getValue(HConstants.CATALOG_FAMILY, MetaTableAccessor.getStartCodeColumn(replicaId)),
 514       Bytes.toBytes(serverName.getStartcode())));
 515     if (checkSeqNum) {
 516       assertTrue(Bytes.equals(
 517         result.getValue(HConstants.CATALOG_FAMILY, MetaTableAccessor.getSeqNumColumn(replicaId)),
 518         Bytes.toBytes(seqNum)));
 519     }
 520   }
 521
 522   public static void assertEmptyMetaLocation(Table meta, byte[] row, int replicaId)
 523       throws IOException {
 524     Get get = new Get(row);
 525     Result result = meta.get(get);
 526     Cell serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 527         MetaTableAccessor.getServerColumn(replicaId));
 528     Cell startCodeCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 529       MetaTableAccessor.getStartCodeColumn(replicaId));
 530     assertNotNull(serverCell);
 531     assertNotNull(startCodeCell);
 532     assertEquals(0, serverCell.getValueLength());
 533     assertEquals(0, startCodeCell.getValueLength());
 534   }
 535
 536   @Test
 537   public void testMetaLocationForRegionReplicasIsRemovedAtTableDeletion() throws IOException {
 538     long regionId = System.currentTimeMillis();
 539     RegionInfo primary = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 540         .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false)
 541         .setRegionId(regionId).setReplicaId(0).build();
 542
 543     Table meta = MetaTableAccessor.getMetaHTable(connection);
 544     try {
 545       List<RegionInfo> regionInfos = Lists.newArrayList(primary);
 546       MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3);
 547       MetaTableAccessor.removeRegionReplicasFromMeta(Sets.newHashSet(primary.getRegionName()), 1, 2,
 548         connection);
 549       Get get = new Get(primary.getRegionName());
 550       Result result = meta.get(get);
 551       for (int replicaId = 0; replicaId < 3; replicaId++) {
 552         Cell serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 553           MetaTableAccessor.getServerColumn(replicaId));
 554         Cell startCodeCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 555           MetaTableAccessor.getStartCodeColumn(replicaId));
 556         Cell stateCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 557           MetaTableAccessor.getRegionStateColumn(replicaId));
 558         Cell snCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 559           MetaTableAccessor.getServerNameColumn(replicaId));
 560         if (replicaId == 0) {
 561           assertNotNull(stateCell);
 562         } else {
 563           assertNull(serverCell);
 564           assertNull(startCodeCell);
 565           assertNull(stateCell);
 566           assertNull(snCell);
 567         }
 568       }
 569     } finally {
 570       meta.close();
 571     }
 572   }
 573
 574   @Test
 575   public void testMetaLocationForRegionReplicasIsAddedAtTableCreation() throws IOException {
 576     long regionId = System.currentTimeMillis();
 577     RegionInfo primary = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 578         .setStartKey(HConstants.EMPTY_START_ROW)
 579         .setEndKey(HConstants.EMPTY_END_ROW)
 580         .setSplit(false)
 581         .setRegionId(regionId)
 582         .setReplicaId(0)
 583         .build();
 584
 585     Table meta = MetaTableAccessor.getMetaHTable(connection);
 586     try {
 587       List<RegionInfo> regionInfos = Lists.newArrayList(primary);
 588       MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3);
 589
 590       assertEmptyMetaLocation(meta, primary.getRegionName(), 1);
 591       assertEmptyMetaLocation(meta, primary.getRegionName(), 2);
 592     } finally {
 593       meta.close();
 594     }
 595   }
 596
 597   @Test
 598   public void testMetaLocationForRegionReplicasIsAddedAtRegionSplit() throws IOException {
 599     long regionId = System.currentTimeMillis();
 600     ServerName serverName0 = ServerName.valueOf("foo", 60010, random.nextLong());
 601     RegionInfo parent = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 602         .setStartKey(HConstants.EMPTY_START_ROW)
 603         .setEndKey(HConstants.EMPTY_END_ROW)
 604         .setSplit(false)
 605         .setRegionId(regionId)
 606         .setReplicaId(0)
 607         .build();
 608
 609     RegionInfo splitA = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 610         .setStartKey(HConstants.EMPTY_START_ROW)
 611         .setEndKey(Bytes.toBytes("a"))
 612         .setSplit(false)
 613         .setRegionId(regionId + 1)
 614         .setReplicaId(0)
 615         .build();
 616     RegionInfo splitB = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 617         .setStartKey(Bytes.toBytes("a"))
 618         .setEndKey(HConstants.EMPTY_END_ROW)
 619         .setSplit(false)
 620         .setRegionId(regionId + 1)
 621         .setReplicaId(0)
 622         .build();
 623
 624     try (Table meta = MetaTableAccessor.getMetaHTable(connection)) {
 625       List<RegionInfo> regionInfos = Lists.newArrayList(parent);
 626       MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3);
 627
 628       MetaTableAccessor.splitRegion(connection, parent, -1L, splitA, splitB, serverName0, 3);
 629
 630       assertEmptyMetaLocation(meta, splitA.getRegionName(), 1);
 631       assertEmptyMetaLocation(meta, splitA.getRegionName(), 2);
 632       assertEmptyMetaLocation(meta, splitB.getRegionName(), 1);
 633       assertEmptyMetaLocation(meta, splitB.getRegionName(), 2);
 634     }
 635   }
 636
 637   @Test
 638   public void testMetaLocationForRegionReplicasIsAddedAtRegionMerge() throws IOException {
 639     long regionId = System.currentTimeMillis();
 640     ServerName serverName0 = ServerName.valueOf("foo", 60010, random.nextLong());
 641
 642     RegionInfo parentA = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 643         .setStartKey(Bytes.toBytes("a"))
 644         .setEndKey(HConstants.EMPTY_END_ROW)
 645         .setSplit(false)
 646         .setRegionId(regionId)
 647         .setReplicaId(0)
 648         .build();
 649
 650     RegionInfo parentB = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 651         .setStartKey(HConstants.EMPTY_START_ROW)
 652         .setEndKey(Bytes.toBytes("a"))
 653         .setSplit(false)
 654         .setRegionId(regionId)
 655         .setReplicaId(0)
 656         .build();
 657     RegionInfo merged = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 658         .setStartKey(HConstants.EMPTY_START_ROW)
 659         .setEndKey(HConstants.EMPTY_END_ROW)
 660         .setSplit(false)
 661         .setRegionId(regionId + 1)
 662         .setReplicaId(0)
 663         .build();
 664
 665     try (Table meta = MetaTableAccessor.getMetaHTable(connection)) {
 666       List<RegionInfo> regionInfos = Lists.newArrayList(parentA, parentB);
 667       MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3);
 668       MetaTableAccessor.mergeRegions(connection, merged, getMapOfRegionsToSeqNum(parentA, parentB),
 669           serverName0, 3);
 670       assertEmptyMetaLocation(meta, merged.getRegionName(), 1);
 671       assertEmptyMetaLocation(meta, merged.getRegionName(), 2);
 672     }
 673   }
 674
 675   private Map<RegionInfo, Long> getMapOfRegionsToSeqNum(RegionInfo ... regions) {
 676     Map<RegionInfo, Long> mids = new HashMap<>(regions.length);
 677     for (RegionInfo region: regions) {
 678       mids.put(region, -1L);
 679     }
 680     return mids;
 681   }
 682
 683   @Test
 684   public void testMetaScanner() throws Exception {
 685     LOG.info("Starting " + name.getMethodName());
 686
 687     final TableName tableName = TableName.valueOf(name.getMethodName());
 688     final byte[] FAMILY = Bytes.toBytes("family");
 689     final byte[][] SPLIT_KEYS =
 690         new byte[][] { Bytes.toBytes("region_a"), Bytes.toBytes("region_b") };
 691
 692     UTIL.createTable(tableName, FAMILY, SPLIT_KEYS);
 693     Table table = connection.getTable(tableName);
 694     // Make sure all the regions are deployed
 695     UTIL.countRows(table);
 696
 697     MetaTableAccessor.Visitor visitor =
 698         mock(MetaTableAccessor.Visitor.class);
 699     doReturn(true).when(visitor).visit((Result) anyObject());
 700
 701     // Scanning the entire table should give us three rows
 702     MetaTableAccessor.scanMetaForTableRegions(connection, visitor, tableName);
 703     verify(visitor, times(3)).visit((Result) anyObject());
 704
 705     // Scanning the table with a specified empty start row should also
 706     // give us three hbase:meta rows
 707     reset(visitor);
 708     doReturn(true).when(visitor).visit((Result) anyObject());
 709     MetaTableAccessor.scanMeta(connection, visitor, tableName, null, 1000);
 710     verify(visitor, times(3)).visit((Result) anyObject());
 711
 712     // Scanning the table starting in the middle should give us two rows:
 713     // region_a and region_b
 714     reset(visitor);
 715     doReturn(true).when(visitor).visit((Result) anyObject());
 716     MetaTableAccessor.scanMeta(connection, visitor, tableName, Bytes.toBytes("region_ac"), 1000);
 717     verify(visitor, times(2)).visit((Result) anyObject());
 718
 719     // Scanning with a limit of 1 should only give us one row
 720     reset(visitor);
 721     doReturn(true).when(visitor).visit((Result) anyObject());
 722     MetaTableAccessor.scanMeta(connection, visitor, tableName, Bytes.toBytes("region_ac"), 1);
 723     verify(visitor, times(1)).visit((Result) anyObject());
 724     table.close();
 725   }
 726
 727   /**
 728    * Tests whether maximum of masters system time versus RSs local system time is used
 729    */
 730   @Test
 731   public void testMastersSystemTimeIsUsedInUpdateLocations() throws IOException {
 732     long regionId = System.currentTimeMillis();
 733     RegionInfo regionInfo = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 734         .setStartKey(HConstants.EMPTY_START_ROW)
 735         .setEndKey(HConstants.EMPTY_END_ROW)
 736         .setSplit(false)
 737         .setRegionId(regionId)
 738         .setReplicaId(0)
 739         .build();
 740
 741     ServerName sn = ServerName.valueOf("bar", 0, 0);
 742     try (Table meta = MetaTableAccessor.getMetaHTable(connection)) {
 743       List<RegionInfo> regionInfos = Lists.newArrayList(regionInfo);
 744       MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 1);
 745
 746       long masterSystemTime = EnvironmentEdgeManager.currentTime() + 123456789;
 747       MetaTableAccessor.updateRegionLocation(connection, regionInfo, sn, 1, masterSystemTime);
 748
 749       Get get = new Get(regionInfo.getRegionName());
 750       Result result = meta.get(get);
 751       Cell serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 752           MetaTableAccessor.getServerColumn(0));
 753       Cell startCodeCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 754         MetaTableAccessor.getStartCodeColumn(0));
 755       Cell seqNumCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 756         MetaTableAccessor.getSeqNumColumn(0));
 757       assertNotNull(serverCell);
 758       assertNotNull(startCodeCell);
 759       assertNotNull(seqNumCell);
 760       assertTrue(serverCell.getValueLength() > 0);
 761       assertTrue(startCodeCell.getValueLength() > 0);
 762       assertTrue(seqNumCell.getValueLength() > 0);
 763       assertEquals(masterSystemTime, serverCell.getTimestamp());
 764       assertEquals(masterSystemTime, startCodeCell.getTimestamp());
 765       assertEquals(masterSystemTime, seqNumCell.getTimestamp());
 766     }
 767   }
 768
 769   @Test
 770   public void testMastersSystemTimeIsUsedInMergeRegions() throws IOException {
 771     long regionId = System.currentTimeMillis();
 772
 773     RegionInfo regionInfoA = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 774         .setStartKey(HConstants.EMPTY_START_ROW)
 775         .setEndKey(new byte[] {'a'})
 776         .setSplit(false)
 777         .setRegionId(regionId)
 778         .setReplicaId(0)
 779         .build();
 780
 781     RegionInfo regionInfoB = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 782         .setStartKey(new byte[] {'a'})
 783         .setEndKey(HConstants.EMPTY_END_ROW)
 784         .setSplit(false)
 785         .setRegionId(regionId)
 786         .setReplicaId(0)
 787         .build();
 788     RegionInfo mergedRegionInfo = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 789         .setStartKey(HConstants.EMPTY_START_ROW)
 790         .setEndKey(HConstants.EMPTY_END_ROW)
 791         .setSplit(false)
 792         .setRegionId(regionId)
 793         .setReplicaId(0)
 794         .build();
 795
 796     ServerName sn = ServerName.valueOf("bar", 0, 0);
 797     try (Table meta = MetaTableAccessor.getMetaHTable(connection)) {
 798       List<RegionInfo> regionInfos = Lists.newArrayList(regionInfoA, regionInfoB);
 799       MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 1);
 800
 801       // write the serverName column with a big current time, but set the masters time as even
 802       // bigger. When region merge deletes the rows for regionA and regionB, the serverName columns
 803       // should not be seen by the following get
 804       long serverNameTime = EnvironmentEdgeManager.currentTime()   + 100000000;
 805       long masterSystemTime = EnvironmentEdgeManager.currentTime() + 123456789;
 806
 807       // write the serverName columns
 808       MetaTableAccessor.updateRegionLocation(connection, regionInfoA, sn, 1, serverNameTime);
 809
 810       // assert that we have the serverName column with expected ts
 811       Get get = new Get(mergedRegionInfo.getRegionName());
 812       Result result = meta.get(get);
 813       Cell serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 814           MetaTableAccessor.getServerColumn(0));
 815       assertNotNull(serverCell);
 816       assertEquals(serverNameTime, serverCell.getTimestamp());
 817
 818       ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
 819       edge.setValue(masterSystemTime);
 820       EnvironmentEdgeManager.injectEdge(edge);
 821       try {
 822         // now merge the regions, effectively deleting the rows for region a and b.
 823         MetaTableAccessor.mergeRegions(connection, mergedRegionInfo,
 824             getMapOfRegionsToSeqNum(regionInfoA, regionInfoB), sn, 1);
 825       } finally {
 826         EnvironmentEdgeManager.reset();
 827       }
 828
 829
 830       result = meta.get(get);
 831       serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 832           MetaTableAccessor.getServerColumn(0));
 833       Cell startCodeCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 834         MetaTableAccessor.getStartCodeColumn(0));
 835       Cell seqNumCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 836         MetaTableAccessor.getSeqNumColumn(0));
 837       assertNull(serverCell);
 838       assertNull(startCodeCell);
 839       assertNull(seqNumCell);
 840     }
 841   }
 842
 843   public static class SpyingRpcSchedulerFactory extends SimpleRpcSchedulerFactory {
 844     @Override
 845     public RpcScheduler create(Configuration conf, PriorityFunction priority, Abortable server) {
 846       final RpcScheduler delegate = super.create(conf, priority, server);
 847       return new SpyingRpcScheduler(delegate);
 848     }
 849   }
 850
 851   public static class SpyingRpcScheduler extends DelegatingRpcScheduler {
 852     long numPriorityCalls = 0;
 853
 854     public SpyingRpcScheduler(RpcScheduler delegate) {
 855       super(delegate);
 856     }
 857
 858     @Override
 859     public boolean dispatch(CallRunner task) throws IOException, InterruptedException {
 860       int priority = task.getRpcCall().getPriority();
 861
 862       if (priority > HConstants.QOS_THRESHOLD) {
 863         numPriorityCalls++;
 864       }
 865       return super.dispatch(task);
 866     }
 867   }
 868
 869   @Test
 870   public void testMetaUpdatesGoToPriorityQueue() throws Exception {
 871     // This test has to be end-to-end, and do the verification from the server side
 872     Configuration c = UTIL.getConfiguration();
 873
 874     c.set(RSRpcServices.REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS,
 875       SpyingRpcSchedulerFactory.class.getName());
 876
 877     // restart so that new config takes place
 878     afterClass();
 879     beforeClass();
 880
 881     final TableName tableName = TableName.valueOf(name.getMethodName());
 882     try (Admin admin = connection.getAdmin();
 883         RegionLocator rl = connection.getRegionLocator(tableName)) {
 884
 885       // create a table and prepare for a manual split
 886       UTIL.createTable(tableName, "cf1");
 887
 888       HRegionLocation loc = rl.getAllRegionLocations().get(0);
 889       RegionInfo parent = loc.getRegion();
 890       long rid = 1000;
 891       byte[] splitKey = Bytes.toBytes("a");
 892       RegionInfo splitA = RegionInfoBuilder.newBuilder(parent.getTable())
 893           .setStartKey(parent.getStartKey())
 894           .setEndKey(splitKey)
 895           .setSplit(false)
 896           .setRegionId(rid)
 897           .build();
 898       RegionInfo splitB = RegionInfoBuilder.newBuilder(parent.getTable())
 899           .setStartKey(splitKey)
 900           .setEndKey(parent.getEndKey())
 901           .setSplit(false)
 902           .setRegionId(rid)
 903           .build();
 904
 905       // find the meta server
 906       MiniHBaseCluster cluster = UTIL.getMiniHBaseCluster();
 907       int rsIndex = cluster.getServerWithMeta();
 908       HRegionServer rs;
 909       if (rsIndex >= 0) {
 910         rs = cluster.getRegionServer(rsIndex);
 911       } else {
 912         // it is in master
 913         rs = cluster.getMaster();
 914       }
 915       SpyingRpcScheduler scheduler = (SpyingRpcScheduler) rs.getRpcServer().getScheduler();
 916       long prevCalls = scheduler.numPriorityCalls;
 917       MetaTableAccessor.splitRegion(connection, parent, -1L, splitA, splitB, loc.getServerName(),
 918         1);
 919
 920       assertTrue(prevCalls < scheduler.numPriorityCalls);
 921     }
 922   }
 923
 924   @Test
 925   public void testEmptyMetaDaughterLocationDuringSplit() throws IOException {
 926     long regionId = System.currentTimeMillis();
 927     ServerName serverName0 = ServerName.valueOf("foo", 60010, random.nextLong());
 928     RegionInfo parent = RegionInfoBuilder.newBuilder(TableName.valueOf("table_foo"))
 929         .setStartKey(HConstants.EMPTY_START_ROW)
 930         .setEndKey(HConstants.EMPTY_END_ROW)
 931         .setSplit(false)
 932         .setRegionId(regionId)
 933         .setReplicaId(0)
 934         .build();
 935     RegionInfo splitA = RegionInfoBuilder.newBuilder(TableName.valueOf("table_foo"))
 936         .setStartKey(HConstants.EMPTY_START_ROW)
 937         .setEndKey(Bytes.toBytes("a"))
 938         .setSplit(false)
 939         .setRegionId(regionId + 1)
 940         .setReplicaId(0)
 941         .build();
 942     RegionInfo splitB = RegionInfoBuilder.newBuilder(TableName.valueOf("table_foo"))
 943         .setStartKey(Bytes.toBytes("a"))
 944         .setEndKey(HConstants.EMPTY_END_ROW)
 945         .setSplit(false)
 946         .setRegionId(regionId + 1)
 947         .setReplicaId(0)
 948         .build();
 949
 950     Table meta = MetaTableAccessor.getMetaHTable(connection);
 951     try {
 952       List<RegionInfo> regionInfos = Lists.newArrayList(parent);
 953       MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3);
 954
 955       MetaTableAccessor.splitRegion(connection, parent, -1L, splitA, splitB,
 956         serverName0, 3);
 957       Get get1 = new Get(splitA.getRegionName());
 958       Result resultA = meta.get(get1);
 959       Cell serverCellA = resultA.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 960         MetaTableAccessor.getServerColumn(splitA.getReplicaId()));
 961       Cell startCodeCellA = resultA.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 962         MetaTableAccessor.getStartCodeColumn(splitA.getReplicaId()));
 963       assertNull(serverCellA);
 964       assertNull(startCodeCellA);
 965
 966       Get get2 = new Get(splitA.getRegionName());
 967       Result resultB = meta.get(get2);
 968       Cell serverCellB = resultB.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 969         MetaTableAccessor.getServerColumn(splitB.getReplicaId()));
 970       Cell startCodeCellB = resultB.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 971         MetaTableAccessor.getStartCodeColumn(splitB.getReplicaId()));
 972       assertNull(serverCellB);
 973       assertNull(startCodeCellB);
 974     } finally {
 975       if (meta != null) {
 976         meta.close();
 977       }
 978     }
 979   }
 980
 981   @Test
 982   public void testScanByRegionEncodedNameExistingRegion() throws Exception {
 983     final TableName tableName = TableName.valueOf("testScanByRegionEncodedNameExistingRegion");
 984     UTIL.createTable(tableName, "cf");
 985     final List<HRegion> regions = UTIL.getHBaseCluster().getRegions(tableName);
 986     final String encodedName = regions.get(0).getRegionInfo().getEncodedName();
 987     final Result result = MetaTableAccessor.scanByRegionEncodedName(UTIL.getConnection(),
 988       encodedName);
 989     assertNotNull(result);
 990     assertTrue(result.advance());
 991     final String resultingRowKey = CellUtil.getCellKeyAsString(result.current());
 992     assertTrue(resultingRowKey.contains(encodedName));
 993     UTIL.deleteTable(tableName);
 994   }
 995
 996   @Test
 997   public void testScanByRegionEncodedNameNonExistingRegion() throws Exception {
 998     final String encodedName = "nonexistingregion";
 999     final Result result = MetaTableAccessor.scanByRegionEncodedName(UTIL.getConnection(),
1000       encodedName);
1001     assertNull(result);
1002   }
1003 }
1004