hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java

   1 /*
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertFalse;
  22 import static org.junit.Assert.assertNotEquals;
  23 import static org.junit.Assert.assertNotNull;
  24 import static org.junit.Assert.assertNull;
  25 import static org.junit.Assert.assertTrue;
  26 import static org.mockito.ArgumentMatchers.anyObject;
  27 import static org.mockito.Mockito.doReturn;
  28 import static org.mockito.Mockito.mock;
  29 import static org.mockito.Mockito.reset;
  30 import static org.mockito.Mockito.times;
  31 import static org.mockito.Mockito.verify;
  32
  33 import java.io.IOException;
  34 import java.util.ArrayList;
  35 import java.util.HashMap;
  36 import java.util.List;
  37 import java.util.Map;
  38 import java.util.Random;
  39 import java.util.concurrent.TimeUnit;
  40 import org.apache.hadoop.conf.Configuration;
  41 import org.apache.hadoop.hbase.client.Admin;
  42 import org.apache.hadoop.hbase.client.Connection;
  43 import org.apache.hadoop.hbase.client.ConnectionFactory;
  44 import org.apache.hadoop.hbase.client.Get;
  45 import org.apache.hadoop.hbase.client.Put;
  46 import org.apache.hadoop.hbase.client.RegionInfo;
  47 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
  48 import org.apache.hadoop.hbase.client.RegionLocator;
  49 import org.apache.hadoop.hbase.client.Result;
  50 import org.apache.hadoop.hbase.client.Table;
  51 import org.apache.hadoop.hbase.ipc.CallRunner;
  52 import org.apache.hadoop.hbase.ipc.DelegatingRpcScheduler;
  53 import org.apache.hadoop.hbase.ipc.PriorityFunction;
  54 import org.apache.hadoop.hbase.ipc.RpcScheduler;
  55 import org.apache.hadoop.hbase.master.HMaster;
  56 import org.apache.hadoop.hbase.regionserver.HRegion;
  57 import org.apache.hadoop.hbase.regionserver.HRegionServer;
  58 import org.apache.hadoop.hbase.regionserver.RSRpcServices;
  59 import org.apache.hadoop.hbase.regionserver.SimpleRpcSchedulerFactory;
  60 import org.apache.hadoop.hbase.testclassification.MediumTests;
  61 import org.apache.hadoop.hbase.testclassification.MiscTests;
  62 import org.apache.hadoop.hbase.util.Bytes;
  63 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
  64 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
  65 import org.apache.hadoop.hbase.util.Pair;
  66 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
  67 import org.junit.AfterClass;
  68 import org.junit.BeforeClass;
  69 import org.junit.ClassRule;
  70 import org.junit.Rule;
  71 import org.junit.Test;
  72 import org.junit.experimental.categories.Category;
  73 import org.junit.rules.TestName;
  74 import org.slf4j.Logger;
  75 import org.slf4j.LoggerFactory;
  76
  77 import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
  78 import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
  79
  80 /**
  81  * Test {@link org.apache.hadoop.hbase.MetaTableAccessor}.
  82  */
  83 @Category({ MiscTests.class, MediumTests.class })
  84 @SuppressWarnings("deprecation")
  85 public class TestMetaTableAccessor {
  86   @ClassRule
  87   public static final HBaseClassTestRule CLASS_RULE =
  88     HBaseClassTestRule.forClass(TestMetaTableAccessor.class);
  89
  90   private static final Logger LOG = LoggerFactory.getLogger(TestMetaTableAccessor.class);
  91   private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
  92   private static Connection connection;
  93   private Random random = new Random();
  94
  95   @Rule
  96   public TestName name = new TestName();
  97
  98   @BeforeClass
  99   public static void beforeClass() throws Exception {
 100     UTIL.startMiniCluster(3);
 101
 102     Configuration c = new Configuration(UTIL.getConfiguration());
 103     // Tests to 4 retries every 5 seconds. Make it try every 1 second so more
 104     // responsive. 1 second is default as is ten retries.
 105     c.setLong("hbase.client.pause", 1000);
 106     c.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 10);
 107     connection = ConnectionFactory.createConnection(c);
 108   }
 109
 110   @AfterClass
 111   public static void afterClass() throws Exception {
 112     connection.close();
 113     UTIL.shutdownMiniCluster();
 114   }
 115
 116   /**
 117    * Test for HBASE-23044.
 118    */
 119   @Test
 120   public void testGetMergeRegions() throws Exception {
 121     TableName tn = TableName.valueOf(this.name.getMethodName());
 122     UTIL.createMultiRegionTable(tn, Bytes.toBytes("CF"), 4);
 123     UTIL.waitTableAvailable(tn);
 124     try (Admin admin = UTIL.getAdmin()) {
 125       List<RegionInfo> regions = admin.getRegions(tn);
 126       assertEquals(4, regions.size());
 127       admin.mergeRegionsAsync(regions.get(0).getRegionName(), regions.get(1).getRegionName(), false)
 128         .get(60, TimeUnit.SECONDS);
 129       admin.mergeRegionsAsync(regions.get(2).getRegionName(), regions.get(3).getRegionName(), false)
 130         .get(60, TimeUnit.SECONDS);
 131
 132       List<RegionInfo> mergedRegions = admin.getRegions(tn);
 133       assertEquals(2, mergedRegions.size());
 134       RegionInfo mergedRegion0 = mergedRegions.get(0);
 135       RegionInfo mergedRegion1 = mergedRegions.get(1);
 136
 137       List<RegionInfo> mergeParents =
 138         MetaTableAccessor.getMergeRegions(connection, mergedRegion0.getRegionName());
 139       assertTrue(mergeParents.contains(regions.get(0)));
 140       assertTrue(mergeParents.contains(regions.get(1)));
 141       mergeParents = MetaTableAccessor.getMergeRegions(connection, mergedRegion1.getRegionName());
 142       assertTrue(mergeParents.contains(regions.get(2)));
 143       assertTrue(mergeParents.contains(regions.get(3)));
 144
 145       // Delete merge qualifiers for mergedRegion0, then cannot getMergeRegions again
 146       MetaTableAccessor.deleteMergeQualifiers(connection, mergedRegion0);
 147       mergeParents = MetaTableAccessor.getMergeRegions(connection, mergedRegion0.getRegionName());
 148       assertNull(mergeParents);
 149
 150       mergeParents = MetaTableAccessor.getMergeRegions(connection, mergedRegion1.getRegionName());
 151       assertTrue(mergeParents.contains(regions.get(2)));
 152       assertTrue(mergeParents.contains(regions.get(3)));
 153     }
 154     UTIL.deleteTable(tn);
 155   }
 156
 157   @Test
 158   public void testAddMergeRegions() throws IOException {
 159     TableName tn = TableName.valueOf(this.name.getMethodName());
 160     Put put = new Put(Bytes.toBytes(this.name.getMethodName()));
 161     List<RegionInfo> ris = new ArrayList<>();
 162     int limit = 10;
 163     byte[] previous = HConstants.EMPTY_START_ROW;
 164     for (int i = 0; i < limit; i++) {
 165       RegionInfo ri =
 166         RegionInfoBuilder.newBuilder(tn).setStartKey(previous).setEndKey(Bytes.toBytes(i)).build();
 167       ris.add(ri);
 168     }
 169     put = MetaTableAccessor.addMergeRegions(put, ris);
 170     List<Cell> cells = put.getFamilyCellMap().get(HConstants.CATALOG_FAMILY);
 171     String previousQualifier = null;
 172     assertEquals(limit, cells.size());
 173     for (Cell cell : cells) {
 174       LOG.info(cell.toString());
 175       String qualifier = Bytes.toString(cell.getQualifierArray());
 176       assertTrue(qualifier.startsWith(HConstants.MERGE_QUALIFIER_PREFIX_STR));
 177       assertNotEquals(qualifier, previousQualifier);
 178       previousQualifier = qualifier;
 179     }
 180   }
 181
 182   @Test
 183   public void testIsMetaWhenAllHealthy() throws InterruptedException {
 184     HMaster m = UTIL.getMiniHBaseCluster().getMaster();
 185     assertTrue(m.waitForMetaOnline());
 186   }
 187
 188   @Test
 189   public void testIsMetaWhenMetaGoesOffline() throws InterruptedException {
 190     HMaster m = UTIL.getMiniHBaseCluster().getMaster();
 191     int index = UTIL.getMiniHBaseCluster().getServerWithMeta();
 192     HRegionServer rsWithMeta = UTIL.getMiniHBaseCluster().getRegionServer(index);
 193     rsWithMeta.abort("TESTING");
 194     assertTrue(m.waitForMetaOnline());
 195   }
 196
 197   /**
 198    * Does {@link MetaTableAccessor#getRegion(Connection, byte[])} and a write against hbase:meta
 199    * while its hosted server is restarted to prove our retrying works.
 200    */
 201   @Test
 202   public void testRetrying() throws IOException, InterruptedException {
 203     final TableName tableName = TableName.valueOf(name.getMethodName());
 204     LOG.info("Started " + tableName);
 205     Table t = UTIL.createMultiRegionTable(tableName, HConstants.CATALOG_FAMILY);
 206     int regionCount = -1;
 207     try (RegionLocator r = UTIL.getConnection().getRegionLocator(tableName)) {
 208       regionCount = r.getStartKeys().length;
 209     }
 210     // Test it works getting a region from just made user table.
 211     final List<RegionInfo> regions = testGettingTableRegions(connection, tableName, regionCount);
 212     MetaTask reader = new MetaTask(connection, "reader") {
 213       @Override
 214       void metaTask() throws Throwable {
 215         testGetRegion(connection, regions.get(0));
 216         LOG.info("Read " + regions.get(0).getEncodedName());
 217       }
 218     };
 219     MetaTask writer = new MetaTask(connection, "writer") {
 220       @Override
 221       void metaTask() throws Throwable {
 222         MetaTableAccessor.addRegionToMeta(connection, regions.get(0));
 223         LOG.info("Wrote " + regions.get(0).getEncodedName());
 224       }
 225     };
 226     reader.start();
 227     writer.start();
 228
 229     // We're gonna check how it takes. If it takes too long, we will consider
 230     // it as a fail. We can't put that in the @Test tag as we want to close
 231     // the threads nicely
 232     final long timeOut = 180000;
 233     long startTime = System.currentTimeMillis();
 234
 235     try {
 236       // Make sure reader and writer are working.
 237       assertTrue(reader.isProgressing());
 238       assertTrue(writer.isProgressing());
 239
 240       // Kill server hosting meta -- twice . See if our reader/writer ride over the
 241       // meta moves. They'll need to retry.
 242       for (int i = 0; i < 2; i++) {
 243         LOG.info("Restart=" + i);
 244         UTIL.ensureSomeRegionServersAvailable(2);
 245         int index = -1;
 246         do {
 247           index = UTIL.getMiniHBaseCluster().getServerWithMeta();
 248         } while (index == -1 && startTime + timeOut < System.currentTimeMillis());
 249
 250         if (index != -1) {
 251           UTIL.getMiniHBaseCluster().abortRegionServer(index);
 252           UTIL.getMiniHBaseCluster().waitOnRegionServer(index);
 253         }
 254       }
 255
 256       assertTrue("reader: " + reader.toString(), reader.isProgressing());
 257       assertTrue("writer: " + writer.toString(), writer.isProgressing());
 258     } catch (IOException e) {
 259       throw e;
 260     } finally {
 261       reader.stop = true;
 262       writer.stop = true;
 263       reader.join();
 264       writer.join();
 265       t.close();
 266     }
 267     long exeTime = System.currentTimeMillis() - startTime;
 268     assertTrue("Timeout: test took " + exeTime / 1000 + " sec", exeTime < timeOut);
 269   }
 270
 271   /**
 272    * Thread that runs a MetaTableAccessor task until asked stop.
 273    */
 274   abstract static class MetaTask extends Thread {
 275     boolean stop = false;
 276     int count = 0;
 277     Throwable t = null;
 278     final Connection connection;
 279
 280     MetaTask(final Connection connection, final String name) {
 281       super(name);
 282       this.connection = connection;
 283     }
 284
 285     @Override
 286     public void run() {
 287       try {
 288         while (!this.stop) {
 289           LOG.info("Before " + this.getName() + ", count=" + this.count);
 290           metaTask();
 291           this.count += 1;
 292           LOG.info("After " + this.getName() + ", count=" + this.count);
 293           Thread.sleep(100);
 294         }
 295       } catch (Throwable t) {
 296         LOG.info(this.getName() + " failed", t);
 297         this.t = t;
 298       }
 299     }
 300
 301     boolean isProgressing() throws InterruptedException {
 302       int currentCount = this.count;
 303       while (currentCount == this.count) {
 304         if (!isAlive()) return false;
 305         if (this.t != null) return false;
 306         Thread.sleep(10);
 307       }
 308       return true;
 309     }
 310
 311     @Override
 312     public String toString() {
 313       return "count=" + this.count + ", t=" + (this.t == null ? "null" : this.t.toString());
 314     }
 315
 316     abstract void metaTask() throws Throwable;
 317   }
 318
 319   @Test
 320   public void testGetRegionsFromMetaTable() throws IOException, InterruptedException {
 321     List<RegionInfo> regions = MetaTableLocator.getMetaRegions(UTIL.getZooKeeperWatcher());
 322     assertTrue(regions.size() >= 1);
 323     assertTrue(MetaTableLocator.getMetaRegionsAndLocations(UTIL.getZooKeeperWatcher()).size() >= 1);
 324   }
 325
 326   @Test
 327   public void testTableExists() throws IOException {
 328     final TableName tableName = TableName.valueOf(name.getMethodName());
 329     assertFalse(MetaTableAccessor.tableExists(connection, tableName));
 330     UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
 331     assertTrue(MetaTableAccessor.tableExists(connection, tableName));
 332     Admin admin = UTIL.getAdmin();
 333     admin.disableTable(tableName);
 334     admin.deleteTable(tableName);
 335     assertFalse(MetaTableAccessor.tableExists(connection, tableName));
 336     assertTrue(MetaTableAccessor.tableExists(connection, TableName.META_TABLE_NAME));
 337     UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
 338     assertTrue(MetaTableAccessor.tableExists(connection, tableName));
 339     admin.disableTable(tableName);
 340     admin.deleteTable(tableName);
 341     assertFalse(MetaTableAccessor.tableExists(connection, tableName));
 342   }
 343
 344   @Test
 345   public void testGetRegion() throws IOException, InterruptedException {
 346     final String name = this.name.getMethodName();
 347     LOG.info("Started " + name);
 348     // Test get on non-existent region.
 349     Pair<RegionInfo, ServerName> pair =
 350       MetaTableAccessor.getRegion(connection, Bytes.toBytes("nonexistent-region"));
 351     assertNull(pair);
 352     LOG.info("Finished " + name);
 353   }
 354
 355   // Test for the optimization made in HBASE-3650
 356   @Test
 357   public void testScanMetaForTable() throws IOException, InterruptedException {
 358     final TableName tableName = TableName.valueOf(name.getMethodName());
 359     LOG.info("Started " + tableName);
 360
 361     /**
 362      * Create 2 tables - testScanMetaForTable - testScanMetaForTablf
 363      **/
 364
 365     UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
 366     // name that is +1 greater than the first one (e+1=f)
 367     TableName greaterName = TableName.valueOf("testScanMetaForTablf");
 368     UTIL.createTable(greaterName, HConstants.CATALOG_FAMILY);
 369
 370     // Now make sure we only get the regions from 1 of the tables at a time
 371
 372     assertEquals(1, MetaTableAccessor.getTableRegions(connection, tableName).size());
 373     assertEquals(1, MetaTableAccessor.getTableRegions(connection, greaterName).size());
 374   }
 375
 376   private static List<RegionInfo> testGettingTableRegions(final Connection connection,
 377     final TableName name, final int regionCount) throws IOException, InterruptedException {
 378     List<RegionInfo> regions = MetaTableAccessor.getTableRegions(connection, name);
 379     assertEquals(regionCount, regions.size());
 380     Pair<RegionInfo, ServerName> pair =
 381       MetaTableAccessor.getRegion(connection, regions.get(0).getRegionName());
 382     assertEquals(regions.get(0).getEncodedName(), pair.getFirst().getEncodedName());
 383     return regions;
 384   }
 385
 386   private static void testGetRegion(final Connection connection, final RegionInfo region)
 387     throws IOException, InterruptedException {
 388     Pair<RegionInfo, ServerName> pair =
 389       MetaTableAccessor.getRegion(connection, region.getRegionName());
 390     assertEquals(region.getEncodedName(), pair.getFirst().getEncodedName());
 391   }
 392
 393   @Test
 394   public void testMetaLocationsForRegionReplicas() throws IOException {
 395     ServerName serverName0 = ServerName.valueOf("foo", 60010, random.nextLong());
 396     ServerName serverName1 = ServerName.valueOf("bar", 60010, random.nextLong());
 397     ServerName serverName100 = ServerName.valueOf("baz", 60010, random.nextLong());
 398
 399     long regionId = System.currentTimeMillis();
 400     RegionInfo primary = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 401       .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false)
 402       .setRegionId(regionId).setReplicaId(0).build();
 403     RegionInfo replica1 = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 404       .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false)
 405       .setRegionId(regionId).setReplicaId(1).build();
 406     RegionInfo replica100 = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 407       .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false)
 408       .setRegionId(regionId).setReplicaId(100).build();
 409
 410     long seqNum0 = random.nextLong();
 411     long seqNum1 = random.nextLong();
 412     long seqNum100 = random.nextLong();
 413
 414     try (Table meta = MetaTableAccessor.getMetaHTable(connection)) {
 415       MetaTableAccessor.updateRegionLocation(connection, primary, serverName0, seqNum0,
 416         EnvironmentEdgeManager.currentTime());
 417
 418       // assert that the server, startcode and seqNum columns are there for the primary region
 419       assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true);
 420
 421       // add replica = 1
 422       MetaTableAccessor.updateRegionLocation(connection, replica1, serverName1, seqNum1,
 423         EnvironmentEdgeManager.currentTime());
 424       // check whether the primary is still there
 425       assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true);
 426       // now check for replica 1
 427       assertMetaLocation(meta, primary.getRegionName(), serverName1, seqNum1, 1, true);
 428
 429       // add replica = 1
 430       MetaTableAccessor.updateRegionLocation(connection, replica100, serverName100, seqNum100,
 431         EnvironmentEdgeManager.currentTime());
 432       // check whether the primary is still there
 433       assertMetaLocation(meta, primary.getRegionName(), serverName0, seqNum0, 0, true);
 434       // check whether the replica 1 is still there
 435       assertMetaLocation(meta, primary.getRegionName(), serverName1, seqNum1, 1, true);
 436       // now check for replica 1
 437       assertMetaLocation(meta, primary.getRegionName(), serverName100, seqNum100, 100, true);
 438     }
 439   }
 440
 441   public static void assertMetaLocation(Table meta, byte[] row, ServerName serverName, long seqNum,
 442     int replicaId, boolean checkSeqNum) throws IOException {
 443     Get get = new Get(row);
 444     Result result = meta.get(get);
 445     assertTrue(Bytes.equals(
 446       result.getValue(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getServerColumn(replicaId)),
 447       Bytes.toBytes(serverName.getAddress().toString())));
 448     assertTrue(Bytes.equals(
 449       result.getValue(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getStartCodeColumn(replicaId)),
 450       Bytes.toBytes(serverName.getStartcode())));
 451     if (checkSeqNum) {
 452       assertTrue(Bytes.equals(
 453         result.getValue(HConstants.CATALOG_FAMILY, CatalogFamilyFormat.getSeqNumColumn(replicaId)),
 454         Bytes.toBytes(seqNum)));
 455     }
 456   }
 457
 458   public static void assertEmptyMetaLocation(Table meta, byte[] row, int replicaId)
 459     throws IOException {
 460     Get get = new Get(row);
 461     Result result = meta.get(get);
 462     Cell serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 463       CatalogFamilyFormat.getServerColumn(replicaId));
 464     Cell startCodeCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 465       CatalogFamilyFormat.getStartCodeColumn(replicaId));
 466     assertNotNull(serverCell);
 467     assertNotNull(startCodeCell);
 468     assertEquals(0, serverCell.getValueLength());
 469     assertEquals(0, startCodeCell.getValueLength());
 470   }
 471
 472   @Test
 473   public void testMetaLocationForRegionReplicasIsRemovedAtTableDeletion() throws IOException {
 474     long regionId = System.currentTimeMillis();
 475     RegionInfo primary = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 476       .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false)
 477       .setRegionId(regionId).setReplicaId(0).build();
 478
 479     Table meta = MetaTableAccessor.getMetaHTable(connection);
 480     try {
 481       List<RegionInfo> regionInfos = Lists.newArrayList(primary);
 482       MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3);
 483       MetaTableAccessor.removeRegionReplicasFromMeta(Sets.newHashSet(primary.getRegionName()), 1, 2,
 484         connection);
 485       Get get = new Get(primary.getRegionName());
 486       Result result = meta.get(get);
 487       for (int replicaId = 0; replicaId < 3; replicaId++) {
 488         Cell serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 489           CatalogFamilyFormat.getServerColumn(replicaId));
 490         Cell startCodeCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 491           CatalogFamilyFormat.getStartCodeColumn(replicaId));
 492         Cell stateCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 493           CatalogFamilyFormat.getRegionStateColumn(replicaId));
 494         Cell snCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 495           CatalogFamilyFormat.getServerNameColumn(replicaId));
 496         if (replicaId == 0) {
 497           assertNotNull(stateCell);
 498         } else {
 499           assertNull(serverCell);
 500           assertNull(startCodeCell);
 501           assertNull(stateCell);
 502           assertNull(snCell);
 503         }
 504       }
 505     } finally {
 506       meta.close();
 507     }
 508   }
 509
 510   @Test
 511   public void testMetaLocationForRegionReplicasIsAddedAtTableCreation() throws IOException {
 512     long regionId = System.currentTimeMillis();
 513     RegionInfo primary = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 514       .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false)
 515       .setRegionId(regionId).setReplicaId(0).build();
 516
 517     Table meta = MetaTableAccessor.getMetaHTable(connection);
 518     try {
 519       List<RegionInfo> regionInfos = Lists.newArrayList(primary);
 520       MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3);
 521
 522       assertEmptyMetaLocation(meta, primary.getRegionName(), 1);
 523       assertEmptyMetaLocation(meta, primary.getRegionName(), 2);
 524     } finally {
 525       meta.close();
 526     }
 527   }
 528
 529   @Test
 530   public void testMetaLocationForRegionReplicasIsAddedAtRegionSplit() throws IOException {
 531     long regionId = System.currentTimeMillis();
 532     ServerName serverName0 = ServerName.valueOf("foo", 60010, random.nextLong());
 533     RegionInfo parent = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 534       .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false)
 535       .setRegionId(regionId).setReplicaId(0).build();
 536
 537     RegionInfo splitA = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 538       .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(Bytes.toBytes("a")).setSplit(false)
 539       .setRegionId(regionId + 1).setReplicaId(0).build();
 540     RegionInfo splitB = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 541       .setStartKey(Bytes.toBytes("a")).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false)
 542       .setRegionId(regionId + 1).setReplicaId(0).build();
 543
 544     try (Table meta = MetaTableAccessor.getMetaHTable(connection)) {
 545       List<RegionInfo> regionInfos = Lists.newArrayList(parent);
 546       MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3);
 547
 548       MetaTableAccessor.splitRegion(connection, parent, -1L, splitA, splitB, serverName0, 3);
 549
 550       assertEmptyMetaLocation(meta, splitA.getRegionName(), 1);
 551       assertEmptyMetaLocation(meta, splitA.getRegionName(), 2);
 552       assertEmptyMetaLocation(meta, splitB.getRegionName(), 1);
 553       assertEmptyMetaLocation(meta, splitB.getRegionName(), 2);
 554     }
 555   }
 556
 557   @Test
 558   public void testMetaLocationForRegionReplicasIsAddedAtRegionMerge() throws IOException {
 559     long regionId = System.currentTimeMillis();
 560     ServerName serverName0 = ServerName.valueOf("foo", 60010, random.nextLong());
 561
 562     RegionInfo parentA = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 563       .setStartKey(Bytes.toBytes("a")).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false)
 564       .setRegionId(regionId).setReplicaId(0).build();
 565
 566     RegionInfo parentB = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 567       .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(Bytes.toBytes("a")).setSplit(false)
 568       .setRegionId(regionId).setReplicaId(0).build();
 569     RegionInfo merged = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 570       .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false)
 571       .setRegionId(regionId + 1).setReplicaId(0).build();
 572
 573     try (Table meta = MetaTableAccessor.getMetaHTable(connection)) {
 574       List<RegionInfo> regionInfos = Lists.newArrayList(parentA, parentB);
 575       MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3);
 576       MetaTableAccessor.mergeRegions(connection, merged, getMapOfRegionsToSeqNum(parentA, parentB),
 577         serverName0, 3);
 578       assertEmptyMetaLocation(meta, merged.getRegionName(), 1);
 579       assertEmptyMetaLocation(meta, merged.getRegionName(), 2);
 580     }
 581   }
 582
 583   private Map<RegionInfo, Long> getMapOfRegionsToSeqNum(RegionInfo... regions) {
 584     Map<RegionInfo, Long> mids = new HashMap<>(regions.length);
 585     for (RegionInfo region : regions) {
 586       mids.put(region, -1L);
 587     }
 588     return mids;
 589   }
 590
 591   @Test
 592   public void testMetaScanner() throws Exception {
 593     LOG.info("Starting " + name.getMethodName());
 594
 595     final TableName tableName = TableName.valueOf(name.getMethodName());
 596     final byte[] FAMILY = Bytes.toBytes("family");
 597     final byte[][] SPLIT_KEYS =
 598       new byte[][] { Bytes.toBytes("region_a"), Bytes.toBytes("region_b") };
 599
 600     UTIL.createTable(tableName, FAMILY, SPLIT_KEYS);
 601     Table table = connection.getTable(tableName);
 602     // Make sure all the regions are deployed
 603     UTIL.countRows(table);
 604
 605     ClientMetaTableAccessor.Visitor visitor = mock(ClientMetaTableAccessor.Visitor.class);
 606     doReturn(true).when(visitor).visit((Result) anyObject());
 607
 608     // Scanning the entire table should give us three rows
 609     MetaTableAccessor.scanMetaForTableRegions(connection, visitor, tableName);
 610     verify(visitor, times(3)).visit((Result) anyObject());
 611
 612     // Scanning the table with a specified empty start row should also
 613     // give us three hbase:meta rows
 614     reset(visitor);
 615     doReturn(true).when(visitor).visit((Result) anyObject());
 616     MetaTableAccessor.scanMeta(connection, visitor, tableName, null, 1000);
 617     verify(visitor, times(3)).visit((Result) anyObject());
 618
 619     // Scanning the table starting in the middle should give us two rows:
 620     // region_a and region_b
 621     reset(visitor);
 622     doReturn(true).when(visitor).visit((Result) anyObject());
 623     MetaTableAccessor.scanMeta(connection, visitor, tableName, Bytes.toBytes("region_ac"), 1000);
 624     verify(visitor, times(2)).visit((Result) anyObject());
 625
 626     // Scanning with a limit of 1 should only give us one row
 627     reset(visitor);
 628     doReturn(true).when(visitor).visit((Result) anyObject());
 629     MetaTableAccessor.scanMeta(connection, visitor, tableName, Bytes.toBytes("region_ac"), 1);
 630     verify(visitor, times(1)).visit((Result) anyObject());
 631     table.close();
 632   }
 633
 634   /**
 635    * Tests whether maximum of masters system time versus RSs local system time is used
 636    */
 637   @Test
 638   public void testMastersSystemTimeIsUsedInUpdateLocations() throws IOException {
 639     long regionId = System.currentTimeMillis();
 640     RegionInfo regionInfo = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 641       .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false)
 642       .setRegionId(regionId).setReplicaId(0).build();
 643
 644     ServerName sn = ServerName.valueOf("bar", 0, 0);
 645     try (Table meta = MetaTableAccessor.getMetaHTable(connection)) {
 646       List<RegionInfo> regionInfos = Lists.newArrayList(regionInfo);
 647       MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 1);
 648
 649       long masterSystemTime = EnvironmentEdgeManager.currentTime() + 123456789;
 650       MetaTableAccessor.updateRegionLocation(connection, regionInfo, sn, 1, masterSystemTime);
 651
 652       Get get = new Get(regionInfo.getRegionName());
 653       Result result = meta.get(get);
 654       Cell serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 655         CatalogFamilyFormat.getServerColumn(0));
 656       Cell startCodeCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 657         CatalogFamilyFormat.getStartCodeColumn(0));
 658       Cell seqNumCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 659         CatalogFamilyFormat.getSeqNumColumn(0));
 660       assertNotNull(serverCell);
 661       assertNotNull(startCodeCell);
 662       assertNotNull(seqNumCell);
 663       assertTrue(serverCell.getValueLength() > 0);
 664       assertTrue(startCodeCell.getValueLength() > 0);
 665       assertTrue(seqNumCell.getValueLength() > 0);
 666       assertEquals(masterSystemTime, serverCell.getTimestamp());
 667       assertEquals(masterSystemTime, startCodeCell.getTimestamp());
 668       assertEquals(masterSystemTime, seqNumCell.getTimestamp());
 669     }
 670   }
 671
 672   @Test
 673   public void testMastersSystemTimeIsUsedInMergeRegions() throws IOException {
 674     long regionId = System.currentTimeMillis();
 675
 676     RegionInfo regionInfoA = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 677       .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(new byte[] { 'a' }).setSplit(false)
 678       .setRegionId(regionId).setReplicaId(0).build();
 679
 680     RegionInfo regionInfoB = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 681       .setStartKey(new byte[] { 'a' }).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false)
 682       .setRegionId(regionId).setReplicaId(0).build();
 683     RegionInfo mergedRegionInfo =
 684       RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
 685         .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false)
 686         .setRegionId(regionId).setReplicaId(0).build();
 687
 688     ServerName sn = ServerName.valueOf("bar", 0, 0);
 689     try (Table meta = MetaTableAccessor.getMetaHTable(connection)) {
 690       List<RegionInfo> regionInfos = Lists.newArrayList(regionInfoA, regionInfoB);
 691       MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 1);
 692
 693       // write the serverName column with a big current time, but set the masters time as even
 694       // bigger. When region merge deletes the rows for regionA and regionB, the serverName columns
 695       // should not be seen by the following get
 696       long serverNameTime = EnvironmentEdgeManager.currentTime() + 100000000;
 697       long masterSystemTime = EnvironmentEdgeManager.currentTime() + 123456789;
 698
 699       // write the serverName columns
 700       MetaTableAccessor.updateRegionLocation(connection, regionInfoA, sn, 1, serverNameTime);
 701
 702       // assert that we have the serverName column with expected ts
 703       Get get = new Get(mergedRegionInfo.getRegionName());
 704       Result result = meta.get(get);
 705       Cell serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 706         CatalogFamilyFormat.getServerColumn(0));
 707       assertNotNull(serverCell);
 708       assertEquals(serverNameTime, serverCell.getTimestamp());
 709
 710       ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
 711       edge.setValue(masterSystemTime);
 712       EnvironmentEdgeManager.injectEdge(edge);
 713       try {
 714         // now merge the regions, effectively deleting the rows for region a and b.
 715         MetaTableAccessor.mergeRegions(connection, mergedRegionInfo,
 716           getMapOfRegionsToSeqNum(regionInfoA, regionInfoB), sn, 1);
 717       } finally {
 718         EnvironmentEdgeManager.reset();
 719       }
 720
 721       result = meta.get(get);
 722       serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 723         CatalogFamilyFormat.getServerColumn(0));
 724       Cell startCodeCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 725         CatalogFamilyFormat.getStartCodeColumn(0));
 726       Cell seqNumCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 727         CatalogFamilyFormat.getSeqNumColumn(0));
 728       assertNull(serverCell);
 729       assertNull(startCodeCell);
 730       assertNull(seqNumCell);
 731     }
 732   }
 733
 734   public static class SpyingRpcSchedulerFactory extends SimpleRpcSchedulerFactory {
 735     @Override
 736     public RpcScheduler create(Configuration conf, PriorityFunction priority, Abortable server) {
 737       final RpcScheduler delegate = super.create(conf, priority, server);
 738       return new SpyingRpcScheduler(delegate);
 739     }
 740   }
 741
 742   public static class SpyingRpcScheduler extends DelegatingRpcScheduler {
 743     long numPriorityCalls = 0;
 744
 745     public SpyingRpcScheduler(RpcScheduler delegate) {
 746       super(delegate);
 747     }
 748
 749     @Override
 750     public boolean dispatch(CallRunner task) throws IOException, InterruptedException {
 751       int priority = task.getRpcCall().getPriority();
 752
 753       if (priority > HConstants.QOS_THRESHOLD) {
 754         numPriorityCalls++;
 755       }
 756       return super.dispatch(task);
 757     }
 758   }
 759
 760   @Test
 761   public void testMetaUpdatesGoToPriorityQueue() throws Exception {
 762     // This test has to be end-to-end, and do the verification from the server side
 763     Configuration c = UTIL.getConfiguration();
 764
 765     c.set(RSRpcServices.REGION_SERVER_RPC_SCHEDULER_FACTORY_CLASS,
 766       SpyingRpcSchedulerFactory.class.getName());
 767
 768     // restart so that new config takes place
 769     afterClass();
 770     beforeClass();
 771
 772     final TableName tableName = TableName.valueOf(name.getMethodName());
 773     try (Admin admin = connection.getAdmin();
 774       RegionLocator rl = connection.getRegionLocator(tableName)) {
 775
 776       // create a table and prepare for a manual split
 777       UTIL.createTable(tableName, "cf1");
 778
 779       HRegionLocation loc = rl.getAllRegionLocations().get(0);
 780       RegionInfo parent = loc.getRegion();
 781       long rid = 1000;
 782       byte[] splitKey = Bytes.toBytes("a");
 783       RegionInfo splitA =
 784         RegionInfoBuilder.newBuilder(parent.getTable()).setStartKey(parent.getStartKey())
 785           .setEndKey(splitKey).setSplit(false).setRegionId(rid).build();
 786       RegionInfo splitB = RegionInfoBuilder.newBuilder(parent.getTable()).setStartKey(splitKey)
 787         .setEndKey(parent.getEndKey()).setSplit(false).setRegionId(rid).build();
 788
 789       // find the meta server
 790       MiniHBaseCluster cluster = UTIL.getMiniHBaseCluster();
 791       int rsIndex = cluster.getServerWithMeta();
 792       HRegionServer rs;
 793       if (rsIndex >= 0) {
 794         rs = cluster.getRegionServer(rsIndex);
 795       } else {
 796         // it is in master
 797         rs = cluster.getMaster();
 798       }
 799       SpyingRpcScheduler scheduler = (SpyingRpcScheduler) rs.getRpcServer().getScheduler();
 800       long prevCalls = scheduler.numPriorityCalls;
 801       MetaTableAccessor.splitRegion(connection, parent, -1L, splitA, splitB, loc.getServerName(),
 802         1);
 803
 804       assertTrue(prevCalls < scheduler.numPriorityCalls);
 805     }
 806   }
 807
 808   @Test
 809   public void testEmptyMetaDaughterLocationDuringSplit() throws IOException {
 810     long regionId = System.currentTimeMillis();
 811     ServerName serverName0 = ServerName.valueOf("foo", 60010, random.nextLong());
 812     RegionInfo parent = RegionInfoBuilder.newBuilder(TableName.valueOf("table_foo"))
 813       .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false)
 814       .setRegionId(regionId).setReplicaId(0).build();
 815     RegionInfo splitA = RegionInfoBuilder.newBuilder(TableName.valueOf("table_foo"))
 816       .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(Bytes.toBytes("a")).setSplit(false)
 817       .setRegionId(regionId + 1).setReplicaId(0).build();
 818     RegionInfo splitB = RegionInfoBuilder.newBuilder(TableName.valueOf("table_foo"))
 819       .setStartKey(Bytes.toBytes("a")).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false)
 820       .setRegionId(regionId + 1).setReplicaId(0).build();
 821
 822     Table meta = MetaTableAccessor.getMetaHTable(connection);
 823     try {
 824       List<RegionInfo> regionInfos = Lists.newArrayList(parent);
 825       MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3);
 826
 827       MetaTableAccessor.splitRegion(connection, parent, -1L, splitA, splitB, serverName0, 3);
 828       Get get1 = new Get(splitA.getRegionName());
 829       Result resultA = meta.get(get1);
 830       Cell serverCellA = resultA.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 831         CatalogFamilyFormat.getServerColumn(splitA.getReplicaId()));
 832       Cell startCodeCellA = resultA.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 833         CatalogFamilyFormat.getStartCodeColumn(splitA.getReplicaId()));
 834       assertNull(serverCellA);
 835       assertNull(startCodeCellA);
 836
 837       Get get2 = new Get(splitA.getRegionName());
 838       Result resultB = meta.get(get2);
 839       Cell serverCellB = resultB.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 840         CatalogFamilyFormat.getServerColumn(splitB.getReplicaId()));
 841       Cell startCodeCellB = resultB.getColumnLatestCell(HConstants.CATALOG_FAMILY,
 842         CatalogFamilyFormat.getStartCodeColumn(splitB.getReplicaId()));
 843       assertNull(serverCellB);
 844       assertNull(startCodeCellB);
 845     } finally {
 846       if (meta != null) {
 847         meta.close();
 848       }
 849     }
 850   }
 851
 852   @Test
 853   public void testScanByRegionEncodedNameExistingRegion() throws Exception {
 854     final TableName tableName = TableName.valueOf("testScanByRegionEncodedNameExistingRegion");
 855     UTIL.createTable(tableName, "cf");
 856     final List<HRegion> regions = UTIL.getHBaseCluster().getRegions(tableName);
 857     final String encodedName = regions.get(0).getRegionInfo().getEncodedName();
 858     final Result result =
 859       MetaTableAccessor.scanByRegionEncodedName(UTIL.getConnection(), encodedName);
 860     assertNotNull(result);
 861     assertTrue(result.advance());
 862     final String resultingRowKey = CellUtil.getCellKeyAsString(result.current());
 863     assertTrue(resultingRowKey.contains(encodedName));
 864     UTIL.deleteTable(tableName);
 865   }
 866
 867   @Test
 868   public void testScanByRegionEncodedNameNonExistingRegion() throws Exception {
 869     final String encodedName = "nonexistingregion";
 870     final Result result =
 871       MetaTableAccessor.scanByRegionEncodedName(UTIL.getConnection(), encodedName);
 872     assertNull(result);
 873   }
 874 }