hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide5.java

   1 /*
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.client;
  19
  20 import static org.junit.Assert.assertArrayEquals;
  21 import static org.junit.Assert.assertEquals;
  22 import static org.junit.Assert.assertFalse;
  23 import static org.junit.Assert.assertNotNull;
  24 import static org.junit.Assert.assertNull;
  25 import static org.junit.Assert.assertTrue;
  26 import static org.junit.Assert.fail;
  27
  28 import java.io.IOException;
  29 import java.util.ArrayList;
  30 import java.util.Arrays;
  31 import java.util.Collection;
  32 import java.util.LinkedList;
  33 import java.util.List;
  34 import java.util.Map;
  35 import java.util.NavigableMap;
  36 import java.util.concurrent.Callable;
  37 import java.util.concurrent.ExecutorService;
  38 import java.util.concurrent.Executors;
  39 import java.util.concurrent.atomic.AtomicReference;
  40 import org.apache.commons.lang3.ArrayUtils;
  41 import org.apache.hadoop.conf.Configuration;
  42 import org.apache.hadoop.fs.Path;
  43 import org.apache.hadoop.hbase.Cell;
  44 import org.apache.hadoop.hbase.CellScanner;
  45 import org.apache.hadoop.hbase.CellUtil;
  46 import org.apache.hadoop.hbase.CompareOperator;
  47 import org.apache.hadoop.hbase.DoNotRetryIOException;
  48 import org.apache.hadoop.hbase.HBaseClassTestRule;
  49 import org.apache.hadoop.hbase.HConstants;
  50 import org.apache.hadoop.hbase.HRegionLocation;
  51 import org.apache.hadoop.hbase.KeyValue;
  52 import org.apache.hadoop.hbase.PrivateCellUtil;
  53 import org.apache.hadoop.hbase.ServerName;
  54 import org.apache.hadoop.hbase.TableName;
  55 import org.apache.hadoop.hbase.TableNameTestRule;
  56 import org.apache.hadoop.hbase.Waiter;
  57 import org.apache.hadoop.hbase.client.Scan.ReadType;
  58 import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
  59 import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
  60 import org.apache.hadoop.hbase.filter.BinaryComparator;
  61 import org.apache.hadoop.hbase.filter.Filter;
  62 import org.apache.hadoop.hbase.filter.FilterList;
  63 import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
  64 import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
  65 import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
  66 import org.apache.hadoop.hbase.filter.QualifierFilter;
  67 import org.apache.hadoop.hbase.filter.RegexStringComparator;
  68 import org.apache.hadoop.hbase.filter.RowFilter;
  69 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
  70 import org.apache.hadoop.hbase.filter.SubstringComparator;
  71 import org.apache.hadoop.hbase.filter.ValueFilter;
  72 import org.apache.hadoop.hbase.io.TimeRange;
  73 import org.apache.hadoop.hbase.io.hfile.BlockCache;
  74 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
  75 import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
  76 import org.apache.hadoop.hbase.regionserver.HRegion;
  77 import org.apache.hadoop.hbase.regionserver.HRegionServer;
  78 import org.apache.hadoop.hbase.regionserver.HStore;
  79 import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
  80 import org.apache.hadoop.hbase.testclassification.ClientTests;
  81 import org.apache.hadoop.hbase.testclassification.LargeTests;
  82 import org.apache.hadoop.hbase.util.Bytes;
  83 import org.apache.hadoop.hbase.util.CommonFSUtils;
  84 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
  85 import org.apache.hadoop.hbase.util.FSUtils;
  86 import org.junit.AfterClass;
  87 import org.junit.ClassRule;
  88 import org.junit.Ignore;
  89 import org.junit.Rule;
  90 import org.junit.Test;
  91 import org.junit.experimental.categories.Category;
  92 import org.junit.runner.RunWith;
  93 import org.junit.runners.Parameterized;
  94 import org.slf4j.Logger;
  95 import org.slf4j.LoggerFactory;
  96
  97 import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
  98 import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto;
  99 import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto.MutationType;
 100 import org.apache.hadoop.hbase.shaded.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService;
 101 import org.apache.hadoop.hbase.shaded.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest;
 102 import org.apache.hadoop.hbase.shaded.protobuf.generated.MultiRowMutationProtos.MutateRowsResponse;
 103
 104 /**
 105  * Run tests that use the HBase clients; {@link Table}.
 106  * Sets up the HBase mini cluster once at start and runs through all client tests.
 107  * Each creates a table named for the method and does its stuff against that.
 108  *
 109  * Parameterized to run with different registry implementations.
 110  */
 111 @Category({LargeTests.class, ClientTests.class})
 112 @SuppressWarnings ("deprecation")
 113 @RunWith(Parameterized.class)
 114 public class TestFromClientSide5 extends FromClientSideBase {
 115   private static final Logger LOG = LoggerFactory.getLogger(TestFromClientSide5.class);
 116
 117   @ClassRule
 118   public static final HBaseClassTestRule CLASS_RULE =
 119       HBaseClassTestRule.forClass(TestFromClientSide5.class);
 120   @Rule
 121   public TableNameTestRule name = new TableNameTestRule();
 122
 123   // To keep the child classes happy.
 124   TestFromClientSide5() {}
 125
 126   public TestFromClientSide5(Class registry, int numHedgedReqs) throws Exception {
 127     initialize(registry, numHedgedReqs, MultiRowMutationEndpoint.class);
 128   }
 129
 130   @Parameterized.Parameters
 131   public static Collection parameters() {
 132     return Arrays.asList(new Object[][] {
 133         { MasterRegistry.class, 1},
 134         { MasterRegistry.class, 2},
 135         { ZKConnectionRegistry.class, 1}
 136     });
 137   }
 138
 139   @AfterClass public static void tearDownAfterClass() throws Exception {
 140     afterClass();
 141   }
 142
 143   @Test
 144   public void testGetClosestRowBefore() throws IOException, InterruptedException {
 145     final TableName tableName = name.getTableName();
 146     final byte[] firstRow = Bytes.toBytes("row111");
 147     final byte[] secondRow = Bytes.toBytes("row222");
 148     final byte[] thirdRow = Bytes.toBytes("row333");
 149     final byte[] forthRow = Bytes.toBytes("row444");
 150     final byte[] beforeFirstRow = Bytes.toBytes("row");
 151     final byte[] beforeSecondRow = Bytes.toBytes("row22");
 152     final byte[] beforeThirdRow = Bytes.toBytes("row33");
 153     final byte[] beforeForthRow = Bytes.toBytes("row44");
 154
 155     try (Table table =
 156         TEST_UTIL.createTable(tableName,
 157           new byte[][] { HConstants.CATALOG_FAMILY, Bytes.toBytes("info2") }, 1, 1024);
 158       RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
 159
 160       // set block size to 64 to making 2 kvs into one block, bypassing the walkForwardInSingleRow
 161       // in Store.rowAtOrBeforeFromStoreFile
 162       String regionName = locator.getAllRegionLocations().get(0).getRegion().getEncodedName();
 163       HRegion region = TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
 164       Put put1 = new Put(firstRow);
 165       Put put2 = new Put(secondRow);
 166       Put put3 = new Put(thirdRow);
 167       Put put4 = new Put(forthRow);
 168       byte[] one = new byte[] { 1 };
 169       byte[] two = new byte[] { 2 };
 170       byte[] three = new byte[] { 3 };
 171       byte[] four = new byte[] { 4 };
 172
 173       put1.addColumn(HConstants.CATALOG_FAMILY, null, one);
 174       put2.addColumn(HConstants.CATALOG_FAMILY, null, two);
 175       put3.addColumn(HConstants.CATALOG_FAMILY, null, three);
 176       put4.addColumn(HConstants.CATALOG_FAMILY, null, four);
 177       table.put(put1);
 178       table.put(put2);
 179       table.put(put3);
 180       table.put(put4);
 181       region.flush(true);
 182
 183       Result result;
 184
 185       // Test before first that null is returned
 186       result = getReverseScanResult(table, beforeFirstRow);
 187       assertNull(result);
 188
 189       // Test at first that first is returned
 190       result = getReverseScanResult(table, firstRow);
 191       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
 192       assertTrue(Bytes.equals(result.getRow(), firstRow));
 193       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));
 194
 195       // Test in between first and second that first is returned
 196       result = getReverseScanResult(table, beforeSecondRow);
 197       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
 198       assertTrue(Bytes.equals(result.getRow(), firstRow));
 199       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));
 200
 201       // Test at second make sure second is returned
 202       result = getReverseScanResult(table, secondRow);
 203       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
 204       assertTrue(Bytes.equals(result.getRow(), secondRow));
 205       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), two));
 206
 207       // Test in second and third, make sure second is returned
 208       result = getReverseScanResult(table, beforeThirdRow);
 209       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
 210       assertTrue(Bytes.equals(result.getRow(), secondRow));
 211       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), two));
 212
 213       // Test at third make sure third is returned
 214       result = getReverseScanResult(table, thirdRow);
 215       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
 216       assertTrue(Bytes.equals(result.getRow(), thirdRow));
 217       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), three));
 218
 219       // Test in third and forth, make sure third is returned
 220       result = getReverseScanResult(table, beforeForthRow);
 221       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
 222       assertTrue(Bytes.equals(result.getRow(), thirdRow));
 223       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), three));
 224
 225       // Test at forth make sure forth is returned
 226       result = getReverseScanResult(table, forthRow);
 227       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
 228       assertTrue(Bytes.equals(result.getRow(), forthRow));
 229       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), four));
 230
 231       // Test after forth make sure forth is returned
 232       result = getReverseScanResult(table, Bytes.add(forthRow, one));
 233       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
 234       assertTrue(Bytes.equals(result.getRow(), forthRow));
 235       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), four));
 236     }
 237   }
 238
 239   private Result getReverseScanResult(Table table, byte[] row) throws IOException {
 240     Scan scan = new Scan().withStartRow(row);
 241     scan.setReadType(ReadType.PREAD);
 242     scan.setReversed(true);
 243     scan.setCaching(1);
 244     scan.addFamily(HConstants.CATALOG_FAMILY);
 245     try (ResultScanner scanner = table.getScanner(scan)) {
 246       return scanner.next();
 247     }
 248   }
 249
 250   /**
 251    * For HBASE-2156
 252    */
 253   @Test
 254   public void testScanVariableReuse() {
 255     Scan scan = new Scan();
 256     scan.addFamily(FAMILY);
 257     scan.addColumn(FAMILY, ROW);
 258
 259     assertEquals(1, scan.getFamilyMap().get(FAMILY).size());
 260
 261     scan = new Scan();
 262     scan.addFamily(FAMILY);
 263
 264     assertNull(scan.getFamilyMap().get(FAMILY));
 265     assertTrue(scan.getFamilyMap().containsKey(FAMILY));
 266   }
 267
 268   @Test
 269   public void testMultiRowMutation() throws Exception {
 270     LOG.info("Starting testMultiRowMutation");
 271     final TableName tableName = name.getTableName();
 272     final byte [] ROW1 = Bytes.toBytes("testRow1");
 273     final byte [] ROW2 = Bytes.toBytes("testRow2");
 274     final byte [] ROW3 = Bytes.toBytes("testRow3");
 275
 276     try (Table t = TEST_UTIL.createTable(tableName, FAMILY)) {
 277       // Add initial data
 278       t.batch(Arrays.asList(
 279         new Put(ROW1).addColumn(FAMILY, QUALIFIER, VALUE),
 280         new Put(ROW2).addColumn(FAMILY, QUALIFIER, Bytes.toBytes(1L)),
 281         new Put(ROW3).addColumn(FAMILY, QUALIFIER, VALUE)
 282       ), new Object[3]);
 283
 284       // Execute MultiRowMutation
 285       Put put = new Put(ROW).addColumn(FAMILY, QUALIFIER, VALUE);
 286       MutationProto m1 = ProtobufUtil.toMutation(MutationType.PUT, put);
 287
 288       Delete delete = new Delete(ROW1);
 289       MutationProto m2 = ProtobufUtil.toMutation(MutationType.DELETE, delete);
 290
 291       Increment increment = new Increment(ROW2).addColumn(FAMILY, QUALIFIER, 1L);
 292       MutationProto m3 = ProtobufUtil.toMutation(MutationType.INCREMENT, increment);
 293
 294       Append append = new Append(ROW3).addColumn(FAMILY, QUALIFIER, VALUE);
 295       MutationProto m4 = ProtobufUtil.toMutation(MutationType.APPEND, append);
 296
 297       MutateRowsRequest.Builder mrmBuilder = MutateRowsRequest.newBuilder();
 298       mrmBuilder.addMutationRequest(m1);
 299       mrmBuilder.addMutationRequest(m2);
 300       mrmBuilder.addMutationRequest(m3);
 301       mrmBuilder.addMutationRequest(m4);
 302
 303       CoprocessorRpcChannel channel = t.coprocessorService(ROW);
 304       MultiRowMutationService.BlockingInterface service =
 305               MultiRowMutationService.newBlockingStub(channel);
 306       MutateRowsResponse response = service.mutateRows(null, mrmBuilder.build());
 307
 308       // Assert
 309       assertTrue(response.getProcessed());
 310
 311       Result r = t.get(new Get(ROW));
 312       assertEquals(Bytes.toString(VALUE), Bytes.toString(r.getValue(FAMILY, QUALIFIER)));
 313
 314       r = t.get(new Get(ROW1));
 315       assertTrue(r.isEmpty());
 316
 317       r = t.get(new Get(ROW2));
 318       assertEquals(2L, Bytes.toLong(r.getValue(FAMILY, QUALIFIER)));
 319
 320       r = t.get(new Get(ROW3));
 321       assertEquals(Bytes.toString(VALUE) + Bytes.toString(VALUE),
 322         Bytes.toString(r.getValue(FAMILY, QUALIFIER)));
 323     }
 324   }
 325
 326   @Test
 327   public void testMultiRowMutationWithSingleConditionWhenConditionMatches() throws Exception {
 328     final TableName tableName = name.getTableName();
 329     final byte [] ROW1 = Bytes.toBytes("testRow1");
 330     final byte [] ROW2 = Bytes.toBytes("testRow2");
 331     final byte [] VALUE1 = Bytes.toBytes("testValue1");
 332     final byte [] VALUE2 = Bytes.toBytes("testValue2");
 333
 334     try (Table t = TEST_UTIL.createTable(tableName, FAMILY)) {
 335       // Add initial data
 336       t.put(new Put(ROW2).addColumn(FAMILY, QUALIFIER, VALUE2));
 337
 338       // Execute MultiRowMutation with conditions
 339       Put put1 = new Put(ROW).addColumn(FAMILY, QUALIFIER, VALUE);
 340       MutationProto m1 = ProtobufUtil.toMutation(MutationType.PUT, put1);
 341       Put put2 = new Put(ROW1).addColumn(FAMILY, QUALIFIER, VALUE1);
 342       MutationProto m2 = ProtobufUtil.toMutation(MutationType.PUT, put2);
 343       Delete delete = new Delete(ROW2);
 344       MutationProto m3 = ProtobufUtil.toMutation(MutationType.DELETE, delete);
 345
 346       MutateRowsRequest.Builder mrmBuilder = MutateRowsRequest.newBuilder();
 347       mrmBuilder.addMutationRequest(m1);
 348       mrmBuilder.addMutationRequest(m2);
 349       mrmBuilder.addMutationRequest(m3);
 350       mrmBuilder.addCondition(ProtobufUtil.toCondition(ROW2, FAMILY, QUALIFIER,
 351         CompareOperator.EQUAL, VALUE2, null));
 352
 353       CoprocessorRpcChannel channel = t.coprocessorService(ROW);
 354       MultiRowMutationService.BlockingInterface service =
 355         MultiRowMutationService.newBlockingStub(channel);
 356       MutateRowsResponse response = service.mutateRows(null, mrmBuilder.build());
 357
 358       // Assert
 359       assertTrue(response.getProcessed());
 360
 361       Result r = t.get(new Get(ROW));
 362       assertEquals(Bytes.toString(VALUE), Bytes.toString(r.getValue(FAMILY, QUALIFIER)));
 363
 364       r = t.get(new Get(ROW1));
 365       assertEquals(Bytes.toString(VALUE1), Bytes.toString(r.getValue(FAMILY, QUALIFIER)));
 366
 367       r = t.get(new Get(ROW2));
 368       assertTrue(r.isEmpty());
 369     }
 370   }
 371
 372   @Test
 373   public void testMultiRowMutationWithSingleConditionWhenConditionNotMatch() throws Exception {
 374     final TableName tableName = name.getTableName();
 375     final byte [] ROW1 = Bytes.toBytes("testRow1");
 376     final byte [] ROW2 = Bytes.toBytes("testRow2");
 377     final byte [] VALUE1 = Bytes.toBytes("testValue1");
 378     final byte [] VALUE2 = Bytes.toBytes("testValue2");
 379
 380     try (Table t = TEST_UTIL.createTable(tableName, FAMILY)) {
 381       // Add initial data
 382       t.put(new Put(ROW2).addColumn(FAMILY, QUALIFIER, VALUE2));
 383
 384       // Execute MultiRowMutation with conditions
 385       Put put1 = new Put(ROW).addColumn(FAMILY, QUALIFIER, VALUE);
 386       MutationProto m1 = ProtobufUtil.toMutation(MutationType.PUT, put1);
 387       Put put2 = new Put(ROW1).addColumn(FAMILY, QUALIFIER, VALUE1);
 388       MutationProto m2 = ProtobufUtil.toMutation(MutationType.PUT, put2);
 389       Delete delete = new Delete(ROW2);
 390       MutationProto m3 = ProtobufUtil.toMutation(MutationType.DELETE, delete);
 391
 392       MutateRowsRequest.Builder mrmBuilder = MutateRowsRequest.newBuilder();
 393       mrmBuilder.addMutationRequest(m1);
 394       mrmBuilder.addMutationRequest(m2);
 395       mrmBuilder.addMutationRequest(m3);
 396       mrmBuilder.addCondition(ProtobufUtil.toCondition(ROW2, FAMILY, QUALIFIER,
 397         CompareOperator.EQUAL, VALUE1, null));
 398
 399       CoprocessorRpcChannel channel = t.coprocessorService(ROW);
 400       MultiRowMutationService.BlockingInterface service =
 401         MultiRowMutationService.newBlockingStub(channel);
 402       MutateRowsResponse response = service.mutateRows(null, mrmBuilder.build());
 403
 404       // Assert
 405       assertFalse(response.getProcessed());
 406
 407       Result r = t.get(new Get(ROW));
 408       assertTrue(r.isEmpty());
 409
 410       r = t.get(new Get(ROW1));
 411       assertTrue(r.isEmpty());
 412
 413       r = t.get(new Get(ROW2));
 414       assertEquals(Bytes.toString(VALUE2), Bytes.toString(r.getValue(FAMILY, QUALIFIER)));
 415     }
 416   }
 417
 418   @Test
 419   public void testMultiRowMutationWithMultipleConditionsWhenConditionsMatch() throws Exception {
 420     final TableName tableName = name.getTableName();
 421     final byte [] ROW1 = Bytes.toBytes("testRow1");
 422     final byte [] ROW2 = Bytes.toBytes("testRow2");
 423     final byte [] VALUE1 = Bytes.toBytes("testValue1");
 424     final byte [] VALUE2 = Bytes.toBytes("testValue2");
 425
 426     try (Table t = TEST_UTIL.createTable(tableName, FAMILY)) {
 427       // Add initial data
 428       t.put(new Put(ROW2).addColumn(FAMILY, QUALIFIER, VALUE2));
 429
 430       // Execute MultiRowMutation with conditions
 431       Put put1 = new Put(ROW).addColumn(FAMILY, QUALIFIER, VALUE);
 432       MutationProto m1 = ProtobufUtil.toMutation(MutationType.PUT, put1);
 433       Put put2 = new Put(ROW1).addColumn(FAMILY, QUALIFIER, VALUE1);
 434       MutationProto m2 = ProtobufUtil.toMutation(MutationType.PUT, put2);
 435       Delete delete = new Delete(ROW2);
 436       MutationProto m3 = ProtobufUtil.toMutation(MutationType.DELETE, delete);
 437
 438       MutateRowsRequest.Builder mrmBuilder = MutateRowsRequest.newBuilder();
 439       mrmBuilder.addMutationRequest(m1);
 440       mrmBuilder.addMutationRequest(m2);
 441       mrmBuilder.addMutationRequest(m3);
 442       mrmBuilder.addCondition(ProtobufUtil.toCondition(ROW, FAMILY, QUALIFIER,
 443         CompareOperator.EQUAL, null, null));
 444       mrmBuilder.addCondition(ProtobufUtil.toCondition(ROW2, FAMILY, QUALIFIER,
 445         CompareOperator.EQUAL, VALUE2, null));
 446
 447       CoprocessorRpcChannel channel = t.coprocessorService(ROW);
 448       MultiRowMutationService.BlockingInterface service =
 449         MultiRowMutationService.newBlockingStub(channel);
 450       MutateRowsResponse response = service.mutateRows(null, mrmBuilder.build());
 451
 452       // Assert
 453       assertTrue(response.getProcessed());
 454
 455       Result r = t.get(new Get(ROW));
 456       assertEquals(Bytes.toString(VALUE), Bytes.toString(r.getValue(FAMILY, QUALIFIER)));
 457
 458       r = t.get(new Get(ROW1));
 459       assertEquals(Bytes.toString(VALUE1), Bytes.toString(r.getValue(FAMILY, QUALIFIER)));
 460
 461       r = t.get(new Get(ROW2));
 462       assertTrue(r.isEmpty());
 463     }
 464   }
 465
 466   @Test
 467   public void testMultiRowMutationWithMultipleConditionsWhenConditionsNotMatch() throws Exception {
 468     final TableName tableName = name.getTableName();
 469     final byte [] ROW1 = Bytes.toBytes("testRow1");
 470     final byte [] ROW2 = Bytes.toBytes("testRow2");
 471     final byte [] VALUE1 = Bytes.toBytes("testValue1");
 472     final byte [] VALUE2 = Bytes.toBytes("testValue2");
 473
 474     try (Table t = TEST_UTIL.createTable(tableName, FAMILY)) {
 475       // Add initial data
 476       t.put(new Put(ROW2).addColumn(FAMILY, QUALIFIER, VALUE2));
 477
 478       // Execute MultiRowMutation with conditions
 479       Put put1 = new Put(ROW).addColumn(FAMILY, QUALIFIER, VALUE);
 480       MutationProto m1 = ProtobufUtil.toMutation(MutationType.PUT, put1);
 481       Put put2 = new Put(ROW1).addColumn(FAMILY, QUALIFIER, VALUE1);
 482       MutationProto m2 = ProtobufUtil.toMutation(MutationType.PUT, put2);
 483       Delete delete = new Delete(ROW2);
 484       MutationProto m3 = ProtobufUtil.toMutation(MutationType.DELETE, delete);
 485
 486       MutateRowsRequest.Builder mrmBuilder = MutateRowsRequest.newBuilder();
 487       mrmBuilder.addMutationRequest(m1);
 488       mrmBuilder.addMutationRequest(m2);
 489       mrmBuilder.addMutationRequest(m3);
 490       mrmBuilder.addCondition(ProtobufUtil.toCondition(ROW1, FAMILY, QUALIFIER,
 491         CompareOperator.EQUAL, null, null));
 492       mrmBuilder.addCondition(ProtobufUtil.toCondition(ROW2, FAMILY, QUALIFIER,
 493         CompareOperator.EQUAL, VALUE1, null));
 494
 495       CoprocessorRpcChannel channel = t.coprocessorService(ROW);
 496       MultiRowMutationService.BlockingInterface service =
 497         MultiRowMutationService.newBlockingStub(channel);
 498       MutateRowsResponse response = service.mutateRows(null, mrmBuilder.build());
 499
 500       // Assert
 501       assertFalse(response.getProcessed());
 502
 503       Result r = t.get(new Get(ROW));
 504       assertTrue(r.isEmpty());
 505
 506       r = t.get(new Get(ROW1));
 507       assertTrue(r.isEmpty());
 508
 509       r = t.get(new Get(ROW2));
 510       assertEquals(Bytes.toString(VALUE2), Bytes.toString(r.getValue(FAMILY, QUALIFIER)));
 511     }
 512   }
 513
 514   @Test
 515   public void testMultiRowMutationWithFilterConditionWhenConditionMatches() throws Exception {
 516     final TableName tableName = name.getTableName();
 517     final byte [] ROW1 = Bytes.toBytes("testRow1");
 518     final byte [] ROW2 = Bytes.toBytes("testRow2");
 519     final byte [] QUALIFIER2 = Bytes.toBytes("testQualifier2");
 520     final byte [] VALUE1 = Bytes.toBytes("testValue1");
 521     final byte [] VALUE2 = Bytes.toBytes("testValue2");
 522     final byte [] VALUE3 = Bytes.toBytes("testValue3");
 523
 524     try (Table t = TEST_UTIL.createTable(tableName, FAMILY)) {
 525       // Add initial data
 526       t.put(new Put(ROW2).addColumn(FAMILY, QUALIFIER, VALUE2)
 527         .addColumn(FAMILY, QUALIFIER2, VALUE3));
 528
 529       // Execute MultiRowMutation with conditions
 530       Put put1 = new Put(ROW).addColumn(FAMILY, QUALIFIER, VALUE);
 531       MutationProto m1 = ProtobufUtil.toMutation(MutationType.PUT, put1);
 532       Put put2 = new Put(ROW1).addColumn(FAMILY, QUALIFIER, VALUE1);
 533       MutationProto m2 = ProtobufUtil.toMutation(MutationType.PUT, put2);
 534       Delete delete = new Delete(ROW2);
 535       MutationProto m3 = ProtobufUtil.toMutation(MutationType.DELETE, delete);
 536
 537       MutateRowsRequest.Builder mrmBuilder = MutateRowsRequest.newBuilder();
 538       mrmBuilder.addMutationRequest(m1);
 539       mrmBuilder.addMutationRequest(m2);
 540       mrmBuilder.addMutationRequest(m3);
 541       mrmBuilder.addCondition(ProtobufUtil.toCondition(ROW2, new FilterList(
 542         new SingleColumnValueFilter(FAMILY, QUALIFIER, CompareOperator.EQUAL, VALUE2),
 543         new SingleColumnValueFilter(FAMILY, QUALIFIER2, CompareOperator.EQUAL, VALUE3)), null));
 544
 545       CoprocessorRpcChannel channel = t.coprocessorService(ROW);
 546       MultiRowMutationService.BlockingInterface service =
 547         MultiRowMutationService.newBlockingStub(channel);
 548       MutateRowsResponse response = service.mutateRows(null, mrmBuilder.build());
 549
 550       // Assert
 551       assertTrue(response.getProcessed());
 552
 553       Result r = t.get(new Get(ROW));
 554       assertEquals(Bytes.toString(VALUE), Bytes.toString(r.getValue(FAMILY, QUALIFIER)));
 555
 556       r = t.get(new Get(ROW1));
 557       assertEquals(Bytes.toString(VALUE1), Bytes.toString(r.getValue(FAMILY, QUALIFIER)));
 558
 559       r = t.get(new Get(ROW2));
 560       assertTrue(r.isEmpty());
 561     }
 562   }
 563
 564   @Test
 565   public void testMultiRowMutationWithFilterConditionWhenConditionNotMatch() throws Exception {
 566     final TableName tableName = name.getTableName();
 567     final byte [] ROW1 = Bytes.toBytes("testRow1");
 568     final byte [] ROW2 = Bytes.toBytes("testRow2");
 569     final byte [] QUALIFIER2 = Bytes.toBytes("testQualifier2");
 570     final byte [] VALUE1 = Bytes.toBytes("testValue1");
 571     final byte [] VALUE2 = Bytes.toBytes("testValue2");
 572     final byte [] VALUE3 = Bytes.toBytes("testValue3");
 573
 574     try (Table t = TEST_UTIL.createTable(tableName, FAMILY)) {
 575       // Add initial data
 576       t.put(new Put(ROW2).addColumn(FAMILY, QUALIFIER, VALUE2)
 577         .addColumn(FAMILY, QUALIFIER2, VALUE3));
 578
 579       // Execute MultiRowMutation with conditions
 580       Put put1 = new Put(ROW).addColumn(FAMILY, QUALIFIER, VALUE);
 581       MutationProto m1 = ProtobufUtil.toMutation(MutationType.PUT, put1);
 582       Put put2 = new Put(ROW1).addColumn(FAMILY, QUALIFIER, VALUE1);
 583       MutationProto m2 = ProtobufUtil.toMutation(MutationType.PUT, put2);
 584       Delete delete = new Delete(ROW2);
 585       MutationProto m3 = ProtobufUtil.toMutation(MutationType.DELETE, delete);
 586
 587       MutateRowsRequest.Builder mrmBuilder = MutateRowsRequest.newBuilder();
 588       mrmBuilder.addMutationRequest(m1);
 589       mrmBuilder.addMutationRequest(m2);
 590       mrmBuilder.addMutationRequest(m3);
 591       mrmBuilder.addCondition(ProtobufUtil.toCondition(ROW2, new FilterList(
 592         new SingleColumnValueFilter(FAMILY, QUALIFIER, CompareOperator.EQUAL, VALUE2),
 593         new SingleColumnValueFilter(FAMILY, QUALIFIER2, CompareOperator.EQUAL, VALUE2)), null));
 594
 595       CoprocessorRpcChannel channel = t.coprocessorService(ROW);
 596       MultiRowMutationService.BlockingInterface service =
 597         MultiRowMutationService.newBlockingStub(channel);
 598       MutateRowsResponse response = service.mutateRows(null, mrmBuilder.build());
 599
 600       // Assert
 601       assertFalse(response.getProcessed());
 602
 603       Result r = t.get(new Get(ROW));
 604       assertTrue(r.isEmpty());
 605
 606       r = t.get(new Get(ROW1));
 607       assertTrue(r.isEmpty());
 608
 609       r = t.get(new Get(ROW2));
 610       assertEquals(Bytes.toString(VALUE2), Bytes.toString(r.getValue(FAMILY, QUALIFIER)));
 611     }
 612   }
 613
 614   @Test
 615   public void testRowMutations() throws Exception {
 616     LOG.info("Starting testRowMutations");
 617     final TableName tableName = name.getTableName();
 618     try (Table t = TEST_UTIL.createTable(tableName, FAMILY)) {
 619       byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b"),
 620         Bytes.toBytes("c"), Bytes.toBytes("d") };
 621
 622       // Test for Put operations
 623       RowMutations arm = new RowMutations(ROW);
 624       Put p = new Put(ROW);
 625       p.addColumn(FAMILY, QUALIFIERS[0], VALUE);
 626       arm.add(p);
 627       Result r = t.mutateRow(arm);
 628       assertTrue(r.getExists());
 629       assertTrue(r.isEmpty());
 630
 631       Get g = new Get(ROW);
 632       r = t.get(g);
 633       assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[0])));
 634
 635       // Test for Put and Delete operations
 636       arm = new RowMutations(ROW);
 637       p = new Put(ROW);
 638       p.addColumn(FAMILY, QUALIFIERS[1], VALUE);
 639       arm.add(p);
 640       Delete d = new Delete(ROW);
 641       d.addColumns(FAMILY, QUALIFIERS[0]);
 642       arm.add(d);
 643       // TODO: Trying mutateRow again. The batch was failing with a one try only.
 644       r = t.mutateRow(arm);
 645       assertTrue(r.getExists());
 646       assertTrue(r.isEmpty());
 647
 648       r = t.get(g);
 649       assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[1])));
 650       assertNull(r.getValue(FAMILY, QUALIFIERS[0]));
 651
 652       // Test for Increment and Append operations
 653       arm = new RowMutations(ROW);
 654       arm.add(Arrays.asList(
 655         new Put(ROW).addColumn(FAMILY, QUALIFIERS[0], VALUE),
 656         new Delete(ROW).addColumns(FAMILY, QUALIFIERS[1]),
 657         new Increment(ROW).addColumn(FAMILY, QUALIFIERS[2], 5L),
 658         new Append(ROW).addColumn(FAMILY, QUALIFIERS[3], Bytes.toBytes("abc"))
 659       ));
 660       r = t.mutateRow(arm);
 661       assertTrue(r.getExists());
 662       assertEquals(5L, Bytes.toLong(r.getValue(FAMILY, QUALIFIERS[2])));
 663       assertEquals("abc", Bytes.toString(r.getValue(FAMILY, QUALIFIERS[3])));
 664
 665       g = new Get(ROW);
 666       r = t.get(g);
 667       assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[0])));
 668       assertNull(r.getValue(FAMILY, QUALIFIERS[1]));
 669       assertEquals(5L, Bytes.toLong(r.getValue(FAMILY, QUALIFIERS[2])));
 670       assertEquals("abc", Bytes.toString(r.getValue(FAMILY, QUALIFIERS[3])));
 671
 672       // Test that we get a region level exception
 673       try {
 674         arm = new RowMutations(ROW);
 675         p = new Put(ROW);
 676         p.addColumn(new byte[] { 'b', 'o', 'g', 'u', 's' }, QUALIFIERS[0], VALUE);
 677         arm.add(p);
 678         t.mutateRow(arm);
 679         fail("Expected NoSuchColumnFamilyException");
 680       } catch (NoSuchColumnFamilyException e) {
 681         return;
 682       } catch (RetriesExhaustedWithDetailsException e) {
 683         for (Throwable rootCause : e.getCauses()) {
 684           if (rootCause instanceof NoSuchColumnFamilyException) {
 685             return;
 686           }
 687         }
 688         throw e;
 689       }
 690     }
 691   }
 692
 693   @Test
 694   public void testBatchAppendWithReturnResultFalse() throws Exception {
 695     LOG.info("Starting testBatchAppendWithReturnResultFalse");
 696     final TableName tableName = name.getTableName();
 697     try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) {
 698       Append append1 = new Append(Bytes.toBytes("row1"));
 699       append1.setReturnResults(false);
 700       append1.addColumn(FAMILY, Bytes.toBytes("f1"), Bytes.toBytes("value1"));
 701       Append append2 = new Append(Bytes.toBytes("row1"));
 702       append2.setReturnResults(false);
 703       append2.addColumn(FAMILY, Bytes.toBytes("f1"), Bytes.toBytes("value2"));
 704       List<Append> appends = new ArrayList<>();
 705       appends.add(append1);
 706       appends.add(append2);
 707       Object[] results = new Object[2];
 708       table.batch(appends, results);
 709       assertEquals(2, results.length);
 710       for (Object r : results) {
 711         Result result = (Result) r;
 712         assertTrue(result.isEmpty());
 713       }
 714     }
 715   }
 716
 717   @Test
 718   public void testAppend() throws Exception {
 719     LOG.info("Starting testAppend");
 720     final TableName tableName = name.getTableName();
 721     try (Table t = TEST_UTIL.createTable(tableName, FAMILY)) {
 722       byte[] v1 = Bytes.toBytes("42");
 723       byte[] v2 = Bytes.toBytes("23");
 724       byte[][] QUALIFIERS = new byte[][]{
 725               Bytes.toBytes("b"), Bytes.toBytes("a"), Bytes.toBytes("c")
 726       };
 727       Append a = new Append(ROW);
 728       a.addColumn(FAMILY, QUALIFIERS[0], v1);
 729       a.addColumn(FAMILY, QUALIFIERS[1], v2);
 730       a.setReturnResults(false);
 731       assertEmptyResult(t.append(a));
 732
 733       a = new Append(ROW);
 734       a.addColumn(FAMILY, QUALIFIERS[0], v2);
 735       a.addColumn(FAMILY, QUALIFIERS[1], v1);
 736       a.addColumn(FAMILY, QUALIFIERS[2], v2);
 737       Result r = t.append(a);
 738       assertEquals(0, Bytes.compareTo(Bytes.add(v1, v2), r.getValue(FAMILY, QUALIFIERS[0])));
 739       assertEquals(0, Bytes.compareTo(Bytes.add(v2, v1), r.getValue(FAMILY, QUALIFIERS[1])));
 740       // QUALIFIERS[2] previously not exist, verify both value and timestamp are correct
 741       assertEquals(0, Bytes.compareTo(v2, r.getValue(FAMILY, QUALIFIERS[2])));
 742       assertEquals(r.getColumnLatestCell(FAMILY, QUALIFIERS[0]).getTimestamp(),
 743               r.getColumnLatestCell(FAMILY, QUALIFIERS[2]).getTimestamp());
 744     }
 745   }
 746   private List<Result> doAppend(final boolean walUsed) throws IOException {
 747     LOG.info("Starting testAppend, walUsed is " + walUsed);
 748     final TableName TABLENAME =
 749             TableName.valueOf(walUsed ? "testAppendWithWAL" : "testAppendWithoutWAL");
 750     try (Table t = TEST_UTIL.createTable(TABLENAME, FAMILY)) {
 751       final byte[] row1 = Bytes.toBytes("c");
 752       final byte[] row2 = Bytes.toBytes("b");
 753       final byte[] row3 = Bytes.toBytes("a");
 754       final byte[] qual = Bytes.toBytes("qual");
 755       Put put_0 = new Put(row2);
 756       put_0.addColumn(FAMILY, qual, Bytes.toBytes("put"));
 757       Put put_1 = new Put(row3);
 758       put_1.addColumn(FAMILY, qual, Bytes.toBytes("put"));
 759       Append append_0 = new Append(row1);
 760       append_0.addColumn(FAMILY, qual, Bytes.toBytes("i"));
 761       Append append_1 = new Append(row1);
 762       append_1.addColumn(FAMILY, qual, Bytes.toBytes("k"));
 763       Append append_2 = new Append(row1);
 764       append_2.addColumn(FAMILY, qual, Bytes.toBytes("e"));
 765       if (!walUsed) {
 766         append_2.setDurability(Durability.SKIP_WAL);
 767       }
 768       Append append_3 = new Append(row1);
 769       append_3.addColumn(FAMILY, qual, Bytes.toBytes("a"));
 770       Scan s = new Scan();
 771       s.setCaching(1);
 772       t.append(append_0);
 773       t.put(put_0);
 774       t.put(put_1);
 775       List<Result> results = new LinkedList<>();
 776       try (ResultScanner scanner = t.getScanner(s)) {
 777         t.append(append_1);
 778         t.append(append_2);
 779         t.append(append_3);
 780         for (Result r : scanner) {
 781           results.add(r);
 782         }
 783       }
 784       TEST_UTIL.deleteTable(TABLENAME);
 785       return results;
 786     }
 787   }
 788
 789   @Test
 790   public void testAppendWithoutWAL() throws Exception {
 791     List<Result> resultsWithWal = doAppend(true);
 792     List<Result> resultsWithoutWal = doAppend(false);
 793     assertEquals(resultsWithWal.size(), resultsWithoutWal.size());
 794     for (int i = 0; i != resultsWithWal.size(); ++i) {
 795       Result resultWithWal = resultsWithWal.get(i);
 796       Result resultWithoutWal = resultsWithoutWal.get(i);
 797       assertEquals(resultWithWal.rawCells().length, resultWithoutWal.rawCells().length);
 798       for (int j = 0; j != resultWithWal.rawCells().length; ++j) {
 799         Cell cellWithWal = resultWithWal.rawCells()[j];
 800         Cell cellWithoutWal = resultWithoutWal.rawCells()[j];
 801         assertArrayEquals(CellUtil.cloneRow(cellWithWal), CellUtil.cloneRow(cellWithoutWal));
 802         assertArrayEquals(CellUtil.cloneFamily(cellWithWal), CellUtil.cloneFamily(cellWithoutWal));
 803         assertArrayEquals(CellUtil.cloneQualifier(cellWithWal),
 804           CellUtil.cloneQualifier(cellWithoutWal));
 805         assertArrayEquals(CellUtil.cloneValue(cellWithWal), CellUtil.cloneValue(cellWithoutWal));
 806       }
 807     }
 808   }
 809
 810   @Test
 811   public void testClientPoolRoundRobin() throws IOException {
 812     final TableName tableName = name.getTableName();
 813
 814     int poolSize = 3;
 815     int numVersions = poolSize * 2;
 816     Configuration conf = TEST_UTIL.getConfiguration();
 817     conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "round-robin");
 818     conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize);
 819
 820     try (Table table =
 821                  TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, Integer.MAX_VALUE)) {
 822
 823       final long ts = EnvironmentEdgeManager.currentTime();
 824       Get get = new Get(ROW);
 825       get.addColumn(FAMILY, QUALIFIER);
 826       get.readAllVersions();
 827
 828       for (int versions = 1; versions <= numVersions; versions++) {
 829         Put put = new Put(ROW);
 830         put.addColumn(FAMILY, QUALIFIER, ts + versions, VALUE);
 831         table.put(put);
 832
 833         Result result = table.get(get);
 834         NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY)
 835                 .get(QUALIFIER);
 836
 837         assertEquals("The number of versions of '" + Bytes.toString(FAMILY) + ":"
 838                 + Bytes.toString(QUALIFIER) + " did not match", versions, navigableMap.size());
 839         for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
 840           assertTrue("The value at time " + entry.getKey()
 841                           + " did not match what was put",
 842                   Bytes.equals(VALUE, entry.getValue()));
 843         }
 844       }
 845     }
 846   }
 847
 848   @Ignore ("Flakey: HBASE-8989") @Test
 849   public void testClientPoolThreadLocal() throws IOException {
 850     final TableName tableName = name.getTableName();
 851
 852     int poolSize = Integer.MAX_VALUE;
 853     int numVersions = 3;
 854     Configuration conf = TEST_UTIL.getConfiguration();
 855     conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "thread-local");
 856     conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize);
 857
 858     try (final Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY },  3)) {
 859
 860       final long ts = EnvironmentEdgeManager.currentTime();
 861       final Get get = new Get(ROW);
 862       get.addColumn(FAMILY, QUALIFIER);
 863       get.readAllVersions();
 864
 865       for (int versions = 1; versions <= numVersions; versions++) {
 866         Put put = new Put(ROW);
 867         put.addColumn(FAMILY, QUALIFIER, ts + versions, VALUE);
 868         table.put(put);
 869
 870         Result result = table.get(get);
 871         NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY)
 872                 .get(QUALIFIER);
 873
 874         assertEquals("The number of versions of '" + Bytes.toString(FAMILY) + ":"
 875                 + Bytes.toString(QUALIFIER) + " did not match", versions, navigableMap.size());
 876         for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
 877           assertTrue("The value at time " + entry.getKey()
 878                           + " did not match what was put",
 879                   Bytes.equals(VALUE, entry.getValue()));
 880         }
 881       }
 882
 883       final Object waitLock = new Object();
 884       ExecutorService executorService = Executors.newFixedThreadPool(numVersions);
 885       final AtomicReference<AssertionError> error = new AtomicReference<>(null);
 886       for (int versions = numVersions; versions < numVersions * 2; versions++) {
 887         final int versionsCopy = versions;
 888         executorService.submit((Callable<Void>) () -> {
 889           try {
 890             Put put = new Put(ROW);
 891             put.addColumn(FAMILY, QUALIFIER, ts + versionsCopy, VALUE);
 892             table.put(put);
 893
 894             Result result = table.get(get);
 895             NavigableMap<Long, byte[]> navigableMap = result.getMap()
 896                     .get(FAMILY).get(QUALIFIER);
 897
 898             assertEquals("The number of versions of '" + Bytes.toString(FAMILY) + ":"
 899                     + Bytes.toString(QUALIFIER) + " did not match " + versionsCopy, versionsCopy,
 900                     navigableMap.size());
 901             for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
 902               assertTrue("The value at time " + entry.getKey()
 903                               + " did not match what was put",
 904                       Bytes.equals(VALUE, entry.getValue()));
 905             }
 906             synchronized (waitLock) {
 907               waitLock.wait();
 908             }
 909           } catch (Exception ignored) {
 910           } catch (AssertionError e) {
 911             // the error happens in a thread, it won't fail the test,
 912             // need to pass it to the caller for proper handling.
 913             error.set(e);
 914             LOG.error(e.toString(), e);
 915           }
 916
 917           return null;
 918         });
 919       }
 920       synchronized (waitLock) {
 921         waitLock.notifyAll();
 922       }
 923       executorService.shutdownNow();
 924       assertNull(error.get());
 925     }
 926   }
 927
 928   @Test
 929   public void testCheckAndPut() throws IOException {
 930     final byte [] anotherrow = Bytes.toBytes("anotherrow");
 931     final byte [] value2 = Bytes.toBytes("abcd");
 932
 933     try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
 934       Put put1 = new Put(ROW);
 935       put1.addColumn(FAMILY, QUALIFIER, VALUE);
 936
 937       // row doesn't exist, so using non-null value should be considered "not match".
 938       boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
 939               .ifEquals(VALUE).thenPut(put1);
 940       assertFalse(ok);
 941
 942       // row doesn't exist, so using "ifNotExists" should be considered "match".
 943       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifNotExists().thenPut(put1);
 944       assertTrue(ok);
 945
 946       // row now exists, so using "ifNotExists" should be considered "not match".
 947       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifNotExists().thenPut(put1);
 948       assertFalse(ok);
 949
 950       Put put2 = new Put(ROW);
 951       put2.addColumn(FAMILY, QUALIFIER, value2);
 952
 953       // row now exists, use the matching value to check
 954       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifEquals(VALUE).thenPut(put2);
 955       assertTrue(ok);
 956
 957       Put put3 = new Put(anotherrow);
 958       put3.addColumn(FAMILY, QUALIFIER, VALUE);
 959
 960       // try to do CheckAndPut on different rows
 961       try {
 962         table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifEquals(value2).thenPut(put3);
 963         fail("trying to check and modify different rows should have failed.");
 964       } catch (Exception ignored) {
 965       }
 966     }
 967   }
 968
 969   @Test
 970   public void testCheckAndMutateWithTimeRange() throws IOException {
 971     try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
 972       final long ts = EnvironmentEdgeManager.currentTime() / 2;
 973       Put put = new Put(ROW);
 974       put.addColumn(FAMILY, QUALIFIER, ts, VALUE);
 975
 976       boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
 977               .ifNotExists()
 978               .thenPut(put);
 979       assertTrue(ok);
 980
 981       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
 982               .timeRange(TimeRange.at(ts + 10000))
 983               .ifEquals(VALUE)
 984               .thenPut(put);
 985       assertFalse(ok);
 986
 987       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
 988               .timeRange(TimeRange.from(ts + 10000))
 989               .ifEquals(VALUE)
 990               .thenPut(put);
 991       assertFalse(ok);
 992
 993       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
 994               .timeRange(TimeRange.between(ts + 10000, ts + 20000))
 995               .ifEquals(VALUE)
 996               .thenPut(put);
 997       assertFalse(ok);
 998
 999       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1000               .timeRange(TimeRange.until(ts))
1001               .ifEquals(VALUE)
1002               .thenPut(put);
1003       assertFalse(ok);
1004
1005       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1006               .timeRange(TimeRange.at(ts))
1007               .ifEquals(VALUE)
1008               .thenPut(put);
1009       assertTrue(ok);
1010
1011       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1012               .timeRange(TimeRange.from(ts))
1013               .ifEquals(VALUE)
1014               .thenPut(put);
1015       assertTrue(ok);
1016
1017       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1018               .timeRange(TimeRange.between(ts, ts + 20000))
1019               .ifEquals(VALUE)
1020               .thenPut(put);
1021       assertTrue(ok);
1022
1023       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1024               .timeRange(TimeRange.until(ts + 10000))
1025               .ifEquals(VALUE)
1026               .thenPut(put);
1027       assertTrue(ok);
1028
1029       RowMutations rm = new RowMutations(ROW)
1030               .add((Mutation) put);
1031       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1032               .timeRange(TimeRange.at(ts + 10000))
1033               .ifEquals(VALUE)
1034               .thenMutate(rm);
1035       assertFalse(ok);
1036
1037       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1038               .timeRange(TimeRange.at(ts))
1039               .ifEquals(VALUE)
1040               .thenMutate(rm);
1041       assertTrue(ok);
1042
1043       Delete delete = new Delete(ROW)
1044               .addColumn(FAMILY, QUALIFIER);
1045
1046       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1047               .timeRange(TimeRange.at(ts + 10000))
1048               .ifEquals(VALUE)
1049               .thenDelete(delete);
1050       assertFalse(ok);
1051
1052       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1053               .timeRange(TimeRange.at(ts))
1054               .ifEquals(VALUE)
1055               .thenDelete(delete);
1056       assertTrue(ok);
1057     }
1058   }
1059
1060   @Test
1061   public void testCheckAndPutWithCompareOp() throws IOException {
1062     final byte [] value1 = Bytes.toBytes("aaaa");
1063     final byte [] value2 = Bytes.toBytes("bbbb");
1064     final byte [] value3 = Bytes.toBytes("cccc");
1065     final byte [] value4 = Bytes.toBytes("dddd");
1066
1067     try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
1068
1069       Put put2 = new Put(ROW);
1070       put2.addColumn(FAMILY, QUALIFIER, value2);
1071
1072       Put put3 = new Put(ROW);
1073       put3.addColumn(FAMILY, QUALIFIER, value3);
1074
1075       // row doesn't exist, so using "ifNotExists" should be considered "match".
1076       boolean ok =
1077               table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifNotExists().thenPut(put2);
1078       assertTrue(ok);
1079
1080       // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL
1081       // turns out "match"
1082       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1083               .ifMatches(CompareOperator.GREATER, value1).thenPut(put2);
1084       assertFalse(ok);
1085       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1086               .ifMatches(CompareOperator.EQUAL, value1).thenPut(put2);
1087       assertFalse(ok);
1088       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1089               .ifMatches(CompareOperator.GREATER_OR_EQUAL, value1).thenPut(put2);
1090       assertFalse(ok);
1091       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1092               .ifMatches(CompareOperator.LESS, value1).thenPut(put2);
1093       assertTrue(ok);
1094       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1095               .ifMatches(CompareOperator.LESS_OR_EQUAL, value1).thenPut(put2);
1096       assertTrue(ok);
1097       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1098               .ifMatches(CompareOperator.NOT_EQUAL, value1).thenPut(put3);
1099       assertTrue(ok);
1100
1101       // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL
1102       // turns out "match"
1103       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1104               .ifMatches(CompareOperator.LESS, value4).thenPut(put3);
1105       assertFalse(ok);
1106       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1107               .ifMatches(CompareOperator.LESS_OR_EQUAL, value4).thenPut(put3);
1108       assertFalse(ok);
1109       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1110               .ifMatches(CompareOperator.EQUAL, value4).thenPut(put3);
1111       assertFalse(ok);
1112       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1113               .ifMatches(CompareOperator.GREATER, value4).thenPut(put3);
1114       assertTrue(ok);
1115       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1116               .ifMatches(CompareOperator.GREATER_OR_EQUAL, value4).thenPut(put3);
1117       assertTrue(ok);
1118       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1119               .ifMatches(CompareOperator.NOT_EQUAL, value4).thenPut(put2);
1120       assertTrue(ok);
1121
1122       // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL
1123       // turns out "match"
1124       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1125               .ifMatches(CompareOperator.GREATER, value2).thenPut(put2);
1126       assertFalse(ok);
1127       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1128               .ifMatches(CompareOperator.NOT_EQUAL, value2).thenPut(put2);
1129       assertFalse(ok);
1130       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1131               .ifMatches(CompareOperator.LESS, value2).thenPut(put2);
1132       assertFalse(ok);
1133       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1134               .ifMatches(CompareOperator.GREATER_OR_EQUAL, value2).thenPut(put2);
1135       assertTrue(ok);
1136       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1137               .ifMatches(CompareOperator.LESS_OR_EQUAL, value2).thenPut(put2);
1138       assertTrue(ok);
1139       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1140               .ifMatches(CompareOperator.EQUAL, value2).thenPut(put3);
1141       assertTrue(ok);
1142     }
1143   }
1144
1145   @Test
1146   public void testCheckAndDelete() throws IOException {
1147     final byte [] value1 = Bytes.toBytes("aaaa");
1148
1149     try (Table table = TEST_UTIL.createTable(name.getTableName(),
1150         FAMILY)) {
1151
1152       Put put = new Put(ROW);
1153       put.addColumn(FAMILY, QUALIFIER, value1);
1154       table.put(put);
1155
1156       Delete delete = new Delete(ROW);
1157       delete.addColumns(FAMILY, QUALIFIER);
1158
1159       boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1160               .ifEquals(value1).thenDelete(delete);
1161       assertTrue(ok);
1162     }
1163   }
1164
1165   @Test
1166   public void testCheckAndDeleteWithCompareOp() throws IOException {
1167     final byte [] value1 = Bytes.toBytes("aaaa");
1168     final byte [] value2 = Bytes.toBytes("bbbb");
1169     final byte [] value3 = Bytes.toBytes("cccc");
1170     final byte [] value4 = Bytes.toBytes("dddd");
1171
1172     try (Table table = TEST_UTIL.createTable(name.getTableName(),
1173         FAMILY)) {
1174
1175       Put put2 = new Put(ROW);
1176       put2.addColumn(FAMILY, QUALIFIER, value2);
1177       table.put(put2);
1178
1179       Put put3 = new Put(ROW);
1180       put3.addColumn(FAMILY, QUALIFIER, value3);
1181
1182       Delete delete = new Delete(ROW);
1183       delete.addColumns(FAMILY, QUALIFIER);
1184
1185       // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL
1186       // turns out "match"
1187       boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1188               .ifMatches(CompareOperator.GREATER, value1).thenDelete(delete);
1189       assertFalse(ok);
1190       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1191               .ifMatches(CompareOperator.EQUAL, value1).thenDelete(delete);
1192       assertFalse(ok);
1193       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1194               .ifMatches(CompareOperator.GREATER_OR_EQUAL, value1).thenDelete(delete);
1195       assertFalse(ok);
1196       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1197               .ifMatches(CompareOperator.LESS, value1).thenDelete(delete);
1198       assertTrue(ok);
1199       table.put(put2);
1200       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1201               .ifMatches(CompareOperator.LESS_OR_EQUAL, value1).thenDelete(delete);
1202       assertTrue(ok);
1203       table.put(put2);
1204       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1205               .ifMatches(CompareOperator.NOT_EQUAL, value1).thenDelete(delete);
1206       assertTrue(ok);
1207
1208       // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL
1209       // turns out "match"
1210       table.put(put3);
1211       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1212               .ifMatches(CompareOperator.LESS, value4).thenDelete(delete);
1213       assertFalse(ok);
1214       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1215               .ifMatches(CompareOperator.LESS_OR_EQUAL, value4).thenDelete(delete);
1216       assertFalse(ok);
1217       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1218               .ifMatches(CompareOperator.EQUAL, value4).thenDelete(delete);
1219       assertFalse(ok);
1220       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1221               .ifMatches(CompareOperator.GREATER, value4).thenDelete(delete);
1222       assertTrue(ok);
1223       table.put(put3);
1224       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1225               .ifMatches(CompareOperator.GREATER_OR_EQUAL, value4).thenDelete(delete);
1226       assertTrue(ok);
1227       table.put(put3);
1228       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1229               .ifMatches(CompareOperator.NOT_EQUAL, value4).thenDelete(delete);
1230       assertTrue(ok);
1231
1232       // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL
1233       // turns out "match"
1234       table.put(put2);
1235       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1236               .ifMatches(CompareOperator.GREATER, value2).thenDelete(delete);
1237       assertFalse(ok);
1238       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1239               .ifMatches(CompareOperator.NOT_EQUAL, value2).thenDelete(delete);
1240       assertFalse(ok);
1241       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1242               .ifMatches(CompareOperator.LESS, value2).thenDelete(delete);
1243       assertFalse(ok);
1244       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1245               .ifMatches(CompareOperator.GREATER_OR_EQUAL, value2).thenDelete(delete);
1246       assertTrue(ok);
1247       table.put(put2);
1248       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1249               .ifMatches(CompareOperator.LESS_OR_EQUAL, value2).thenDelete(delete);
1250       assertTrue(ok);
1251       table.put(put2);
1252       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
1253               .ifMatches(CompareOperator.EQUAL, value2).thenDelete(delete);
1254       assertTrue(ok);
1255     }
1256   }
1257
1258   /**
1259   * Test ScanMetrics
1260   */
1261   @Test
1262   @SuppressWarnings({"unused", "checkstyle:EmptyBlock"})
1263   public void testScanMetrics() throws Exception {
1264     final TableName tableName = name.getTableName();
1265
1266     // Set up test table:
1267     // Create table:
1268     try (Table ht = TEST_UTIL.createMultiRegionTable(tableName, FAMILY)) {
1269       int numOfRegions;
1270       try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
1271         numOfRegions = r.getStartKeys().length;
1272       }
1273       // Create 3 rows in the table, with rowkeys starting with "zzz*" so that
1274       // scan are forced to hit all the regions.
1275       Put put1 = new Put(Bytes.toBytes("zzz1"));
1276       put1.addColumn(FAMILY, QUALIFIER, VALUE);
1277       Put put2 = new Put(Bytes.toBytes("zzz2"));
1278       put2.addColumn(FAMILY, QUALIFIER, VALUE);
1279       Put put3 = new Put(Bytes.toBytes("zzz3"));
1280       put3.addColumn(FAMILY, QUALIFIER, VALUE);
1281       ht.put(Arrays.asList(put1, put2, put3));
1282
1283       Scan scan1 = new Scan();
1284       int numRecords = 0;
1285       try (ResultScanner scanner = ht.getScanner(scan1)) {
1286         for (Result result : scanner) {
1287           numRecords++;
1288         }
1289
1290         LOG.info("test data has {} records.", numRecords);
1291
1292         // by default, scan metrics collection is turned off
1293         assertNull(scanner.getScanMetrics());
1294       }
1295
1296       // turn on scan metrics
1297       Scan scan2 = new Scan();
1298       scan2.setScanMetricsEnabled(true);
1299       scan2.setCaching(numRecords + 1);
1300       try (ResultScanner scanner = ht.getScanner(scan2)) {
1301         for (Result result : scanner.next(numRecords - 1)) {
1302         }
1303         assertNotNull(scanner.getScanMetrics());
1304       }
1305
1306       // set caching to 1, because metrics are collected in each roundtrip only
1307       scan2 = new Scan();
1308       scan2.setScanMetricsEnabled(true);
1309       scan2.setCaching(1);
1310       try (ResultScanner scanner = ht.getScanner(scan2)) {
1311         // per HBASE-5717, this should still collect even if you don't run all the way to
1312         // the end of the scanner. So this is asking for 2 of the 3 rows we inserted.
1313         for (Result result : scanner.next(numRecords - 1)) {
1314         }
1315         ScanMetrics scanMetrics = scanner.getScanMetrics();
1316         assertEquals("Did not access all the regions in the table", numOfRegions,
1317           scanMetrics.countOfRegions.get());
1318       }
1319
1320       // check byte counters
1321       scan2 = new Scan();
1322       scan2.setScanMetricsEnabled(true);
1323       scan2.setCaching(1);
1324       try (ResultScanner scanner = ht.getScanner(scan2)) {
1325         int numBytes = 0;
1326         for (Result result : scanner) {
1327           for (Cell cell : result.listCells()) {
1328             numBytes += PrivateCellUtil.estimatedSerializedSizeOf(cell);
1329           }
1330         }
1331         ScanMetrics scanMetrics = scanner.getScanMetrics();
1332         assertEquals("Did not count the result bytes", numBytes,
1333           scanMetrics.countOfBytesInResults.get());
1334       }
1335
1336       // check byte counters on a small scan
1337       scan2 = new Scan();
1338       scan2.setScanMetricsEnabled(true);
1339       scan2.setCaching(1);
1340       scan2.setReadType(ReadType.PREAD);
1341       try (ResultScanner scanner = ht.getScanner(scan2)) {
1342         int numBytes = 0;
1343         for (Result result : scanner) {
1344           for (Cell cell : result.listCells()) {
1345             numBytes += PrivateCellUtil.estimatedSerializedSizeOf(cell);
1346           }
1347         }
1348         ScanMetrics scanMetrics = scanner.getScanMetrics();
1349         assertEquals("Did not count the result bytes", numBytes,
1350           scanMetrics.countOfBytesInResults.get());
1351       }
1352
1353       // now, test that the metrics are still collected even if you don't call close, but do
1354       // run past the end of all the records
1355       /** There seems to be a timing issue here.  Comment out for now. Fix when time.
1356        Scan scanWithoutClose = new Scan();
1357        scanWithoutClose.setCaching(1);
1358        scanWithoutClose.setScanMetricsEnabled(true);
1359        ResultScanner scannerWithoutClose = ht.getScanner(scanWithoutClose);
1360        for (Result result : scannerWithoutClose.next(numRecords + 1)) {
1361        }
1362        ScanMetrics scanMetricsWithoutClose = getScanMetrics(scanWithoutClose);
1363        assertEquals("Did not access all the regions in the table", numOfRegions,
1364        scanMetricsWithoutClose.countOfRegions.get());
1365        */
1366
1367       // finally,
1368       // test that the metrics are collected correctly if you both run past all the records,
1369       // AND close the scanner
1370       Scan scanWithClose = new Scan();
1371       // make sure we can set caching up to the number of a scanned values
1372       scanWithClose.setCaching(numRecords);
1373       scanWithClose.setScanMetricsEnabled(true);
1374       try (ResultScanner scannerWithClose = ht.getScanner(scanWithClose)) {
1375         for (Result result : scannerWithClose.next(numRecords + 1)) {
1376         }
1377         scannerWithClose.close();
1378         ScanMetrics scanMetricsWithClose = scannerWithClose.getScanMetrics();
1379         assertEquals("Did not access all the regions in the table", numOfRegions,
1380           scanMetricsWithClose.countOfRegions.get());
1381       }
1382     } finally {
1383       TEST_UTIL.deleteTable(tableName);
1384     }
1385   }
1386
1387   /**
1388    * Tests that cache on write works all the way up from the client-side.
1389    *
1390    * Performs inserts, flushes, and compactions, verifying changes in the block
1391    * cache along the way.
1392    */
1393   @Test
1394   public void testCacheOnWriteEvictOnClose() throws Exception {
1395     final TableName tableName = name.getTableName();
1396     byte [] data = Bytes.toBytes("data");
1397     try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) {
1398       try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
1399         // get the block cache and region
1400         String regionName = locator.getAllRegionLocations().get(0).getRegion().getEncodedName();
1401
1402         HRegion region = TEST_UTIL.getRSForFirstRegionInTable(tableName)
1403                 .getRegion(regionName);
1404         HStore store = region.getStores().iterator().next();
1405         CacheConfig cacheConf = store.getCacheConfig();
1406         cacheConf.setCacheDataOnWrite(true);
1407         cacheConf.setEvictOnClose(true);
1408         BlockCache cache = cacheConf.getBlockCache().get();
1409
1410         // establish baseline stats
1411         long startBlockCount = cache.getBlockCount();
1412         long startBlockHits = cache.getStats().getHitCount();
1413         long startBlockMiss = cache.getStats().getMissCount();
1414
1415         // wait till baseline is stable, (minimal 500 ms)
1416         for (int i = 0; i < 5; i++) {
1417           Thread.sleep(100);
1418           if (startBlockCount != cache.getBlockCount()
1419                   || startBlockHits != cache.getStats().getHitCount()
1420                   || startBlockMiss != cache.getStats().getMissCount()) {
1421             startBlockCount = cache.getBlockCount();
1422             startBlockHits = cache.getStats().getHitCount();
1423             startBlockMiss = cache.getStats().getMissCount();
1424             i = -1;
1425           }
1426         }
1427
1428         // insert data
1429         Put put = new Put(ROW);
1430         put.addColumn(FAMILY, QUALIFIER, data);
1431         table.put(put);
1432         assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data));
1433
1434         // data was in memstore so don't expect any changes
1435         assertEquals(startBlockCount, cache.getBlockCount());
1436         assertEquals(startBlockHits, cache.getStats().getHitCount());
1437         assertEquals(startBlockMiss, cache.getStats().getMissCount());
1438
1439         // flush the data
1440         LOG.debug("Flushing cache");
1441         region.flush(true);
1442
1443         // expect two more blocks in cache - DATA and ROOT_INDEX
1444         // , no change in hits/misses
1445         long expectedBlockCount = startBlockCount + 2;
1446         long expectedBlockHits = startBlockHits;
1447         long expectedBlockMiss = startBlockMiss;
1448         assertEquals(expectedBlockCount, cache.getBlockCount());
1449         assertEquals(expectedBlockHits, cache.getStats().getHitCount());
1450         assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
1451         // read the data and expect same blocks, one new hit, no misses
1452         assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data));
1453         assertEquals(expectedBlockCount, cache.getBlockCount());
1454         assertEquals(++expectedBlockHits, cache.getStats().getHitCount());
1455         assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
1456         // insert a second column, read the row, no new blocks, one new hit
1457         byte[] QUALIFIER2 = Bytes.add(QUALIFIER, QUALIFIER);
1458         byte[] data2 = Bytes.add(data, data);
1459         put = new Put(ROW);
1460         put.addColumn(FAMILY, QUALIFIER2, data2);
1461         table.put(put);
1462         Result r = table.get(new Get(ROW));
1463         assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data));
1464         assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2));
1465         assertEquals(expectedBlockCount, cache.getBlockCount());
1466         assertEquals(++expectedBlockHits, cache.getStats().getHitCount());
1467         assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
1468         // flush, one new block
1469         System.out.println("Flushing cache");
1470         region.flush(true);
1471
1472         // + 1 for Index Block, +1 for data block
1473         expectedBlockCount += 2;
1474         assertEquals(expectedBlockCount, cache.getBlockCount());
1475         assertEquals(expectedBlockHits, cache.getStats().getHitCount());
1476         assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
1477         // compact, net minus two blocks, two hits, no misses
1478         System.out.println("Compacting");
1479         assertEquals(2, store.getStorefilesCount());
1480         store.triggerMajorCompaction();
1481         region.compact(true);
1482         store.closeAndArchiveCompactedFiles();
1483         waitForStoreFileCount(store, 1, 10000); // wait 10 seconds max
1484         assertEquals(1, store.getStorefilesCount());
1485         // evicted two data blocks and two index blocks and compaction does not cache new blocks
1486         expectedBlockCount = 0;
1487         assertEquals(expectedBlockCount, cache.getBlockCount());
1488         expectedBlockHits += 2;
1489         assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
1490         assertEquals(expectedBlockHits, cache.getStats().getHitCount());
1491         // read the row, this should be a cache miss because we don't cache data
1492         // blocks on compaction
1493         r = table.get(new Get(ROW));
1494         assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data));
1495         assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2));
1496         expectedBlockCount += 1; // cached one data block
1497         assertEquals(expectedBlockCount, cache.getBlockCount());
1498         assertEquals(expectedBlockHits, cache.getStats().getHitCount());
1499         assertEquals(++expectedBlockMiss, cache.getStats().getMissCount());
1500       }
1501     }
1502   }
1503
1504   private void waitForStoreFileCount(HStore store, int count, int timeout)
1505       throws InterruptedException {
1506     long start = EnvironmentEdgeManager.currentTime();
1507     while (start + timeout > EnvironmentEdgeManager.currentTime() &&
1508         store.getStorefilesCount() != count) {
1509       Thread.sleep(100);
1510     }
1511     System.out.println("start=" + start + ", now=" + EnvironmentEdgeManager.currentTime() +
1512       ", cur=" + store.getStorefilesCount());
1513     assertEquals(count, store.getStorefilesCount());
1514   }
1515
1516   /**
1517    * Tests the non cached version of getRegionLocator by moving a region.
1518    */
1519   @Test
1520   public void testNonCachedGetRegionLocation() throws Exception {
1521     // Test Initialization.
1522     final TableName tableName = name.getTableName();
1523     byte [] family1 = Bytes.toBytes("f1");
1524     byte [] family2 = Bytes.toBytes("f2");
1525     try (Table ignored = TEST_UTIL.createTable(tableName, new byte[][] {family1, family2}, 10);
1526         Admin admin = TEST_UTIL.getAdmin();
1527         RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
1528       List<HRegionLocation> allRegionLocations = locator.getAllRegionLocations();
1529       assertEquals(1, allRegionLocations.size());
1530       RegionInfo regionInfo = allRegionLocations.get(0).getRegion();
1531       ServerName addrBefore = allRegionLocations.get(0).getServerName();
1532       // Verify region location before move.
1533       HRegionLocation addrCache = locator.getRegionLocation(regionInfo.getStartKey(), false);
1534       HRegionLocation addrNoCache = locator.getRegionLocation(regionInfo.getStartKey(),  true);
1535
1536       assertEquals(addrBefore.getPort(), addrCache.getPort());
1537       assertEquals(addrBefore.getPort(), addrNoCache.getPort());
1538
1539
1540       // Make sure more than one server.
1541       if (TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size() <= 1) {
1542         TEST_UTIL.getMiniHBaseCluster().startRegionServer();
1543         Waiter.waitFor(TEST_UTIL.getConfiguration(), 30000, new Waiter.Predicate<Exception>() {
1544           @Override public boolean evaluate() throws Exception {
1545             return TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size() > 1;
1546           }
1547         });
1548       }
1549
1550       ServerName addrAfter = null;
1551       // Now move the region to a different server.
1552       for (int i = 0; i < TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size();
1553            i++) {
1554         HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(i);
1555         ServerName addr = regionServer.getServerName();
1556         if (addr.getPort() != addrBefore.getPort()) {
1557           admin.move(regionInfo.getEncodedNameAsBytes(), addr);
1558           // Wait for the region to move.
1559           Thread.sleep(5000);
1560           addrAfter = addr;
1561           break;
1562         }
1563       }
1564
1565       // Verify the region was moved.
1566       addrCache = locator.getRegionLocation(regionInfo.getStartKey(), false);
1567       addrNoCache = locator.getRegionLocation(regionInfo.getStartKey(), true);
1568       assertNotNull(addrAfter);
1569       assertTrue(addrAfter.getPort() != addrCache.getPort());
1570       assertEquals(addrAfter.getPort(), addrNoCache.getPort());
1571     }
1572   }
1573
1574   /**
1575    * Tests getRegionsInRange by creating some regions over which a range of
1576    * keys spans; then changing the key range.
1577    */
1578   @Test
1579   public void testGetRegionsInRange() throws Exception {
1580     // Test Initialization.
1581     byte [] startKey = Bytes.toBytes("ddc");
1582     byte [] endKey = Bytes.toBytes("mmm");
1583     TableName tableName = name.getTableName();
1584     TEST_UTIL.createMultiRegionTable(tableName, new byte[][] { FAMILY }, 10);
1585
1586     int numOfRegions;
1587     try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
1588       numOfRegions = r.getStartKeys().length;
1589     }
1590     assertEquals(26, numOfRegions);
1591
1592     // Get the regions in this range
1593     List<HRegionLocation> regionsList = getRegionsInRange(tableName, startKey, endKey);
1594     assertEquals(10, regionsList.size());
1595
1596     // Change the start key
1597     startKey = Bytes.toBytes("fff");
1598     regionsList = getRegionsInRange(tableName, startKey, endKey);
1599     assertEquals(7, regionsList.size());
1600
1601     // Change the end key
1602     endKey = Bytes.toBytes("nnn");
1603     regionsList = getRegionsInRange(tableName, startKey, endKey);
1604     assertEquals(8, regionsList.size());
1605
1606     // Empty start key
1607     regionsList = getRegionsInRange(tableName, HConstants.EMPTY_START_ROW, endKey);
1608     assertEquals(13, regionsList.size());
1609
1610     // Empty end key
1611     regionsList = getRegionsInRange(tableName, startKey, HConstants.EMPTY_END_ROW);
1612     assertEquals(21, regionsList.size());
1613
1614     // Both start and end keys empty
1615     regionsList = getRegionsInRange(tableName, HConstants.EMPTY_START_ROW,
1616         HConstants.EMPTY_END_ROW);
1617     assertEquals(26, regionsList.size());
1618
1619     // Change the end key to somewhere in the last block
1620     endKey = Bytes.toBytes("zzz1");
1621     regionsList = getRegionsInRange(tableName, startKey, endKey);
1622     assertEquals(21, regionsList.size());
1623
1624     // Change the start key to somewhere in the first block
1625     startKey = Bytes.toBytes("aac");
1626     regionsList = getRegionsInRange(tableName, startKey, endKey);
1627     assertEquals(26, regionsList.size());
1628
1629     // Make start and end key the same
1630     startKey = Bytes.toBytes("ccc");
1631     endKey = Bytes.toBytes("ccc");
1632     regionsList = getRegionsInRange(tableName, startKey, endKey);
1633     assertEquals(1, regionsList.size());
1634   }
1635
1636   private List<HRegionLocation> getRegionsInRange(TableName tableName, byte[] startKey,
1637       byte[] endKey) throws IOException {
1638     List<HRegionLocation> regionsInRange = new ArrayList<>();
1639     byte[] currentKey = startKey;
1640     final boolean endKeyIsEndOfTable = Bytes.equals(endKey, HConstants.EMPTY_END_ROW);
1641     try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
1642       do {
1643         HRegionLocation regionLocation = r.getRegionLocation(currentKey);
1644         regionsInRange.add(regionLocation);
1645         currentKey = regionLocation.getRegion().getEndKey();
1646       } while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW)
1647           && (endKeyIsEndOfTable || Bytes.compareTo(currentKey, endKey) < 0));
1648       return regionsInRange;
1649     }
1650   }
1651
1652   @Test
1653   public void testJira6912() throws Exception {
1654     final TableName tableName = name.getTableName();
1655     try (Table foo = TEST_UTIL.createTable(tableName, new byte[][] {FAMILY}, 10)) {
1656
1657       List<Put> puts = new ArrayList<>();
1658       for (int i = 0; i != 100; i++) {
1659         Put put = new Put(Bytes.toBytes(i));
1660         put.addColumn(FAMILY, FAMILY, Bytes.toBytes(i));
1661         puts.add(put);
1662       }
1663       foo.put(puts);
1664       // If i comment this out it works
1665       TEST_UTIL.flush();
1666
1667       Scan scan = new Scan();
1668       scan.withStartRow(Bytes.toBytes(1));
1669       scan.withStopRow(Bytes.toBytes(3));
1670       scan.addColumn(FAMILY, FAMILY);
1671       scan.setFilter(new RowFilter(CompareOperator.NOT_EQUAL,
1672               new BinaryComparator(Bytes.toBytes(1))));
1673
1674       try (ResultScanner scanner = foo.getScanner(scan)) {
1675         Result[] bar = scanner.next(100);
1676         assertEquals(1, bar.length);
1677       }
1678     }
1679   }
1680
1681   @Test
1682   public void testScan_NullQualifier() throws IOException {
1683     try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
1684       Put put = new Put(ROW);
1685       put.addColumn(FAMILY, QUALIFIER, VALUE);
1686       table.put(put);
1687
1688       put = new Put(ROW);
1689       put.addColumn(FAMILY, null, VALUE);
1690       table.put(put);
1691       LOG.info("Row put");
1692
1693       Scan scan = new Scan();
1694       scan.addColumn(FAMILY, null);
1695
1696       ResultScanner scanner = table.getScanner(scan);
1697       Result[] bar = scanner.next(100);
1698       assertEquals(1, bar.length);
1699       assertEquals(1, bar[0].size());
1700
1701       scan = new Scan();
1702       scan.addFamily(FAMILY);
1703
1704       scanner = table.getScanner(scan);
1705       bar = scanner.next(100);
1706       assertEquals(1, bar.length);
1707       assertEquals(2, bar[0].size());
1708     }
1709   }
1710
1711   @Test
1712   public void testNegativeTimestamp() throws IOException {
1713     try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
1714
1715       try {
1716         Put put = new Put(ROW, -1);
1717         put.addColumn(FAMILY, QUALIFIER, VALUE);
1718         table.put(put);
1719         fail("Negative timestamps should not have been allowed");
1720       } catch (IllegalArgumentException ex) {
1721         assertTrue(ex.getMessage().contains("negative"));
1722       }
1723
1724       try {
1725         Put put = new Put(ROW);
1726         long ts = -1;
1727         put.addColumn(FAMILY, QUALIFIER, ts, VALUE);
1728         table.put(put);
1729         fail("Negative timestamps should not have been allowed");
1730       } catch (IllegalArgumentException ex) {
1731         assertTrue(ex.getMessage().contains("negative"));
1732       }
1733
1734       try {
1735         Delete delete = new Delete(ROW, -1);
1736         table.delete(delete);
1737         fail("Negative timestamps should not have been allowed");
1738       } catch (IllegalArgumentException ex) {
1739         assertTrue(ex.getMessage().contains("negative"));
1740       }
1741
1742       try {
1743         Delete delete = new Delete(ROW);
1744         delete.addFamily(FAMILY, -1);
1745         table.delete(delete);
1746         fail("Negative timestamps should not have been allowed");
1747       } catch (IllegalArgumentException ex) {
1748         assertTrue(ex.getMessage().contains("negative"));
1749       }
1750
1751       try {
1752         Scan scan = new Scan();
1753         scan.setTimeRange(-1, 1);
1754         table.getScanner(scan);
1755         fail("Negative timestamps should not have been allowed");
1756       } catch (IllegalArgumentException ex) {
1757         assertTrue(ex.getMessage().contains("negative"));
1758       }
1759
1760       // KeyValue should allow negative timestamps for backwards compat. Otherwise, if the user
1761       // already has negative timestamps in cluster data, HBase won't be able to handle that
1762       try {
1763         new KeyValue(Bytes.toBytes(42), Bytes.toBytes(42), Bytes.toBytes(42), -1,
1764                 Bytes.toBytes(42));
1765       } catch (IllegalArgumentException ex) {
1766         fail("KeyValue SHOULD allow negative timestamps");
1767       }
1768
1769     }
1770   }
1771
1772   @Test
1773   public void testRawScanRespectsVersions() throws Exception {
1774     final TableName tableName = name.getTableName();
1775     try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) {
1776       byte[] row = Bytes.toBytes("row");
1777
1778       // put the same row 4 times, with different values
1779       Put p = new Put(row);
1780       p.addColumn(FAMILY, QUALIFIER, 10, VALUE);
1781       table.put(p);
1782       p = new Put(row);
1783       p.addColumn(FAMILY, QUALIFIER, 11, ArrayUtils.add(VALUE, (byte) 2));
1784       table.put(p);
1785
1786       p = new Put(row);
1787       p.addColumn(FAMILY, QUALIFIER, 12, ArrayUtils.add(VALUE, (byte) 3));
1788       table.put(p);
1789
1790       p = new Put(row);
1791       p.addColumn(FAMILY, QUALIFIER, 13, ArrayUtils.add(VALUE, (byte) 4));
1792       table.put(p);
1793
1794       int versions = 4;
1795       Scan s = new Scan().withStartRow(row);
1796       // get all the possible versions
1797       s.readAllVersions();
1798       s.setRaw(true);
1799
1800       try (ResultScanner scanner = table.getScanner(s)) {
1801         int count = 0;
1802         for (Result r : scanner) {
1803           assertEquals("Found an unexpected number of results for the row!", versions,
1804                   r.listCells().size());
1805           count++;
1806         }
1807         assertEquals("Found more than a single row when raw scanning the table with a single row!",
1808                 1, count);
1809       }
1810
1811       // then if we decrease the number of versions, but keep the scan raw, we should see exactly
1812       // that number of versions
1813       versions = 2;
1814       s.readVersions(versions);
1815       try (ResultScanner scanner = table.getScanner(s)) {
1816         int count = 0;
1817         for (Result r : scanner) {
1818           assertEquals("Found an unexpected number of results for the row!", versions,
1819                   r.listCells().size());
1820           count++;
1821         }
1822         assertEquals("Found more than a single row when raw scanning the table with a single row!",
1823                 1, count);
1824       }
1825
1826       // finally, if we turn off raw scanning, but max out the number of versions, we should go back
1827       // to seeing just three
1828       versions = 3;
1829       s.readVersions(versions);
1830       try (ResultScanner scanner = table.getScanner(s)) {
1831         int count = 0;
1832         for (Result r : scanner) {
1833           assertEquals("Found an unexpected number of results for the row!", versions,
1834                   r.listCells().size());
1835           count++;
1836         }
1837         assertEquals("Found more than a single row when raw scanning the table with a single row!",
1838                 1, count);
1839       }
1840
1841     }
1842     TEST_UTIL.deleteTable(tableName);
1843   }
1844
1845   @Test
1846   public void testEmptyFilterList() throws Exception {
1847     // Test Initialization.
1848     final TableName tableName = name.getTableName();
1849     try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) {
1850
1851       // Insert one row each region
1852       Put put = new Put(Bytes.toBytes("row"));
1853       put.addColumn(FAMILY, QUALIFIER, VALUE);
1854       table.put(put);
1855
1856       List<Result> scanResults = new LinkedList<>();
1857       Scan scan = new Scan();
1858       scan.setFilter(new FilterList());
1859       try (ResultScanner scanner = table.getScanner(scan)) {
1860         for (Result r : scanner) {
1861           scanResults.add(r);
1862         }
1863       }
1864       assertEquals(1, scanResults.size());
1865       Get g = new Get(Bytes.toBytes("row"));
1866       g.setFilter(new FilterList());
1867       Result getResult = table.get(g);
1868       Result scanResult = scanResults.get(0);
1869       assertEquals(scanResult.rawCells().length, getResult.rawCells().length);
1870       for (int i = 0; i != scanResult.rawCells().length; ++i) {
1871         Cell scanCell = scanResult.rawCells()[i];
1872         Cell getCell = getResult.rawCells()[i];
1873         assertEquals(0, Bytes.compareTo(CellUtil.cloneRow(scanCell),
1874                 CellUtil.cloneRow(getCell)));
1875         assertEquals(0, Bytes.compareTo(CellUtil.cloneFamily(scanCell),
1876                 CellUtil.cloneFamily(getCell)));
1877         assertEquals(0, Bytes.compareTo(CellUtil.cloneQualifier(scanCell),
1878                 CellUtil.cloneQualifier(getCell)));
1879         assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(scanCell),
1880                 CellUtil.cloneValue(getCell)));
1881       }
1882     }
1883   }
1884
1885   @Test
1886   public void testSmallScan() throws Exception {
1887     // Test Initialization.
1888     final TableName tableName = name.getTableName();
1889     try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) {
1890
1891       // Insert one row each region
1892       int insertNum = 10;
1893       for (int i = 0; i < 10; i++) {
1894         Put put = new Put(Bytes.toBytes("row" + String.format("%03d", i)));
1895         put.addColumn(FAMILY, QUALIFIER, VALUE);
1896         table.put(put);
1897       }
1898
1899       // normal scan
1900       try (ResultScanner scanner = table.getScanner(new Scan())) {
1901         int count = 0;
1902         for (Result r : scanner) {
1903           assertFalse(r.isEmpty());
1904           count++;
1905         }
1906         assertEquals(insertNum, count);
1907       }
1908
1909       // small scan
1910       Scan scan = new Scan().withStartRow(HConstants.EMPTY_START_ROW)
1911         .withStopRow(HConstants.EMPTY_END_ROW, true);
1912       scan.setReadType(ReadType.PREAD);
1913       scan.setCaching(2);
1914       try (ResultScanner scanner = table.getScanner(scan)) {
1915         int count = 0;
1916         for (Result r : scanner) {
1917           assertFalse(r.isEmpty());
1918           count++;
1919         }
1920         assertEquals(insertNum, count);
1921       }
1922     }
1923   }
1924
1925   @Test
1926   public void testSuperSimpleWithReverseScan() throws Exception {
1927     final TableName tableName = name.getTableName();
1928     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
1929       Put put = new Put(Bytes.toBytes("0-b11111-0000000000000000000"));
1930       put.addColumn(FAMILY, QUALIFIER, VALUE);
1931       ht.put(put);
1932       put = new Put(Bytes.toBytes("0-b11111-0000000000000000002"));
1933       put.addColumn(FAMILY, QUALIFIER, VALUE);
1934       ht.put(put);
1935       put = new Put(Bytes.toBytes("0-b11111-0000000000000000004"));
1936       put.addColumn(FAMILY, QUALIFIER, VALUE);
1937       ht.put(put);
1938       put = new Put(Bytes.toBytes("0-b11111-0000000000000000006"));
1939       put.addColumn(FAMILY, QUALIFIER, VALUE);
1940       ht.put(put);
1941       put = new Put(Bytes.toBytes("0-b11111-0000000000000000008"));
1942       put.addColumn(FAMILY, QUALIFIER, VALUE);
1943       ht.put(put);
1944       put = new Put(Bytes.toBytes("0-b22222-0000000000000000001"));
1945       put.addColumn(FAMILY, QUALIFIER, VALUE);
1946       ht.put(put);
1947       put = new Put(Bytes.toBytes("0-b22222-0000000000000000003"));
1948       put.addColumn(FAMILY, QUALIFIER, VALUE);
1949       ht.put(put);
1950       put = new Put(Bytes.toBytes("0-b22222-0000000000000000005"));
1951       put.addColumn(FAMILY, QUALIFIER, VALUE);
1952       ht.put(put);
1953       put = new Put(Bytes.toBytes("0-b22222-0000000000000000007"));
1954       put.addColumn(FAMILY, QUALIFIER, VALUE);
1955       ht.put(put);
1956       put = new Put(Bytes.toBytes("0-b22222-0000000000000000009"));
1957       put.addColumn(FAMILY, QUALIFIER, VALUE);
1958       ht.put(put);
1959       Scan scan = new Scan().withStartRow(Bytes.toBytes("0-b11111-9223372036854775807"))
1960         .withStopRow(Bytes.toBytes("0-b11111-0000000000000000000"), true);
1961       scan.setReversed(true);
1962       try (ResultScanner scanner = ht.getScanner(scan)) {
1963         Result result = scanner.next();
1964         assertTrue(Bytes.equals(result.getRow(),
1965                 Bytes.toBytes("0-b11111-0000000000000000008")));
1966       }
1967     }
1968   }
1969
1970   @Test
1971   public void testFiltersWithReverseScan() throws Exception {
1972     final TableName tableName = name.getTableName();
1973     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
1974       byte[][] ROWS = makeN(ROW, 10);
1975       byte[][] QUALIFIERS = {Bytes.toBytes("col0-<d2v1>-<d3v2>"),
1976               Bytes.toBytes("col1-<d2v1>-<d3v2>"),
1977               Bytes.toBytes("col2-<d2v1>-<d3v2>"),
1978               Bytes.toBytes("col3-<d2v1>-<d3v2>"),
1979               Bytes.toBytes("col4-<d2v1>-<d3v2>"),
1980               Bytes.toBytes("col5-<d2v1>-<d3v2>"),
1981               Bytes.toBytes("col6-<d2v1>-<d3v2>"),
1982               Bytes.toBytes("col7-<d2v1>-<d3v2>"),
1983               Bytes.toBytes("col8-<d2v1>-<d3v2>"),
1984               Bytes.toBytes("col9-<d2v1>-<d3v2>")};
1985       for (int i = 0; i < 10; i++) {
1986         Put put = new Put(ROWS[i]);
1987         put.addColumn(FAMILY, QUALIFIERS[i], VALUE);
1988         ht.put(put);
1989       }
1990       Scan scan = new Scan();
1991       scan.setReversed(true);
1992       scan.addFamily(FAMILY);
1993       Filter filter = new QualifierFilter(CompareOperator.EQUAL,
1994               new RegexStringComparator("col[1-5]"));
1995       scan.setFilter(filter);
1996       try (ResultScanner scanner = ht.getScanner(scan)) {
1997         int expectedIndex = 5;
1998         for (Result result : scanner) {
1999           assertEquals(1, result.size());
2000           Cell c = result.rawCells()[0];
2001           assertTrue(Bytes.equals(c.getRowArray(), c.getRowOffset(), c.getRowLength(),
2002                   ROWS[expectedIndex], 0, ROWS[expectedIndex].length));
2003           assertTrue(Bytes.equals(c.getQualifierArray(), c.getQualifierOffset(),
2004                   c.getQualifierLength(), QUALIFIERS[expectedIndex], 0,
2005                   QUALIFIERS[expectedIndex].length));
2006           expectedIndex--;
2007         }
2008         assertEquals(0, expectedIndex);
2009       }
2010     }
2011   }
2012
2013   @Test
2014   public void testKeyOnlyFilterWithReverseScan() throws Exception {
2015     final TableName tableName = name.getTableName();
2016     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
2017       byte[][] ROWS = makeN(ROW, 10);
2018       byte[][] QUALIFIERS = {Bytes.toBytes("col0-<d2v1>-<d3v2>"),
2019               Bytes.toBytes("col1-<d2v1>-<d3v2>"),
2020               Bytes.toBytes("col2-<d2v1>-<d3v2>"),
2021               Bytes.toBytes("col3-<d2v1>-<d3v2>"),
2022               Bytes.toBytes("col4-<d2v1>-<d3v2>"),
2023               Bytes.toBytes("col5-<d2v1>-<d3v2>"),
2024               Bytes.toBytes("col6-<d2v1>-<d3v2>"),
2025               Bytes.toBytes("col7-<d2v1>-<d3v2>"),
2026               Bytes.toBytes("col8-<d2v1>-<d3v2>"),
2027               Bytes.toBytes("col9-<d2v1>-<d3v2>")};
2028       for (int i = 0; i < 10; i++) {
2029         Put put = new Put(ROWS[i]);
2030         put.addColumn(FAMILY, QUALIFIERS[i], VALUE);
2031         ht.put(put);
2032       }
2033       Scan scan = new Scan();
2034       scan.setReversed(true);
2035       scan.addFamily(FAMILY);
2036       Filter filter = new KeyOnlyFilter(true);
2037       scan.setFilter(filter);
2038       try (ResultScanner ignored = ht.getScanner(scan)) {
2039         int count = 0;
2040         for (Result result : ht.getScanner(scan)) {
2041           assertEquals(1, result.size());
2042           assertEquals(Bytes.SIZEOF_INT, result.rawCells()[0].getValueLength());
2043           assertEquals(VALUE.length, Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0])));
2044           count++;
2045         }
2046         assertEquals(10, count);
2047       }
2048     }
2049   }
2050
2051   /**
2052    * Test simple table and non-existent row cases.
2053    */
2054   @Test
2055   public void testSimpleMissingWithReverseScan() throws Exception {
2056     final TableName tableName = name.getTableName();
2057     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
2058       byte[][] ROWS = makeN(ROW, 4);
2059
2060       // Try to get a row on an empty table
2061       Scan scan = new Scan();
2062       scan.setReversed(true);
2063       Result result = getSingleScanResult(ht, scan);
2064       assertNullResult(result);
2065
2066       scan = new Scan().withStartRow(ROWS[0]);
2067       scan.setReversed(true);
2068       result = getSingleScanResult(ht, scan);
2069       assertNullResult(result);
2070
2071       scan = new Scan().withStartRow(ROWS[0]).withStopRow(ROWS[1], true);
2072       scan.setReversed(true);
2073       result = getSingleScanResult(ht, scan);
2074       assertNullResult(result);
2075
2076       scan = new Scan();
2077       scan.setReversed(true);
2078       scan.addFamily(FAMILY);
2079       result = getSingleScanResult(ht, scan);
2080       assertNullResult(result);
2081
2082       scan = new Scan();
2083       scan.setReversed(true);
2084       scan.addColumn(FAMILY, QUALIFIER);
2085       result = getSingleScanResult(ht, scan);
2086       assertNullResult(result);
2087
2088       // Insert a row
2089
2090       Put put = new Put(ROWS[2]);
2091       put.addColumn(FAMILY, QUALIFIER, VALUE);
2092       ht.put(put);
2093
2094       // Make sure we can scan the row
2095       scan = new Scan();
2096       scan.setReversed(true);
2097       result = getSingleScanResult(ht, scan);
2098       assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
2099
2100       scan = new Scan().withStartRow(ROWS[3]).withStopRow(ROWS[0], true);
2101       scan.setReversed(true);
2102       result = getSingleScanResult(ht, scan);
2103       assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
2104
2105       scan = new Scan().withStartRow(ROWS[2]).withStopRow(ROWS[1], true);
2106       scan.setReversed(true);
2107       result = getSingleScanResult(ht, scan);
2108       assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
2109
2110       // Try to scan empty rows around it
2111       // Introduced MemStore#shouldSeekForReverseScan to fix the following
2112       scan = new Scan().withStartRow(ROWS[1]);
2113       scan.setReversed(true);
2114       result = getSingleScanResult(ht, scan);
2115       assertNullResult(result);
2116     }
2117   }
2118
2119   @Test
2120   public void testNullWithReverseScan() throws Exception {
2121     final TableName tableName = name.getTableName();
2122     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
2123       // Null qualifier (should work)
2124       Put put = new Put(ROW);
2125       put.addColumn(FAMILY, null, VALUE);
2126       ht.put(put);
2127       scanTestNull(ht, ROW, FAMILY, VALUE, true);
2128       Delete delete = new Delete(ROW);
2129       delete.addColumns(FAMILY, null);
2130       ht.delete(delete);
2131     }
2132
2133     // Use a new table
2134     try (Table ht =
2135        TEST_UTIL.createTable(TableName.valueOf(name.getTableName().toString() + "2"), FAMILY)) {
2136       // Empty qualifier, byte[0] instead of null (should work)
2137       Put put = new Put(ROW);
2138       put.addColumn(FAMILY, HConstants.EMPTY_BYTE_ARRAY, VALUE);
2139       ht.put(put);
2140       scanTestNull(ht, ROW, FAMILY, VALUE, true);
2141       TEST_UTIL.flush();
2142       scanTestNull(ht, ROW, FAMILY, VALUE, true);
2143       Delete delete = new Delete(ROW);
2144       delete.addColumns(FAMILY, HConstants.EMPTY_BYTE_ARRAY);
2145       ht.delete(delete);
2146       // Null value
2147       put = new Put(ROW);
2148       put.addColumn(FAMILY, QUALIFIER, null);
2149       ht.put(put);
2150       Scan scan = new Scan();
2151       scan.setReversed(true);
2152       scan.addColumn(FAMILY, QUALIFIER);
2153       Result result = getSingleScanResult(ht, scan);
2154       assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
2155     }
2156   }
2157
2158   @Test
2159   @SuppressWarnings("checkstyle:MethodLength")
2160   public void testDeletesWithReverseScan() throws Exception {
2161     final TableName tableName = name.getTableName();
2162     byte[][] ROWS = makeNAscii(ROW, 6);
2163     byte[][] FAMILIES = makeNAscii(FAMILY, 3);
2164     byte[][] VALUES = makeN(VALUE, 5);
2165     long[] ts = { 1000, 2000, 3000, 4000, 5000 };
2166     try (Table ht = TEST_UTIL.createTable(tableName, FAMILIES, 3)) {
2167
2168       Put put = new Put(ROW);
2169       put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
2170       put.addColumn(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]);
2171       ht.put(put);
2172
2173       Delete delete = new Delete(ROW);
2174       delete.addFamily(FAMILIES[0], ts[0]);
2175       ht.delete(delete);
2176
2177       Scan scan = new Scan().withStartRow(ROW);
2178       scan.setReversed(true);
2179       scan.addFamily(FAMILIES[0]);
2180       scan.readVersions(Integer.MAX_VALUE);
2181       Result result = getSingleScanResult(ht, scan);
2182       assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[]{ts[1]},
2183               new byte[][]{VALUES[1]}, 0, 0);
2184
2185       // Test delete latest version
2186       put = new Put(ROW);
2187       put.addColumn(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
2188       put.addColumn(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]);
2189       put.addColumn(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]);
2190       put.addColumn(FAMILIES[0], null, ts[4], VALUES[4]);
2191       put.addColumn(FAMILIES[0], null, ts[2], VALUES[2]);
2192       put.addColumn(FAMILIES[0], null, ts[3], VALUES[3]);
2193       ht.put(put);
2194
2195       delete = new Delete(ROW);
2196       delete.addColumn(FAMILIES[0], QUALIFIER); // ts[4]
2197       ht.delete(delete);
2198
2199       scan = new Scan().withStartRow(ROW);
2200       scan.setReversed(true);
2201       scan.addColumn(FAMILIES[0], QUALIFIER);
2202       scan.readVersions(Integer.MAX_VALUE);
2203       result = getSingleScanResult(ht, scan);
2204       assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[]{ts[1],
2205               ts[2], ts[3]}, new byte[][]{VALUES[1], VALUES[2], VALUES[3]}, 0, 2);
2206
2207       // Test for HBASE-1847
2208       delete = new Delete(ROW);
2209       delete.addColumn(FAMILIES[0], null);
2210       ht.delete(delete);
2211
2212       // Cleanup null qualifier
2213       delete = new Delete(ROW);
2214       delete.addColumns(FAMILIES[0], null);
2215       ht.delete(delete);
2216
2217       // Expected client behavior might be that you can re-put deleted values
2218       // But alas, this is not to be. We can't put them back in either case.
2219
2220       put = new Put(ROW);
2221       put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
2222       put.addColumn(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
2223       ht.put(put);
2224
2225       // The Scanner returns the previous values, the expected-naive-unexpected
2226       // behavior
2227
2228       scan = new Scan().withStartRow(ROW);
2229       scan.setReversed(true);
2230       scan.addFamily(FAMILIES[0]);
2231       scan.readVersions(Integer.MAX_VALUE);
2232       result = getSingleScanResult(ht, scan);
2233       assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[]{ts[1],
2234               ts[2], ts[3]}, new byte[][]{VALUES[1], VALUES[2], VALUES[3]}, 0, 2);
2235
2236       // Test deleting an entire family from one row but not the other various
2237       // ways
2238
2239       put = new Put(ROWS[0]);
2240       put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
2241       put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
2242       put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
2243       put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
2244       ht.put(put);
2245
2246       put = new Put(ROWS[1]);
2247       put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
2248       put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
2249       put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
2250       put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
2251       ht.put(put);
2252
2253       put = new Put(ROWS[2]);
2254       put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
2255       put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
2256       put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
2257       put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
2258       ht.put(put);
2259
2260       delete = new Delete(ROWS[0]);
2261       delete.addFamily(FAMILIES[2]);
2262       ht.delete(delete);
2263
2264       delete = new Delete(ROWS[1]);
2265       delete.addColumns(FAMILIES[1], QUALIFIER);
2266       ht.delete(delete);
2267
2268       delete = new Delete(ROWS[2]);
2269       delete.addColumn(FAMILIES[1], QUALIFIER);
2270       delete.addColumn(FAMILIES[1], QUALIFIER);
2271       delete.addColumn(FAMILIES[2], QUALIFIER);
2272       ht.delete(delete);
2273
2274       scan = new Scan().withStartRow(ROWS[0]);
2275       scan.setReversed(true);
2276       scan.addFamily(FAMILIES[1]);
2277       scan.addFamily(FAMILIES[2]);
2278       scan.readVersions(Integer.MAX_VALUE);
2279       result = getSingleScanResult(ht, scan);
2280       assertEquals("Expected 2 keys but received " + result.size(), 2, result.size());
2281       assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER, new long[]{ts[0],
2282               ts[1]}, new byte[][]{VALUES[0], VALUES[1]}, 0, 1);
2283
2284       scan = new Scan().withStartRow(ROWS[1]);
2285       scan.setReversed(true);
2286       scan.addFamily(FAMILIES[1]);
2287       scan.addFamily(FAMILIES[2]);
2288       scan.readVersions(Integer.MAX_VALUE);
2289       result = getSingleScanResult(ht, scan);
2290       assertEquals("Expected 2 keys but received " + result.size(), 2, result.size());
2291
2292       scan = new Scan().withStartRow(ROWS[2]);
2293       scan.setReversed(true);
2294       scan.addFamily(FAMILIES[1]);
2295       scan.addFamily(FAMILIES[2]);
2296       scan.readVersions(Integer.MAX_VALUE);
2297       result = getSingleScanResult(ht, scan);
2298       assertEquals(1, result.size());
2299       assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
2300               new long[]{ts[2]}, new byte[][]{VALUES[2]}, 0, 0);
2301
2302       // Test if we delete the family first in one row (HBASE-1541)
2303
2304       delete = new Delete(ROWS[3]);
2305       delete.addFamily(FAMILIES[1]);
2306       ht.delete(delete);
2307
2308       put = new Put(ROWS[3]);
2309       put.addColumn(FAMILIES[2], QUALIFIER, VALUES[0]);
2310       ht.put(put);
2311
2312       put = new Put(ROWS[4]);
2313       put.addColumn(FAMILIES[1], QUALIFIER, VALUES[1]);
2314       put.addColumn(FAMILIES[2], QUALIFIER, VALUES[2]);
2315       ht.put(put);
2316
2317       scan = new Scan().withStartRow(ROWS[4]);
2318       scan.setReversed(true);
2319       scan.addFamily(FAMILIES[1]);
2320       scan.addFamily(FAMILIES[2]);
2321       scan.readVersions(Integer.MAX_VALUE);
2322       ResultScanner scanner = ht.getScanner(scan);
2323       result = scanner.next();
2324       assertEquals("Expected 2 keys but received " + result.size(), 2, result.size());
2325       assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[4]));
2326       assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[1]), ROWS[4]));
2327       assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[1]));
2328       assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[1]), VALUES[2]));
2329       result = scanner.next();
2330       assertEquals("Expected 1 key but received " + result.size(), 1, result.size());
2331       assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[3]));
2332       assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[0]));
2333       scanner.close();
2334     }
2335   }
2336
2337   /**
2338    * Tests reversed scan under multi regions
2339    */
2340   @Test
2341   public void testReversedScanUnderMultiRegions() throws Exception {
2342     // Test Initialization.
2343     final TableName tableName = name.getTableName();
2344     byte[] maxByteArray = ConnectionUtils.MAX_BYTE_ARRAY;
2345     byte[][] splitRows = new byte[][] { Bytes.toBytes("005"),
2346         Bytes.add(Bytes.toBytes("005"), Bytes.multiple(maxByteArray, 16)),
2347         Bytes.toBytes("006"),
2348         Bytes.add(Bytes.toBytes("006"), Bytes.multiple(maxByteArray, 8)),
2349         Bytes.toBytes("007"),
2350         Bytes.add(Bytes.toBytes("007"), Bytes.multiple(maxByteArray, 4)),
2351         Bytes.toBytes("008"), Bytes.multiple(maxByteArray, 2) };
2352     try (Table table = TEST_UTIL.createTable(tableName, FAMILY, splitRows)) {
2353       TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
2354
2355       try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
2356         assertEquals(splitRows.length + 1, l.getAllRegionLocations().size());
2357       }
2358       // Insert one row each region
2359       int insertNum = splitRows.length;
2360       for (byte[] splitRow : splitRows) {
2361         Put put = new Put(splitRow);
2362         put.addColumn(FAMILY, QUALIFIER, VALUE);
2363         table.put(put);
2364       }
2365
2366       // scan forward
2367       try (ResultScanner scanner = table.getScanner(new Scan())) {
2368         int count = 0;
2369         for (Result r : scanner) {
2370           assertFalse(r.isEmpty());
2371           count++;
2372         }
2373         assertEquals(insertNum, count);
2374       }
2375
2376       // scan backward
2377       Scan scan = new Scan();
2378       scan.setReversed(true);
2379       try (ResultScanner scanner = table.getScanner(scan)) {
2380         int count = 0;
2381         byte[] lastRow = null;
2382         for (Result r : scanner) {
2383           assertFalse(r.isEmpty());
2384           count++;
2385           byte[] thisRow = r.getRow();
2386           if (lastRow != null) {
2387             assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
2388                             + ",this row=" + Bytes.toString(thisRow),
2389                     Bytes.compareTo(thisRow, lastRow) < 0);
2390           }
2391           lastRow = thisRow;
2392         }
2393         assertEquals(insertNum, count);
2394       }
2395     }
2396   }
2397
2398   /**
2399    * Tests reversed scan under multi regions
2400    */
2401   @Test
2402   public void testSmallReversedScanUnderMultiRegions() throws Exception {
2403     // Test Initialization.
2404     final TableName tableName = name.getTableName();
2405     byte[][] splitRows = new byte[][]{
2406         Bytes.toBytes("000"), Bytes.toBytes("002"), Bytes.toBytes("004"),
2407         Bytes.toBytes("006"), Bytes.toBytes("008"), Bytes.toBytes("010")};
2408     try (Table table = TEST_UTIL.createTable(tableName, FAMILY, splitRows)) {
2409       TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
2410
2411       try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
2412         assertEquals(splitRows.length + 1, l.getAllRegionLocations().size());
2413       }
2414       for (byte[] splitRow : splitRows) {
2415         Put put = new Put(splitRow);
2416         put.addColumn(FAMILY, QUALIFIER, VALUE);
2417         table.put(put);
2418
2419         byte[] nextRow = Bytes.copy(splitRow);
2420         nextRow[nextRow.length - 1]++;
2421
2422         put = new Put(nextRow);
2423         put.addColumn(FAMILY, QUALIFIER, VALUE);
2424         table.put(put);
2425       }
2426
2427       // scan forward
2428       try (ResultScanner scanner = table.getScanner(new Scan())) {
2429         int count = 0;
2430         for (Result r : scanner) {
2431           assertTrue(!r.isEmpty());
2432           count++;
2433         }
2434         assertEquals(12, count);
2435       }
2436
2437       reverseScanTest(table, ReadType.STREAM);
2438       reverseScanTest(table, ReadType.PREAD);
2439       reverseScanTest(table, ReadType.DEFAULT);
2440     }
2441   }
2442
2443   private void reverseScanTest(Table table, ReadType readType) throws IOException {
2444     // scan backward
2445     Scan scan = new Scan();
2446     scan.setReversed(true);
2447     try (ResultScanner scanner = table.getScanner(scan)) {
2448       int count = 0;
2449       byte[] lastRow = null;
2450       for (Result r : scanner) {
2451         assertTrue(!r.isEmpty());
2452         count++;
2453         byte[] thisRow = r.getRow();
2454         if (lastRow != null) {
2455           assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
2456                           + ",this row=" + Bytes.toString(thisRow),
2457                   Bytes.compareTo(thisRow, lastRow) < 0);
2458         }
2459         lastRow = thisRow;
2460       }
2461       assertEquals(12, count);
2462     }
2463
2464     scan = new Scan();
2465     scan.setReadType(readType);
2466     scan.setReversed(true);
2467     scan.withStartRow(Bytes.toBytes("002"));
2468     try (ResultScanner scanner = table.getScanner(scan)) {
2469       int count = 0;
2470       byte[] lastRow = null;
2471       for (Result r : scanner) {
2472         assertTrue(!r.isEmpty());
2473         count++;
2474         byte[] thisRow = r.getRow();
2475         if (lastRow != null) {
2476           assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
2477                           + ",this row=" + Bytes.toString(thisRow),
2478                   Bytes.compareTo(thisRow, lastRow) < 0);
2479         }
2480         lastRow = thisRow;
2481       }
2482       assertEquals(3, count); // 000 001 002
2483     }
2484
2485     scan = new Scan();
2486     scan.setReadType(readType);
2487     scan.setReversed(true);
2488     scan.withStartRow(Bytes.toBytes("002"));
2489     scan.withStopRow(Bytes.toBytes("000"));
2490     try (ResultScanner scanner = table.getScanner(scan)) {
2491       int count = 0;
2492       byte[] lastRow = null;
2493       for (Result r : scanner) {
2494         assertFalse(r.isEmpty());
2495         count++;
2496         byte[] thisRow = r.getRow();
2497         if (lastRow != null) {
2498           assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
2499                           + ",this row=" + Bytes.toString(thisRow),
2500                   Bytes.compareTo(thisRow, lastRow) < 0);
2501         }
2502         lastRow = thisRow;
2503       }
2504       assertEquals(2, count); // 001 002
2505     }
2506
2507     scan = new Scan();
2508     scan.setReadType(readType);
2509     scan.setReversed(true);
2510     scan.withStartRow(Bytes.toBytes("001"));
2511     try (ResultScanner scanner = table.getScanner(scan)) {
2512       int count = 0;
2513       byte[] lastRow = null;
2514       for (Result r : scanner) {
2515         assertFalse(r.isEmpty());
2516         count++;
2517         byte[] thisRow = r.getRow();
2518         if (lastRow != null) {
2519           assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
2520                           + ",this row=" + Bytes.toString(thisRow),
2521                   Bytes.compareTo(thisRow, lastRow) < 0);
2522         }
2523         lastRow = thisRow;
2524       }
2525       assertEquals(2, count); // 000 001
2526     }
2527
2528     scan = new Scan();
2529     scan.setReadType(readType);
2530     scan.setReversed(true);
2531     scan.withStartRow(Bytes.toBytes("000"));
2532     try (ResultScanner scanner = table.getScanner(scan)) {
2533       int count = 0;
2534       byte[] lastRow = null;
2535       for (Result r : scanner) {
2536         assertFalse(r.isEmpty());
2537         count++;
2538         byte[] thisRow = r.getRow();
2539         if (lastRow != null) {
2540           assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
2541                           + ",this row=" + Bytes.toString(thisRow),
2542                   Bytes.compareTo(thisRow, lastRow) < 0);
2543         }
2544         lastRow = thisRow;
2545       }
2546       assertEquals(1, count); // 000
2547     }
2548
2549     scan = new Scan();
2550     scan.setReadType(readType);
2551     scan.setReversed(true);
2552     scan.withStartRow(Bytes.toBytes("006"));
2553     scan.withStopRow(Bytes.toBytes("002"));
2554     try (ResultScanner scanner = table.getScanner(scan)) {
2555       int count = 0;
2556       byte[] lastRow = null;
2557       for (Result r : scanner) {
2558         assertFalse(r.isEmpty());
2559         count++;
2560         byte[] thisRow = r.getRow();
2561         if (lastRow != null) {
2562           assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
2563                           + ",this row=" + Bytes.toString(thisRow),
2564                   Bytes.compareTo(thisRow, lastRow) < 0);
2565         }
2566         lastRow = thisRow;
2567       }
2568       assertEquals(4, count); // 003 004 005 006
2569     }
2570   }
2571
2572   @Test
2573   public void testFilterAllRecords() throws IOException {
2574     Scan scan = new Scan();
2575     scan.setBatch(1);
2576     scan.setCaching(1);
2577     // Filter out any records
2578     scan.setFilter(new FilterList(new FirstKeyOnlyFilter(), new InclusiveStopFilter(new byte[0])));
2579     try (Table table = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME)) {
2580       try (ResultScanner s = table.getScanner(scan)) {
2581         assertNull(s.next());
2582       }
2583     }
2584   }
2585
2586   @Test
2587   public void testCellSizeLimit() throws IOException {
2588     final TableName tableName = name.getTableName();
2589     TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
2590       .setValue(HRegion.HBASE_MAX_CELL_SIZE_KEY, Integer.toString(10 * 1024))
2591       .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).build();
2592     try (Admin admin = TEST_UTIL.getAdmin()) {
2593       admin.createTable(tableDescriptor);
2594     }
2595     // Will succeed
2596     try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
2597       t.put(new Put(ROW).addColumn(FAMILY, QUALIFIER, Bytes.toBytes(0L)));
2598       t.increment(new Increment(ROW).addColumn(FAMILY, QUALIFIER, 1L));
2599     }
2600     // Will succeed
2601     try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
2602       t.put(new Put(ROW).addColumn(FAMILY, QUALIFIER, new byte[9*1024]));
2603     }
2604     // Will fail
2605     try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
2606       try {
2607         t.put(new Put(ROW).addColumn(FAMILY, QUALIFIER, new byte[10 * 1024]));
2608         fail("Oversize cell failed to trigger exception");
2609       } catch (IOException e) {
2610         // expected
2611       }
2612       try {
2613         t.append(new Append(ROW).addColumn(FAMILY, QUALIFIER, new byte[2 * 1024]));
2614         fail("Oversize cell failed to trigger exception");
2615       } catch (IOException e) {
2616         // expected
2617       }
2618     }
2619   }
2620
2621   @Test
2622   public void testCellSizeNoLimit() throws IOException {
2623     final TableName tableName = name.getTableName();
2624     TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
2625       .setValue(HRegion.HBASE_MAX_CELL_SIZE_KEY, Integer.toString(0))
2626       .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).build();
2627
2628     try (Admin admin = TEST_UTIL.getAdmin()) {
2629       admin.createTable(tableDescriptor);
2630     }
2631
2632     // Will succeed
2633     try (Table ht = TEST_UTIL.getConnection().getTable(tableName)) {
2634       ht.put(new Put(ROW).addColumn(FAMILY, QUALIFIER,  new byte[HRegion.DEFAULT_MAX_CELL_SIZE -
2635         1024]));
2636       ht.append(new Append(ROW).addColumn(FAMILY, QUALIFIER, new byte[1024 + 1]));
2637     }
2638   }
2639
2640   @Test
2641   public void testDeleteSpecifiedVersionOfSpecifiedColumn() throws Exception {
2642     final TableName tableName = name.getTableName();
2643
2644     byte[][] VALUES = makeN(VALUE, 5);
2645     long[] ts = {1000, 2000, 3000, 4000, 5000};
2646
2647     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 5)) {
2648
2649       Put put = new Put(ROW);
2650       // Put version 1000,2000,3000,4000 of column FAMILY:QUALIFIER
2651       for (int t = 0; t < 4; t++) {
2652         put.addColumn(FAMILY, QUALIFIER, ts[t], VALUES[t]);
2653       }
2654       ht.put(put);
2655
2656       Delete delete = new Delete(ROW);
2657       // Delete version 3000 of column FAMILY:QUALIFIER
2658       delete.addColumn(FAMILY, QUALIFIER, ts[2]);
2659       ht.delete(delete);
2660
2661       Get get = new Get(ROW);
2662       get.addColumn(FAMILY, QUALIFIER);
2663       get.readVersions(Integer.MAX_VALUE);
2664       Result result = ht.get(get);
2665       // verify version 1000,2000,4000 remains for column FAMILY:QUALIFIER
2666       assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[0], ts[1], ts[3]}, new byte[][]{
2667               VALUES[0], VALUES[1], VALUES[3]}, 0, 2);
2668
2669       delete = new Delete(ROW);
2670       // Delete a version 5000 of column FAMILY:QUALIFIER which didn't exist
2671       delete.addColumn(FAMILY, QUALIFIER, ts[4]);
2672       ht.delete(delete);
2673
2674       get = new Get(ROW);
2675       get.addColumn(FAMILY, QUALIFIER);
2676       get.readVersions(Integer.MAX_VALUE);
2677       result = ht.get(get);
2678       // verify version 1000,2000,4000 remains for column FAMILY:QUALIFIER
2679       assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[0], ts[1], ts[3]}, new byte[][]{
2680               VALUES[0], VALUES[1], VALUES[3]}, 0, 2);
2681     }
2682   }
2683
2684   @Test
2685   public void testDeleteLatestVersionOfSpecifiedColumn() throws Exception {
2686     final TableName tableName = name.getTableName();
2687     byte[][] VALUES = makeN(VALUE, 5);
2688     long[] ts = {1000, 2000, 3000, 4000, 5000};
2689     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 5)) {
2690       Put put = new Put(ROW);
2691       // Put version 1000,2000,3000,4000 of column FAMILY:QUALIFIER
2692       for (int t = 0; t < 4; t++) {
2693         put.addColumn(FAMILY, QUALIFIER, ts[t], VALUES[t]);
2694       }
2695       ht.put(put);
2696
2697       Delete delete = new Delete(ROW);
2698       // Delete latest version of column FAMILY:QUALIFIER
2699       delete.addColumn(FAMILY, QUALIFIER);
2700       ht.delete(delete);
2701
2702       Get get = new Get(ROW);
2703       get.addColumn(FAMILY, QUALIFIER);
2704       get.readVersions(Integer.MAX_VALUE);
2705       Result result = ht.get(get);
2706       // verify version 1000,2000,3000 remains for column FAMILY:QUALIFIER
2707       assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[0], ts[1], ts[2]}, new byte[][]{
2708               VALUES[0], VALUES[1], VALUES[2]}, 0, 2);
2709
2710       delete = new Delete(ROW);
2711       // Delete two latest version of column FAMILY:QUALIFIER
2712       delete.addColumn(FAMILY, QUALIFIER);
2713       delete.addColumn(FAMILY, QUALIFIER);
2714       ht.delete(delete);
2715
2716       get = new Get(ROW);
2717       get.addColumn(FAMILY, QUALIFIER);
2718       get.readVersions(Integer.MAX_VALUE);
2719       result = ht.get(get);
2720       // verify version 1000 remains for column FAMILY:QUALIFIER
2721       assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[0]}, new byte[][]{VALUES[0]},
2722               0, 0);
2723
2724       put = new Put(ROW);
2725       // Put a version 5000 of column FAMILY:QUALIFIER
2726       put.addColumn(FAMILY, QUALIFIER, ts[4], VALUES[4]);
2727       ht.put(put);
2728
2729       get = new Get(ROW);
2730       get.addColumn(FAMILY, QUALIFIER);
2731       get.readVersions(Integer.MAX_VALUE);
2732       result = ht.get(get);
2733       // verify version 1000,5000 remains for column FAMILY:QUALIFIER
2734       assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[0], ts[4]}, new byte[][]{
2735               VALUES[0], VALUES[4]}, 0, 1);
2736     }
2737   }
2738
2739   /**
2740    * Test for HBASE-17125
2741    */
2742   @Test
2743   public void testReadWithFilter() throws Exception {
2744     final TableName tableName = name.getTableName();
2745     try (Table table = TEST_UTIL.createTable(tableName, FAMILY, 3)) {
2746
2747       byte[] VALUEA = Bytes.toBytes("value-a");
2748       byte[] VALUEB = Bytes.toBytes("value-b");
2749       long[] ts = {1000, 2000, 3000, 4000};
2750
2751       Put put = new Put(ROW);
2752       // Put version 1000,2000,3000,4000 of column FAMILY:QUALIFIER
2753       for (int t = 0; t <= 3; t++) {
2754         if (t <= 1) {
2755           put.addColumn(FAMILY, QUALIFIER, ts[t], VALUEA);
2756         } else {
2757           put.addColumn(FAMILY, QUALIFIER, ts[t], VALUEB);
2758         }
2759       }
2760       table.put(put);
2761
2762       Scan scan =
2763               new Scan().setFilter(new ValueFilter(CompareOperator.EQUAL,
2764                       new SubstringComparator("value-a")))
2765                       .readVersions(3);
2766       ResultScanner scanner = table.getScanner(scan);
2767       Result result = scanner.next();
2768       // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
2769       assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0,
2770               0);
2771
2772       Get get =
2773               new Get(ROW)
2774                       .setFilter(new ValueFilter(CompareOperator.EQUAL,
2775                               new SubstringComparator("value-a")))
2776                       .readVersions(3);
2777       result = table.get(get);
2778       // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
2779       assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0,
2780               0);
2781
2782       // Test with max versions 1, it should still read ts[1]
2783       scan =
2784               new Scan().setFilter(new ValueFilter(CompareOperator.EQUAL,
2785                       new SubstringComparator("value-a")))
2786                       .readVersions(1);
2787       scanner = table.getScanner(scan);
2788       result = scanner.next();
2789       // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
2790       assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0,
2791               0);
2792
2793       // Test with max versions 1, it should still read ts[1]
2794       get =
2795               new Get(ROW)
2796                       .setFilter(new ValueFilter(CompareOperator.EQUAL,
2797                               new SubstringComparator("value-a")))
2798                       .readVersions(1);
2799       result = table.get(get);
2800       // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
2801       assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0,
2802               0);
2803
2804       // Test with max versions 5, it should still read ts[1]
2805       scan =
2806               new Scan().setFilter(new ValueFilter(CompareOperator.EQUAL,
2807                       new SubstringComparator("value-a")))
2808                       .readVersions(5);
2809       scanner = table.getScanner(scan);
2810       result = scanner.next();
2811       // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
2812       assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0,
2813               0);
2814
2815       // Test with max versions 5, it should still read ts[1]
2816       get =
2817               new Get(ROW)
2818                       .setFilter(new ValueFilter(CompareOperator.EQUAL,
2819                               new SubstringComparator("value-a")))
2820                       .readVersions(5);
2821       result = table.get(get);
2822       // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
2823       assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0,
2824               0);
2825     }
2826   }
2827
2828   @Test
2829   public void testCellUtilTypeMethods() throws IOException {
2830     final TableName tableName = name.getTableName();
2831     try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) {
2832
2833       final byte[] row = Bytes.toBytes("p");
2834       Put p = new Put(row);
2835       p.addColumn(FAMILY, QUALIFIER, VALUE);
2836       table.put(p);
2837
2838       try (ResultScanner scanner = table.getScanner(new Scan())) {
2839         Result result = scanner.next();
2840         assertNotNull(result);
2841         CellScanner cs = result.cellScanner();
2842         assertTrue(cs.advance());
2843         Cell c = cs.current();
2844         assertTrue(CellUtil.isPut(c));
2845         assertFalse(CellUtil.isDelete(c));
2846         assertFalse(cs.advance());
2847         assertNull(scanner.next());
2848       }
2849
2850       Delete d = new Delete(row);
2851       d.addColumn(FAMILY, QUALIFIER);
2852       table.delete(d);
2853
2854       Scan scan = new Scan();
2855       scan.setRaw(true);
2856       try (ResultScanner scanner = table.getScanner(scan)) {
2857         Result result = scanner.next();
2858         assertNotNull(result);
2859         CellScanner cs = result.cellScanner();
2860         assertTrue(cs.advance());
2861
2862         // First cell should be the delete (masking the Put)
2863         Cell c = cs.current();
2864         assertTrue("Cell should be a Delete: " + c, CellUtil.isDelete(c));
2865         assertFalse("Cell should not be a Put: " + c, CellUtil.isPut(c));
2866
2867         // Second cell should be the original Put
2868         assertTrue(cs.advance());
2869         c = cs.current();
2870         assertFalse("Cell should not be a Delete: " + c, CellUtil.isDelete(c));
2871         assertTrue("Cell should be a Put: " + c, CellUtil.isPut(c));
2872
2873         // No more cells in this row
2874         assertFalse(cs.advance());
2875
2876         // No more results in this scan
2877         assertNull(scanner.next());
2878       }
2879     }
2880   }
2881
2882   @Test(expected = DoNotRetryIOException.class)
2883   public void testCreateTableWithZeroRegionReplicas() throws Exception {
2884     TableName tableName = name.getTableName();
2885     TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
2886         .setColumnFamily(ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("cf")))
2887         .setRegionReplication(0)
2888         .build();
2889
2890     TEST_UTIL.getAdmin().createTable(desc);
2891   }
2892
2893   @Test(expected = DoNotRetryIOException.class)
2894   public void testModifyTableWithZeroRegionReplicas() throws Exception {
2895     TableName tableName = name.getTableName();
2896     TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
2897         .setColumnFamily(ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("cf")))
2898         .build();
2899
2900     TEST_UTIL.getAdmin().createTable(desc);
2901     TableDescriptor newDesc = TableDescriptorBuilder.newBuilder(desc)
2902         .setRegionReplication(0)
2903         .build();
2904
2905     TEST_UTIL.getAdmin().modifyTable(newDesc);
2906   }
2907
2908   @Test(timeout = 60000)
2909   public void testModifyTableWithMemstoreData() throws Exception {
2910     TableName tableName = name.getTableName();
2911     createTableAndValidateTableSchemaModification(tableName, true);
2912   }
2913
2914   @Test(timeout = 60000)
2915   public void testDeleteCFWithMemstoreData() throws Exception {
2916     TableName tableName = name.getTableName();
2917     createTableAndValidateTableSchemaModification(tableName, false);
2918   }
2919
2920   /**
2921    * Create table and validate online schema modification
2922    * @param tableName Table name
2923    * @param modifyTable Modify table if true otherwise delete column family
2924    * @throws IOException in case of failures
2925    */
2926   private void createTableAndValidateTableSchemaModification(TableName tableName,
2927       boolean modifyTable) throws Exception {
2928     Admin admin = TEST_UTIL.getAdmin();
2929     // Create table with two Cfs
2930     byte[] cf1 = Bytes.toBytes("cf1");
2931     byte[] cf2 = Bytes.toBytes("cf2");
2932     TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName)
2933         .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf1))
2934         .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf2)).build();
2935     admin.createTable(tableDesc);
2936
2937     Table t = TEST_UTIL.getConnection().getTable(tableName);
2938     // Insert few records and flush the table
2939     t.put(new Put(ROW).addColumn(cf1, QUALIFIER, Bytes.toBytes("val1")));
2940     t.put(new Put(ROW).addColumn(cf2, QUALIFIER, Bytes.toBytes("val2")));
2941     admin.flush(tableName);
2942     Path tableDir = CommonFSUtils.getTableDir(TEST_UTIL.getDefaultRootDirPath(), tableName);
2943     List<Path> regionDirs = FSUtils.getRegionDirs(TEST_UTIL.getTestFileSystem(), tableDir);
2944     assertEquals(1, regionDirs.size());
2945     List<Path> familyDirs = FSUtils.getFamilyDirs(TEST_UTIL.getTestFileSystem(), regionDirs.get(0));
2946     assertEquals(2, familyDirs.size());
2947
2948     // Insert record but dont flush the table
2949     t.put(new Put(ROW).addColumn(cf1, QUALIFIER, Bytes.toBytes("val2")));
2950     t.put(new Put(ROW).addColumn(cf2, QUALIFIER, Bytes.toBytes("val2")));
2951
2952     if (modifyTable) {
2953       tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).removeColumnFamily(cf2).build();
2954       admin.modifyTable(tableDesc);
2955     } else {
2956       admin.deleteColumnFamily(tableName, cf2);
2957     }
2958     // After table modification or delete family there should be only one CF in FS
2959     familyDirs = FSUtils.getFamilyDirs(TEST_UTIL.getTestFileSystem(), regionDirs.get(0));
2960     assertEquals("CF dir count should be 1, but was " + familyDirs.size(), 1, familyDirs.size());
2961   }
2962 }