hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java

   1 /*
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.client;
  19
  20 import static org.apache.hadoop.hbase.HBaseTestingUtility.countRows;
  21 import static org.hamcrest.CoreMatchers.instanceOf;
  22 import static org.junit.Assert.assertArrayEquals;
  23 import static org.junit.Assert.assertEquals;
  24 import static org.junit.Assert.assertFalse;
  25 import static org.junit.Assert.assertNotNull;
  26 import static org.junit.Assert.assertNull;
  27 import static org.junit.Assert.assertSame;
  28 import static org.junit.Assert.assertThat;
  29 import static org.junit.Assert.assertTrue;
  30 import static org.junit.Assert.fail;
  31 import java.io.IOException;
  32 import java.util.ArrayList;
  33 import java.util.Arrays;
  34 import java.util.Collection;
  35 import java.util.EnumSet;
  36 import java.util.HashMap;
  37 import java.util.HashSet;
  38 import java.util.Iterator;
  39 import java.util.LinkedList;
  40 import java.util.List;
  41 import java.util.Map;
  42 import java.util.NavigableMap;
  43 import java.util.concurrent.Callable;
  44 import java.util.concurrent.ExecutorService;
  45 import java.util.concurrent.Executors;
  46 import java.util.concurrent.atomic.AtomicReference;
  47 import org.apache.commons.lang3.ArrayUtils;
  48 import org.apache.hadoop.conf.Configuration;
  49 import org.apache.hadoop.fs.Path;
  50 import org.apache.hadoop.hbase.Cell;
  51 import org.apache.hadoop.hbase.CellScanner;
  52 import org.apache.hadoop.hbase.CellUtil;
  53 import org.apache.hadoop.hbase.ClusterMetrics.Option;
  54 import org.apache.hadoop.hbase.CompareOperator;
  55 import org.apache.hadoop.hbase.DoNotRetryIOException;
  56 import org.apache.hadoop.hbase.HBaseClassTestRule;
  57 import org.apache.hadoop.hbase.HBaseTestingUtility;
  58 import org.apache.hadoop.hbase.HColumnDescriptor;
  59 import org.apache.hadoop.hbase.HConstants;
  60 import org.apache.hadoop.hbase.HRegionLocation;
  61 import org.apache.hadoop.hbase.HTableDescriptor;
  62 import org.apache.hadoop.hbase.KeepDeletedCells;
  63 import org.apache.hadoop.hbase.KeyValue;
  64 import org.apache.hadoop.hbase.MiniHBaseCluster;
  65 import org.apache.hadoop.hbase.PrivateCellUtil;
  66 import org.apache.hadoop.hbase.ServerName;
  67 import org.apache.hadoop.hbase.StartMiniClusterOption;
  68 import org.apache.hadoop.hbase.TableName;
  69 import org.apache.hadoop.hbase.TableNameTestRule;
  70 import org.apache.hadoop.hbase.Waiter;
  71 import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
  72 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
  73 import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
  74 import org.apache.hadoop.hbase.filter.BinaryComparator;
  75 import org.apache.hadoop.hbase.filter.Filter;
  76 import org.apache.hadoop.hbase.filter.FilterList;
  77 import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
  78 import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
  79 import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
  80 import org.apache.hadoop.hbase.filter.LongComparator;
  81 import org.apache.hadoop.hbase.filter.PrefixFilter;
  82 import org.apache.hadoop.hbase.filter.QualifierFilter;
  83 import org.apache.hadoop.hbase.filter.RegexStringComparator;
  84 import org.apache.hadoop.hbase.filter.RowFilter;
  85 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
  86 import org.apache.hadoop.hbase.filter.SubstringComparator;
  87 import org.apache.hadoop.hbase.filter.ValueFilter;
  88 import org.apache.hadoop.hbase.filter.WhileMatchFilter;
  89 import org.apache.hadoop.hbase.io.TimeRange;
  90 import org.apache.hadoop.hbase.io.hfile.BlockCache;
  91 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
  92 import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
  93 import org.apache.hadoop.hbase.master.LoadBalancer;
  94 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
  95 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
  96 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
  97 import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService;
  98 import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest;
  99 import org.apache.hadoop.hbase.regionserver.HRegion;
 100 import org.apache.hadoop.hbase.regionserver.HRegionServer;
 101 import org.apache.hadoop.hbase.regionserver.HStore;
 102 import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
 103 import org.apache.hadoop.hbase.testclassification.ClientTests;
 104 import org.apache.hadoop.hbase.testclassification.LargeTests;
 105 import org.apache.hadoop.hbase.util.Bytes;
 106 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 107 import org.apache.hadoop.hbase.util.FSUtils;
 108 import org.apache.hadoop.hbase.util.NonRepeatedEnvironmentEdge;
 109 import org.apache.hadoop.hbase.util.TableDescriptorChecker;
 110 import org.junit.AfterClass;
 111 import org.junit.Assume;
 112 import org.junit.ClassRule;
 113 import org.junit.Ignore;
 114 import org.junit.Rule;
 115 import org.junit.Test;
 116 import org.junit.experimental.categories.Category;
 117 import org.junit.runner.RunWith;
 118 import org.junit.runners.Parameterized;
 119 import org.slf4j.Logger;
 120 import org.slf4j.LoggerFactory;
 121 import org.apache.hbase.thirdparty.com.google.common.base.Preconditions;
 122
 123 /**
 124  * Run tests that use the HBase clients; {@link Table}.
 125  * Sets up the HBase mini cluster once at start and runs through all client tests.
 126  * Each creates a table named for the method and does its stuff against that.
 127  *
 128  * Parameterized to run with different registry implementations.
 129  */
 130 @Category({LargeTests.class, ClientTests.class})
 131 @SuppressWarnings ("deprecation")
 132 @RunWith(Parameterized.class)
 133 public class TestFromClientSide {
 134
 135   @ClassRule
 136   public static final HBaseClassTestRule CLASS_RULE =
 137       HBaseClassTestRule.forClass(TestFromClientSide.class);
 138
 139   // NOTE: Increment tests were moved to their own class, TestIncrementsFromClientSide.
 140   private static final Logger LOG = LoggerFactory.getLogger(TestFromClientSide.class);
 141   protected static HBaseTestingUtility TEST_UTIL;
 142   private static byte [] ROW = Bytes.toBytes("testRow");
 143   private static byte [] FAMILY = Bytes.toBytes("testFamily");
 144   private static final byte[] INVALID_FAMILY = Bytes.toBytes("invalidTestFamily");
 145   private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
 146   private static byte [] VALUE = Bytes.toBytes("testValue");
 147   protected static int SLAVES = 3;
 148
 149   @Rule public TableNameTestRule name = new TableNameTestRule();
 150
 151   // To keep the child classes happy.
 152   TestFromClientSide() {}
 153
 154   public TestFromClientSide(Class registry, int numHedgedReqs) throws Exception {
 155     initialize(registry, numHedgedReqs, MultiRowMutationEndpoint.class);
 156   }
 157
 158   @Parameterized.Parameters
 159   public static Collection parameters() {
 160     return Arrays.asList(new Object[][] {
 161         { MasterRegistry.class, 1},
 162         { MasterRegistry.class, 2},
 163         { ZKConnectionRegistry.class, 1}
 164     });
 165   }
 166
 167   /**
 168    * JUnit does not provide an easy way to run a hook after each parameterized run. Without that
 169    * there is no easy way to restart the test cluster after each parameterized run. Annotation
 170    * BeforeParam does not work either because it runs before parameterization and hence does not
 171    * have access to the test parameters (which is weird).
 172    *
 173    * This *hack* checks if the current instance of test cluster configuration has the passed
 174    * parameterized configs. In such a case, we can just reuse the cluster for test and do not need
 175    * to initialize from scratch. While this is a hack, it saves a ton of time for the full
 176    * test and de-flakes it.
 177    */
 178   private static boolean isSameParameterizedCluster(Class registryImpl, int numHedgedReqs) {
 179     if (TEST_UTIL == null) {
 180       return false;
 181     }
 182     Configuration conf = TEST_UTIL.getConfiguration();
 183     Class confClass = conf.getClass(
 184         HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY, ZKConnectionRegistry.class);
 185     int hedgedReqConfig = conf.getInt(HConstants.HBASE_RPCS_HEDGED_REQS_FANOUT_KEY,
 186         HConstants.HBASE_RPCS_HEDGED_REQS_FANOUT_DEFAULT);
 187     return confClass.getName().equals(registryImpl.getName()) && numHedgedReqs == hedgedReqConfig;
 188   }
 189
 190   protected static final void initialize(Class registryImpl, int numHedgedReqs, Class<?>... cps)
 191       throws Exception {
 192     // initialize() is called for every unit test, however we only want to reset the cluster state
 193     // at the end of every parameterized run.
 194     if (isSameParameterizedCluster(registryImpl, numHedgedReqs)) {
 195       return;
 196     }
 197     // Uncomment the following lines if more verbosity is needed for
 198     // debugging (see HBASE-12285 for details).
 199     // ((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
 200     // ((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
 201     // ((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
 202     // make sure that we do not get the same ts twice, see HBASE-19731 for more details.
 203     EnvironmentEdgeManager.injectEdge(new NonRepeatedEnvironmentEdge());
 204     if (TEST_UTIL != null) {
 205       // We reached end of a parameterized run, clean up.
 206       TEST_UTIL.shutdownMiniCluster();
 207     }
 208     TEST_UTIL = new HBaseTestingUtility();
 209     Configuration conf = TEST_UTIL.getConfiguration();
 210     conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
 211       Arrays.stream(cps).map(Class::getName).toArray(String[]::new));
 212     conf.setBoolean(TableDescriptorChecker.TABLE_SANITY_CHECKS, true); // enable for below tests
 213     conf.setClass(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY, registryImpl,
 214         ConnectionRegistry.class);
 215     if (numHedgedReqs == 1) {
 216       conf.setBoolean(HConstants.MASTER_REGISTRY_ENABLE_HEDGED_READS_KEY, false);
 217     } else {
 218       Preconditions.checkArgument(numHedgedReqs > 1);
 219       conf.setBoolean(HConstants.MASTER_REGISTRY_ENABLE_HEDGED_READS_KEY, true);
 220     }
 221     conf.setInt(HConstants.HBASE_RPCS_HEDGED_REQS_FANOUT_KEY, numHedgedReqs);
 222     StartMiniClusterOption.Builder builder = StartMiniClusterOption.builder();
 223     // Multiple masters needed only when hedged reads for master registry are enabled.
 224     builder.numMasters(numHedgedReqs > 1 ? 3 : 1).numRegionServers(SLAVES);
 225     TEST_UTIL.startMiniCluster(builder.build());
 226   }
 227
 228   @AfterClass
 229   public static void tearDownAfterClass() throws Exception {
 230     if (TEST_UTIL != null) {
 231       TEST_UTIL.shutdownMiniCluster();
 232     }
 233   }
 234
 235   /**
 236    * Test append result when there are duplicate rpc request.
 237    */
 238   @Test
 239   public void testDuplicateAppend() throws Exception {
 240     HTableDescriptor hdt = TEST_UTIL.createTableDescriptor(name.getTableName(),
 241       HColumnDescriptor.DEFAULT_MIN_VERSIONS, 3, HConstants.FOREVER,
 242       HColumnDescriptor.DEFAULT_KEEP_DELETED);
 243     Map<String, String> kvs = new HashMap<>();
 244     kvs.put(SleepAtFirstRpcCall.SLEEP_TIME_CONF_KEY, "2000");
 245     hdt.addCoprocessor(SleepAtFirstRpcCall.class.getName(), null, 1, kvs);
 246     TEST_UTIL.createTable(hdt, new byte[][] { ROW }).close();
 247
 248     Configuration c = new Configuration(TEST_UTIL.getConfiguration());
 249     c.setInt(HConstants.HBASE_CLIENT_PAUSE, 50);
 250     // Client will retry because rpc timeout is small than the sleep time of first rpc call
 251     c.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 1500);
 252
 253     try (Connection connection = ConnectionFactory.createConnection(c);
 254         Table table = connection.getTableBuilder(name.getTableName(), null)
 255           .setOperationTimeout(3 * 1000).build()) {
 256       Append append = new Append(ROW);
 257       append.addColumn(HBaseTestingUtility.fam1, QUALIFIER, VALUE);
 258       Result result = table.append(append);
 259
 260       // Verify expected result
 261       Cell[] cells = result.rawCells();
 262       assertEquals(1, cells.length);
 263       assertKey(cells[0], ROW, HBaseTestingUtility.fam1, QUALIFIER, VALUE);
 264
 265       // Verify expected result again
 266       Result readResult = table.get(new Get(ROW));
 267       cells = readResult.rawCells();
 268       assertEquals(1, cells.length);
 269       assertKey(cells[0], ROW, HBaseTestingUtility.fam1, QUALIFIER, VALUE);
 270     }
 271   }
 272
 273   /**
 274    * Basic client side validation of HBASE-4536
 275    */
 276   @Test
 277   public void testKeepDeletedCells() throws Exception {
 278     final TableName tableName = name.getTableName();
 279     final byte[] FAMILY = Bytes.toBytes("family");
 280     final byte[] C0 = Bytes.toBytes("c0");
 281
 282     final byte[] T1 = Bytes.toBytes("T1");
 283     final byte[] T2 = Bytes.toBytes("T2");
 284     final byte[] T3 = Bytes.toBytes("T3");
 285     ColumnFamilyDescriptor familyDescriptor =
 286       new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(FAMILY)
 287         .setKeepDeletedCells(KeepDeletedCells.TRUE)
 288         .setMaxVersions(3);
 289
 290     TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
 291       new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
 292     tableDescriptor.setColumnFamily(familyDescriptor);
 293     TEST_UTIL.getAdmin().createTable(tableDescriptor);
 294     try (Table h = TEST_UTIL.getConnection().getTable(tableName)) {
 295       long ts = System.currentTimeMillis();
 296       Put p = new Put(T1, ts);
 297       p.addColumn(FAMILY, C0, T1);
 298       h.put(p);
 299       p = new Put(T1, ts + 2);
 300       p.addColumn(FAMILY, C0, T2);
 301       h.put(p);
 302       p = new Put(T1, ts + 4);
 303       p.addColumn(FAMILY, C0, T3);
 304       h.put(p);
 305
 306       Delete d = new Delete(T1, ts + 3);
 307       h.delete(d);
 308
 309       d = new Delete(T1, ts + 3);
 310       d.addColumns(FAMILY, C0, ts + 3);
 311       h.delete(d);
 312
 313       Get g = new Get(T1);
 314       // does *not* include the delete
 315       g.setTimeRange(0, ts + 3);
 316       Result r = h.get(g);
 317       assertArrayEquals(T2, r.getValue(FAMILY, C0));
 318
 319       Scan s = new Scan(T1);
 320       s.setTimeRange(0, ts + 3);
 321       s.setMaxVersions();
 322       ResultScanner scanner = h.getScanner(s);
 323       Cell[] kvs = scanner.next().rawCells();
 324       assertArrayEquals(T2, CellUtil.cloneValue(kvs[0]));
 325       assertArrayEquals(T1, CellUtil.cloneValue(kvs[1]));
 326       scanner.close();
 327
 328       s = new Scan(T1);
 329       s.setRaw(true);
 330       s.setMaxVersions();
 331       scanner = h.getScanner(s);
 332       kvs = scanner.next().rawCells();
 333       assertTrue(PrivateCellUtil.isDeleteFamily(kvs[0]));
 334       assertArrayEquals(T3, CellUtil.cloneValue(kvs[1]));
 335       assertTrue(CellUtil.isDelete(kvs[2]));
 336       assertArrayEquals(T2, CellUtil.cloneValue(kvs[3]));
 337       assertArrayEquals(T1, CellUtil.cloneValue(kvs[4]));
 338       scanner.close();
 339     }
 340   }
 341
 342   /**
 343    * Basic client side validation of HBASE-10118
 344    */
 345   @Test
 346   public void testPurgeFutureDeletes() throws Exception {
 347     final TableName tableName = name.getTableName();
 348     final byte[] ROW = Bytes.toBytes("row");
 349     final byte[] FAMILY = Bytes.toBytes("family");
 350     final byte[] COLUMN = Bytes.toBytes("column");
 351     final byte[] VALUE = Bytes.toBytes("value");
 352
 353     try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) {
 354       // future timestamp
 355       long ts = System.currentTimeMillis() * 2;
 356       Put put = new Put(ROW, ts);
 357       put.addColumn(FAMILY, COLUMN, VALUE);
 358       table.put(put);
 359
 360       Get get = new Get(ROW);
 361       Result result = table.get(get);
 362       assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
 363
 364       Delete del = new Delete(ROW);
 365       del.addColumn(FAMILY, COLUMN, ts);
 366       table.delete(del);
 367
 368       get = new Get(ROW);
 369       result = table.get(get);
 370       assertNull(result.getValue(FAMILY, COLUMN));
 371
 372       // major compaction, purged future deletes
 373       TEST_UTIL.getAdmin().flush(tableName);
 374       TEST_UTIL.getAdmin().majorCompact(tableName);
 375
 376       // waiting for the major compaction to complete
 377       TEST_UTIL.waitFor(6000, new Waiter.Predicate<IOException>() {
 378         @Override
 379         public boolean evaluate() throws IOException {
 380           return TEST_UTIL.getAdmin().getCompactionState(tableName) == CompactionState.NONE;
 381         }
 382       });
 383
 384       put = new Put(ROW, ts);
 385       put.addColumn(FAMILY, COLUMN, VALUE);
 386       table.put(put);
 387
 388       get = new Get(ROW);
 389       result = table.get(get);
 390       assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
 391     }
 392   }
 393
 394   /**
 395    * Verifies that getConfiguration returns the same Configuration object used
 396    * to create the HTable instance.
 397    */
 398   @Test
 399   public void testGetConfiguration() throws Exception {
 400     final TableName tableName = name.getTableName();
 401     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") };
 402     Configuration conf = TEST_UTIL.getConfiguration();
 403     try (Table table = TEST_UTIL.createTable(tableName, FAMILIES)) {
 404       assertSame(conf, table.getConfiguration());
 405     }
 406   }
 407
 408   /**
 409    * Test from client side of an involved filter against a multi family that
 410    * involves deletes.
 411    */
 412   @Test
 413   public void testWeirdCacheBehaviour() throws Exception {
 414     final TableName tableName = name.getTableName();
 415     byte [][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),
 416         Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),
 417         Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
 418     try (Table ht = TEST_UTIL.createTable(tableName, FAMILIES)) {
 419       String value = "this is the value";
 420       String value2 = "this is some other value";
 421       String keyPrefix1 = TEST_UTIL.getRandomUUID().toString();
 422       String keyPrefix2 = TEST_UTIL.getRandomUUID().toString();
 423       String keyPrefix3 = TEST_UTIL.getRandomUUID().toString();
 424       putRows(ht, 3, value, keyPrefix1);
 425       putRows(ht, 3, value, keyPrefix2);
 426       putRows(ht, 3, value, keyPrefix3);
 427       putRows(ht, 3, value2, keyPrefix1);
 428       putRows(ht, 3, value2, keyPrefix2);
 429       putRows(ht, 3, value2, keyPrefix3);
 430       try (Table table = TEST_UTIL.getConnection().getTable(tableName)) {
 431         System.out.println("Checking values for key: " + keyPrefix1);
 432         assertEquals("Got back incorrect number of rows from scan", 3,
 433                 getNumberOfRows(keyPrefix1, value2, table));
 434         System.out.println("Checking values for key: " + keyPrefix2);
 435         assertEquals("Got back incorrect number of rows from scan", 3,
 436                 getNumberOfRows(keyPrefix2, value2, table));
 437         System.out.println("Checking values for key: " + keyPrefix3);
 438         assertEquals("Got back incorrect number of rows from scan", 3,
 439                 getNumberOfRows(keyPrefix3, value2, table));
 440         deleteColumns(ht, value2, keyPrefix1);
 441         deleteColumns(ht, value2, keyPrefix2);
 442         deleteColumns(ht, value2, keyPrefix3);
 443         System.out.println("Starting important checks.....");
 444         assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1,
 445                 0, getNumberOfRows(keyPrefix1, value2, table));
 446         assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2,
 447                 0, getNumberOfRows(keyPrefix2, value2, table));
 448         assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3,
 449                 0, getNumberOfRows(keyPrefix3, value2, table));
 450       }
 451     }
 452   }
 453
 454   private void deleteColumns(Table ht, String value, String keyPrefix)
 455       throws IOException {
 456     ResultScanner scanner = buildScanner(keyPrefix, value, ht);
 457     Iterator<Result> it = scanner.iterator();
 458     int count = 0;
 459     while (it.hasNext()) {
 460       Result result = it.next();
 461       Delete delete = new Delete(result.getRow());
 462       delete.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
 463       ht.delete(delete);
 464       count++;
 465     }
 466     assertEquals("Did not perform correct number of deletes", 3, count);
 467   }
 468
 469   private int getNumberOfRows(String keyPrefix, String value, Table ht)
 470       throws Exception {
 471     ResultScanner resultScanner = buildScanner(keyPrefix, value, ht);
 472     Iterator<Result> scanner = resultScanner.iterator();
 473     int numberOfResults = 0;
 474     while (scanner.hasNext()) {
 475       Result result = scanner.next();
 476       System.out.println("Got back key: " + Bytes.toString(result.getRow()));
 477       for (Cell kv : result.rawCells()) {
 478         System.out.println("kv=" + kv.toString() + ", "
 479             + Bytes.toString(CellUtil.cloneValue(kv)));
 480       }
 481       numberOfResults++;
 482     }
 483     return numberOfResults;
 484   }
 485
 486   private ResultScanner buildScanner(String keyPrefix, String value, Table ht)
 487       throws IOException {
 488     // OurFilterList allFilters = new OurFilterList();
 489     FilterList allFilters = new FilterList(/* FilterList.Operator.MUST_PASS_ALL */);
 490     allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
 491     SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes
 492         .toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOperator.EQUAL, Bytes
 493         .toBytes(value));
 494     filter.setFilterIfMissing(true);
 495     allFilters.addFilter(filter);
 496
 497     // allFilters.addFilter(new
 498     // RowExcludingSingleColumnValueFilter(Bytes.toBytes("trans-tags"),
 499     // Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value)));
 500
 501     Scan scan = new Scan();
 502     scan.addFamily(Bytes.toBytes("trans-blob"));
 503     scan.addFamily(Bytes.toBytes("trans-type"));
 504     scan.addFamily(Bytes.toBytes("trans-date"));
 505     scan.addFamily(Bytes.toBytes("trans-tags"));
 506     scan.addFamily(Bytes.toBytes("trans-group"));
 507     scan.setFilter(allFilters);
 508
 509     return ht.getScanner(scan);
 510   }
 511
 512   private void putRows(Table ht, int numRows, String value, String key)
 513       throws IOException {
 514     for (int i = 0; i < numRows; i++) {
 515       String row = key + "_" + TEST_UTIL.getRandomUUID().toString();
 516       System.out.println(String.format("Saving row: %s, with value %s", row,
 517           value));
 518       Put put = new Put(Bytes.toBytes(row));
 519       put.setDurability(Durability.SKIP_WAL);
 520       put.addColumn(Bytes.toBytes("trans-blob"), null, Bytes
 521           .toBytes("value for blob"));
 522       put.addColumn(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
 523       put.addColumn(Bytes.toBytes("trans-date"), null, Bytes
 524           .toBytes("20090921010101999"));
 525       put.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes
 526           .toBytes(value));
 527       put.addColumn(Bytes.toBytes("trans-group"), null, Bytes
 528           .toBytes("adhocTransactionGroupId"));
 529       ht.put(put);
 530     }
 531   }
 532
 533   /**
 534    * Test filters when multiple regions.  It does counts.  Needs eye-balling of
 535    * logs to ensure that we're not scanning more regions that we're supposed to.
 536    * Related to the TestFilterAcrossRegions over in the o.a.h.h.filter package.
 537    */
 538   @Test
 539   public void testFilterAcrossMultipleRegions()
 540       throws IOException, InterruptedException {
 541     final TableName tableName = name.getTableName();
 542     try (Table t = TEST_UTIL.createTable(tableName, FAMILY)) {
 543       int rowCount = TEST_UTIL.loadTable(t, FAMILY, false);
 544       assertRowCount(t, rowCount);
 545       // Split the table.  Should split on a reasonable key; 'lqj'
 546       List<HRegionLocation> regions = splitTable(t);
 547       assertRowCount(t, rowCount);
 548       // Get end key of first region.
 549       byte[] endKey = regions.get(0).getRegion().getEndKey();
 550       // Count rows with a filter that stops us before passed 'endKey'.
 551       // Should be count of rows in first region.
 552       int endKeyCount = countRows(t, createScanWithRowFilter(endKey));
 553       assertTrue(endKeyCount < rowCount);
 554
 555       // How do I know I did not got to second region?  Thats tough.  Can't really
 556       // do that in client-side region test.  I verified by tracing in debugger.
 557       // I changed the messages that come out when set to DEBUG so should see
 558       // when scanner is done. Says "Finished with scanning..." with region name.
 559       // Check that its finished in right region.
 560
 561       // New test.  Make it so scan goes into next region by one and then two.
 562       // Make sure count comes out right.
 563       byte[] key = new byte[]{endKey[0], endKey[1], (byte) (endKey[2] + 1)};
 564       int plusOneCount = countRows(t, createScanWithRowFilter(key));
 565       assertEquals(endKeyCount + 1, plusOneCount);
 566       key = new byte[]{endKey[0], endKey[1], (byte) (endKey[2] + 2)};
 567       int plusTwoCount = countRows(t, createScanWithRowFilter(key));
 568       assertEquals(endKeyCount + 2, plusTwoCount);
 569
 570       // New test.  Make it so I scan one less than endkey.
 571       key = new byte[]{endKey[0], endKey[1], (byte) (endKey[2] - 1)};
 572       int minusOneCount = countRows(t, createScanWithRowFilter(key));
 573       assertEquals(endKeyCount - 1, minusOneCount);
 574       // For above test... study logs.  Make sure we do "Finished with scanning.."
 575       // in first region and that we do not fall into the next region.
 576
 577       key = new byte[] { 'a', 'a', 'a' };
 578       int countBBB = countRows(t, createScanWithRowFilter(key, null, CompareOperator.EQUAL));
 579       assertEquals(1, countBBB);
 580
 581       int countGreater =
 582         countRows(t, createScanWithRowFilter(endKey, null, CompareOperator.GREATER_OR_EQUAL));
 583       // Because started at start of table.
 584       assertEquals(0, countGreater);
 585       countGreater =
 586         countRows(t, createScanWithRowFilter(endKey, endKey, CompareOperator.GREATER_OR_EQUAL));
 587       assertEquals(rowCount - endKeyCount, countGreater);
 588     }
 589   }
 590
 591   /*
 592    * @param key
 593    * @return Scan with RowFilter that does LESS than passed key.
 594    */
 595   private Scan createScanWithRowFilter(final byte [] key) {
 596     return createScanWithRowFilter(key, null, CompareOperator.LESS);
 597   }
 598
 599   /*
 600    * @param key
 601    * @param op
 602    * @param startRow
 603    * @return Scan with RowFilter that does CompareOp op on passed key.
 604    */
 605   private Scan createScanWithRowFilter(final byte [] key,
 606       final byte [] startRow, CompareOperator op) {
 607     // Make sure key is of some substance... non-null and > than first key.
 608     assertTrue(key != null && key.length > 0 &&
 609       Bytes.BYTES_COMPARATOR.compare(key, new byte [] {'a', 'a', 'a'}) >= 0);
 610     LOG.info("Key=" + Bytes.toString(key));
 611     Scan s = startRow == null? new Scan(): new Scan(startRow);
 612     Filter f = new RowFilter(op, new BinaryComparator(key));
 613     f = new WhileMatchFilter(f);
 614     s.setFilter(f);
 615     return s;
 616   }
 617
 618   private void assertRowCount(final Table t, final int expected) throws IOException {
 619     assertEquals(expected, countRows(t, new Scan()));
 620   }
 621
 622   /**
 623    * Split table into multiple regions.
 624    * @param t Table to split.
 625    * @return Map of regions to servers.
 626    */
 627   private List<HRegionLocation> splitTable(final Table t) throws IOException {
 628     // Split this table in two.
 629     Admin admin = TEST_UTIL.getAdmin();
 630     admin.split(t.getName());
 631     admin.close();
 632     List<HRegionLocation> regions = waitOnSplit(t);
 633     assertTrue(regions.size() > 1);
 634     return regions;
 635   }
 636
 637   /*
 638    * Wait on table split.  May return because we waited long enough on the split
 639    * and it didn't happen.  Caller should check.
 640    * @param t
 641    * @return Map of table regions; caller needs to check table actually split.
 642    */
 643   private List<HRegionLocation> waitOnSplit(final Table t) throws IOException {
 644     try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(t.getName())) {
 645       List<HRegionLocation> regions = locator.getAllRegionLocations();
 646       int originalCount = regions.size();
 647       for (int i = 0; i < TEST_UTIL.getConfiguration().getInt("hbase.test.retries", 30); i++) {
 648         Thread.currentThread();
 649         try {
 650           Thread.sleep(1000);
 651         } catch (InterruptedException e) {
 652           e.printStackTrace();
 653         }
 654         regions = locator.getAllRegionLocations();
 655         if (regions.size() > originalCount) {
 656           break;
 657         }
 658       }
 659       return regions;
 660     }
 661   }
 662
 663   @Test
 664   public void testSuperSimple() throws Exception {
 665     final TableName tableName = name.getTableName();
 666     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
 667       Put put = new Put(ROW);
 668       put.addColumn(FAMILY, QUALIFIER, VALUE);
 669       ht.put(put);
 670       Scan scan = new Scan();
 671       scan.addColumn(FAMILY, tableName.toBytes());
 672       ResultScanner scanner = ht.getScanner(scan);
 673       Result result = scanner.next();
 674       assertTrue("Expected null result", result == null);
 675       scanner.close();
 676     }
 677   }
 678
 679   @Test
 680   public void testMaxKeyValueSize() throws Exception {
 681     final TableName tableName = name.getTableName();
 682     Configuration conf = TEST_UTIL.getConfiguration();
 683     String oldMaxSize = conf.get(ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY);
 684     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
 685       byte[] value = new byte[4 * 1024 * 1024];
 686       Put put = new Put(ROW);
 687       put.addColumn(FAMILY, QUALIFIER, value);
 688       ht.put(put);
 689
 690       try {
 691         TEST_UTIL.getConfiguration().setInt(
 692                 ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY, 2 * 1024 * 1024);
 693         // Create new table so we pick up the change in Configuration.
 694         try (Connection connection =
 695                      ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) {
 696           try (Table t = connection.getTable(TableName.valueOf(FAMILY))) {
 697             put = new Put(ROW);
 698             put.addColumn(FAMILY, QUALIFIER, value);
 699             t.put(put);
 700           }
 701         }
 702         fail("Inserting a too large KeyValue worked, should throw exception");
 703       } catch (Exception e) {
 704       }
 705     }
 706     conf.set(ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY, oldMaxSize);
 707   }
 708
 709   @Test
 710   public void testFilters() throws Exception {
 711     final TableName tableName = name.getTableName();
 712     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
 713       byte[][] ROWS = makeN(ROW, 10);
 714       byte[][] QUALIFIERS = {
 715               Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
 716               Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
 717               Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
 718               Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
 719               Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>")
 720       };
 721       for (int i = 0; i < 10; i++) {
 722         Put put = new Put(ROWS[i]);
 723         put.setDurability(Durability.SKIP_WAL);
 724         put.addColumn(FAMILY, QUALIFIERS[i], VALUE);
 725         ht.put(put);
 726       }
 727       Scan scan = new Scan();
 728       scan.addFamily(FAMILY);
 729       Filter filter = new QualifierFilter(CompareOperator.EQUAL,
 730               new RegexStringComparator("col[1-5]"));
 731       scan.setFilter(filter);
 732       try (ResultScanner scanner = ht.getScanner(scan)) {
 733         int expectedIndex = 1;
 734         for (Result result : scanner) {
 735           assertEquals(1, result.size());
 736           assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[expectedIndex]));
 737           assertTrue(Bytes.equals(CellUtil.cloneQualifier(result.rawCells()[0]),
 738                   QUALIFIERS[expectedIndex]));
 739           expectedIndex++;
 740         }
 741         assertEquals(6, expectedIndex);
 742       }
 743     }
 744   }
 745
 746   @Test
 747   public void testFilterWithLongCompartor() throws Exception {
 748     final TableName tableName = name.getTableName();
 749     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
 750       byte[][] ROWS = makeN(ROW, 10);
 751       byte[][] values = new byte[10][];
 752       for (int i = 0; i < 10; i++) {
 753         values[i] = Bytes.toBytes(100L * i);
 754       }
 755       for (int i = 0; i < 10; i++) {
 756         Put put = new Put(ROWS[i]);
 757         put.setDurability(Durability.SKIP_WAL);
 758         put.addColumn(FAMILY, QUALIFIER, values[i]);
 759         ht.put(put);
 760       }
 761       Scan scan = new Scan();
 762       scan.addFamily(FAMILY);
 763       Filter filter = new SingleColumnValueFilter(FAMILY, QUALIFIER, CompareOperator.GREATER,
 764               new LongComparator(500));
 765       scan.setFilter(filter);
 766       try (ResultScanner scanner = ht.getScanner(scan)) {
 767         int expectedIndex = 0;
 768         for (Result result : scanner) {
 769           assertEquals(1, result.size());
 770           assertTrue(Bytes.toLong(result.getValue(FAMILY, QUALIFIER)) > 500);
 771           expectedIndex++;
 772         }
 773         assertEquals(4, expectedIndex);
 774       }
 775     }
 776   }
 777
 778   @Test
 779   public void testKeyOnlyFilter() throws Exception {
 780     final TableName tableName = name.getTableName();
 781     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
 782       byte[][] ROWS = makeN(ROW, 10);
 783       byte[][] QUALIFIERS = {
 784               Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
 785               Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
 786               Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
 787               Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
 788               Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>")
 789       };
 790       for (int i = 0; i < 10; i++) {
 791         Put put = new Put(ROWS[i]);
 792         put.setDurability(Durability.SKIP_WAL);
 793         put.addColumn(FAMILY, QUALIFIERS[i], VALUE);
 794         ht.put(put);
 795       }
 796       Scan scan = new Scan();
 797       scan.addFamily(FAMILY);
 798       Filter filter = new KeyOnlyFilter(true);
 799       scan.setFilter(filter);
 800       try (ResultScanner scanner = ht.getScanner(scan)) {
 801         int count = 0;
 802         for (Result result : scanner) {
 803           assertEquals(1, result.size());
 804           assertEquals(Bytes.SIZEOF_INT, result.rawCells()[0].getValueLength());
 805           assertEquals(VALUE.length, Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0])));
 806           count++;
 807         }
 808         assertEquals(10, count);
 809       }
 810     }
 811   }
 812
 813   /**
 814    * Test simple table and non-existent row cases.
 815    */
 816   @Test
 817   public void testSimpleMissing() throws Exception {
 818     final TableName tableName = name.getTableName();
 819     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
 820       byte[][] ROWS = makeN(ROW, 4);
 821
 822       // Try to get a row on an empty table
 823       Get get = new Get(ROWS[0]);
 824       Result result = ht.get(get);
 825       assertEmptyResult(result);
 826
 827       get = new Get(ROWS[0]);
 828       get.addFamily(FAMILY);
 829       result = ht.get(get);
 830       assertEmptyResult(result);
 831
 832       get = new Get(ROWS[0]);
 833       get.addColumn(FAMILY, QUALIFIER);
 834       result = ht.get(get);
 835       assertEmptyResult(result);
 836
 837       Scan scan = new Scan();
 838       result = getSingleScanResult(ht, scan);
 839       assertNullResult(result);
 840
 841       scan = new Scan(ROWS[0]);
 842       result = getSingleScanResult(ht, scan);
 843       assertNullResult(result);
 844
 845       scan = new Scan(ROWS[0], ROWS[1]);
 846       result = getSingleScanResult(ht, scan);
 847       assertNullResult(result);
 848
 849       scan = new Scan();
 850       scan.addFamily(FAMILY);
 851       result = getSingleScanResult(ht, scan);
 852       assertNullResult(result);
 853
 854       scan = new Scan();
 855       scan.addColumn(FAMILY, QUALIFIER);
 856       result = getSingleScanResult(ht, scan);
 857       assertNullResult(result);
 858
 859       // Insert a row
 860
 861       Put put = new Put(ROWS[2]);
 862       put.addColumn(FAMILY, QUALIFIER, VALUE);
 863       ht.put(put);
 864
 865       // Try to get empty rows around it
 866
 867       get = new Get(ROWS[1]);
 868       result = ht.get(get);
 869       assertEmptyResult(result);
 870
 871       get = new Get(ROWS[0]);
 872       get.addFamily(FAMILY);
 873       result = ht.get(get);
 874       assertEmptyResult(result);
 875
 876       get = new Get(ROWS[3]);
 877       get.addColumn(FAMILY, QUALIFIER);
 878       result = ht.get(get);
 879       assertEmptyResult(result);
 880
 881       // Try to scan empty rows around it
 882
 883       scan = new Scan(ROWS[3]);
 884       result = getSingleScanResult(ht, scan);
 885       assertNullResult(result);
 886
 887       scan = new Scan(ROWS[0], ROWS[2]);
 888       result = getSingleScanResult(ht, scan);
 889       assertNullResult(result);
 890
 891       // Make sure we can actually get the row
 892
 893       get = new Get(ROWS[2]);
 894       result = ht.get(get);
 895       assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
 896
 897       get = new Get(ROWS[2]);
 898       get.addFamily(FAMILY);
 899       result = ht.get(get);
 900       assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
 901
 902       get = new Get(ROWS[2]);
 903       get.addColumn(FAMILY, QUALIFIER);
 904       result = ht.get(get);
 905       assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
 906
 907       // Make sure we can scan the row
 908
 909       scan = new Scan();
 910       result = getSingleScanResult(ht, scan);
 911       assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
 912
 913       scan = new Scan(ROWS[0], ROWS[3]);
 914       result = getSingleScanResult(ht, scan);
 915       assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
 916
 917       scan = new Scan(ROWS[2], ROWS[3]);
 918       result = getSingleScanResult(ht, scan);
 919       assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
 920     }
 921   }
 922
 923   /**
 924    * Test basic puts, gets, scans, and deletes for a single row
 925    * in a multiple family table.
 926    */
 927   @Test
 928   public void testSingleRowMultipleFamily() throws Exception {
 929     final TableName tableName = name.getTableName();
 930     byte [][] ROWS = makeN(ROW, 3);
 931     byte [][] FAMILIES = makeNAscii(FAMILY, 10);
 932     byte [][] QUALIFIERS = makeN(QUALIFIER, 10);
 933     byte [][] VALUES = makeN(VALUE, 10);
 934
 935     try (Table ht = TEST_UTIL.createTable(tableName, FAMILIES)) {
 936
 937       Get get;
 938       Scan scan;
 939       Delete delete;
 940       Put put;
 941       Result result;
 942
 943       ////////////////////////////////////////////////////////////////////////////
 944       // Insert one column to one family
 945       ////////////////////////////////////////////////////////////////////////////
 946
 947       put = new Put(ROWS[0]);
 948       put.addColumn(FAMILIES[4], QUALIFIERS[0], VALUES[0]);
 949       ht.put(put);
 950
 951       // Get the single column
 952       getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
 953
 954       // Scan the single column
 955       scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
 956
 957       // Get empty results around inserted column
 958       getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
 959
 960       // Scan empty results around inserted column
 961       scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
 962
 963       ////////////////////////////////////////////////////////////////////////////
 964       // Flush memstore and run same tests from storefiles
 965       ////////////////////////////////////////////////////////////////////////////
 966
 967       TEST_UTIL.flush();
 968
 969       // Redo get and scan tests from storefile
 970       getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
 971       scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
 972       getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
 973       scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
 974
 975       ////////////////////////////////////////////////////////////////////////////
 976       // Now, Test reading from memstore and storefiles at once
 977       ////////////////////////////////////////////////////////////////////////////
 978
 979       // Insert multiple columns to two other families
 980       put = new Put(ROWS[0]);
 981       put.addColumn(FAMILIES[2], QUALIFIERS[2], VALUES[2]);
 982       put.addColumn(FAMILIES[2], QUALIFIERS[4], VALUES[4]);
 983       put.addColumn(FAMILIES[4], QUALIFIERS[4], VALUES[4]);
 984       put.addColumn(FAMILIES[6], QUALIFIERS[6], VALUES[6]);
 985       put.addColumn(FAMILIES[6], QUALIFIERS[7], VALUES[7]);
 986       put.addColumn(FAMILIES[7], QUALIFIERS[7], VALUES[7]);
 987       put.addColumn(FAMILIES[9], QUALIFIERS[0], VALUES[0]);
 988       ht.put(put);
 989
 990       // Get multiple columns across multiple families and get empties around it
 991       singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
 992
 993       // Scan multiple columns across multiple families and scan empties around it
 994       singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
 995
 996       ////////////////////////////////////////////////////////////////////////////
 997       // Flush the table again
 998       ////////////////////////////////////////////////////////////////////////////
 999
1000       TEST_UTIL.flush();
1001
1002       // Redo tests again
1003       singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
1004       singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
1005
1006       // Insert more data to memstore
1007       put = new Put(ROWS[0]);
1008       put.addColumn(FAMILIES[6], QUALIFIERS[5], VALUES[5]);
1009       put.addColumn(FAMILIES[6], QUALIFIERS[8], VALUES[8]);
1010       put.addColumn(FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1011       put.addColumn(FAMILIES[4], QUALIFIERS[3], VALUES[3]);
1012       ht.put(put);
1013
1014       ////////////////////////////////////////////////////////////////////////////
1015       // Delete a storefile column
1016       ////////////////////////////////////////////////////////////////////////////
1017       delete = new Delete(ROWS[0]);
1018       delete.addColumns(FAMILIES[6], QUALIFIERS[7]);
1019       ht.delete(delete);
1020
1021       // Try to get deleted column
1022       get = new Get(ROWS[0]);
1023       get.addColumn(FAMILIES[6], QUALIFIERS[7]);
1024       result = ht.get(get);
1025       assertEmptyResult(result);
1026
1027       // Try to scan deleted column
1028       scan = new Scan();
1029       scan.addColumn(FAMILIES[6], QUALIFIERS[7]);
1030       result = getSingleScanResult(ht, scan);
1031       assertNullResult(result);
1032
1033       // Make sure we can still get a column before it and after it
1034       get = new Get(ROWS[0]);
1035       get.addColumn(FAMILIES[6], QUALIFIERS[6]);
1036       result = ht.get(get);
1037       assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1038
1039       get = new Get(ROWS[0]);
1040       get.addColumn(FAMILIES[6], QUALIFIERS[8]);
1041       result = ht.get(get);
1042       assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]);
1043
1044       // Make sure we can still scan a column before it and after it
1045       scan = new Scan();
1046       scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1047       result = getSingleScanResult(ht, scan);
1048       assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1049
1050       scan = new Scan();
1051       scan.addColumn(FAMILIES[6], QUALIFIERS[8]);
1052       result = getSingleScanResult(ht, scan);
1053       assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]);
1054
1055       ////////////////////////////////////////////////////////////////////////////
1056       // Delete a memstore column
1057       ////////////////////////////////////////////////////////////////////////////
1058       delete = new Delete(ROWS[0]);
1059       delete.addColumns(FAMILIES[6], QUALIFIERS[8]);
1060       ht.delete(delete);
1061
1062       // Try to get deleted column
1063       get = new Get(ROWS[0]);
1064       get.addColumn(FAMILIES[6], QUALIFIERS[8]);
1065       result = ht.get(get);
1066       assertEmptyResult(result);
1067
1068       // Try to scan deleted column
1069       scan = new Scan();
1070       scan.addColumn(FAMILIES[6], QUALIFIERS[8]);
1071       result = getSingleScanResult(ht, scan);
1072       assertNullResult(result);
1073
1074       // Make sure we can still get a column before it and after it
1075       get = new Get(ROWS[0]);
1076       get.addColumn(FAMILIES[6], QUALIFIERS[6]);
1077       result = ht.get(get);
1078       assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1079
1080       get = new Get(ROWS[0]);
1081       get.addColumn(FAMILIES[6], QUALIFIERS[9]);
1082       result = ht.get(get);
1083       assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1084
1085       // Make sure we can still scan a column before it and after it
1086       scan = new Scan();
1087       scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1088       result = getSingleScanResult(ht, scan);
1089       assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1090
1091       scan = new Scan();
1092       scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
1093       result = getSingleScanResult(ht, scan);
1094       assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1095
1096       ////////////////////////////////////////////////////////////////////////////
1097       // Delete joint storefile/memstore family
1098       ////////////////////////////////////////////////////////////////////////////
1099
1100       delete = new Delete(ROWS[0]);
1101       delete.addFamily(FAMILIES[4]);
1102       ht.delete(delete);
1103
1104       // Try to get storefile column in deleted family
1105       get = new Get(ROWS[0]);
1106       get.addColumn(FAMILIES[4], QUALIFIERS[4]);
1107       result = ht.get(get);
1108       assertEmptyResult(result);
1109
1110       // Try to get memstore column in deleted family
1111       get = new Get(ROWS[0]);
1112       get.addColumn(FAMILIES[4], QUALIFIERS[3]);
1113       result = ht.get(get);
1114       assertEmptyResult(result);
1115
1116       // Try to get deleted family
1117       get = new Get(ROWS[0]);
1118       get.addFamily(FAMILIES[4]);
1119       result = ht.get(get);
1120       assertEmptyResult(result);
1121
1122       // Try to scan storefile column in deleted family
1123       scan = new Scan();
1124       scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
1125       result = getSingleScanResult(ht, scan);
1126       assertNullResult(result);
1127
1128       // Try to scan memstore column in deleted family
1129       scan = new Scan();
1130       scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
1131       result = getSingleScanResult(ht, scan);
1132       assertNullResult(result);
1133
1134       // Try to scan deleted family
1135       scan = new Scan();
1136       scan.addFamily(FAMILIES[4]);
1137       result = getSingleScanResult(ht, scan);
1138       assertNullResult(result);
1139
1140       // Make sure we can still get another family
1141       get = new Get(ROWS[0]);
1142       get.addColumn(FAMILIES[2], QUALIFIERS[2]);
1143       result = ht.get(get);
1144       assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
1145
1146       get = new Get(ROWS[0]);
1147       get.addColumn(FAMILIES[6], QUALIFIERS[9]);
1148       result = ht.get(get);
1149       assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1150
1151       // Make sure we can still scan another family
1152       scan = new Scan();
1153       scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1154       result = getSingleScanResult(ht, scan);
1155       assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1156
1157       scan = new Scan();
1158       scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
1159       result = getSingleScanResult(ht, scan);
1160       assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1161
1162       ////////////////////////////////////////////////////////////////////////////
1163       // Flush everything and rerun delete tests
1164       ////////////////////////////////////////////////////////////////////////////
1165
1166       TEST_UTIL.flush();
1167
1168       // Try to get storefile column in deleted family
1169       get = new Get(ROWS[0]);
1170       get.addColumn(FAMILIES[4], QUALIFIERS[4]);
1171       result = ht.get(get);
1172       assertEmptyResult(result);
1173
1174       // Try to get memstore column in deleted family
1175       get = new Get(ROWS[0]);
1176       get.addColumn(FAMILIES[4], QUALIFIERS[3]);
1177       result = ht.get(get);
1178       assertEmptyResult(result);
1179
1180       // Try to get deleted family
1181       get = new Get(ROWS[0]);
1182       get.addFamily(FAMILIES[4]);
1183       result = ht.get(get);
1184       assertEmptyResult(result);
1185
1186       // Try to scan storefile column in deleted family
1187       scan = new Scan();
1188       scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
1189       result = getSingleScanResult(ht, scan);
1190       assertNullResult(result);
1191
1192       // Try to scan memstore column in deleted family
1193       scan = new Scan();
1194       scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
1195       result = getSingleScanResult(ht, scan);
1196       assertNullResult(result);
1197
1198       // Try to scan deleted family
1199       scan = new Scan();
1200       scan.addFamily(FAMILIES[4]);
1201       result = getSingleScanResult(ht, scan);
1202       assertNullResult(result);
1203
1204       // Make sure we can still get another family
1205       get = new Get(ROWS[0]);
1206       get.addColumn(FAMILIES[2], QUALIFIERS[2]);
1207       result = ht.get(get);
1208       assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
1209
1210       get = new Get(ROWS[0]);
1211       get.addColumn(FAMILIES[6], QUALIFIERS[9]);
1212       result = ht.get(get);
1213       assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1214
1215       // Make sure we can still scan another family
1216       scan = new Scan();
1217       scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1218       result = getSingleScanResult(ht, scan);
1219       assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1220
1221       scan = new Scan();
1222       scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
1223       result = getSingleScanResult(ht, scan);
1224       assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1225     }
1226   }
1227
1228   @Test(expected = NullPointerException.class)
1229   public void testNullTableName() throws IOException {
1230     // Null table name (should NOT work)
1231     TEST_UTIL.createTable((TableName)null, FAMILY);
1232     fail("Creating a table with null name passed, should have failed");
1233   }
1234
1235   @Test(expected = IllegalArgumentException.class)
1236   public void testNullFamilyName() throws IOException {
1237     final TableName tableName = name.getTableName();
1238
1239     // Null family (should NOT work)
1240     TEST_UTIL.createTable(tableName, new byte[][]{null});
1241     fail("Creating a table with a null family passed, should fail");
1242   }
1243
1244   @Test
1245   public void testNullRowAndQualifier() throws Exception {
1246     final TableName tableName = name.getTableName();
1247
1248     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
1249
1250       // Null row (should NOT work)
1251       try {
1252         Put put = new Put((byte[]) null);
1253         put.addColumn(FAMILY, QUALIFIER, VALUE);
1254         ht.put(put);
1255         fail("Inserting a null row worked, should throw exception");
1256       } catch (Exception e) {
1257       }
1258
1259       // Null qualifier (should work)
1260       {
1261         Put put = new Put(ROW);
1262         put.addColumn(FAMILY, null, VALUE);
1263         ht.put(put);
1264
1265         getTestNull(ht, ROW, FAMILY, VALUE);
1266
1267         scanTestNull(ht, ROW, FAMILY, VALUE);
1268
1269         Delete delete = new Delete(ROW);
1270         delete.addColumns(FAMILY, null);
1271         ht.delete(delete);
1272
1273         Get get = new Get(ROW);
1274         Result result = ht.get(get);
1275         assertEmptyResult(result);
1276       }
1277     }
1278   }
1279
1280   @Test
1281   public void testNullEmptyQualifier() throws Exception {
1282     final TableName tableName = name.getTableName();
1283
1284     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
1285
1286       // Empty qualifier, byte[0] instead of null (should work)
1287       try {
1288         Put put = new Put(ROW);
1289         put.addColumn(FAMILY, HConstants.EMPTY_BYTE_ARRAY, VALUE);
1290         ht.put(put);
1291
1292         getTestNull(ht, ROW, FAMILY, VALUE);
1293
1294         scanTestNull(ht, ROW, FAMILY, VALUE);
1295
1296         // Flush and try again
1297
1298         TEST_UTIL.flush();
1299
1300         getTestNull(ht, ROW, FAMILY, VALUE);
1301
1302         scanTestNull(ht, ROW, FAMILY, VALUE);
1303
1304         Delete delete = new Delete(ROW);
1305         delete.addColumns(FAMILY, HConstants.EMPTY_BYTE_ARRAY);
1306         ht.delete(delete);
1307
1308         Get get = new Get(ROW);
1309         Result result = ht.get(get);
1310         assertEmptyResult(result);
1311
1312       } catch (Exception e) {
1313         throw new IOException("Using a row with null qualifier should not throw exception");
1314       }
1315     }
1316   }
1317
1318   @Test
1319   public void testNullValue() throws IOException {
1320     final TableName tableName = name.getTableName();
1321
1322     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
1323       // Null value
1324       try {
1325         Put put = new Put(ROW);
1326         put.addColumn(FAMILY, QUALIFIER, null);
1327         ht.put(put);
1328
1329         Get get = new Get(ROW);
1330         get.addColumn(FAMILY, QUALIFIER);
1331         Result result = ht.get(get);
1332         assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
1333
1334         Scan scan = new Scan();
1335         scan.addColumn(FAMILY, QUALIFIER);
1336         result = getSingleScanResult(ht, scan);
1337         assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
1338
1339         Delete delete = new Delete(ROW);
1340         delete.addColumns(FAMILY, QUALIFIER);
1341         ht.delete(delete);
1342
1343         get = new Get(ROW);
1344         result = ht.get(get);
1345         assertEmptyResult(result);
1346
1347       } catch (Exception e) {
1348         throw new IOException("Null values should be allowed, but threw exception");
1349       }
1350     }
1351   }
1352
1353   @Test
1354   public void testNullQualifier() throws Exception {
1355     final TableName tableName = name.getTableName();
1356     try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) {
1357
1358       // Work for Put
1359       Put put = new Put(ROW);
1360       put.addColumn(FAMILY, null, VALUE);
1361       table.put(put);
1362
1363       // Work for Get, Scan
1364       getTestNull(table, ROW, FAMILY, VALUE);
1365       scanTestNull(table, ROW, FAMILY, VALUE);
1366
1367       // Work for Delete
1368       Delete delete = new Delete(ROW);
1369       delete.addColumns(FAMILY, null);
1370       table.delete(delete);
1371
1372       Get get = new Get(ROW);
1373       Result result = table.get(get);
1374       assertEmptyResult(result);
1375
1376       // Work for Increment/Append
1377       Increment increment = new Increment(ROW);
1378       increment.addColumn(FAMILY, null, 1L);
1379       table.increment(increment);
1380       getTestNull(table, ROW, FAMILY, 1L);
1381
1382       table.incrementColumnValue(ROW, FAMILY, null, 1L);
1383       getTestNull(table, ROW, FAMILY, 2L);
1384
1385       delete = new Delete(ROW);
1386       delete.addColumns(FAMILY, null);
1387       table.delete(delete);
1388
1389       Append append = new Append(ROW);
1390       append.addColumn(FAMILY, null, VALUE);
1391       table.append(append);
1392       getTestNull(table, ROW, FAMILY, VALUE);
1393
1394       // Work for checkAndMutate using thenPut, thenMutate and thenDelete
1395       put = new Put(ROW);
1396       put.addColumn(FAMILY, null, Bytes.toBytes("checkAndPut"));
1397       table.put(put);
1398       table.checkAndMutate(ROW, FAMILY).ifEquals(VALUE).thenPut(put);
1399
1400       RowMutations mutate = new RowMutations(ROW);
1401       mutate.add(new Put(ROW).addColumn(FAMILY, null, Bytes.toBytes("checkAndMutate")));
1402       table.checkAndMutate(ROW, FAMILY).ifEquals(Bytes.toBytes("checkAndPut")).thenMutate(mutate);
1403
1404       delete = new Delete(ROW);
1405       delete.addColumns(FAMILY, null);
1406       table.checkAndMutate(ROW, FAMILY).ifEquals(Bytes.toBytes("checkAndMutate"))
1407               .thenDelete(delete);
1408     }
1409   }
1410
1411   @Test
1412   public void testVersions() throws Exception {
1413     final TableName tableName = name.getTableName();
1414
1415     long [] STAMPS = makeStamps(20);
1416     byte [][] VALUES = makeNAscii(VALUE, 20);
1417
1418     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
1419
1420       // Insert 4 versions of same column
1421       Put put = new Put(ROW);
1422       put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1423       put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1424       put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1425       put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1426       ht.put(put);
1427
1428       // Verify we can get each one properly
1429       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1430       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1431       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1432       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1433       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1434       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1435       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1436       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1437
1438       // Verify we don't accidentally get others
1439       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1440       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1441       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1442       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1443       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1444       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1445
1446       // Ensure maxVersions in query is respected
1447       Get get = new Get(ROW);
1448       get.addColumn(FAMILY, QUALIFIER);
1449       get.readVersions(2);
1450       Result result = ht.get(get);
1451       assertNResult(result, ROW, FAMILY, QUALIFIER,
1452           new long [] {STAMPS[4], STAMPS[5]},
1453           new byte[][] {VALUES[4], VALUES[5]},
1454           0, 1);
1455
1456       Scan scan = new Scan(ROW);
1457       scan.addColumn(FAMILY, QUALIFIER);
1458       scan.setMaxVersions(2);
1459       result = getSingleScanResult(ht, scan);
1460       assertNResult(result, ROW, FAMILY, QUALIFIER,
1461               new long[]{STAMPS[4], STAMPS[5]},
1462               new byte[][]{VALUES[4], VALUES[5]},
1463               0, 1);
1464
1465       // Flush and redo
1466
1467       TEST_UTIL.flush();
1468
1469       // Verify we can get each one properly
1470       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1471       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1472       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1473       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1474       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1475       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1476       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1477       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1478
1479       // Verify we don't accidentally get others
1480       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1481       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1482       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1483       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1484       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1485       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1486
1487       // Ensure maxVersions in query is respected
1488       get = new Get(ROW);
1489       get.addColumn(FAMILY, QUALIFIER);
1490       get.readVersions(2);
1491       result = ht.get(get);
1492       assertNResult(result, ROW, FAMILY, QUALIFIER,
1493           new long [] {STAMPS[4], STAMPS[5]},
1494           new byte[][] {VALUES[4], VALUES[5]},
1495           0, 1);
1496
1497       scan = new Scan(ROW);
1498       scan.addColumn(FAMILY, QUALIFIER);
1499       scan.setMaxVersions(2);
1500       result = getSingleScanResult(ht, scan);
1501       assertNResult(result, ROW, FAMILY, QUALIFIER,
1502               new long[]{STAMPS[4], STAMPS[5]},
1503               new byte[][]{VALUES[4], VALUES[5]},
1504               0, 1);
1505
1506
1507       // Add some memstore and retest
1508
1509       // Insert 4 more versions of same column and a dupe
1510       put = new Put(ROW);
1511       put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
1512       put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
1513       put.addColumn(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1514       put.addColumn(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
1515       ht.put(put);
1516
1517       // Ensure maxVersions in query is respected
1518       get = new Get(ROW);
1519       get.addColumn(FAMILY, QUALIFIER);
1520       get.readAllVersions();
1521       result = ht.get(get);
1522       assertNResult(result, ROW, FAMILY, QUALIFIER,
1523           new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7],
1524                   STAMPS[8]},
1525           new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7],
1526                   VALUES[8]},
1527           0, 7);
1528
1529       scan = new Scan(ROW);
1530       scan.addColumn(FAMILY, QUALIFIER);
1531       scan.setMaxVersions();
1532       result = getSingleScanResult(ht, scan);
1533       assertNResult(result, ROW, FAMILY, QUALIFIER,
1534           new long[]{STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7],
1535                   STAMPS[8]},
1536           new byte[][]{VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7],
1537                   VALUES[8]},0, 7);
1538
1539       get = new Get(ROW);
1540       get.readAllVersions();
1541       result = ht.get(get);
1542       assertNResult(result, ROW, FAMILY, QUALIFIER,
1543           new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7],
1544                   STAMPS[8]},
1545           new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7],
1546                   VALUES[8]},
1547           0, 7);
1548
1549       scan = new Scan(ROW);
1550       scan.setMaxVersions();
1551       result = getSingleScanResult(ht, scan);
1552       assertNResult(result, ROW, FAMILY, QUALIFIER,
1553           new long[]{STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7],
1554                   STAMPS[8]},
1555           new byte[][]{VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7],
1556                   VALUES[8]},0, 7);
1557
1558       // Verify we can get each one properly
1559       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1560       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1561       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1562       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1563       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1564       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1565       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1566       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1567
1568       // Verify we don't accidentally get others
1569       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1570       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
1571       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1572       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
1573
1574       // Ensure maxVersions of table is respected
1575
1576       TEST_UTIL.flush();
1577
1578       // Insert 4 more versions of same column and a dupe
1579       put = new Put(ROW);
1580       put.addColumn(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
1581       put.addColumn(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
1582       put.addColumn(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
1583       put.addColumn(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
1584       ht.put(put);
1585
1586       get = new Get(ROW);
1587       get.addColumn(FAMILY, QUALIFIER);
1588       get.readVersions(Integer.MAX_VALUE);
1589       result = ht.get(get);
1590       assertNResult(result, ROW, FAMILY, QUALIFIER,
1591           new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9],
1592                   STAMPS[11], STAMPS[13], STAMPS[15]},
1593           new byte[][] {VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9],
1594                   VALUES[11], VALUES[13], VALUES[15]},
1595           0, 9);
1596
1597       scan = new Scan(ROW);
1598       scan.addColumn(FAMILY, QUALIFIER);
1599       scan.setMaxVersions(Integer.MAX_VALUE);
1600       result = getSingleScanResult(ht, scan);
1601       assertNResult(result, ROW, FAMILY, QUALIFIER,
1602           new long[]{STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9],
1603                   STAMPS[11], STAMPS[13], STAMPS[15]},
1604           new byte[][]{VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9],
1605                   VALUES[11], VALUES[13], VALUES[15]},0, 9);
1606
1607       // Delete a version in the memstore and a version in a storefile
1608       Delete delete = new Delete(ROW);
1609       delete.addColumn(FAMILY, QUALIFIER, STAMPS[11]);
1610       delete.addColumn(FAMILY, QUALIFIER, STAMPS[7]);
1611       ht.delete(delete);
1612
1613       // Test that it's gone
1614       get = new Get(ROW);
1615       get.addColumn(FAMILY, QUALIFIER);
1616       get.readVersions(Integer.MAX_VALUE);
1617       result = ht.get(get);
1618       assertNResult(result, ROW, FAMILY, QUALIFIER,
1619           new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8],
1620                   STAMPS[9], STAMPS[13], STAMPS[15]},
1621           new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8],
1622                   VALUES[9], VALUES[13], VALUES[15]},
1623           0, 9);
1624
1625       scan = new Scan(ROW);
1626       scan.addColumn(FAMILY, QUALIFIER);
1627       scan.setMaxVersions(Integer.MAX_VALUE);
1628       result = getSingleScanResult(ht, scan);
1629       assertNResult(result, ROW, FAMILY, QUALIFIER,
1630           new long[]{STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8],
1631                   STAMPS[9], STAMPS[13], STAMPS[15]},
1632           new byte[][]{VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8],
1633                   VALUES[9], VALUES[13], VALUES[15]},0, 9);
1634     }
1635   }
1636
1637   @Test
1638   @SuppressWarnings("checkstyle:MethodLength")
1639   public void testVersionLimits() throws Exception {
1640     final TableName tableName = name.getTableName();
1641     byte [][] FAMILIES = makeNAscii(FAMILY, 3);
1642     int [] LIMITS = {1,3,5};
1643     long [] STAMPS = makeStamps(10);
1644     byte [][] VALUES = makeNAscii(VALUE, 10);
1645     try (Table ht = TEST_UTIL.createTable(tableName, FAMILIES, LIMITS)) {
1646
1647       // Insert limit + 1 on each family
1648       Put put = new Put(ROW);
1649       put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[0], VALUES[0]);
1650       put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[1], VALUES[1]);
1651       put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[0], VALUES[0]);
1652       put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[1], VALUES[1]);
1653       put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[2], VALUES[2]);
1654       put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[3], VALUES[3]);
1655       put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[0], VALUES[0]);
1656       put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[1], VALUES[1]);
1657       put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[2], VALUES[2]);
1658       put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[3], VALUES[3]);
1659       put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[4], VALUES[4]);
1660       put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[5], VALUES[5]);
1661       put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[6], VALUES[6]);
1662       ht.put(put);
1663
1664       // Verify we only get the right number out of each
1665
1666       // Family0
1667
1668       Get get = new Get(ROW);
1669       get.addColumn(FAMILIES[0], QUALIFIER);
1670       get.readVersions(Integer.MAX_VALUE);
1671       Result result = ht.get(get);
1672       assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1673           new long [] {STAMPS[1]},
1674           new byte[][] {VALUES[1]},
1675           0, 0);
1676
1677       get = new Get(ROW);
1678       get.addFamily(FAMILIES[0]);
1679       get.readVersions(Integer.MAX_VALUE);
1680       result = ht.get(get);
1681       assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1682           new long [] {STAMPS[1]},
1683           new byte[][] {VALUES[1]},
1684           0, 0);
1685
1686       Scan scan = new Scan(ROW);
1687       scan.addColumn(FAMILIES[0], QUALIFIER);
1688       scan.setMaxVersions(Integer.MAX_VALUE);
1689       result = getSingleScanResult(ht, scan);
1690       assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1691               new long[]{STAMPS[1]},
1692               new byte[][]{VALUES[1]},
1693               0, 0);
1694
1695       scan = new Scan(ROW);
1696       scan.addFamily(FAMILIES[0]);
1697       scan.setMaxVersions(Integer.MAX_VALUE);
1698       result = getSingleScanResult(ht, scan);
1699       assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1700               new long[]{STAMPS[1]},
1701               new byte[][]{VALUES[1]},
1702               0, 0);
1703
1704       // Family1
1705
1706       get = new Get(ROW);
1707       get.addColumn(FAMILIES[1], QUALIFIER);
1708       get.readVersions(Integer.MAX_VALUE);
1709       result = ht.get(get);
1710       assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1711           new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1712           new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1713           0, 2);
1714
1715       get = new Get(ROW);
1716       get.addFamily(FAMILIES[1]);
1717       get.readVersions(Integer.MAX_VALUE);
1718       result = ht.get(get);
1719       assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1720           new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1721           new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1722           0, 2);
1723
1724       scan = new Scan(ROW);
1725       scan.addColumn(FAMILIES[1], QUALIFIER);
1726       scan.setMaxVersions(Integer.MAX_VALUE);
1727       result = getSingleScanResult(ht, scan);
1728       assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1729               new long[]{STAMPS[1], STAMPS[2], STAMPS[3]},
1730               new byte[][]{VALUES[1], VALUES[2], VALUES[3]},
1731               0, 2);
1732
1733       scan = new Scan(ROW);
1734       scan.addFamily(FAMILIES[1]);
1735       scan.setMaxVersions(Integer.MAX_VALUE);
1736       result = getSingleScanResult(ht, scan);
1737       assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1738               new long[]{STAMPS[1], STAMPS[2], STAMPS[3]},
1739               new byte[][]{VALUES[1], VALUES[2], VALUES[3]},
1740               0, 2);
1741
1742       // Family2
1743
1744       get = new Get(ROW);
1745       get.addColumn(FAMILIES[2], QUALIFIER);
1746       get.readVersions(Integer.MAX_VALUE);
1747       result = ht.get(get);
1748       assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1749           new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1750           new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1751           0, 4);
1752
1753       get = new Get(ROW);
1754       get.addFamily(FAMILIES[2]);
1755       get.readVersions(Integer.MAX_VALUE);
1756       result = ht.get(get);
1757       assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1758           new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1759           new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1760           0, 4);
1761
1762       scan = new Scan(ROW);
1763       scan.addColumn(FAMILIES[2], QUALIFIER);
1764       scan.setMaxVersions(Integer.MAX_VALUE);
1765       result = getSingleScanResult(ht, scan);
1766       assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1767               new long[]{STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1768               new byte[][]{VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1769               0, 4);
1770
1771       scan = new Scan(ROW);
1772       scan.addFamily(FAMILIES[2]);
1773       scan.setMaxVersions(Integer.MAX_VALUE);
1774       result = getSingleScanResult(ht, scan);
1775       assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1776               new long[]{STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1777               new byte[][]{VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1778               0, 4);
1779
1780       // Try all families
1781
1782       get = new Get(ROW);
1783       get.readVersions(Integer.MAX_VALUE);
1784       result = ht.get(get);
1785       assertTrue("Expected 9 keys but received " + result.size(),
1786           result.size() == 9);
1787
1788       get = new Get(ROW);
1789       get.addFamily(FAMILIES[0]);
1790       get.addFamily(FAMILIES[1]);
1791       get.addFamily(FAMILIES[2]);
1792       get.readVersions(Integer.MAX_VALUE);
1793       result = ht.get(get);
1794       assertTrue("Expected 9 keys but received " + result.size(),
1795           result.size() == 9);
1796
1797       get = new Get(ROW);
1798       get.addColumn(FAMILIES[0], QUALIFIER);
1799       get.addColumn(FAMILIES[1], QUALIFIER);
1800       get.addColumn(FAMILIES[2], QUALIFIER);
1801       get.readVersions(Integer.MAX_VALUE);
1802       result = ht.get(get);
1803       assertTrue("Expected 9 keys but received " + result.size(),
1804           result.size() == 9);
1805
1806       scan = new Scan(ROW);
1807       scan.setMaxVersions(Integer.MAX_VALUE);
1808       result = getSingleScanResult(ht, scan);
1809       assertTrue("Expected 9 keys but received " + result.size(),
1810               result.size() == 9);
1811
1812       scan = new Scan(ROW);
1813       scan.setMaxVersions(Integer.MAX_VALUE);
1814       scan.addFamily(FAMILIES[0]);
1815       scan.addFamily(FAMILIES[1]);
1816       scan.addFamily(FAMILIES[2]);
1817       result = getSingleScanResult(ht, scan);
1818       assertTrue("Expected 9 keys but received " + result.size(),
1819               result.size() == 9);
1820
1821       scan = new Scan(ROW);
1822       scan.setMaxVersions(Integer.MAX_VALUE);
1823       scan.addColumn(FAMILIES[0], QUALIFIER);
1824       scan.addColumn(FAMILIES[1], QUALIFIER);
1825       scan.addColumn(FAMILIES[2], QUALIFIER);
1826       result = getSingleScanResult(ht, scan);
1827       assertTrue("Expected 9 keys but received " + result.size(),
1828               result.size() == 9);
1829     }
1830   }
1831
1832   @Test
1833   public void testDeleteFamilyVersion() throws Exception {
1834     try (Admin admin = TEST_UTIL.getAdmin()) {
1835       final TableName tableName = name.getTableName();
1836
1837       byte[][] QUALIFIERS = makeNAscii(QUALIFIER, 1);
1838       byte[][] VALUES = makeN(VALUE, 5);
1839       long[] ts = {1000, 2000, 3000, 4000, 5000};
1840
1841       try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 5)) {
1842
1843         Put put = new Put(ROW);
1844         for (int q = 0; q < 1; q++) {
1845           for (int t = 0; t < 5; t++) {
1846             put.addColumn(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
1847           }
1848         }
1849         ht.put(put);
1850         admin.flush(tableName);
1851
1852         Delete delete = new Delete(ROW);
1853         delete.addFamilyVersion(FAMILY, ts[1]);  // delete version '2000'
1854         delete.addFamilyVersion(FAMILY, ts[3]);  // delete version '4000'
1855         ht.delete(delete);
1856         admin.flush(tableName);
1857
1858         for (int i = 0; i < 1; i++) {
1859           Get get = new Get(ROW);
1860           get.addColumn(FAMILY, QUALIFIERS[i]);
1861           get.readVersions(Integer.MAX_VALUE);
1862           Result result = ht.get(get);
1863           // verify version '1000'/'3000'/'5000' remains for all columns
1864           assertNResult(result, ROW, FAMILY, QUALIFIERS[i],
1865                   new long[]{ts[0], ts[2], ts[4]},
1866                   new byte[][]{VALUES[0], VALUES[2], VALUES[4]},
1867                   0, 2);
1868         }
1869       }
1870     }
1871   }
1872
1873   @Test
1874   public void testDeleteFamilyVersionWithOtherDeletes() throws Exception {
1875     final TableName tableName = name.getTableName();
1876
1877     byte [][] QUALIFIERS = makeNAscii(QUALIFIER, 5);
1878     byte [][] VALUES = makeN(VALUE, 5);
1879     long [] ts = {1000, 2000, 3000, 4000, 5000};
1880
1881     try (Admin admin = TEST_UTIL.getAdmin();
1882         Table ht = TEST_UTIL.createTable(tableName, FAMILY, 5)) {
1883       Put put = null;
1884       Result result = null;
1885       Get get = null;
1886       Delete delete = null;
1887
1888       // 1. put on ROW
1889       put = new Put(ROW);
1890       for (int q = 0; q < 5; q++) {
1891         for (int t = 0; t < 5; t++) {
1892           put.addColumn(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
1893         }
1894       }
1895       ht.put(put);
1896       admin.flush(tableName);
1897
1898       // 2. put on ROWS[0]
1899       byte[] ROW2 = Bytes.toBytes("myRowForTest");
1900       put = new Put(ROW2);
1901       for (int q = 0; q < 5; q++) {
1902         for (int t = 0; t < 5; t++) {
1903           put.addColumn(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
1904         }
1905       }
1906       ht.put(put);
1907       admin.flush(tableName);
1908
1909       // 3. delete on ROW
1910       delete = new Delete(ROW);
1911       // delete version <= 2000 of all columns
1912       // note: addFamily must be the first since it will mask
1913       // the subsequent other type deletes!
1914       delete.addFamily(FAMILY, ts[1]);
1915       // delete version '4000' of all columns
1916       delete.addFamilyVersion(FAMILY, ts[3]);
1917       // delete version <= 3000 of column 0
1918       delete.addColumns(FAMILY, QUALIFIERS[0], ts[2]);
1919       // delete version <= 5000 of column 2
1920       delete.addColumns(FAMILY, QUALIFIERS[2], ts[4]);
1921       // delete version 5000 of column 4
1922       delete.addColumn(FAMILY, QUALIFIERS[4], ts[4]);
1923       ht.delete(delete);
1924       admin.flush(tableName);
1925
1926       // 4. delete on ROWS[0]
1927       delete = new Delete(ROW2);
1928       delete.addFamilyVersion(FAMILY, ts[1]);  // delete version '2000'
1929       delete.addFamilyVersion(FAMILY, ts[3]);  // delete version '4000'
1930       ht.delete(delete);
1931       admin.flush(tableName);
1932
1933       // 5. check ROW
1934       get = new Get(ROW);
1935       get.addColumn(FAMILY, QUALIFIERS[0]);
1936       get.readVersions(Integer.MAX_VALUE);
1937       result = ht.get(get);
1938       assertNResult(result, ROW, FAMILY, QUALIFIERS[0],
1939               new long[]{ts[4]},
1940               new byte[][]{VALUES[4]},
1941               0, 0);
1942
1943       get = new Get(ROW);
1944       get.addColumn(FAMILY, QUALIFIERS[1]);
1945       get.readVersions(Integer.MAX_VALUE);
1946       result = ht.get(get);
1947       assertNResult(result, ROW, FAMILY, QUALIFIERS[1],
1948               new long[]{ts[2], ts[4]},
1949               new byte[][]{VALUES[2], VALUES[4]},
1950               0, 1);
1951
1952       get = new Get(ROW);
1953       get.addColumn(FAMILY, QUALIFIERS[2]);
1954       get.readVersions(Integer.MAX_VALUE);
1955       result = ht.get(get);
1956       assertEquals(0, result.size());
1957
1958       get = new Get(ROW);
1959       get.addColumn(FAMILY, QUALIFIERS[3]);
1960       get.readVersions(Integer.MAX_VALUE);
1961       result = ht.get(get);
1962       assertNResult(result, ROW, FAMILY, QUALIFIERS[3],
1963               new long[]{ts[2], ts[4]},
1964               new byte[][]{VALUES[2], VALUES[4]},
1965               0, 1);
1966
1967       get = new Get(ROW);
1968       get.addColumn(FAMILY, QUALIFIERS[4]);
1969       get.readVersions(Integer.MAX_VALUE);
1970       result = ht.get(get);
1971       assertNResult(result, ROW, FAMILY, QUALIFIERS[4],
1972               new long[]{ts[2]},
1973               new byte[][]{VALUES[2]},
1974               0, 0);
1975
1976       // 6. check ROWS[0]
1977       for (int i = 0; i < 5; i++) {
1978         get = new Get(ROW2);
1979         get.addColumn(FAMILY, QUALIFIERS[i]);
1980         get.readVersions(Integer.MAX_VALUE);
1981         result = ht.get(get);
1982         // verify version '1000'/'3000'/'5000' remains for all columns
1983         assertNResult(result, ROW2, FAMILY, QUALIFIERS[i],
1984                 new long[]{ts[0], ts[2], ts[4]},
1985                 new byte[][]{VALUES[0], VALUES[2], VALUES[4]},
1986                 0, 2);
1987       }
1988     }
1989   }
1990
1991   @Test
1992   public void testDeleteWithFailed() throws Exception {
1993     final TableName tableName = name.getTableName();
1994
1995     byte [][] FAMILIES = makeNAscii(FAMILY, 3);
1996     byte [][] VALUES = makeN(VALUE, 5);
1997     long [] ts = {1000, 2000, 3000, 4000, 5000};
1998
1999     try (Table ht = TEST_UTIL.createTable(tableName, FAMILIES, 3)) {
2000       Put put = new Put(ROW);
2001       put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
2002       ht.put(put);
2003
2004       // delete wrong family
2005       Delete delete = new Delete(ROW);
2006       delete.addFamily(FAMILIES[1], ts[0]);
2007       ht.delete(delete);
2008
2009       Get get = new Get(ROW);
2010       get.addFamily(FAMILIES[0]);
2011       get.readAllVersions();
2012       Result result = ht.get(get);
2013       assertTrue(Bytes.equals(result.getValue(FAMILIES[0], QUALIFIER), VALUES[0]));
2014     }
2015   }
2016
2017   @Test
2018   public void testDeletes() throws Exception {
2019     final TableName tableName = name.getTableName();
2020
2021     byte [][] ROWS = makeNAscii(ROW, 6);
2022     byte [][] FAMILIES = makeNAscii(FAMILY, 3);
2023     byte [][] VALUES = makeN(VALUE, 5);
2024     long [] ts = {1000, 2000, 3000, 4000, 5000};
2025
2026     try (Table ht = TEST_UTIL.createTable(tableName, FAMILIES, 3)) {
2027
2028       Put put = new Put(ROW);
2029       put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
2030       put.addColumn(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]);
2031       ht.put(put);
2032
2033       Delete delete = new Delete(ROW);
2034       delete.addFamily(FAMILIES[0], ts[0]);
2035       ht.delete(delete);
2036
2037       Get get = new Get(ROW);
2038       get.addFamily(FAMILIES[0]);
2039       get.readVersions(Integer.MAX_VALUE);
2040       Result result = ht.get(get);
2041       assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2042           new long [] {ts[1]},
2043           new byte[][] {VALUES[1]},
2044           0, 0);
2045
2046       Scan scan = new Scan(ROW);
2047       scan.addFamily(FAMILIES[0]);
2048       scan.setMaxVersions(Integer.MAX_VALUE);
2049       result = getSingleScanResult(ht, scan);
2050       assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2051               new long[]{ts[1]},
2052               new byte[][]{VALUES[1]},
2053               0, 0);
2054
2055       // Test delete latest version
2056       put = new Put(ROW);
2057       put.addColumn(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
2058       put.addColumn(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]);
2059       put.addColumn(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]);
2060       put.addColumn(FAMILIES[0], null, ts[4], VALUES[4]);
2061       put.addColumn(FAMILIES[0], null, ts[2], VALUES[2]);
2062       put.addColumn(FAMILIES[0], null, ts[3], VALUES[3]);
2063       ht.put(put);
2064
2065       delete = new Delete(ROW);
2066       delete.addColumn(FAMILIES[0], QUALIFIER); // ts[4]
2067       ht.delete(delete);
2068
2069       get = new Get(ROW);
2070       get.addColumn(FAMILIES[0], QUALIFIER);
2071       get.readVersions(Integer.MAX_VALUE);
2072       result = ht.get(get);
2073       assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2074           new long [] {ts[1], ts[2], ts[3]},
2075           new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
2076           0, 2);
2077
2078       scan = new Scan(ROW);
2079       scan.addColumn(FAMILIES[0], QUALIFIER);
2080       scan.setMaxVersions(Integer.MAX_VALUE);
2081       result = getSingleScanResult(ht, scan);
2082       assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2083               new long[]{ts[1], ts[2], ts[3]},
2084               new byte[][]{VALUES[1], VALUES[2], VALUES[3]},
2085               0, 2);
2086
2087       // Test for HBASE-1847
2088       delete = new Delete(ROW);
2089       delete.addColumn(FAMILIES[0], null);
2090       ht.delete(delete);
2091
2092       // Cleanup null qualifier
2093       delete = new Delete(ROW);
2094       delete.addColumns(FAMILIES[0], null);
2095       ht.delete(delete);
2096
2097       // Expected client behavior might be that you can re-put deleted values
2098       // But alas, this is not to be.  We can't put them back in either case.
2099
2100       put = new Put(ROW);
2101       put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]); // 1000
2102       put.addColumn(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]); // 5000
2103       ht.put(put);
2104
2105
2106       // It used to be due to the internal implementation of Get, that
2107       // the Get() call would return ts[4] UNLIKE the Scan below. With
2108       // the switch to using Scan for Get this is no longer the case.
2109       get = new Get(ROW);
2110       get.addFamily(FAMILIES[0]);
2111       get.readVersions(Integer.MAX_VALUE);
2112       result = ht.get(get);
2113       assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2114           new long [] {ts[1], ts[2], ts[3]},
2115           new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
2116           0, 2);
2117
2118       // The Scanner returns the previous values, the expected-naive-unexpected behavior
2119
2120       scan = new Scan(ROW);
2121       scan.addFamily(FAMILIES[0]);
2122       scan.setMaxVersions(Integer.MAX_VALUE);
2123       result = getSingleScanResult(ht, scan);
2124       assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2125               new long[]{ts[1], ts[2], ts[3]},
2126               new byte[][]{VALUES[1], VALUES[2], VALUES[3]},
2127               0, 2);
2128
2129       // Test deleting an entire family from one row but not the other various ways
2130
2131       put = new Put(ROWS[0]);
2132       put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
2133       put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
2134       put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
2135       put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
2136       ht.put(put);
2137
2138       put = new Put(ROWS[1]);
2139       put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
2140       put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
2141       put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
2142       put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
2143       ht.put(put);
2144
2145       put = new Put(ROWS[2]);
2146       put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
2147       put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
2148       put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
2149       put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
2150       ht.put(put);
2151
2152       // Assert that above went in.
2153       get = new Get(ROWS[2]);
2154       get.addFamily(FAMILIES[1]);
2155       get.addFamily(FAMILIES[2]);
2156       get.readVersions(Integer.MAX_VALUE);
2157       result = ht.get(get);
2158       assertTrue("Expected 4 key but received " + result.size() + ": " + result,
2159           result.size() == 4);
2160
2161       delete = new Delete(ROWS[0]);
2162       delete.addFamily(FAMILIES[2]);
2163       ht.delete(delete);
2164
2165       delete = new Delete(ROWS[1]);
2166       delete.addColumns(FAMILIES[1], QUALIFIER);
2167       ht.delete(delete);
2168
2169       delete = new Delete(ROWS[2]);
2170       delete.addColumn(FAMILIES[1], QUALIFIER);
2171       delete.addColumn(FAMILIES[1], QUALIFIER);
2172       delete.addColumn(FAMILIES[2], QUALIFIER);
2173       ht.delete(delete);
2174
2175       get = new Get(ROWS[0]);
2176       get.addFamily(FAMILIES[1]);
2177       get.addFamily(FAMILIES[2]);
2178       get.readVersions(Integer.MAX_VALUE);
2179       result = ht.get(get);
2180       assertTrue("Expected 2 keys but received " + result.size(),
2181           result.size() == 2);
2182       assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER,
2183           new long [] {ts[0], ts[1]},
2184           new byte[][] {VALUES[0], VALUES[1]},
2185           0, 1);
2186
2187       scan = new Scan(ROWS[0]);
2188       scan.addFamily(FAMILIES[1]);
2189       scan.addFamily(FAMILIES[2]);
2190       scan.setMaxVersions(Integer.MAX_VALUE);
2191       result = getSingleScanResult(ht, scan);
2192       assertTrue("Expected 2 keys but received " + result.size(),
2193               result.size() == 2);
2194       assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER,
2195               new long[]{ts[0], ts[1]},
2196               new byte[][]{VALUES[0], VALUES[1]},
2197               0, 1);
2198
2199       get = new Get(ROWS[1]);
2200       get.addFamily(FAMILIES[1]);
2201       get.addFamily(FAMILIES[2]);
2202       get.readVersions(Integer.MAX_VALUE);
2203       result = ht.get(get);
2204       assertTrue("Expected 2 keys but received " + result.size(),
2205           result.size() == 2);
2206
2207       scan = new Scan(ROWS[1]);
2208       scan.addFamily(FAMILIES[1]);
2209       scan.addFamily(FAMILIES[2]);
2210       scan.setMaxVersions(Integer.MAX_VALUE);
2211       result = getSingleScanResult(ht, scan);
2212       assertTrue("Expected 2 keys but received " + result.size(),
2213               result.size() == 2);
2214
2215       get = new Get(ROWS[2]);
2216       get.addFamily(FAMILIES[1]);
2217       get.addFamily(FAMILIES[2]);
2218       get.readVersions(Integer.MAX_VALUE);
2219       result = ht.get(get);
2220       assertEquals(1, result.size());
2221       assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
2222           new long [] {ts[2]},
2223           new byte[][] {VALUES[2]},
2224           0, 0);
2225
2226       scan = new Scan(ROWS[2]);
2227       scan.addFamily(FAMILIES[1]);
2228       scan.addFamily(FAMILIES[2]);
2229       scan.setMaxVersions(Integer.MAX_VALUE);
2230       result = getSingleScanResult(ht, scan);
2231       assertEquals(1, result.size());
2232       assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
2233               new long[]{ts[2]},
2234               new byte[][]{VALUES[2]},
2235               0, 0);
2236
2237       // Test if we delete the family first in one row (HBASE-1541)
2238
2239       delete = new Delete(ROWS[3]);
2240       delete.addFamily(FAMILIES[1]);
2241       ht.delete(delete);
2242
2243       put = new Put(ROWS[3]);
2244       put.addColumn(FAMILIES[2], QUALIFIER, VALUES[0]);
2245       ht.put(put);
2246
2247       put = new Put(ROWS[4]);
2248       put.addColumn(FAMILIES[1], QUALIFIER, VALUES[1]);
2249       put.addColumn(FAMILIES[2], QUALIFIER, VALUES[2]);
2250       ht.put(put);
2251
2252       get = new Get(ROWS[3]);
2253       get.addFamily(FAMILIES[1]);
2254       get.addFamily(FAMILIES[2]);
2255       get.readVersions(Integer.MAX_VALUE);
2256       result = ht.get(get);
2257       assertTrue("Expected 1 key but received " + result.size(),
2258           result.size() == 1);
2259
2260       get = new Get(ROWS[4]);
2261       get.addFamily(FAMILIES[1]);
2262       get.addFamily(FAMILIES[2]);
2263       get.readVersions(Integer.MAX_VALUE);
2264       result = ht.get(get);
2265       assertTrue("Expected 2 keys but received " + result.size(),
2266           result.size() == 2);
2267
2268       scan = new Scan(ROWS[3]);
2269       scan.addFamily(FAMILIES[1]);
2270       scan.addFamily(FAMILIES[2]);
2271       scan.setMaxVersions(Integer.MAX_VALUE);
2272       ResultScanner scanner = ht.getScanner(scan);
2273       result = scanner.next();
2274       assertTrue("Expected 1 key but received " + result.size(),
2275               result.size() == 1);
2276       assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[3]));
2277       assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[0]));
2278       result = scanner.next();
2279       assertTrue("Expected 2 keys but received " + result.size(),
2280               result.size() == 2);
2281       assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[4]));
2282       assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[1]), ROWS[4]));
2283       assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[1]));
2284       assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[1]), VALUES[2]));
2285       scanner.close();
2286
2287       // Add test of bulk deleting.
2288       for (int i = 0; i < 10; i++) {
2289         byte[] bytes = Bytes.toBytes(i);
2290         put = new Put(bytes);
2291         put.setDurability(Durability.SKIP_WAL);
2292         put.addColumn(FAMILIES[0], QUALIFIER, bytes);
2293         ht.put(put);
2294       }
2295       for (int i = 0; i < 10; i++) {
2296         byte[] bytes = Bytes.toBytes(i);
2297         get = new Get(bytes);
2298         get.addFamily(FAMILIES[0]);
2299         result = ht.get(get);
2300         assertTrue(result.size() == 1);
2301       }
2302       ArrayList<Delete> deletes = new ArrayList<>();
2303       for (int i = 0; i < 10; i++) {
2304         byte[] bytes = Bytes.toBytes(i);
2305         delete = new Delete(bytes);
2306         delete.addFamily(FAMILIES[0]);
2307         deletes.add(delete);
2308       }
2309       ht.delete(deletes);
2310       for (int i = 0; i < 10; i++) {
2311         byte[] bytes = Bytes.toBytes(i);
2312         get = new Get(bytes);
2313         get.addFamily(FAMILIES[0]);
2314         result = ht.get(get);
2315         assertTrue(result.isEmpty());
2316       }
2317     }
2318   }
2319
2320   /**
2321    * Test batch operations with combination of valid and invalid args
2322    */
2323   @Test
2324   public void testBatchOperationsWithErrors() throws Exception {
2325     final TableName tableName = name.getTableName();
2326     try (Table foo = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, 10)) {
2327
2328       int NUM_OPS = 100;
2329
2330       // 1.1 Put with no column families (local validation, runtime exception)
2331       List<Put> puts = new ArrayList<Put>(NUM_OPS);
2332       for (int i = 0; i != NUM_OPS; i++) {
2333         Put put = new Put(Bytes.toBytes(i));
2334         puts.add(put);
2335       }
2336
2337       try {
2338         foo.put(puts);
2339         fail();
2340       } catch (IllegalArgumentException e) {
2341         // expected
2342         assertEquals(NUM_OPS, puts.size());
2343       }
2344
2345       // 1.2 Put with invalid column family
2346       puts.clear();
2347       for (int i = 0; i < NUM_OPS; i++) {
2348         Put put = new Put(Bytes.toBytes(i));
2349         put.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY, Bytes.toBytes(i));
2350         puts.add(put);
2351       }
2352
2353       try {
2354         foo.put(puts);
2355         fail();
2356       } catch (RetriesExhaustedException e) {
2357         // expected
2358         assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class));
2359       }
2360
2361       // 2.1 Get non-existent rows
2362       List<Get> gets = new ArrayList<>(NUM_OPS);
2363       for (int i = 0; i < NUM_OPS; i++) {
2364         Get get = new Get(Bytes.toBytes(i));
2365         gets.add(get);
2366       }
2367       Result[] getsResult = foo.get(gets);
2368       assertNotNull(getsResult);
2369       assertEquals(NUM_OPS, getsResult.length);
2370       for (int i = 0; i < NUM_OPS; i++) {
2371         Result getResult = getsResult[i];
2372         if (i % 2 == 0) {
2373           assertFalse(getResult.isEmpty());
2374         } else {
2375           assertTrue(getResult.isEmpty());
2376         }
2377       }
2378
2379       // 2.2 Get with invalid column family
2380       gets.clear();
2381       for (int i = 0; i < NUM_OPS; i++) {
2382         Get get = new Get(Bytes.toBytes(i));
2383         get.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY);
2384         gets.add(get);
2385       }
2386       try {
2387         foo.get(gets);
2388         fail();
2389       } catch (RetriesExhaustedException e) {
2390         // expected
2391         assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class));
2392       }
2393
2394       // 3.1 Delete with invalid column family
2395       List<Delete> deletes = new ArrayList<>(NUM_OPS);
2396       for (int i = 0; i < NUM_OPS; i++) {
2397         Delete delete = new Delete(Bytes.toBytes(i));
2398         delete.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY);
2399         deletes.add(delete);
2400       }
2401       try {
2402         foo.delete(deletes);
2403         fail();
2404       } catch (RetriesExhaustedException e) {
2405         // expected
2406         assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class));
2407       }
2408
2409       // all valid rows should have been deleted
2410       gets.clear();
2411       for (int i = 0; i < NUM_OPS; i++) {
2412         Get get = new Get(Bytes.toBytes(i));
2413         gets.add(get);
2414       }
2415       getsResult = foo.get(gets);
2416       assertNotNull(getsResult);
2417       assertEquals(NUM_OPS, getsResult.length);
2418       for (Result getResult : getsResult) {
2419         assertTrue(getResult.isEmpty());
2420       }
2421
2422       // 3.2 Delete non-existent rows
2423       deletes.clear();
2424       for (int i = 0; i < NUM_OPS; i++) {
2425         Delete delete = new Delete(Bytes.toBytes(i));
2426         deletes.add(delete);
2427       }
2428       foo.delete(deletes);
2429     }
2430   }
2431
2432   //
2433   // JIRA Testers
2434   //
2435
2436   /**
2437    * HBASE-867
2438    *    If millions of columns in a column family, hbase scanner won't come up
2439    *
2440    *    Test will create numRows rows, each with numColsPerRow columns
2441    *    (1 version each), and attempt to scan them all.
2442    *
2443    *    To test at scale, up numColsPerRow to the millions
2444    *    (have not gotten that to work running as junit though)
2445    */
2446   @Test
2447   public void testJiraTest867() throws Exception {
2448     int numRows = 10;
2449     int numColsPerRow = 2000;
2450
2451     final TableName tableName = name.getTableName();
2452
2453     byte [][] ROWS = makeN(ROW, numRows);
2454     byte [][] QUALIFIERS = makeN(QUALIFIER, numColsPerRow);
2455
2456     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
2457
2458       // Insert rows
2459
2460       for (int i = 0; i < numRows; i++) {
2461         Put put = new Put(ROWS[i]);
2462         put.setDurability(Durability.SKIP_WAL);
2463         for (int j = 0; j < numColsPerRow; j++) {
2464           put.addColumn(FAMILY, QUALIFIERS[j], QUALIFIERS[j]);
2465         }
2466         assertTrue("Put expected to contain " + numColsPerRow + " columns but " +
2467                 "only contains " + put.size(), put.size() == numColsPerRow);
2468         ht.put(put);
2469       }
2470
2471       // Get a row
2472       Get get = new Get(ROWS[numRows - 1]);
2473       Result result = ht.get(get);
2474       assertNumKeys(result, numColsPerRow);
2475       Cell[] keys = result.rawCells();
2476       for (int i = 0; i < result.size(); i++) {
2477         assertKey(keys[i], ROWS[numRows - 1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2478       }
2479
2480       // Scan the rows
2481       Scan scan = new Scan();
2482       try (ResultScanner scanner = ht.getScanner(scan)) {
2483         int rowCount = 0;
2484         while ((result = scanner.next()) != null) {
2485           assertNumKeys(result, numColsPerRow);
2486           Cell[] kvs = result.rawCells();
2487           for (int i = 0; i < numColsPerRow; i++) {
2488             assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2489           }
2490           rowCount++;
2491         }
2492         assertTrue("Expected to scan " + numRows + " rows but actually scanned "
2493                 + rowCount + " rows", rowCount == numRows);
2494       }
2495
2496       // flush and try again
2497
2498       TEST_UTIL.flush();
2499
2500       // Get a row
2501       get = new Get(ROWS[numRows - 1]);
2502       result = ht.get(get);
2503       assertNumKeys(result, numColsPerRow);
2504       keys = result.rawCells();
2505       for (int i = 0; i < result.size(); i++) {
2506         assertKey(keys[i], ROWS[numRows - 1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2507       }
2508
2509       // Scan the rows
2510       scan = new Scan();
2511       try (ResultScanner scanner = ht.getScanner(scan)) {
2512         int rowCount = 0;
2513         while ((result = scanner.next()) != null) {
2514           assertNumKeys(result, numColsPerRow);
2515           Cell[] kvs = result.rawCells();
2516           for (int i = 0; i < numColsPerRow; i++) {
2517             assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2518           }
2519           rowCount++;
2520         }
2521         assertTrue("Expected to scan " + numRows + " rows but actually scanned "
2522                 + rowCount + " rows", rowCount == numRows);
2523       }
2524     }
2525   }
2526
2527   /**
2528    * HBASE-861
2529    *    get with timestamp will return a value if there is a version with an
2530    *    earlier timestamp
2531    */
2532   @Test
2533   public void testJiraTest861() throws Exception {
2534     final TableName tableName = name.getTableName();
2535     byte [][] VALUES = makeNAscii(VALUE, 7);
2536     long [] STAMPS = makeStamps(7);
2537
2538     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
2539
2540       // Insert three versions
2541
2542       Put put = new Put(ROW);
2543       put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2544       put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2545       put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2546       ht.put(put);
2547
2548       // Get the middle value
2549       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2550
2551       // Try to get one version before (expect fail)
2552       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2553
2554       // Try to get one version after (expect fail)
2555       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2556
2557       // Try same from storefile
2558       TEST_UTIL.flush();
2559       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2560       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2561       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2562
2563       // Insert two more versions surrounding others, into memstore
2564       put = new Put(ROW);
2565       put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2566       put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
2567       ht.put(put);
2568
2569       // Check we can get everything we should and can't get what we shouldn't
2570       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2571       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2572       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2573       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2574       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2575       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2576       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
2577
2578       // Try same from two storefiles
2579       TEST_UTIL.flush();
2580       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2581       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2582       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2583       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2584       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2585       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2586       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
2587     }
2588   }
2589
2590   /**
2591    * HBASE-33
2592    *    Add a HTable get/obtainScanner method that retrieves all versions of a
2593    *    particular column and row between two timestamps
2594    */
2595   @Test
2596   public void testJiraTest33() throws Exception {
2597     final TableName tableName = name.getTableName();
2598     byte [][] VALUES = makeNAscii(VALUE, 7);
2599     long [] STAMPS = makeStamps(7);
2600
2601     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
2602
2603       // Insert lots versions
2604
2605       Put put = new Put(ROW);
2606       put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2607       put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2608       put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2609       put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2610       put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2611       put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2612       ht.put(put);
2613
2614       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2615       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2616       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2617       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2618
2619       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2620       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2621       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2622       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2623
2624       // Try same from storefile
2625       TEST_UTIL.flush();
2626
2627       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2628       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2629       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2630       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2631
2632       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2633       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2634       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2635       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2636     }
2637   }
2638
2639   /**
2640    * HBASE-1014
2641    *    commit(BatchUpdate) method should return timestamp
2642    */
2643   @Test
2644   public void testJiraTest1014() throws Exception {
2645     final TableName tableName = name.getTableName();
2646
2647     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
2648
2649       long manualStamp = 12345;
2650
2651       // Insert lots versions
2652
2653       Put put = new Put(ROW);
2654       put.addColumn(FAMILY, QUALIFIER, manualStamp, VALUE);
2655       ht.put(put);
2656
2657       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, manualStamp, VALUE);
2658       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp - 1);
2659       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp + 1);
2660     }
2661   }
2662
2663   /**
2664    * HBASE-1182
2665    *    Scan for columns > some timestamp
2666    */
2667   @Test
2668   public void testJiraTest1182() throws Exception {
2669     final TableName tableName = name.getTableName();
2670     byte [][] VALUES = makeNAscii(VALUE, 7);
2671     long [] STAMPS = makeStamps(7);
2672
2673     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
2674
2675       // Insert lots versions
2676
2677       Put put = new Put(ROW);
2678       put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2679       put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2680       put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2681       put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2682       put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2683       put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2684       ht.put(put);
2685
2686       getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2687       getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2688       getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2689
2690       scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2691       scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2692       scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2693
2694       // Try same from storefile
2695       TEST_UTIL.flush();
2696
2697       getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2698       getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2699       getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2700
2701       scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2702       scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2703       scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2704     }
2705   }
2706
2707   /**
2708    * HBASE-52
2709    *    Add a means of scanning over all versions
2710    */
2711   @Test
2712   public void testJiraTest52() throws Exception {
2713     final TableName tableName = name.getTableName();
2714     byte [][] VALUES = makeNAscii(VALUE, 7);
2715     long [] STAMPS = makeStamps(7);
2716
2717     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
2718
2719       // Insert lots versions
2720
2721       Put put = new Put(ROW);
2722       put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2723       put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2724       put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2725       put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2726       put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2727       put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2728       ht.put(put);
2729
2730       getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2731
2732       scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2733
2734       // Try same from storefile
2735       TEST_UTIL.flush();
2736
2737       getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2738
2739       scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2740     }
2741   }
2742
2743   //
2744   // Bulk Testers
2745   //
2746
2747   private void getVersionRangeAndVerifyGreaterThan(Table ht, byte [] row,
2748       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
2749       int start, int end) throws IOException {
2750     Get get = new Get(row);
2751     get.addColumn(family, qualifier);
2752     get.readVersions(Integer.MAX_VALUE);
2753     get.setTimeRange(stamps[start+1], Long.MAX_VALUE);
2754     Result result = ht.get(get);
2755     assertNResult(result, row, family, qualifier, stamps, values, start+1, end);
2756   }
2757
2758   private void getVersionRangeAndVerify(Table ht, byte [] row, byte [] family,
2759       byte [] qualifier, long [] stamps, byte [][] values, int start, int end) throws IOException {
2760     Get get = new Get(row);
2761     get.addColumn(family, qualifier);
2762     get.readVersions(Integer.MAX_VALUE);
2763     get.setTimeRange(stamps[start], stamps[end]+1);
2764     Result result = ht.get(get);
2765     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2766   }
2767
2768   private void getAllVersionsAndVerify(Table ht, byte [] row, byte [] family,
2769       byte [] qualifier, long [] stamps, byte [][] values, int start, int end) throws IOException {
2770     Get get = new Get(row);
2771     get.addColumn(family, qualifier);
2772     get.readVersions(Integer.MAX_VALUE);
2773     Result result = ht.get(get);
2774     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2775   }
2776
2777   private void scanVersionRangeAndVerifyGreaterThan(Table ht, byte [] row,
2778       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
2779       int start, int end) throws IOException {
2780     Scan scan = new Scan(row);
2781     scan.addColumn(family, qualifier);
2782     scan.setMaxVersions(Integer.MAX_VALUE);
2783     scan.setTimeRange(stamps[start+1], Long.MAX_VALUE);
2784     Result result = getSingleScanResult(ht, scan);
2785     assertNResult(result, row, family, qualifier, stamps, values, start+1, end);
2786   }
2787
2788   private void scanVersionRangeAndVerify(Table ht, byte [] row, byte [] family,
2789       byte [] qualifier, long [] stamps, byte [][] values, int start, int end) throws IOException {
2790     Scan scan = new Scan(row);
2791     scan.addColumn(family, qualifier);
2792     scan.setMaxVersions(Integer.MAX_VALUE);
2793     scan.setTimeRange(stamps[start], stamps[end]+1);
2794     Result result = getSingleScanResult(ht, scan);
2795     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2796   }
2797
2798   private void scanAllVersionsAndVerify(Table ht, byte [] row, byte [] family,
2799       byte [] qualifier, long [] stamps, byte [][] values, int start, int end) throws IOException {
2800     Scan scan = new Scan(row);
2801     scan.addColumn(family, qualifier);
2802     scan.setMaxVersions(Integer.MAX_VALUE);
2803     Result result = getSingleScanResult(ht, scan);
2804     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2805   }
2806
2807   private void getVersionAndVerify(Table ht, byte [] row, byte [] family,
2808       byte [] qualifier, long stamp, byte [] value) throws Exception {
2809     Get get = new Get(row);
2810     get.addColumn(family, qualifier);
2811     get.setTimestamp(stamp);
2812     get.readVersions(Integer.MAX_VALUE);
2813     Result result = ht.get(get);
2814     assertSingleResult(result, row, family, qualifier, stamp, value);
2815   }
2816
2817   private void getVersionAndVerifyMissing(Table ht, byte [] row, byte [] family,
2818       byte [] qualifier, long stamp) throws Exception {
2819     Get get = new Get(row);
2820     get.addColumn(family, qualifier);
2821     get.setTimestamp(stamp);
2822     get.readVersions(Integer.MAX_VALUE);
2823     Result result = ht.get(get);
2824     assertEmptyResult(result);
2825   }
2826
2827   private void scanVersionAndVerify(Table ht, byte [] row, byte [] family,
2828       byte [] qualifier, long stamp, byte [] value) throws Exception {
2829     Scan scan = new Scan(row);
2830     scan.addColumn(family, qualifier);
2831     scan.setTimestamp(stamp);
2832     scan.setMaxVersions(Integer.MAX_VALUE);
2833     Result result = getSingleScanResult(ht, scan);
2834     assertSingleResult(result, row, family, qualifier, stamp, value);
2835   }
2836
2837   private void scanVersionAndVerifyMissing(Table ht, byte [] row,
2838       byte [] family, byte [] qualifier, long stamp) throws Exception {
2839     Scan scan = new Scan(row);
2840     scan.addColumn(family, qualifier);
2841     scan.setTimestamp(stamp);
2842     scan.setMaxVersions(Integer.MAX_VALUE);
2843     Result result = getSingleScanResult(ht, scan);
2844     assertNullResult(result);
2845   }
2846
2847   private void getTestNull(Table ht, byte [] row, byte [] family, byte [] value) throws Exception {
2848     Get get = new Get(row);
2849     get.addColumn(family, null);
2850     Result result = ht.get(get);
2851     assertSingleResult(result, row, family, null, value);
2852
2853     get = new Get(row);
2854     get.addColumn(family, HConstants.EMPTY_BYTE_ARRAY);
2855     result = ht.get(get);
2856     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2857
2858     get = new Get(row);
2859     get.addFamily(family);
2860     result = ht.get(get);
2861     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2862
2863     get = new Get(row);
2864     result = ht.get(get);
2865     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2866
2867   }
2868
2869   private void getTestNull(Table ht, byte[] row, byte[] family, long value) throws Exception {
2870     Get get = new Get(row);
2871     get.addColumn(family, null);
2872     Result result = ht.get(get);
2873     assertSingleResult(result, row, family, null, value);
2874
2875     get = new Get(row);
2876     get.addColumn(family, HConstants.EMPTY_BYTE_ARRAY);
2877     result = ht.get(get);
2878     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2879
2880     get = new Get(row);
2881     get.addFamily(family);
2882     result = ht.get(get);
2883     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2884
2885     get = new Get(row);
2886     result = ht.get(get);
2887     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2888   }
2889
2890   private void scanTestNull(Table ht, byte[] row, byte[] family, byte[] value)
2891       throws Exception {
2892     scanTestNull(ht, row, family, value, false);
2893   }
2894
2895   private void scanTestNull(Table ht, byte[] row, byte[] family, byte[] value,
2896       boolean isReversedScan) throws Exception {
2897
2898     Scan scan = new Scan();
2899     scan.setReversed(isReversedScan);
2900     scan.addColumn(family, null);
2901     Result result = getSingleScanResult(ht, scan);
2902     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2903
2904     scan = new Scan();
2905     scan.setReversed(isReversedScan);
2906     scan.addColumn(family, HConstants.EMPTY_BYTE_ARRAY);
2907     result = getSingleScanResult(ht, scan);
2908     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2909
2910     scan = new Scan();
2911     scan.setReversed(isReversedScan);
2912     scan.addFamily(family);
2913     result = getSingleScanResult(ht, scan);
2914     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2915
2916     scan = new Scan();
2917     scan.setReversed(isReversedScan);
2918     result = getSingleScanResult(ht, scan);
2919     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2920
2921   }
2922
2923   private void singleRowGetTest(Table ht, byte [][] ROWS, byte [][] FAMILIES,
2924       byte [][] QUALIFIERS, byte [][] VALUES) throws Exception {
2925     // Single column from memstore
2926     Get get = new Get(ROWS[0]);
2927     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
2928     Result result = ht.get(get);
2929     assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]);
2930
2931     // Single column from storefile
2932     get = new Get(ROWS[0]);
2933     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
2934     result = ht.get(get);
2935     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
2936
2937     // Single column from storefile, family match
2938     get = new Get(ROWS[0]);
2939     get.addFamily(FAMILIES[7]);
2940     result = ht.get(get);
2941     assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]);
2942
2943     // Two columns, one from memstore one from storefile, same family,
2944     // wildcard match
2945     get = new Get(ROWS[0]);
2946     get.addFamily(FAMILIES[4]);
2947     result = ht.get(get);
2948     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2949         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2950
2951     // Two columns, one from memstore one from storefile, same family,
2952     // explicit match
2953     get = new Get(ROWS[0]);
2954     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
2955     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
2956     result = ht.get(get);
2957     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2958         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2959
2960     // Three column, one from memstore two from storefile, different families,
2961     // wildcard match
2962     get = new Get(ROWS[0]);
2963     get.addFamily(FAMILIES[4]);
2964     get.addFamily(FAMILIES[7]);
2965     result = ht.get(get);
2966     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2967         new int [][] { {4, 0, 0}, {4, 4, 4}, {7, 7, 7} });
2968
2969     // Multiple columns from everywhere storefile, many family, wildcard
2970     get = new Get(ROWS[0]);
2971     get.addFamily(FAMILIES[2]);
2972     get.addFamily(FAMILIES[4]);
2973     get.addFamily(FAMILIES[6]);
2974     get.addFamily(FAMILIES[7]);
2975     result = ht.get(get);
2976     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2977         new int [][] {
2978           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
2979         });
2980
2981     // Multiple columns from everywhere storefile, many family, wildcard
2982     get = new Get(ROWS[0]);
2983     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
2984     get.addColumn(FAMILIES[2], QUALIFIERS[4]);
2985     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
2986     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
2987     get.addColumn(FAMILIES[6], QUALIFIERS[6]);
2988     get.addColumn(FAMILIES[6], QUALIFIERS[7]);
2989     get.addColumn(FAMILIES[7], QUALIFIERS[7]);
2990     get.addColumn(FAMILIES[7], QUALIFIERS[8]);
2991     result = ht.get(get);
2992     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2993         new int [][] {
2994           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
2995         });
2996
2997     // Everything
2998     get = new Get(ROWS[0]);
2999     result = ht.get(get);
3000     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
3001         new int [][] {
3002           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}, {9, 0, 0}
3003         });
3004
3005     // Get around inserted columns
3006
3007     get = new Get(ROWS[1]);
3008     result = ht.get(get);
3009     assertEmptyResult(result);
3010
3011     get = new Get(ROWS[0]);
3012     get.addColumn(FAMILIES[4], QUALIFIERS[3]);
3013     get.addColumn(FAMILIES[2], QUALIFIERS[3]);
3014     result = ht.get(get);
3015     assertEmptyResult(result);
3016
3017   }
3018
3019   private void singleRowScanTest(Table ht, byte [][] ROWS, byte [][] FAMILIES,
3020       byte [][] QUALIFIERS, byte [][] VALUES) throws Exception {
3021     // Single column from memstore
3022     Scan scan = new Scan();
3023     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
3024     Result result = getSingleScanResult(ht, scan);
3025     assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]);
3026
3027     // Single column from storefile
3028     scan = new Scan();
3029     scan.addColumn(FAMILIES[2], QUALIFIERS[2]);
3030     result = getSingleScanResult(ht, scan);
3031     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
3032
3033     // Single column from storefile, family match
3034     scan = new Scan();
3035     scan.addFamily(FAMILIES[7]);
3036     result = getSingleScanResult(ht, scan);
3037     assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]);
3038
3039     // Two columns, one from memstore one from storefile, same family,
3040     // wildcard match
3041     scan = new Scan();
3042     scan.addFamily(FAMILIES[4]);
3043     result = getSingleScanResult(ht, scan);
3044     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
3045         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
3046
3047     // Two columns, one from memstore one from storefile, same family,
3048     // explicit match
3049     scan = new Scan();
3050     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
3051     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
3052     result = getSingleScanResult(ht, scan);
3053     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
3054         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
3055
3056     // Three column, one from memstore two from storefile, different families,
3057     // wildcard match
3058     scan = new Scan();
3059     scan.addFamily(FAMILIES[4]);
3060     scan.addFamily(FAMILIES[7]);
3061     result = getSingleScanResult(ht, scan);
3062     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
3063         new int [][] { {4, 0, 0}, {4, 4, 4}, {7, 7, 7} });
3064
3065     // Multiple columns from everywhere storefile, many family, wildcard
3066     scan = new Scan();
3067     scan.addFamily(FAMILIES[2]);
3068     scan.addFamily(FAMILIES[4]);
3069     scan.addFamily(FAMILIES[6]);
3070     scan.addFamily(FAMILIES[7]);
3071     result = getSingleScanResult(ht, scan);
3072     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
3073         new int [][] {
3074           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
3075         });
3076
3077     // Multiple columns from everywhere storefile, many family, wildcard
3078     scan = new Scan();
3079     scan.addColumn(FAMILIES[2], QUALIFIERS[2]);
3080     scan.addColumn(FAMILIES[2], QUALIFIERS[4]);
3081     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
3082     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
3083     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
3084     scan.addColumn(FAMILIES[6], QUALIFIERS[7]);
3085     scan.addColumn(FAMILIES[7], QUALIFIERS[7]);
3086     scan.addColumn(FAMILIES[7], QUALIFIERS[8]);
3087     result = getSingleScanResult(ht, scan);
3088     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
3089         new int [][] {
3090           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
3091         });
3092
3093     // Everything
3094     scan = new Scan();
3095     result = getSingleScanResult(ht, scan);
3096     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
3097         new int [][] {
3098           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}, {9, 0, 0}
3099         });
3100
3101     // Scan around inserted columns
3102
3103     scan = new Scan(ROWS[1]);
3104     result = getSingleScanResult(ht, scan);
3105     assertNullResult(result);
3106
3107     scan = new Scan();
3108     scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
3109     scan.addColumn(FAMILIES[2], QUALIFIERS[3]);
3110     result = getSingleScanResult(ht, scan);
3111     assertNullResult(result);
3112   }
3113
3114   /**
3115    * Verify a single column using gets.
3116    * Expects family and qualifier arrays to be valid for at least
3117    * the range:  idx-2 < idx < idx+2
3118    */
3119   private void getVerifySingleColumn(Table ht, byte [][] ROWS, int ROWIDX, byte [][] FAMILIES,
3120       int FAMILYIDX, byte [][] QUALIFIERS, int QUALIFIERIDX, byte [][] VALUES, int VALUEIDX)
3121       throws Exception {
3122     Get get = new Get(ROWS[ROWIDX]);
3123     Result result = ht.get(get);
3124     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3125         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3126
3127     get = new Get(ROWS[ROWIDX]);
3128     get.addFamily(FAMILIES[FAMILYIDX]);
3129     result = ht.get(get);
3130     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3131         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3132
3133     get = new Get(ROWS[ROWIDX]);
3134     get.addFamily(FAMILIES[FAMILYIDX-2]);
3135     get.addFamily(FAMILIES[FAMILYIDX]);
3136     get.addFamily(FAMILIES[FAMILYIDX+2]);
3137     result = ht.get(get);
3138     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3139         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3140
3141     get = new Get(ROWS[ROWIDX]);
3142     get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[0]);
3143     result = ht.get(get);
3144     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3145         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3146
3147     get = new Get(ROWS[ROWIDX]);
3148     get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[1]);
3149     get.addFamily(FAMILIES[FAMILYIDX]);
3150     result = ht.get(get);
3151     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3152         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3153
3154     get = new Get(ROWS[ROWIDX]);
3155     get.addFamily(FAMILIES[FAMILYIDX]);
3156     get.addColumn(FAMILIES[FAMILYIDX+1], QUALIFIERS[1]);
3157     get.addColumn(FAMILIES[FAMILYIDX-2], QUALIFIERS[1]);
3158     get.addFamily(FAMILIES[FAMILYIDX-1]);
3159     get.addFamily(FAMILIES[FAMILYIDX+2]);
3160     result = ht.get(get);
3161     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3162         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3163
3164   }
3165
3166
3167   /**
3168    * Verify a single column using scanners.
3169    * Expects family and qualifier arrays to be valid for at least
3170    * the range:  idx-2 to idx+2
3171    * Expects row array to be valid for at least idx to idx+2
3172    */
3173   private void scanVerifySingleColumn(Table ht, byte [][] ROWS, int ROWIDX, byte [][] FAMILIES,
3174       int FAMILYIDX, byte [][] QUALIFIERS, int QUALIFIERIDX, byte [][] VALUES, int VALUEIDX)
3175       throws Exception {
3176     Scan scan = new Scan();
3177     Result result = getSingleScanResult(ht, scan);
3178     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3179         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3180
3181     scan = new Scan(ROWS[ROWIDX]);
3182     result = getSingleScanResult(ht, scan);
3183     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3184         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3185
3186     scan = new Scan(ROWS[ROWIDX], ROWS[ROWIDX+1]);
3187     result = getSingleScanResult(ht, scan);
3188     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3189         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3190
3191     scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX+1]);
3192     result = getSingleScanResult(ht, scan);
3193     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3194         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3195
3196     scan = new Scan();
3197     scan.addFamily(FAMILIES[FAMILYIDX]);
3198     result = getSingleScanResult(ht, scan);
3199     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3200         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3201
3202     scan = new Scan();
3203     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]);
3204     result = getSingleScanResult(ht, scan);
3205     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3206         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3207
3208     scan = new Scan();
3209     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX+1]);
3210     scan.addFamily(FAMILIES[FAMILYIDX]);
3211     result = getSingleScanResult(ht, scan);
3212     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3213         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3214
3215     scan = new Scan();
3216     scan.addColumn(FAMILIES[FAMILYIDX-1], QUALIFIERS[QUALIFIERIDX+1]);
3217     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]);
3218     scan.addFamily(FAMILIES[FAMILYIDX+1]);
3219     result = getSingleScanResult(ht, scan);
3220     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3221         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3222
3223   }
3224
3225   /**
3226    * Verify we do not read any values by accident around a single column
3227    * Same requirements as getVerifySingleColumn
3228    */
3229   private void getVerifySingleEmpty(Table ht, byte [][] ROWS, int ROWIDX, byte [][] FAMILIES,
3230       int FAMILYIDX, byte [][] QUALIFIERS, int QUALIFIERIDX) throws Exception {
3231     Get get = new Get(ROWS[ROWIDX]);
3232     get.addFamily(FAMILIES[4]);
3233     get.addColumn(FAMILIES[4], QUALIFIERS[1]);
3234     Result result = ht.get(get);
3235     assertEmptyResult(result);
3236
3237     get = new Get(ROWS[ROWIDX]);
3238     get.addFamily(FAMILIES[4]);
3239     get.addColumn(FAMILIES[4], QUALIFIERS[2]);
3240     result = ht.get(get);
3241     assertEmptyResult(result);
3242
3243     get = new Get(ROWS[ROWIDX]);
3244     get.addFamily(FAMILIES[3]);
3245     get.addColumn(FAMILIES[4], QUALIFIERS[2]);
3246     get.addFamily(FAMILIES[5]);
3247     result = ht.get(get);
3248     assertEmptyResult(result);
3249
3250     get = new Get(ROWS[ROWIDX+1]);
3251     result = ht.get(get);
3252     assertEmptyResult(result);
3253
3254   }
3255
3256   private void scanVerifySingleEmpty(Table ht, byte [][] ROWS, int ROWIDX, byte [][] FAMILIES,
3257       int FAMILYIDX, byte [][] QUALIFIERS, int QUALIFIERIDX) throws Exception {
3258     Scan scan = new Scan(ROWS[ROWIDX+1]);
3259     Result result = getSingleScanResult(ht, scan);
3260     assertNullResult(result);
3261
3262     scan = new Scan(ROWS[ROWIDX+1],ROWS[ROWIDX+2]);
3263     result = getSingleScanResult(ht, scan);
3264     assertNullResult(result);
3265
3266     scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX]);
3267     result = getSingleScanResult(ht, scan);
3268     assertNullResult(result);
3269
3270     scan = new Scan();
3271     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX+1]);
3272     scan.addFamily(FAMILIES[FAMILYIDX-1]);
3273     result = getSingleScanResult(ht, scan);
3274     assertNullResult(result);
3275
3276   }
3277
3278   //
3279   // Verifiers
3280   //
3281
3282   private void assertKey(Cell key, byte [] row, byte [] family, byte [] qualifier, byte [] value) {
3283     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3284         "Got row [" + Bytes.toString(CellUtil.cloneRow(key)) +"]",
3285         equals(row, CellUtil.cloneRow(key)));
3286     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3287         "Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
3288         equals(family, CellUtil.cloneFamily(key)));
3289     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3290         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key)) + "]",
3291         equals(qualifier, CellUtil.cloneQualifier(key)));
3292     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
3293         "Got value [" + Bytes.toString(CellUtil.cloneValue(key)) + "]",
3294         equals(value, CellUtil.cloneValue(key)));
3295   }
3296
3297   static void assertIncrementKey(Cell key, byte [] row, byte [] family,
3298       byte [] qualifier, long value) {
3299     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3300         "Got row [" + Bytes.toString(CellUtil.cloneRow(key)) +"]",
3301         equals(row, CellUtil.cloneRow(key)));
3302     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3303         "Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
3304         equals(family, CellUtil.cloneFamily(key)));
3305     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3306         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key)) + "]",
3307         equals(qualifier, CellUtil.cloneQualifier(key)));
3308     assertTrue("Expected value [" + value + "] " +
3309         "Got value [" + Bytes.toLong(CellUtil.cloneValue(key)) + "]",
3310         Bytes.toLong(CellUtil.cloneValue(key)) == value);
3311   }
3312
3313   private void assertNumKeys(Result result, int n) throws Exception {
3314     assertTrue("Expected " + n + " keys but got " + result.size(),
3315         result.size() == n);
3316   }
3317
3318   private void assertNResult(Result result, byte [] row,
3319       byte [][] families, byte [][] qualifiers, byte [][] values, int [][] idxs) {
3320     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3321         "Got row [" + Bytes.toString(result.getRow()) +"]",
3322         equals(row, result.getRow()));
3323     assertTrue("Expected " + idxs.length + " keys but result contains "
3324         + result.size(), result.size() == idxs.length);
3325
3326     Cell [] keys = result.rawCells();
3327
3328     for(int i=0;i<keys.length;i++) {
3329       byte [] family = families[idxs[i][0]];
3330       byte [] qualifier = qualifiers[idxs[i][1]];
3331       byte [] value = values[idxs[i][2]];
3332       Cell key = keys[i];
3333
3334       byte[] famb = CellUtil.cloneFamily(key);
3335       byte[] qualb = CellUtil.cloneQualifier(key);
3336       byte[] valb = CellUtil.cloneValue(key);
3337       assertTrue("(" + i + ") Expected family [" + Bytes.toString(family)
3338           + "] " + "Got family [" + Bytes.toString(famb) + "]",
3339           equals(family, famb));
3340       assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier)
3341           + "] " + "Got qualifier [" + Bytes.toString(qualb) + "]",
3342           equals(qualifier, qualb));
3343       assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] "
3344           + "Got value [" + Bytes.toString(valb) + "]",
3345           equals(value, valb));
3346     }
3347   }
3348
3349   private void assertNResult(Result result, byte [] row,
3350       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
3351       int start, int end) {
3352     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3353         "Got row [" + Bytes.toString(result.getRow()) +"]",
3354         equals(row, result.getRow()));
3355     int expectedResults = end - start + 1;
3356     assertEquals(expectedResults, result.size());
3357
3358     Cell[] keys = result.rawCells();
3359
3360     for (int i=0; i<keys.length; i++) {
3361       byte [] value = values[end-i];
3362       long ts = stamps[end-i];
3363       Cell key = keys[i];
3364
3365       assertTrue("(" + i + ") Expected family [" + Bytes.toString(family)
3366           + "] " + "Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
3367           CellUtil.matchingFamily(key, family));
3368       assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier)
3369           + "] " + "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key))+ "]",
3370           CellUtil.matchingQualifier(key, qualifier));
3371       assertTrue("Expected ts [" + ts + "] " +
3372           "Got ts [" + key.getTimestamp() + "]", ts == key.getTimestamp());
3373       assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] "
3374           + "Got value [" + Bytes.toString(CellUtil.cloneValue(key)) + "]",
3375           CellUtil.matchingValue(key,  value));
3376     }
3377   }
3378
3379   /**
3380    * Validate that result contains two specified keys, exactly.
3381    * It is assumed key A sorts before key B.
3382    */
3383   private void assertDoubleResult(Result result, byte [] row,
3384       byte [] familyA, byte [] qualifierA, byte [] valueA,
3385       byte [] familyB, byte [] qualifierB, byte [] valueB) {
3386     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3387         "Got row [" + Bytes.toString(result.getRow()) +"]",
3388         equals(row, result.getRow()));
3389     assertTrue("Expected two keys but result contains " + result.size(),
3390         result.size() == 2);
3391     Cell [] kv = result.rawCells();
3392     Cell kvA = kv[0];
3393     assertTrue("(A) Expected family [" + Bytes.toString(familyA) + "] " +
3394         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kvA)) + "]",
3395         equals(familyA, CellUtil.cloneFamily(kvA)));
3396     assertTrue("(A) Expected qualifier [" + Bytes.toString(qualifierA) + "] " +
3397         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kvA)) + "]",
3398         equals(qualifierA, CellUtil.cloneQualifier(kvA)));
3399     assertTrue("(A) Expected value [" + Bytes.toString(valueA) + "] " +
3400         "Got value [" + Bytes.toString(CellUtil.cloneValue(kvA)) + "]",
3401         equals(valueA, CellUtil.cloneValue(kvA)));
3402     Cell kvB = kv[1];
3403     assertTrue("(B) Expected family [" + Bytes.toString(familyB) + "] " +
3404         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kvB)) + "]",
3405         equals(familyB, CellUtil.cloneFamily(kvB)));
3406     assertTrue("(B) Expected qualifier [" + Bytes.toString(qualifierB) + "] " +
3407         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kvB)) + "]",
3408         equals(qualifierB, CellUtil.cloneQualifier(kvB)));
3409     assertTrue("(B) Expected value [" + Bytes.toString(valueB) + "] " +
3410         "Got value [" + Bytes.toString(CellUtil.cloneValue(kvB)) + "]",
3411         equals(valueB, CellUtil.cloneValue(kvB)));
3412   }
3413
3414   private void assertSingleResult(Result result, byte [] row, byte [] family,
3415       byte [] qualifier, byte [] value) {
3416     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3417         "Got row [" + Bytes.toString(result.getRow()) +"]",
3418         equals(row, result.getRow()));
3419     assertTrue("Expected a single key but result contains " + result.size(),
3420         result.size() == 1);
3421     Cell kv = result.rawCells()[0];
3422     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3423         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kv)) + "]",
3424         equals(family, CellUtil.cloneFamily(kv)));
3425     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3426         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kv)) + "]",
3427         equals(qualifier, CellUtil.cloneQualifier(kv)));
3428     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
3429         "Got value [" + Bytes.toString(CellUtil.cloneValue(kv)) + "]",
3430         equals(value, CellUtil.cloneValue(kv)));
3431   }
3432
3433   private void assertSingleResult(Result result, byte[] row, byte[] family, byte[] qualifier,
3434       long value) throws Exception {
3435     assertTrue(
3436       "Expected row [" + Bytes.toString(row) + "] " + "Got row [" + Bytes.toString(result.getRow())
3437           + "]", equals(row, result.getRow()));
3438     assertTrue("Expected a single key but result contains " + result.size(), result.size() == 1);
3439     Cell kv = result.rawCells()[0];
3440     assertTrue(
3441       "Expected family [" + Bytes.toString(family) + "] " + "Got family ["
3442           + Bytes.toString(CellUtil.cloneFamily(kv)) + "]",
3443       equals(family, CellUtil.cloneFamily(kv)));
3444     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " + "Got qualifier ["
3445         + Bytes.toString(CellUtil.cloneQualifier(kv)) + "]",
3446       equals(qualifier, CellUtil.cloneQualifier(kv)));
3447     assertTrue(
3448       "Expected value [" + value + "] " + "Got value [" + Bytes.toLong(CellUtil.cloneValue(kv))
3449           + "]", value == Bytes.toLong(CellUtil.cloneValue(kv)));
3450   }
3451
3452   private void assertSingleResult(Result result, byte [] row, byte [] family,
3453       byte [] qualifier, long ts, byte [] value) {
3454     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3455         "Got row [" + Bytes.toString(result.getRow()) +"]",
3456         equals(row, result.getRow()));
3457     assertTrue("Expected a single key but result contains " + result.size(),
3458         result.size() == 1);
3459     Cell kv = result.rawCells()[0];
3460     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3461         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kv)) + "]",
3462         equals(family, CellUtil.cloneFamily(kv)));
3463     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3464         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kv)) + "]",
3465         equals(qualifier, CellUtil.cloneQualifier(kv)));
3466     assertTrue("Expected ts [" + ts + "] " +
3467         "Got ts [" + kv.getTimestamp() + "]", ts == kv.getTimestamp());
3468     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
3469         "Got value [" + Bytes.toString(CellUtil.cloneValue(kv)) + "]",
3470         equals(value, CellUtil.cloneValue(kv)));
3471   }
3472
3473   private void assertEmptyResult(Result result) throws Exception {
3474     assertTrue("expected an empty result but result contains " +
3475         result.size() + " keys", result.isEmpty());
3476   }
3477
3478   private void assertNullResult(Result result) throws Exception {
3479     assertTrue("expected null result but received a non-null result",
3480         result == null);
3481   }
3482
3483   //
3484   // Helpers
3485   //
3486
3487   private Result getSingleScanResult(Table ht, Scan scan) throws IOException {
3488     ResultScanner scanner = ht.getScanner(scan);
3489     Result result = scanner.next();
3490     scanner.close();
3491     return result;
3492   }
3493
3494   private byte [][] makeNAscii(byte [] base, int n) {
3495     if(n > 256) {
3496       return makeNBig(base, n);
3497     }
3498     byte [][] ret = new byte[n][];
3499     for(int i=0;i<n;i++) {
3500       byte [] tail = Bytes.toBytes(Integer.toString(i));
3501       ret[i] = Bytes.add(base, tail);
3502     }
3503     return ret;
3504   }
3505
3506   private byte [][] makeN(byte [] base, int n) {
3507     if (n > 256) {
3508       return makeNBig(base, n);
3509     }
3510     byte [][] ret = new byte[n][];
3511     for(int i=0;i<n;i++) {
3512       ret[i] = Bytes.add(base, new byte[]{(byte)i});
3513     }
3514     return ret;
3515   }
3516
3517   private byte [][] makeNBig(byte [] base, int n) {
3518     byte [][] ret = new byte[n][];
3519     for(int i=0;i<n;i++) {
3520       int byteA = (i % 256);
3521       int byteB = (i >> 8);
3522       ret[i] = Bytes.add(base, new byte[]{(byte)byteB,(byte)byteA});
3523     }
3524     return ret;
3525   }
3526
3527   private long [] makeStamps(int n) {
3528     long [] stamps = new long[n];
3529     for (int i = 0; i < n; i++) {
3530       stamps[i] = i+1L;
3531     }
3532     return stamps;
3533   }
3534
3535   static boolean equals(byte [] left, byte [] right) {
3536     if (left == null && right == null) return true;
3537     if (left == null && right.length == 0) return true;
3538     if (right == null && left.length == 0) return true;
3539     return Bytes.equals(left, right);
3540   }
3541
3542   @Test
3543   public void testDuplicateVersions() throws Exception {
3544     final TableName tableName = name.getTableName();
3545
3546     long [] STAMPS = makeStamps(20);
3547     byte [][] VALUES = makeNAscii(VALUE, 20);
3548
3549     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
3550
3551       // Insert 4 versions of same column
3552       Put put = new Put(ROW);
3553       put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3554       put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3555       put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3556       put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3557       ht.put(put);
3558
3559       // Verify we can get each one properly
3560       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3561       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3562       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3563       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3564       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3565       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3566       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3567       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3568
3569       // Verify we don't accidentally get others
3570       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3571       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3572       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3573       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3574       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3575       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3576
3577       // Ensure maxVersions in query is respected
3578       Get get = new Get(ROW);
3579       get.addColumn(FAMILY, QUALIFIER);
3580       get.readVersions(2);
3581       Result result = ht.get(get);
3582       assertNResult(result, ROW, FAMILY, QUALIFIER,
3583           new long [] {STAMPS[4], STAMPS[5]},
3584           new byte[][] {VALUES[4], VALUES[5]},
3585           0, 1);
3586
3587       Scan scan = new Scan(ROW);
3588       scan.addColumn(FAMILY, QUALIFIER);
3589       scan.setMaxVersions(2);
3590       result = getSingleScanResult(ht, scan);
3591       assertNResult(result, ROW, FAMILY, QUALIFIER,
3592               new long[]{STAMPS[4], STAMPS[5]},
3593               new byte[][]{VALUES[4], VALUES[5]},
3594               0, 1);
3595
3596       // Flush and redo
3597
3598       TEST_UTIL.flush();
3599
3600       // Verify we can get each one properly
3601       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3602       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3603       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3604       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3605       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3606       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3607       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3608       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3609
3610       // Verify we don't accidentally get others
3611       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3612       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3613       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3614       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3615       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3616       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3617
3618       // Ensure maxVersions in query is respected
3619       get = new Get(ROW);
3620       get.addColumn(FAMILY, QUALIFIER);
3621       get.readVersions(2);
3622       result = ht.get(get);
3623       assertNResult(result, ROW, FAMILY, QUALIFIER,
3624           new long [] {STAMPS[4], STAMPS[5]},
3625           new byte[][] {VALUES[4], VALUES[5]},
3626           0, 1);
3627
3628       scan = new Scan(ROW);
3629       scan.addColumn(FAMILY, QUALIFIER);
3630       scan.setMaxVersions(2);
3631       result = getSingleScanResult(ht, scan);
3632       assertNResult(result, ROW, FAMILY, QUALIFIER,
3633               new long[]{STAMPS[4], STAMPS[5]},
3634               new byte[][]{VALUES[4], VALUES[5]},
3635               0, 1);
3636
3637
3638       // Add some memstore and retest
3639
3640       // Insert 4 more versions of same column and a dupe
3641       put = new Put(ROW);
3642       put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
3643       put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3644       put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
3645       put.addColumn(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3646       put.addColumn(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
3647       ht.put(put);
3648
3649       // Ensure maxVersions in query is respected
3650       get = new Get(ROW);
3651       get.addColumn(FAMILY, QUALIFIER);
3652       get.readVersions(7);
3653       result = ht.get(get);
3654       assertNResult(result, ROW, FAMILY, QUALIFIER,
3655           new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3656           new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
3657                   VALUES[8]},
3658           0, 6);
3659
3660       scan = new Scan(ROW);
3661       scan.addColumn(FAMILY, QUALIFIER);
3662       scan.setMaxVersions(7);
3663       result = getSingleScanResult(ht, scan);
3664       assertNResult(result, ROW, FAMILY, QUALIFIER,
3665         new long[]{STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3666         new byte[][]{VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3667         0, 6);
3668
3669       get = new Get(ROW);
3670       get.readVersions(7);
3671       result = ht.get(get);
3672       assertNResult(result, ROW, FAMILY, QUALIFIER,
3673           new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3674           new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
3675                   VALUES[8]},
3676           0, 6);
3677
3678       scan = new Scan(ROW);
3679       scan.setMaxVersions(7);
3680       result = getSingleScanResult(ht, scan);
3681       assertNResult(result, ROW, FAMILY, QUALIFIER,
3682         new long[]{STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3683         new byte[][]{VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3684         0, 6);
3685
3686       // Verify we can get each one properly
3687       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3688       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3689       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3690       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3691       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3692       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3693       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3694       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3695
3696       // Verify we don't accidentally get others
3697       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3698       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
3699       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3700       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
3701
3702       // Ensure maxVersions of table is respected
3703
3704       TEST_UTIL.flush();
3705
3706       // Insert 4 more versions of same column and a dupe
3707       put = new Put(ROW);
3708       put.addColumn(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
3709       put.addColumn(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
3710       put.addColumn(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
3711       put.addColumn(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
3712       ht.put(put);
3713
3714       get = new Get(ROW);
3715       get.addColumn(FAMILY, QUALIFIER);
3716       get.readVersions(Integer.MAX_VALUE);
3717       result = ht.get(get);
3718       assertNResult(result, ROW, FAMILY, QUALIFIER,
3719           new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9],
3720                   STAMPS[11], STAMPS[13], STAMPS[15]},
3721           new byte[][] {VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8],
3722                   VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
3723           0, 9);
3724
3725       scan = new Scan(ROW);
3726       scan.addColumn(FAMILY, QUALIFIER);
3727       scan.setMaxVersions(Integer.MAX_VALUE);
3728       result = getSingleScanResult(ht, scan);
3729       assertNResult(result, ROW, FAMILY, QUALIFIER,
3730           new long[]{STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9],
3731                   STAMPS[11], STAMPS[13], STAMPS[15]},
3732           new byte[][]{VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9],
3733                   VALUES[11], VALUES[13], VALUES[15]},0, 9);
3734
3735       // Delete a version in the memstore and a version in a storefile
3736       Delete delete = new Delete(ROW);
3737       delete.addColumn(FAMILY, QUALIFIER, STAMPS[11]);
3738       delete.addColumn(FAMILY, QUALIFIER, STAMPS[7]);
3739       ht.delete(delete);
3740
3741       // Test that it's gone
3742       get = new Get(ROW);
3743       get.addColumn(FAMILY, QUALIFIER);
3744       get.readVersions(Integer.MAX_VALUE);
3745       result = ht.get(get);
3746       assertNResult(result, ROW, FAMILY, QUALIFIER,
3747           new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8],
3748                   STAMPS[9], STAMPS[13], STAMPS[15]},
3749           new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6],
3750                   VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
3751           0, 9);
3752
3753       scan = new Scan(ROW);
3754       scan.addColumn(FAMILY, QUALIFIER);
3755       scan.setMaxVersions(Integer.MAX_VALUE);
3756       result = getSingleScanResult(ht, scan);
3757       assertNResult(result, ROW, FAMILY, QUALIFIER,
3758           new long[]{STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8],
3759                   STAMPS[9], STAMPS[13], STAMPS[15]},
3760           new byte[][]{VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8],
3761                   VALUES[9], VALUES[13], VALUES[15]},0,9);
3762     }
3763   }
3764
3765   @Test
3766   public void testUpdates() throws Exception {
3767     final TableName tableName = name.getTableName();
3768     try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
3769
3770       // Write a column with values at timestamp 1, 2 and 3
3771       byte[] row = Bytes.toBytes("row1");
3772       byte[] qualifier = Bytes.toBytes("myCol");
3773       Put put = new Put(row);
3774       put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3775       hTable.put(put);
3776
3777       put = new Put(row);
3778       put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3779       hTable.put(put);
3780
3781       put = new Put(row);
3782       put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3783       hTable.put(put);
3784
3785       Get get = new Get(row);
3786       get.addColumn(FAMILY, qualifier);
3787       get.readAllVersions();
3788
3789       // Check that the column indeed has the right values at timestamps 1 and
3790       // 2
3791       Result result = hTable.get(get);
3792       NavigableMap<Long, byte[]> navigableMap =
3793               result.getMap().get(FAMILY).get(qualifier);
3794       assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3795       assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3796
3797       // Update the value at timestamp 1
3798       put = new Put(row);
3799       put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3800       hTable.put(put);
3801
3802       // Update the value at timestamp 2
3803       put = new Put(row);
3804       put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3805       hTable.put(put);
3806
3807       // Check that the values at timestamp 2 and 1 got updated
3808       result = hTable.get(get);
3809       navigableMap = result.getMap().get(FAMILY).get(qualifier);
3810       assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3811       assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3812     }
3813   }
3814
3815   @Test
3816   public void testUpdatesWithMajorCompaction() throws Exception {
3817     final TableName tableName = name.getTableName();
3818     try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10);
3819         Admin admin = TEST_UTIL.getAdmin()) {
3820
3821       // Write a column with values at timestamp 1, 2 and 3
3822       byte[] row = Bytes.toBytes("row2");
3823       byte[] qualifier = Bytes.toBytes("myCol");
3824       Put put = new Put(row);
3825       put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3826       hTable.put(put);
3827
3828       put = new Put(row);
3829       put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3830       hTable.put(put);
3831
3832       put = new Put(row);
3833       put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3834       hTable.put(put);
3835
3836       Get get = new Get(row);
3837       get.addColumn(FAMILY, qualifier);
3838       get.readAllVersions();
3839
3840       // Check that the column indeed has the right values at timestamps 1 and
3841       // 2
3842       Result result = hTable.get(get);
3843       NavigableMap<Long, byte[]> navigableMap =
3844               result.getMap().get(FAMILY).get(qualifier);
3845       assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3846       assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3847
3848       // Trigger a major compaction
3849       admin.flush(tableName);
3850       admin.majorCompact(tableName);
3851       Thread.sleep(6000);
3852
3853       // Update the value at timestamp 1
3854       put = new Put(row);
3855       put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3856       hTable.put(put);
3857
3858       // Update the value at timestamp 2
3859       put = new Put(row);
3860       put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3861       hTable.put(put);
3862
3863       // Trigger a major compaction
3864       admin.flush(tableName);
3865       admin.majorCompact(tableName);
3866       Thread.sleep(6000);
3867
3868       // Check that the values at timestamp 2 and 1 got updated
3869       result = hTable.get(get);
3870       navigableMap = result.getMap().get(FAMILY).get(qualifier);
3871       assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3872       assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3873     }
3874   }
3875
3876   @Test
3877   public void testMajorCompactionBetweenTwoUpdates() throws Exception {
3878     final TableName tableName = name.getTableName();
3879     try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10);
3880         Admin admin = TEST_UTIL.getAdmin()) {
3881
3882       // Write a column with values at timestamp 1, 2 and 3
3883       byte[] row = Bytes.toBytes("row3");
3884       byte[] qualifier = Bytes.toBytes("myCol");
3885       Put put = new Put(row);
3886       put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3887       hTable.put(put);
3888
3889       put = new Put(row);
3890       put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3891       hTable.put(put);
3892
3893       put = new Put(row);
3894       put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3895       hTable.put(put);
3896
3897       Get get = new Get(row);
3898       get.addColumn(FAMILY, qualifier);
3899       get.readAllVersions();
3900
3901       // Check that the column indeed has the right values at timestamps 1 and
3902       // 2
3903       Result result = hTable.get(get);
3904       NavigableMap<Long, byte[]> navigableMap =
3905               result.getMap().get(FAMILY).get(qualifier);
3906       assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3907       assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3908
3909       // Trigger a major compaction
3910       admin.flush(tableName);
3911       admin.majorCompact(tableName);
3912       Thread.sleep(6000);
3913
3914       // Update the value at timestamp 1
3915       put = new Put(row);
3916       put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3917       hTable.put(put);
3918
3919       // Trigger a major compaction
3920       admin.flush(tableName);
3921       admin.majorCompact(tableName);
3922       Thread.sleep(6000);
3923
3924       // Update the value at timestamp 2
3925       put = new Put(row);
3926       put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3927       hTable.put(put);
3928
3929       // Trigger a major compaction
3930       admin.flush(tableName);
3931       admin.majorCompact(tableName);
3932       Thread.sleep(6000);
3933
3934       // Check that the values at timestamp 2 and 1 got updated
3935       result = hTable.get(get);
3936       navigableMap = result.getMap().get(FAMILY).get(qualifier);
3937
3938       assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3939       assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3940     }
3941   }
3942
3943   @Test
3944   public void testGet_EmptyTable() throws IOException {
3945     try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
3946       Get get = new Get(ROW);
3947       get.addFamily(FAMILY);
3948       Result r = table.get(get);
3949       assertTrue(r.isEmpty());
3950     }
3951   }
3952
3953   @Test
3954   public void testGet_NullQualifier() throws IOException {
3955     try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
3956       Put put = new Put(ROW);
3957       put.addColumn(FAMILY, QUALIFIER, VALUE);
3958       table.put(put);
3959
3960       put = new Put(ROW);
3961       put.addColumn(FAMILY, null, VALUE);
3962       table.put(put);
3963       LOG.info("Row put");
3964
3965       Get get = new Get(ROW);
3966       get.addColumn(FAMILY, null);
3967       Result r = table.get(get);
3968       assertEquals(1, r.size());
3969
3970       get = new Get(ROW);
3971       get.addFamily(FAMILY);
3972       r = table.get(get);
3973       assertEquals(2, r.size());
3974     }
3975   }
3976
3977   @Test
3978   public void testGet_NonExistentRow() throws IOException {
3979     try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
3980       Put put = new Put(ROW);
3981       put.addColumn(FAMILY, QUALIFIER, VALUE);
3982       table.put(put);
3983       LOG.info("Row put");
3984
3985       Get get = new Get(ROW);
3986       get.addFamily(FAMILY);
3987       Result r = table.get(get);
3988       assertFalse(r.isEmpty());
3989       System.out.println("Row retrieved successfully");
3990
3991       byte[] missingrow = Bytes.toBytes("missingrow");
3992       get = new Get(missingrow);
3993       get.addFamily(FAMILY);
3994       r = table.get(get);
3995       assertTrue(r.isEmpty());
3996       LOG.info("Row missing as it should be");
3997     }
3998   }
3999
4000   @Test
4001   public void testPut() throws IOException {
4002     final byte [] CONTENTS_FAMILY = Bytes.toBytes("contents");
4003     final byte [] SMALL_FAMILY = Bytes.toBytes("smallfam");
4004     final byte [] row1 = Bytes.toBytes("row1");
4005     final byte [] row2 = Bytes.toBytes("row2");
4006     final byte [] value = Bytes.toBytes("abcd");
4007     try (Table table = TEST_UTIL.createTable(name.getTableName(),
4008         new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) {
4009       Put put = new Put(row1);
4010       put.addColumn(CONTENTS_FAMILY, null, value);
4011       table.put(put);
4012
4013       put = new Put(row2);
4014       put.addColumn(CONTENTS_FAMILY, null, value);
4015
4016       assertEquals(1, put.size());
4017       assertEquals(1, put.getFamilyCellMap().get(CONTENTS_FAMILY).size());
4018
4019       // KeyValue v1 expectation.  Cast for now until we go all Cell all the time. TODO
4020       KeyValue kv = (KeyValue) put.getFamilyCellMap().get(CONTENTS_FAMILY).get(0);
4021
4022       assertTrue(Bytes.equals(CellUtil.cloneFamily(kv), CONTENTS_FAMILY));
4023       // will it return null or an empty byte array?
4024       assertTrue(Bytes.equals(CellUtil.cloneQualifier(kv), new byte[0]));
4025
4026       assertTrue(Bytes.equals(CellUtil.cloneValue(kv), value));
4027
4028       table.put(put);
4029
4030       Scan scan = new Scan();
4031       scan.addColumn(CONTENTS_FAMILY, null);
4032       try (ResultScanner scanner = table.getScanner(scan)) {
4033         for (Result r : scanner) {
4034           for (Cell key : r.rawCells()) {
4035             System.out.println(Bytes.toString(r.getRow()) + ": " + key.toString());
4036           }
4037         }
4038       }
4039     }
4040   }
4041
4042   @Test
4043   public void testPutNoCF() throws IOException {
4044     final byte[] BAD_FAM = Bytes.toBytes("BAD_CF");
4045     final byte[] VAL = Bytes.toBytes(100);
4046     try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
4047       boolean caughtNSCFE = false;
4048
4049       try {
4050         Put p = new Put(ROW);
4051         p.addColumn(BAD_FAM, QUALIFIER, VAL);
4052         table.put(p);
4053       } catch (Exception e) {
4054         caughtNSCFE = e instanceof NoSuchColumnFamilyException;
4055       }
4056       assertTrue("Should throw NoSuchColumnFamilyException", caughtNSCFE);
4057     }
4058   }
4059
4060   @Test
4061   public void testRowsPut() throws IOException {
4062     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
4063     final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
4064     final int NB_BATCH_ROWS = 10;
4065     final byte[] value = Bytes.toBytes("abcd");
4066     try (Table table = TEST_UTIL.createTable(name.getTableName(),
4067       new byte[][] {CONTENTS_FAMILY, SMALL_FAMILY })) {
4068       ArrayList<Put> rowsUpdate = new ArrayList<Put>();
4069       for (int i = 0; i < NB_BATCH_ROWS; i++) {
4070         byte[] row = Bytes.toBytes("row" + i);
4071         Put put = new Put(row);
4072         put.setDurability(Durability.SKIP_WAL);
4073         put.addColumn(CONTENTS_FAMILY, null, value);
4074         rowsUpdate.add(put);
4075       }
4076       table.put(rowsUpdate);
4077       Scan scan = new Scan();
4078       scan.addFamily(CONTENTS_FAMILY);
4079       try (ResultScanner scanner = table.getScanner(scan)) {
4080         int nbRows = 0;
4081         for (@SuppressWarnings("unused")
4082                 Result row : scanner) {
4083           nbRows++;
4084         }
4085         assertEquals(NB_BATCH_ROWS, nbRows);
4086       }
4087     }
4088   }
4089
4090   @Test
4091   public void testRowsPutBufferedManyManyFlushes() throws IOException {
4092     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
4093     final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
4094     final byte[] value = Bytes.toBytes("abcd");
4095     final int NB_BATCH_ROWS = 10;
4096     try (Table table = TEST_UTIL.createTable(name.getTableName(),
4097         new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) {
4098       ArrayList<Put> rowsUpdate = new ArrayList<Put>();
4099       for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
4100         byte[] row = Bytes.toBytes("row" + i);
4101         Put put = new Put(row);
4102         put.setDurability(Durability.SKIP_WAL);
4103         put.addColumn(CONTENTS_FAMILY, null, value);
4104         rowsUpdate.add(put);
4105       }
4106       table.put(rowsUpdate);
4107
4108       Scan scan = new Scan();
4109       scan.addFamily(CONTENTS_FAMILY);
4110       try (ResultScanner scanner = table.getScanner(scan)) {
4111         int nbRows = 0;
4112         for (@SuppressWarnings("unused")
4113                 Result row : scanner) {
4114           nbRows++;
4115         }
4116         assertEquals(NB_BATCH_ROWS * 10, nbRows);
4117       }
4118     }
4119   }
4120
4121   @Test
4122   public void testAddKeyValue() throws IOException {
4123     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
4124     final byte[] value = Bytes.toBytes("abcd");
4125     final byte[] row1 = Bytes.toBytes("row1");
4126     final byte[] row2 = Bytes.toBytes("row2");
4127     byte[] qualifier = Bytes.toBytes("qf1");
4128     Put put = new Put(row1);
4129
4130     // Adding KeyValue with the same row
4131     KeyValue kv = new KeyValue(row1, CONTENTS_FAMILY, qualifier, value);
4132     boolean ok = true;
4133     try {
4134       put.add(kv);
4135     } catch (IOException e) {
4136       ok = false;
4137     }
4138     assertEquals(true, ok);
4139
4140     // Adding KeyValue with the different row
4141     kv = new KeyValue(row2, CONTENTS_FAMILY, qualifier, value);
4142     ok = false;
4143     try {
4144       put.add(kv);
4145     } catch (IOException e) {
4146       ok = true;
4147     }
4148     assertEquals(true, ok);
4149   }
4150
4151   /**
4152    * test for HBASE-737
4153    */
4154   @Test
4155   public void testHBase737 () throws IOException {
4156     final byte [] FAM1 = Bytes.toBytes("fam1");
4157     final byte [] FAM2 = Bytes.toBytes("fam2");
4158     // Open table
4159     try (Table table = TEST_UTIL.createTable(name.getTableName(),
4160       new byte [][] {FAM1, FAM2})) {
4161       // Insert some values
4162       Put put = new Put(ROW);
4163       put.addColumn(FAM1, Bytes.toBytes("letters"), Bytes.toBytes("abcdefg"));
4164       table.put(put);
4165       try {
4166         Thread.sleep(1000);
4167       } catch (InterruptedException i) {
4168         //ignore
4169       }
4170
4171       put = new Put(ROW);
4172       put.addColumn(FAM1, Bytes.toBytes("numbers"), Bytes.toBytes("123456"));
4173       table.put(put);
4174
4175       try {
4176         Thread.sleep(1000);
4177       } catch (InterruptedException i) {
4178         //ignore
4179       }
4180
4181       put = new Put(ROW);
4182       put.addColumn(FAM2, Bytes.toBytes("letters"), Bytes.toBytes("hijklmnop"));
4183       table.put(put);
4184
4185       long[] times = new long[3];
4186
4187       // First scan the memstore
4188
4189       Scan scan = new Scan();
4190       scan.addFamily(FAM1);
4191       scan.addFamily(FAM2);
4192       try (ResultScanner s = table.getScanner(scan)) {
4193         int index = 0;
4194         Result r = null;
4195         while ((r = s.next()) != null) {
4196           for (Cell key : r.rawCells()) {
4197             times[index++] = key.getTimestamp();
4198           }
4199         }
4200       }
4201       for (int i = 0; i < times.length - 1; i++) {
4202         for (int j = i + 1; j < times.length; j++) {
4203           assertTrue(times[j] > times[i]);
4204         }
4205       }
4206
4207       // Flush data to disk and try again
4208       TEST_UTIL.flush();
4209
4210       // Reset times
4211       for (int i = 0; i < times.length; i++) {
4212         times[i] = 0;
4213       }
4214
4215       try {
4216         Thread.sleep(1000);
4217       } catch (InterruptedException i) {
4218         //ignore
4219       }
4220       scan = new Scan();
4221       scan.addFamily(FAM1);
4222       scan.addFamily(FAM2);
4223       try (ResultScanner s = table.getScanner(scan)) {
4224         int index = 0;
4225         Result r = null;
4226         while ((r = s.next()) != null) {
4227           for (Cell key : r.rawCells()) {
4228             times[index++] = key.getTimestamp();
4229           }
4230         }
4231         for (int i = 0; i < times.length - 1; i++) {
4232           for (int j = i + 1; j < times.length; j++) {
4233             assertTrue(times[j] > times[i]);
4234           }
4235         }
4236       }
4237     }
4238   }
4239
4240   @Test
4241   public void testListTables() throws IOException, InterruptedException {
4242     final String testTableName = name.getTableName().toString();
4243     final TableName tableName1 = TableName.valueOf(testTableName + "1");
4244     final TableName tableName2 = TableName.valueOf(testTableName + "2");
4245     final TableName tableName3 = TableName.valueOf(testTableName + "3");
4246     TableName [] tables = new TableName[] { tableName1, tableName2, tableName3 };
4247     for (int i = 0; i < tables.length; i++) {
4248       TEST_UTIL.createTable(tables[i], FAMILY);
4249     }
4250     try (Admin admin = TEST_UTIL.getAdmin()) {
4251       List<TableDescriptor> ts = admin.listTableDescriptors();
4252       HashSet<TableDescriptor> result = new HashSet<>(ts);
4253       int size = result.size();
4254       assertTrue(size >= tables.length);
4255       for (int i = 0; i < tables.length && i < size; i++) {
4256         boolean found = false;
4257         for (int j = 0; j < ts.size(); j++) {
4258           if (ts.get(j).getTableName().equals(tables[i])) {
4259             found = true;
4260             break;
4261           }
4262         }
4263         assertTrue("Not found: " + tables[i], found);
4264       }
4265     }
4266   }
4267
4268   /**
4269    * simple test that just executes parts of the client
4270    * API that accept a pre-created Connection instance
4271    */
4272   @Test
4273   public void testUnmanagedHConnection() throws IOException {
4274     final TableName tableName = name.getTableName();
4275     TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
4276     try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
4277         Table t = conn.getTable(tableName);
4278         Admin admin = conn.getAdmin()) {
4279       assertTrue(admin.tableExists(tableName));
4280       assertTrue(t.get(new Get(ROW)).isEmpty());
4281     }
4282   }
4283
4284   /**
4285    * test of that unmanaged HConnections are able to reconnect
4286    * properly (see HBASE-5058)
4287    */
4288   @Test
4289   public void testUnmanagedHConnectionReconnect() throws Exception {
4290     Configuration conf = TEST_UTIL.getConfiguration();
4291     Class registryImpl = conf.getClass(
4292         HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY, ZKConnectionRegistry.class);
4293     // This test does not make sense for MasterRegistry since it stops the only master in the
4294     // cluster and starts a new master without populating the underlying config for the connection.
4295     Assume.assumeFalse(registryImpl.equals(MasterRegistry.class));
4296     final TableName tableName = name.getTableName();
4297     TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
4298     try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) {
4299       try (Table t = conn.getTable(tableName); Admin admin = conn.getAdmin()) {
4300         assertTrue(admin.tableExists(tableName));
4301         assertTrue(t.get(new Get(ROW)).isEmpty());
4302       }
4303
4304       // stop the master
4305       MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
4306       cluster.stopMaster(0, false);
4307       cluster.waitOnMaster(0);
4308
4309       // start up a new master
4310       cluster.startMaster();
4311       assertTrue(cluster.waitForActiveAndReadyMaster());
4312
4313       // test that the same unmanaged connection works with a new
4314       // Admin and can connect to the new master;
4315       boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration());
4316       try (Admin admin = conn.getAdmin()) {
4317         assertTrue(admin.tableExists(tableName));
4318         assertTrue(admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics()
4319           .size() == SLAVES + (tablesOnMaster ? 1 : 0));
4320       }
4321     }
4322   }
4323
4324   @Test
4325   public void testMiscHTableStuff() throws IOException {
4326     final String testTableName = name.getTableName().toString();
4327     final TableName tableAname = TableName.valueOf(testTableName + "A");
4328     final TableName tableBname = TableName.valueOf(testTableName + "B");
4329     final byte[] attrName = Bytes.toBytes("TESTATTR");
4330     final byte[] attrValue = Bytes.toBytes("somevalue");
4331     byte[] value = Bytes.toBytes("value");
4332
4333     try (Table a = TEST_UTIL.createTable(tableAname, HConstants.CATALOG_FAMILY);
4334         Table b = TEST_UTIL.createTable(tableBname, HConstants.CATALOG_FAMILY)) {
4335       Put put = new Put(ROW);
4336       put.addColumn(HConstants.CATALOG_FAMILY, null, value);
4337       a.put(put);
4338
4339       // open a new connection to A and a connection to b
4340       try (Table newA = TEST_UTIL.getConnection().getTable(tableAname)) {
4341
4342         // copy data from A to B
4343         Scan scan = new Scan();
4344         scan.addFamily(HConstants.CATALOG_FAMILY);
4345         try (ResultScanner s = newA.getScanner(scan)) {
4346           for (Result r : s) {
4347             put = new Put(r.getRow());
4348             put.setDurability(Durability.SKIP_WAL);
4349             for (Cell kv : r.rawCells()) {
4350               put.add(kv);
4351             }
4352             b.put(put);
4353           }
4354         }
4355       }
4356
4357       // Opening a new connection to A will cause the tables to be reloaded
4358       try (Table anotherA = TEST_UTIL.getConnection().getTable(tableAname)) {
4359         Get get = new Get(ROW);
4360         get.addFamily(HConstants.CATALOG_FAMILY);
4361         anotherA.get(get);
4362       }
4363
4364       // We can still access A through newA because it has the table information
4365       // cached. And if it needs to recalibrate, that will cause the information
4366       // to be reloaded.
4367
4368       // Test user metadata
4369       Admin admin = TEST_UTIL.getAdmin();
4370       // make a modifiable descriptor
4371       HTableDescriptor desc = new HTableDescriptor(a.getDescriptor());
4372       // offline the table
4373       admin.disableTable(tableAname);
4374       // add a user attribute to HTD
4375       desc.setValue(attrName, attrValue);
4376       // add a user attribute to HCD
4377       for (HColumnDescriptor c : desc.getFamilies()) {
4378         c.setValue(attrName, attrValue);
4379       }
4380       // update metadata for all regions of this table
4381       admin.modifyTable(desc);
4382       // enable the table
4383       admin.enableTable(tableAname);
4384
4385       // Test that attribute changes were applied
4386       desc = new HTableDescriptor(a.getDescriptor());
4387       assertEquals("wrong table descriptor returned", desc.getTableName(), tableAname);
4388       // check HTD attribute
4389       value = desc.getValue(attrName);
4390       assertFalse("missing HTD attribute value", value == null);
4391       assertFalse("HTD attribute value is incorrect", Bytes.compareTo(value, attrValue) != 0);
4392       // check HCD attribute
4393       for (HColumnDescriptor c : desc.getFamilies()) {
4394         value = c.getValue(attrName);
4395         assertFalse("missing HCD attribute value", value == null);
4396         assertFalse("HCD attribute value is incorrect", Bytes.compareTo(value, attrValue) != 0);
4397       }
4398     }
4399   }
4400
4401   @Test
4402   public void testGetClosestRowBefore() throws IOException, InterruptedException {
4403     final TableName tableName = name.getTableName();
4404     final byte[] firstRow = Bytes.toBytes("row111");
4405     final byte[] secondRow = Bytes.toBytes("row222");
4406     final byte[] thirdRow = Bytes.toBytes("row333");
4407     final byte[] forthRow = Bytes.toBytes("row444");
4408     final byte[] beforeFirstRow = Bytes.toBytes("row");
4409     final byte[] beforeSecondRow = Bytes.toBytes("row22");
4410     final byte[] beforeThirdRow = Bytes.toBytes("row33");
4411     final byte[] beforeForthRow = Bytes.toBytes("row44");
4412
4413     try (Table table =
4414         TEST_UTIL.createTable(tableName,
4415           new byte[][] { HConstants.CATALOG_FAMILY, Bytes.toBytes("info2") }, 1, 1024);
4416       RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
4417
4418       // set block size to 64 to making 2 kvs into one block, bypassing the walkForwardInSingleRow
4419       // in Store.rowAtOrBeforeFromStoreFile
4420       String regionName = locator.getAllRegionLocations().get(0).getRegion().getEncodedName();
4421       HRegion region = TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
4422       Put put1 = new Put(firstRow);
4423       Put put2 = new Put(secondRow);
4424       Put put3 = new Put(thirdRow);
4425       Put put4 = new Put(forthRow);
4426       byte[] one = new byte[] { 1 };
4427       byte[] two = new byte[] { 2 };
4428       byte[] three = new byte[] { 3 };
4429       byte[] four = new byte[] { 4 };
4430
4431       put1.addColumn(HConstants.CATALOG_FAMILY, null, one);
4432       put2.addColumn(HConstants.CATALOG_FAMILY, null, two);
4433       put3.addColumn(HConstants.CATALOG_FAMILY, null, three);
4434       put4.addColumn(HConstants.CATALOG_FAMILY, null, four);
4435       table.put(put1);
4436       table.put(put2);
4437       table.put(put3);
4438       table.put(put4);
4439       region.flush(true);
4440
4441       Result result;
4442
4443       // Test before first that null is returned
4444       result = getReverseScanResult(table, beforeFirstRow,
4445         HConstants.CATALOG_FAMILY);
4446       assertNull(result);
4447
4448       // Test at first that first is returned
4449       result = getReverseScanResult(table, firstRow, HConstants.CATALOG_FAMILY);
4450       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4451       assertTrue(Bytes.equals(result.getRow(), firstRow));
4452       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));
4453
4454       // Test in between first and second that first is returned
4455       result = getReverseScanResult(table, beforeSecondRow, HConstants.CATALOG_FAMILY);
4456       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4457       assertTrue(Bytes.equals(result.getRow(), firstRow));
4458       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));
4459
4460       // Test at second make sure second is returned
4461       result = getReverseScanResult(table, secondRow, HConstants.CATALOG_FAMILY);
4462       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4463       assertTrue(Bytes.equals(result.getRow(), secondRow));
4464       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), two));
4465
4466       // Test in second and third, make sure second is returned
4467       result = getReverseScanResult(table, beforeThirdRow, HConstants.CATALOG_FAMILY);
4468       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4469       assertTrue(Bytes.equals(result.getRow(), secondRow));
4470       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), two));
4471
4472       // Test at third make sure third is returned
4473       result = getReverseScanResult(table, thirdRow, HConstants.CATALOG_FAMILY);
4474       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4475       assertTrue(Bytes.equals(result.getRow(), thirdRow));
4476       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), three));
4477
4478       // Test in third and forth, make sure third is returned
4479       result = getReverseScanResult(table, beforeForthRow, HConstants.CATALOG_FAMILY);
4480       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4481       assertTrue(Bytes.equals(result.getRow(), thirdRow));
4482       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), three));
4483
4484       // Test at forth make sure forth is returned
4485       result = getReverseScanResult(table, forthRow, HConstants.CATALOG_FAMILY);
4486       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4487       assertTrue(Bytes.equals(result.getRow(), forthRow));
4488       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), four));
4489
4490       // Test after forth make sure forth is returned
4491       result = getReverseScanResult(table, Bytes.add(forthRow, one), HConstants.CATALOG_FAMILY);
4492       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4493       assertTrue(Bytes.equals(result.getRow(), forthRow));
4494       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), four));
4495     }
4496   }
4497
4498   private Result getReverseScanResult(Table table, byte[] row, byte[] fam) throws IOException {
4499     Scan scan = new Scan(row);
4500     scan.setSmall(true);
4501     scan.setReversed(true);
4502     scan.setCaching(1);
4503     scan.addFamily(fam);
4504     try (ResultScanner scanner = table.getScanner(scan)) {
4505       return scanner.next();
4506     }
4507   }
4508
4509   /**
4510    * For HBASE-2156
4511    */
4512   @Test
4513   public void testScanVariableReuse() throws Exception {
4514     Scan scan = new Scan();
4515     scan.addFamily(FAMILY);
4516     scan.addColumn(FAMILY, ROW);
4517
4518     assertTrue(scan.getFamilyMap().get(FAMILY).size() == 1);
4519
4520     scan = new Scan();
4521     scan.addFamily(FAMILY);
4522
4523     assertTrue(scan.getFamilyMap().get(FAMILY) == null);
4524     assertTrue(scan.getFamilyMap().containsKey(FAMILY));
4525   }
4526
4527   @Test
4528   public void testMultiRowMutation() throws Exception {
4529     LOG.info("Starting testMultiRowMutation");
4530     final TableName tableName = name.getTableName();
4531     final byte [] ROW1 = Bytes.toBytes("testRow1");
4532
4533     try (Table t = TEST_UTIL.createTable(tableName, FAMILY)) {
4534       Put p = new Put(ROW);
4535       p.addColumn(FAMILY, QUALIFIER, VALUE);
4536       MutationProto m1 = ProtobufUtil.toMutation(MutationType.PUT, p);
4537
4538       p = new Put(ROW1);
4539       p.addColumn(FAMILY, QUALIFIER, VALUE);
4540       MutationProto m2 = ProtobufUtil.toMutation(MutationType.PUT, p);
4541
4542       MutateRowsRequest.Builder mrmBuilder = MutateRowsRequest.newBuilder();
4543       mrmBuilder.addMutationRequest(m1);
4544       mrmBuilder.addMutationRequest(m2);
4545       MutateRowsRequest mrm = mrmBuilder.build();
4546       CoprocessorRpcChannel channel = t.coprocessorService(ROW);
4547       MultiRowMutationService.BlockingInterface service =
4548               MultiRowMutationService.newBlockingStub(channel);
4549       service.mutateRows(null, mrm);
4550       Get g = new Get(ROW);
4551       Result r = t.get(g);
4552       assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)));
4553       g = new Get(ROW1);
4554       r = t.get(g);
4555       assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)));
4556     }
4557   }
4558
4559   @Test
4560   public void testRowMutation() throws Exception {
4561     LOG.info("Starting testRowMutation");
4562     final TableName tableName = name.getTableName();
4563     try (Table t = TEST_UTIL.createTable(tableName, FAMILY)) {
4564       byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b") };
4565       RowMutations arm = new RowMutations(ROW);
4566       Put p = new Put(ROW);
4567       p.addColumn(FAMILY, QUALIFIERS[0], VALUE);
4568       arm.add(p);
4569       t.mutateRow(arm);
4570
4571       Get g = new Get(ROW);
4572       Result r = t.get(g);
4573       assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[0])));
4574
4575       arm = new RowMutations(ROW);
4576       p = new Put(ROW);
4577       p.addColumn(FAMILY, QUALIFIERS[1], VALUE);
4578       arm.add(p);
4579       Delete d = new Delete(ROW);
4580       d.addColumns(FAMILY, QUALIFIERS[0]);
4581       arm.add(d);
4582       // TODO: Trying mutateRow again. The batch was failing with a one try only.
4583       t.mutateRow(arm);
4584       r = t.get(g);
4585       assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[1])));
4586       assertNull(r.getValue(FAMILY, QUALIFIERS[0]));
4587
4588       // Test that we get a region level exception
4589       try {
4590         arm = new RowMutations(ROW);
4591         p = new Put(ROW);
4592         p.addColumn(new byte[] { 'b', 'o', 'g', 'u', 's' }, QUALIFIERS[0], VALUE);
4593         arm.add(p);
4594         t.mutateRow(arm);
4595         fail("Expected NoSuchColumnFamilyException");
4596       } catch (NoSuchColumnFamilyException e) {
4597         return;
4598       } catch (RetriesExhaustedWithDetailsException e) {
4599         for (Throwable rootCause : e.getCauses()) {
4600           if (rootCause instanceof NoSuchColumnFamilyException) {
4601             return;
4602           }
4603         }
4604         throw e;
4605       }
4606     }
4607   }
4608
4609   @Test
4610   public void testBatchAppendWithReturnResultFalse() throws Exception {
4611     LOG.info("Starting testBatchAppendWithReturnResultFalse");
4612     final TableName tableName = name.getTableName();
4613     try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) {
4614       Append append1 = new Append(Bytes.toBytes("row1"));
4615       append1.setReturnResults(false);
4616       append1.addColumn(FAMILY, Bytes.toBytes("f1"), Bytes.toBytes("value1"));
4617       Append append2 = new Append(Bytes.toBytes("row1"));
4618       append2.setReturnResults(false);
4619       append2.addColumn(FAMILY, Bytes.toBytes("f1"), Bytes.toBytes("value2"));
4620       List<Append> appends = new ArrayList<>();
4621       appends.add(append1);
4622       appends.add(append2);
4623       Object[] results = new Object[2];
4624       table.batch(appends, results);
4625       assertTrue(results.length == 2);
4626       for (Object r : results) {
4627         Result result = (Result) r;
4628         assertTrue(result.isEmpty());
4629       }
4630     }
4631   }
4632
4633   @Test
4634   public void testAppend() throws Exception {
4635     LOG.info("Starting testAppend");
4636     final TableName tableName = name.getTableName();
4637     try (Table t = TEST_UTIL.createTable(tableName, FAMILY)) {
4638       byte[] v1 = Bytes.toBytes("42");
4639       byte[] v2 = Bytes.toBytes("23");
4640       byte[][] QUALIFIERS = new byte[][]{
4641               Bytes.toBytes("b"), Bytes.toBytes("a"), Bytes.toBytes("c")
4642       };
4643       Append a = new Append(ROW);
4644       a.addColumn(FAMILY, QUALIFIERS[0], v1);
4645       a.addColumn(FAMILY, QUALIFIERS[1], v2);
4646       a.setReturnResults(false);
4647       assertEmptyResult(t.append(a));
4648
4649       a = new Append(ROW);
4650       a.addColumn(FAMILY, QUALIFIERS[0], v2);
4651       a.addColumn(FAMILY, QUALIFIERS[1], v1);
4652       a.addColumn(FAMILY, QUALIFIERS[2], v2);
4653       Result r = t.append(a);
4654       assertEquals(0, Bytes.compareTo(Bytes.add(v1, v2), r.getValue(FAMILY, QUALIFIERS[0])));
4655       assertEquals(0, Bytes.compareTo(Bytes.add(v2, v1), r.getValue(FAMILY, QUALIFIERS[1])));
4656       // QUALIFIERS[2] previously not exist, verify both value and timestamp are correct
4657       assertEquals(0, Bytes.compareTo(v2, r.getValue(FAMILY, QUALIFIERS[2])));
4658       assertEquals(r.getColumnLatestCell(FAMILY, QUALIFIERS[0]).getTimestamp(),
4659               r.getColumnLatestCell(FAMILY, QUALIFIERS[2]).getTimestamp());
4660     }
4661   }
4662   private List<Result> doAppend(final boolean walUsed) throws IOException {
4663     LOG.info("Starting testAppend, walUsed is " + walUsed);
4664     final TableName TABLENAME =
4665             TableName.valueOf(walUsed ? "testAppendWithWAL" : "testAppendWithoutWAL");
4666     try (Table t = TEST_UTIL.createTable(TABLENAME, FAMILY)) {
4667       final byte[] row1 = Bytes.toBytes("c");
4668       final byte[] row2 = Bytes.toBytes("b");
4669       final byte[] row3 = Bytes.toBytes("a");
4670       final byte[] qual = Bytes.toBytes("qual");
4671       Put put_0 = new Put(row2);
4672       put_0.addColumn(FAMILY, qual, Bytes.toBytes("put"));
4673       Put put_1 = new Put(row3);
4674       put_1.addColumn(FAMILY, qual, Bytes.toBytes("put"));
4675       Append append_0 = new Append(row1);
4676       append_0.addColumn(FAMILY, qual, Bytes.toBytes("i"));
4677       Append append_1 = new Append(row1);
4678       append_1.addColumn(FAMILY, qual, Bytes.toBytes("k"));
4679       Append append_2 = new Append(row1);
4680       append_2.addColumn(FAMILY, qual, Bytes.toBytes("e"));
4681       if (!walUsed) {
4682         append_2.setDurability(Durability.SKIP_WAL);
4683       }
4684       Append append_3 = new Append(row1);
4685       append_3.addColumn(FAMILY, qual, Bytes.toBytes("a"));
4686       Scan s = new Scan();
4687       s.setCaching(1);
4688       t.append(append_0);
4689       t.put(put_0);
4690       t.put(put_1);
4691       List<Result> results = new LinkedList<>();
4692       try (ResultScanner scanner = t.getScanner(s)) {
4693         t.append(append_1);
4694         t.append(append_2);
4695         t.append(append_3);
4696         for (Result r : scanner) {
4697           results.add(r);
4698         }
4699       }
4700       TEST_UTIL.deleteTable(TABLENAME);
4701       return results;
4702     }
4703   }
4704
4705   @Test
4706   public void testAppendWithoutWAL() throws Exception {
4707     List<Result> resultsWithWal = doAppend(true);
4708     List<Result> resultsWithoutWal = doAppend(false);
4709     assertEquals(resultsWithWal.size(), resultsWithoutWal.size());
4710     for (int i = 0; i != resultsWithWal.size(); ++i) {
4711       Result resultWithWal = resultsWithWal.get(i);
4712       Result resultWithoutWal = resultsWithoutWal.get(i);
4713       assertEquals(resultWithWal.rawCells().length, resultWithoutWal.rawCells().length);
4714       for (int j = 0; j != resultWithWal.rawCells().length; ++j) {
4715         Cell cellWithWal = resultWithWal.rawCells()[j];
4716         Cell cellWithoutWal = resultWithoutWal.rawCells()[j];
4717         assertArrayEquals(CellUtil.cloneRow(cellWithWal), CellUtil.cloneRow(cellWithoutWal));
4718         assertArrayEquals(CellUtil.cloneFamily(cellWithWal), CellUtil.cloneFamily(cellWithoutWal));
4719         assertArrayEquals(CellUtil.cloneQualifier(cellWithWal),
4720           CellUtil.cloneQualifier(cellWithoutWal));
4721         assertArrayEquals(CellUtil.cloneValue(cellWithWal), CellUtil.cloneValue(cellWithoutWal));
4722       }
4723     }
4724   }
4725
4726   @Test
4727   public void testClientPoolRoundRobin() throws IOException {
4728     final TableName tableName = name.getTableName();
4729
4730     int poolSize = 3;
4731     int numVersions = poolSize * 2;
4732     Configuration conf = TEST_UTIL.getConfiguration();
4733     conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "round-robin");
4734     conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize);
4735
4736     try (Table table =
4737                  TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, Integer.MAX_VALUE)) {
4738
4739       final long ts = EnvironmentEdgeManager.currentTime();
4740       Get get = new Get(ROW);
4741       get.addColumn(FAMILY, QUALIFIER);
4742       get.readAllVersions();
4743
4744       for (int versions = 1; versions <= numVersions; versions++) {
4745         Put put = new Put(ROW);
4746         put.addColumn(FAMILY, QUALIFIER, ts + versions, VALUE);
4747         table.put(put);
4748
4749         Result result = table.get(get);
4750         NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY)
4751                 .get(QUALIFIER);
4752
4753         assertEquals("The number of versions of '" + Bytes.toString(FAMILY) + ":"
4754                 + Bytes.toString(QUALIFIER) + " did not match", versions, navigableMap.size());
4755         for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
4756           assertTrue("The value at time " + entry.getKey()
4757                           + " did not match what was put",
4758                   Bytes.equals(VALUE, entry.getValue()));
4759         }
4760       }
4761     }
4762   }
4763
4764   @Ignore ("Flakey: HBASE-8989") @Test
4765   public void testClientPoolThreadLocal() throws IOException {
4766     final TableName tableName = name.getTableName();
4767
4768     int poolSize = Integer.MAX_VALUE;
4769     int numVersions = 3;
4770     Configuration conf = TEST_UTIL.getConfiguration();
4771     conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "thread-local");
4772     conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize);
4773
4774     try (final Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY },  3)) {
4775
4776       final long ts = EnvironmentEdgeManager.currentTime();
4777       final Get get = new Get(ROW);
4778       get.addColumn(FAMILY, QUALIFIER);
4779       get.readAllVersions();
4780
4781       for (int versions = 1; versions <= numVersions; versions++) {
4782         Put put = new Put(ROW);
4783         put.addColumn(FAMILY, QUALIFIER, ts + versions, VALUE);
4784         table.put(put);
4785
4786         Result result = table.get(get);
4787         NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY)
4788                 .get(QUALIFIER);
4789
4790         assertEquals("The number of versions of '" + Bytes.toString(FAMILY) + ":"
4791                 + Bytes.toString(QUALIFIER) + " did not match", versions, navigableMap.size());
4792         for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
4793           assertTrue("The value at time " + entry.getKey()
4794                           + " did not match what was put",
4795                   Bytes.equals(VALUE, entry.getValue()));
4796         }
4797       }
4798
4799       final Object waitLock = new Object();
4800       ExecutorService executorService = Executors.newFixedThreadPool(numVersions);
4801       final AtomicReference<AssertionError> error = new AtomicReference<>(null);
4802       for (int versions = numVersions; versions < numVersions * 2; versions++) {
4803         final int versionsCopy = versions;
4804         executorService.submit(new Callable<Void>() {
4805           @Override
4806           public Void call() {
4807             try {
4808               Put put = new Put(ROW);
4809               put.addColumn(FAMILY, QUALIFIER, ts + versionsCopy, VALUE);
4810               table.put(put);
4811
4812               Result result = table.get(get);
4813               NavigableMap<Long, byte[]> navigableMap = result.getMap()
4814                       .get(FAMILY).get(QUALIFIER);
4815
4816               assertEquals("The number of versions of '" + Bytes.toString(FAMILY) + ":"
4817                       + Bytes.toString(QUALIFIER) + " did not match " + versionsCopy, versionsCopy,
4818                       navigableMap.size());
4819               for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
4820                 assertTrue("The value at time " + entry.getKey()
4821                                 + " did not match what was put",
4822                         Bytes.equals(VALUE, entry.getValue()));
4823               }
4824               synchronized (waitLock) {
4825                 waitLock.wait();
4826               }
4827             } catch (Exception e) {
4828             } catch (AssertionError e) {
4829               // the error happens in a thread, it won't fail the test,
4830               // need to pass it to the caller for proper handling.
4831               error.set(e);
4832               LOG.error(e.toString(), e);
4833             }
4834
4835             return null;
4836           }
4837         });
4838       }
4839       synchronized (waitLock) {
4840         waitLock.notifyAll();
4841       }
4842       executorService.shutdownNow();
4843       assertNull(error.get());
4844     }
4845   }
4846
4847   @Test
4848   public void testCheckAndPut() throws IOException {
4849     final byte [] anotherrow = Bytes.toBytes("anotherrow");
4850     final byte [] value2 = Bytes.toBytes("abcd");
4851
4852     try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
4853       Put put1 = new Put(ROW);
4854       put1.addColumn(FAMILY, QUALIFIER, VALUE);
4855
4856       // row doesn't exist, so using non-null value should be considered "not match".
4857       boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4858               .ifEquals(VALUE).thenPut(put1);
4859       assertFalse(ok);
4860
4861       // row doesn't exist, so using "ifNotExists" should be considered "match".
4862       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifNotExists().thenPut(put1);
4863       assertTrue(ok);
4864
4865       // row now exists, so using "ifNotExists" should be considered "not match".
4866       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifNotExists().thenPut(put1);
4867       assertFalse(ok);
4868
4869       Put put2 = new Put(ROW);
4870       put2.addColumn(FAMILY, QUALIFIER, value2);
4871
4872       // row now exists, use the matching value to check
4873       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifEquals(VALUE).thenPut(put2);
4874       assertTrue(ok);
4875
4876       Put put3 = new Put(anotherrow);
4877       put3.addColumn(FAMILY, QUALIFIER, VALUE);
4878
4879       // try to do CheckAndPut on different rows
4880       try {
4881         table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifEquals(value2).thenPut(put3);
4882         fail("trying to check and modify different rows should have failed.");
4883       } catch (Exception e) {
4884       }
4885     }
4886   }
4887
4888   @Test
4889   public void testCheckAndMutateWithTimeRange() throws IOException {
4890     try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
4891       final long ts = System.currentTimeMillis() / 2;
4892       Put put = new Put(ROW);
4893       put.addColumn(FAMILY, QUALIFIER, ts, VALUE);
4894
4895       boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4896               .ifNotExists()
4897               .thenPut(put);
4898       assertTrue(ok);
4899
4900       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4901               .timeRange(TimeRange.at(ts + 10000))
4902               .ifEquals(VALUE)
4903               .thenPut(put);
4904       assertFalse(ok);
4905
4906       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4907               .timeRange(TimeRange.from(ts + 10000))
4908               .ifEquals(VALUE)
4909               .thenPut(put);
4910       assertFalse(ok);
4911
4912       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4913               .timeRange(TimeRange.between(ts + 10000, ts + 20000))
4914               .ifEquals(VALUE)
4915               .thenPut(put);
4916       assertFalse(ok);
4917
4918       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4919               .timeRange(TimeRange.until(ts))
4920               .ifEquals(VALUE)
4921               .thenPut(put);
4922       assertFalse(ok);
4923
4924       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4925               .timeRange(TimeRange.at(ts))
4926               .ifEquals(VALUE)
4927               .thenPut(put);
4928       assertTrue(ok);
4929
4930       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4931               .timeRange(TimeRange.from(ts))
4932               .ifEquals(VALUE)
4933               .thenPut(put);
4934       assertTrue(ok);
4935
4936       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4937               .timeRange(TimeRange.between(ts, ts + 20000))
4938               .ifEquals(VALUE)
4939               .thenPut(put);
4940       assertTrue(ok);
4941
4942       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4943               .timeRange(TimeRange.until(ts + 10000))
4944               .ifEquals(VALUE)
4945               .thenPut(put);
4946       assertTrue(ok);
4947
4948       RowMutations rm = new RowMutations(ROW)
4949               .add((Mutation) put);
4950       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4951               .timeRange(TimeRange.at(ts + 10000))
4952               .ifEquals(VALUE)
4953               .thenMutate(rm);
4954       assertFalse(ok);
4955
4956       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4957               .timeRange(TimeRange.at(ts))
4958               .ifEquals(VALUE)
4959               .thenMutate(rm);
4960       assertTrue(ok);
4961
4962       Delete delete = new Delete(ROW)
4963               .addColumn(FAMILY, QUALIFIER);
4964
4965       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4966               .timeRange(TimeRange.at(ts + 10000))
4967               .ifEquals(VALUE)
4968               .thenDelete(delete);
4969       assertFalse(ok);
4970
4971       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4972               .timeRange(TimeRange.at(ts))
4973               .ifEquals(VALUE)
4974               .thenDelete(delete);
4975       assertTrue(ok);
4976     }
4977   }
4978
4979   @Test
4980   public void testCheckAndPutWithCompareOp() throws IOException {
4981     final byte [] value1 = Bytes.toBytes("aaaa");
4982     final byte [] value2 = Bytes.toBytes("bbbb");
4983     final byte [] value3 = Bytes.toBytes("cccc");
4984     final byte [] value4 = Bytes.toBytes("dddd");
4985
4986     try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
4987
4988       Put put2 = new Put(ROW);
4989       put2.addColumn(FAMILY, QUALIFIER, value2);
4990
4991       Put put3 = new Put(ROW);
4992       put3.addColumn(FAMILY, QUALIFIER, value3);
4993
4994       // row doesn't exist, so using "ifNotExists" should be considered "match".
4995       boolean ok =
4996               table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifNotExists().thenPut(put2);
4997       assertTrue(ok);
4998
4999       // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL
5000       // turns out "match"
5001       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5002               .ifMatches(CompareOperator.GREATER, value1).thenPut(put2);
5003       assertFalse(ok);
5004       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5005               .ifMatches(CompareOperator.EQUAL, value1).thenPut(put2);
5006       assertFalse(ok);
5007       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5008               .ifMatches(CompareOperator.GREATER_OR_EQUAL, value1).thenPut(put2);
5009       assertFalse(ok);
5010       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5011               .ifMatches(CompareOperator.LESS, value1).thenPut(put2);
5012       assertTrue(ok);
5013       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5014               .ifMatches(CompareOperator.LESS_OR_EQUAL, value1).thenPut(put2);
5015       assertTrue(ok);
5016       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5017               .ifMatches(CompareOperator.NOT_EQUAL, value1).thenPut(put3);
5018       assertTrue(ok);
5019
5020       // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL
5021       // turns out "match"
5022       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5023               .ifMatches(CompareOperator.LESS, value4).thenPut(put3);
5024       assertFalse(ok);
5025       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5026               .ifMatches(CompareOperator.LESS_OR_EQUAL, value4).thenPut(put3);
5027       assertFalse(ok);
5028       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5029               .ifMatches(CompareOperator.EQUAL, value4).thenPut(put3);
5030       assertFalse(ok);
5031       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5032               .ifMatches(CompareOperator.GREATER, value4).thenPut(put3);
5033       assertTrue(ok);
5034       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5035               .ifMatches(CompareOperator.GREATER_OR_EQUAL, value4).thenPut(put3);
5036       assertTrue(ok);
5037       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5038               .ifMatches(CompareOperator.NOT_EQUAL, value4).thenPut(put2);
5039       assertTrue(ok);
5040
5041       // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL
5042       // turns out "match"
5043       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5044               .ifMatches(CompareOperator.GREATER, value2).thenPut(put2);
5045       assertFalse(ok);
5046       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5047               .ifMatches(CompareOperator.NOT_EQUAL, value2).thenPut(put2);
5048       assertFalse(ok);
5049       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5050               .ifMatches(CompareOperator.LESS, value2).thenPut(put2);
5051       assertFalse(ok);
5052       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5053               .ifMatches(CompareOperator.GREATER_OR_EQUAL, value2).thenPut(put2);
5054       assertTrue(ok);
5055       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5056               .ifMatches(CompareOperator.LESS_OR_EQUAL, value2).thenPut(put2);
5057       assertTrue(ok);
5058       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5059               .ifMatches(CompareOperator.EQUAL, value2).thenPut(put3);
5060       assertTrue(ok);
5061     }
5062   }
5063
5064   @Test
5065   public void testCheckAndDelete() throws IOException {
5066     final byte [] value1 = Bytes.toBytes("aaaa");
5067
5068     try (Table table = TEST_UTIL.createTable(name.getTableName(),
5069         FAMILY)) {
5070
5071       Put put = new Put(ROW);
5072       put.addColumn(FAMILY, QUALIFIER, value1);
5073       table.put(put);
5074
5075       Delete delete = new Delete(ROW);
5076       delete.addColumns(FAMILY, QUALIFIER);
5077
5078       boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5079               .ifEquals(value1).thenDelete(delete);
5080       assertTrue(ok);
5081     }
5082   }
5083
5084   @Test
5085   public void testCheckAndDeleteWithCompareOp() throws IOException {
5086     final byte [] value1 = Bytes.toBytes("aaaa");
5087     final byte [] value2 = Bytes.toBytes("bbbb");
5088     final byte [] value3 = Bytes.toBytes("cccc");
5089     final byte [] value4 = Bytes.toBytes("dddd");
5090
5091     try (Table table = TEST_UTIL.createTable(name.getTableName(),
5092         FAMILY)) {
5093
5094       Put put2 = new Put(ROW);
5095       put2.addColumn(FAMILY, QUALIFIER, value2);
5096       table.put(put2);
5097
5098       Put put3 = new Put(ROW);
5099       put3.addColumn(FAMILY, QUALIFIER, value3);
5100
5101       Delete delete = new Delete(ROW);
5102       delete.addColumns(FAMILY, QUALIFIER);
5103
5104       // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL
5105       // turns out "match"
5106       boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5107               .ifMatches(CompareOperator.GREATER, value1).thenDelete(delete);
5108       assertFalse(ok);
5109       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5110               .ifMatches(CompareOperator.EQUAL, value1).thenDelete(delete);
5111       assertFalse(ok);
5112       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5113               .ifMatches(CompareOperator.GREATER_OR_EQUAL, value1).thenDelete(delete);
5114       assertFalse(ok);
5115       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5116               .ifMatches(CompareOperator.LESS, value1).thenDelete(delete);
5117       assertTrue(ok);
5118       table.put(put2);
5119       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5120               .ifMatches(CompareOperator.LESS_OR_EQUAL, value1).thenDelete(delete);
5121       assertTrue(ok);
5122       table.put(put2);
5123       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5124               .ifMatches(CompareOperator.NOT_EQUAL, value1).thenDelete(delete);
5125       assertTrue(ok);
5126
5127       // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL
5128       // turns out "match"
5129       table.put(put3);
5130       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5131               .ifMatches(CompareOperator.LESS, value4).thenDelete(delete);
5132       assertFalse(ok);
5133       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5134               .ifMatches(CompareOperator.LESS_OR_EQUAL, value4).thenDelete(delete);
5135       assertFalse(ok);
5136       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5137               .ifMatches(CompareOperator.EQUAL, value4).thenDelete(delete);
5138       assertFalse(ok);
5139       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5140               .ifMatches(CompareOperator.GREATER, value4).thenDelete(delete);
5141       assertTrue(ok);
5142       table.put(put3);
5143       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5144               .ifMatches(CompareOperator.GREATER_OR_EQUAL, value4).thenDelete(delete);
5145       assertTrue(ok);
5146       table.put(put3);
5147       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5148               .ifMatches(CompareOperator.NOT_EQUAL, value4).thenDelete(delete);
5149       assertTrue(ok);
5150
5151       // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL
5152       // turns out "match"
5153       table.put(put2);
5154       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5155               .ifMatches(CompareOperator.GREATER, value2).thenDelete(delete);
5156       assertFalse(ok);
5157       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5158               .ifMatches(CompareOperator.NOT_EQUAL, value2).thenDelete(delete);
5159       assertFalse(ok);
5160       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5161               .ifMatches(CompareOperator.LESS, value2).thenDelete(delete);
5162       assertFalse(ok);
5163       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5164               .ifMatches(CompareOperator.GREATER_OR_EQUAL, value2).thenDelete(delete);
5165       assertTrue(ok);
5166       table.put(put2);
5167       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5168               .ifMatches(CompareOperator.LESS_OR_EQUAL, value2).thenDelete(delete);
5169       assertTrue(ok);
5170       table.put(put2);
5171       ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5172               .ifMatches(CompareOperator.EQUAL, value2).thenDelete(delete);
5173       assertTrue(ok);
5174     }
5175   }
5176
5177   /**
5178   * Test ScanMetrics
5179   */
5180   @Test
5181   @SuppressWarnings({"unused", "checkstyle:EmptyBlock"})
5182   public void testScanMetrics() throws Exception {
5183     final TableName tableName = name.getTableName();
5184
5185     // Set up test table:
5186     // Create table:
5187     try (Table ht = TEST_UTIL.createMultiRegionTable(tableName, FAMILY)) {
5188       int numOfRegions = -1;
5189       try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
5190         numOfRegions = r.getStartKeys().length;
5191       }
5192       // Create 3 rows in the table, with rowkeys starting with "zzz*" so that
5193       // scan are forced to hit all the regions.
5194       Put put1 = new Put(Bytes.toBytes("zzz1"));
5195       put1.addColumn(FAMILY, QUALIFIER, VALUE);
5196       Put put2 = new Put(Bytes.toBytes("zzz2"));
5197       put2.addColumn(FAMILY, QUALIFIER, VALUE);
5198       Put put3 = new Put(Bytes.toBytes("zzz3"));
5199       put3.addColumn(FAMILY, QUALIFIER, VALUE);
5200       ht.put(Arrays.asList(put1, put2, put3));
5201
5202       Scan scan1 = new Scan();
5203       int numRecords = 0;
5204       try (ResultScanner scanner = ht.getScanner(scan1)) {
5205         for (Result result : scanner) {
5206           numRecords++;
5207         }
5208
5209         LOG.info("test data has " + numRecords + " records.");
5210
5211         // by default, scan metrics collection is turned off
5212         assertEquals(null, scanner.getScanMetrics());
5213       }
5214
5215       // turn on scan metrics
5216       Scan scan2 = new Scan();
5217       scan2.setScanMetricsEnabled(true);
5218       scan2.setCaching(numRecords + 1);
5219       try (ResultScanner scanner = ht.getScanner(scan2)) {
5220         for (Result result : scanner.next(numRecords - 1)) {
5221         }
5222         scanner.close();
5223         // closing the scanner will set the metrics.
5224         assertNotNull(scanner.getScanMetrics());
5225       }
5226
5227       // set caching to 1, because metrics are collected in each roundtrip only
5228       scan2 = new Scan();
5229       scan2.setScanMetricsEnabled(true);
5230       scan2.setCaching(1);
5231       try (ResultScanner scanner = ht.getScanner(scan2)) {
5232         // per HBASE-5717, this should still collect even if you don't run all the way to
5233         // the end of the scanner. So this is asking for 2 of the 3 rows we inserted.
5234         for (Result result : scanner.next(numRecords - 1)) {
5235         }
5236         ScanMetrics scanMetrics = scanner.getScanMetrics();
5237         assertEquals("Did not access all the regions in the table", numOfRegions,
5238                 scanMetrics.countOfRegions.get());
5239       }
5240
5241       // check byte counters
5242       scan2 = new Scan();
5243       scan2.setScanMetricsEnabled(true);
5244       scan2.setCaching(1);
5245       try (ResultScanner scanner = ht.getScanner(scan2)) {
5246         int numBytes = 0;
5247         for (Result result : scanner.next(1)) {
5248           for (Cell cell : result.listCells()) {
5249             numBytes += PrivateCellUtil.estimatedSerializedSizeOf(cell);
5250           }
5251         }
5252         scanner.close();
5253         ScanMetrics scanMetrics = scanner.getScanMetrics();
5254         assertEquals("Did not count the result bytes", numBytes,
5255                 scanMetrics.countOfBytesInResults.get());
5256       }
5257
5258       // check byte counters on a small scan
5259       scan2 = new Scan();
5260       scan2.setScanMetricsEnabled(true);
5261       scan2.setCaching(1);
5262       scan2.setSmall(true);
5263       try (ResultScanner scanner = ht.getScanner(scan2)) {
5264         int numBytes = 0;
5265         for (Result result : scanner.next(1)) {
5266           for (Cell cell : result.listCells()) {
5267             numBytes += PrivateCellUtil.estimatedSerializedSizeOf(cell);
5268           }
5269         }
5270         scanner.close();
5271         ScanMetrics scanMetrics = scanner.getScanMetrics();
5272         assertEquals("Did not count the result bytes", numBytes,
5273                 scanMetrics.countOfBytesInResults.get());
5274       }
5275
5276       // now, test that the metrics are still collected even if you don't call close, but do
5277       // run past the end of all the records
5278       /** There seems to be a timing issue here.  Comment out for now. Fix when time.
5279        Scan scanWithoutClose = new Scan();
5280        scanWithoutClose.setCaching(1);
5281        scanWithoutClose.setScanMetricsEnabled(true);
5282        ResultScanner scannerWithoutClose = ht.getScanner(scanWithoutClose);
5283        for (Result result : scannerWithoutClose.next(numRecords + 1)) {
5284        }
5285        ScanMetrics scanMetricsWithoutClose = getScanMetrics(scanWithoutClose);
5286        assertEquals("Did not access all the regions in the table", numOfRegions,
5287        scanMetricsWithoutClose.countOfRegions.get());
5288        */
5289
5290       // finally,
5291       // test that the metrics are collected correctly if you both run past all the records,
5292       // AND close the scanner
5293       Scan scanWithClose = new Scan();
5294       // make sure we can set caching up to the number of a scanned values
5295       scanWithClose.setCaching(numRecords);
5296       scanWithClose.setScanMetricsEnabled(true);
5297       try (ResultScanner scannerWithClose = ht.getScanner(scanWithClose)) {
5298         for (Result result : scannerWithClose.next(numRecords + 1)) {
5299         }
5300         scannerWithClose.close();
5301         ScanMetrics scanMetricsWithClose = scannerWithClose.getScanMetrics();
5302         assertEquals("Did not access all the regions in the table", numOfRegions,
5303                 scanMetricsWithClose.countOfRegions.get());
5304       }
5305     }
5306   }
5307
5308   /**
5309    * Tests that cache on write works all the way up from the client-side.
5310    *
5311    * Performs inserts, flushes, and compactions, verifying changes in the block
5312    * cache along the way.
5313    */
5314   @Test
5315   public void testCacheOnWriteEvictOnClose() throws Exception {
5316     final TableName tableName = name.getTableName();
5317     byte [] data = Bytes.toBytes("data");
5318     try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) {
5319       try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
5320         // get the block cache and region
5321         String regionName = locator.getAllRegionLocations().get(0).getRegion().getEncodedName();
5322
5323         HRegion region = TEST_UTIL.getRSForFirstRegionInTable(tableName)
5324                 .getRegion(regionName);
5325         HStore store = region.getStores().iterator().next();
5326         CacheConfig cacheConf = store.getCacheConfig();
5327         cacheConf.setCacheDataOnWrite(true);
5328         cacheConf.setEvictOnClose(true);
5329         BlockCache cache = cacheConf.getBlockCache().get();
5330
5331         // establish baseline stats
5332         long startBlockCount = cache.getBlockCount();
5333         long startBlockHits = cache.getStats().getHitCount();
5334         long startBlockMiss = cache.getStats().getMissCount();
5335
5336         // wait till baseline is stable, (minimal 500 ms)
5337         for (int i = 0; i < 5; i++) {
5338           Thread.sleep(100);
5339           if (startBlockCount != cache.getBlockCount()
5340                   || startBlockHits != cache.getStats().getHitCount()
5341                   || startBlockMiss != cache.getStats().getMissCount()) {
5342             startBlockCount = cache.getBlockCount();
5343             startBlockHits = cache.getStats().getHitCount();
5344             startBlockMiss = cache.getStats().getMissCount();
5345             i = -1;
5346           }
5347         }
5348
5349         // insert data
5350         Put put = new Put(ROW);
5351         put.addColumn(FAMILY, QUALIFIER, data);
5352         table.put(put);
5353         assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data));
5354
5355         // data was in memstore so don't expect any changes
5356         assertEquals(startBlockCount, cache.getBlockCount());
5357         assertEquals(startBlockHits, cache.getStats().getHitCount());
5358         assertEquals(startBlockMiss, cache.getStats().getMissCount());
5359
5360         // flush the data
5361         LOG.debug("Flushing cache");
5362         region.flush(true);
5363
5364         // expect two more blocks in cache - DATA and ROOT_INDEX
5365         // , no change in hits/misses
5366         long expectedBlockCount = startBlockCount + 2;
5367         long expectedBlockHits = startBlockHits;
5368         long expectedBlockMiss = startBlockMiss;
5369         assertEquals(expectedBlockCount, cache.getBlockCount());
5370         assertEquals(expectedBlockHits, cache.getStats().getHitCount());
5371         assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5372         // read the data and expect same blocks, one new hit, no misses
5373         assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data));
5374         assertEquals(expectedBlockCount, cache.getBlockCount());
5375         assertEquals(++expectedBlockHits, cache.getStats().getHitCount());
5376         assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5377         // insert a second column, read the row, no new blocks, one new hit
5378         byte[] QUALIFIER2 = Bytes.add(QUALIFIER, QUALIFIER);
5379         byte[] data2 = Bytes.add(data, data);
5380         put = new Put(ROW);
5381         put.addColumn(FAMILY, QUALIFIER2, data2);
5382         table.put(put);
5383         Result r = table.get(new Get(ROW));
5384         assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data));
5385         assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2));
5386         assertEquals(expectedBlockCount, cache.getBlockCount());
5387         assertEquals(++expectedBlockHits, cache.getStats().getHitCount());
5388         assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5389         // flush, one new block
5390         System.out.println("Flushing cache");
5391         region.flush(true);
5392
5393         // + 1 for Index Block, +1 for data block
5394         expectedBlockCount += 2;
5395         assertEquals(expectedBlockCount, cache.getBlockCount());
5396         assertEquals(expectedBlockHits, cache.getStats().getHitCount());
5397         assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5398         // compact, net minus two blocks, two hits, no misses
5399         System.out.println("Compacting");
5400         assertEquals(2, store.getStorefilesCount());
5401         store.triggerMajorCompaction();
5402         region.compact(true);
5403         store.closeAndArchiveCompactedFiles();
5404         waitForStoreFileCount(store, 1, 10000); // wait 10 seconds max
5405         assertEquals(1, store.getStorefilesCount());
5406         // evicted two data blocks and two index blocks and compaction does not cache new blocks
5407         expectedBlockCount = 0;
5408         assertEquals(expectedBlockCount, cache.getBlockCount());
5409         expectedBlockHits += 2;
5410         assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5411         assertEquals(expectedBlockHits, cache.getStats().getHitCount());
5412         // read the row, this should be a cache miss because we don't cache data
5413         // blocks on compaction
5414         r = table.get(new Get(ROW));
5415         assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data));
5416         assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2));
5417         expectedBlockCount += 1; // cached one data block
5418         assertEquals(expectedBlockCount, cache.getBlockCount());
5419         assertEquals(expectedBlockHits, cache.getStats().getHitCount());
5420         assertEquals(++expectedBlockMiss, cache.getStats().getMissCount());
5421       }
5422     }
5423   }
5424
5425   private void waitForStoreFileCount(HStore store, int count, int timeout)
5426       throws InterruptedException {
5427     long start = System.currentTimeMillis();
5428     while (start + timeout > System.currentTimeMillis() && store.getStorefilesCount() != count) {
5429       Thread.sleep(100);
5430     }
5431     System.out.println("start=" + start + ", now=" + System.currentTimeMillis() + ", cur=" +
5432         store.getStorefilesCount());
5433     assertEquals(count, store.getStorefilesCount());
5434   }
5435
5436   @Test
5437   /**
5438    * Tests the non cached version of getRegionLocator by moving a region.
5439    */
5440   public void testNonCachedGetRegionLocation() throws Exception {
5441     // Test Initialization.
5442     final TableName tableName = name.getTableName();
5443     byte [] family1 = Bytes.toBytes("f1");
5444     byte [] family2 = Bytes.toBytes("f2");
5445     try (Table table = TEST_UTIL.createTable(tableName, new byte[][] {family1, family2}, 10);
5446         Admin admin = TEST_UTIL.getAdmin();
5447         RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
5448       List<HRegionLocation> allRegionLocations = locator.getAllRegionLocations();
5449       assertEquals(1, allRegionLocations.size());
5450       RegionInfo regionInfo = allRegionLocations.get(0).getRegion();
5451       ServerName addrBefore = allRegionLocations.get(0).getServerName();
5452       // Verify region location before move.
5453       HRegionLocation addrCache = locator.getRegionLocation(regionInfo.getStartKey(), false);
5454       HRegionLocation addrNoCache = locator.getRegionLocation(regionInfo.getStartKey(),  true);
5455
5456       assertEquals(addrBefore.getPort(), addrCache.getPort());
5457       assertEquals(addrBefore.getPort(), addrNoCache.getPort());
5458
5459       ServerName addrAfter = null;
5460       // Now move the region to a different server.
5461       for (int i = 0; i < SLAVES; i++) {
5462         HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(i);
5463         ServerName addr = regionServer.getServerName();
5464         if (addr.getPort() != addrBefore.getPort()) {
5465           admin.move(regionInfo.getEncodedNameAsBytes(), addr);
5466           // Wait for the region to move.
5467           Thread.sleep(5000);
5468           addrAfter = addr;
5469           break;
5470         }
5471       }
5472
5473       // Verify the region was moved.
5474       addrCache = locator.getRegionLocation(regionInfo.getStartKey(), false);
5475       addrNoCache = locator.getRegionLocation(regionInfo.getStartKey(), true);
5476       assertNotNull(addrAfter);
5477       assertTrue(addrAfter.getPort() != addrCache.getPort());
5478       assertEquals(addrAfter.getPort(), addrNoCache.getPort());
5479     }
5480   }
5481
5482   @Test
5483   /**
5484    * Tests getRegionsInRange by creating some regions over which a range of
5485    * keys spans; then changing the key range.
5486    */
5487   public void testGetRegionsInRange() throws Exception {
5488     // Test Initialization.
5489     byte [] startKey = Bytes.toBytes("ddc");
5490     byte [] endKey = Bytes.toBytes("mmm");
5491     TableName tableName = name.getTableName();
5492     TEST_UTIL.createMultiRegionTable(tableName, new byte[][] { FAMILY }, 10);
5493
5494     int numOfRegions = -1;
5495     try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
5496       numOfRegions = r.getStartKeys().length;
5497     }
5498     assertEquals(26, numOfRegions);
5499
5500     // Get the regions in this range
5501     List<HRegionLocation> regionsList = getRegionsInRange(tableName, startKey, endKey);
5502     assertEquals(10, regionsList.size());
5503
5504     // Change the start key
5505     startKey = Bytes.toBytes("fff");
5506     regionsList = getRegionsInRange(tableName, startKey, endKey);
5507     assertEquals(7, regionsList.size());
5508
5509     // Change the end key
5510     endKey = Bytes.toBytes("nnn");
5511     regionsList = getRegionsInRange(tableName, startKey, endKey);
5512     assertEquals(8, regionsList.size());
5513
5514     // Empty start key
5515     regionsList = getRegionsInRange(tableName, HConstants.EMPTY_START_ROW, endKey);
5516     assertEquals(13, regionsList.size());
5517
5518     // Empty end key
5519     regionsList = getRegionsInRange(tableName, startKey, HConstants.EMPTY_END_ROW);
5520     assertEquals(21, regionsList.size());
5521
5522     // Both start and end keys empty
5523     regionsList = getRegionsInRange(tableName, HConstants.EMPTY_START_ROW,
5524         HConstants.EMPTY_END_ROW);
5525     assertEquals(26, regionsList.size());
5526
5527     // Change the end key to somewhere in the last block
5528     endKey = Bytes.toBytes("zzz1");
5529     regionsList = getRegionsInRange(tableName, startKey, endKey);
5530     assertEquals(21, regionsList.size());
5531
5532     // Change the start key to somewhere in the first block
5533     startKey = Bytes.toBytes("aac");
5534     regionsList = getRegionsInRange(tableName, startKey, endKey);
5535     assertEquals(26, regionsList.size());
5536
5537     // Make start and end key the same
5538     startKey = endKey = Bytes.toBytes("ccc");
5539     regionsList = getRegionsInRange(tableName, startKey, endKey);
5540     assertEquals(1, regionsList.size());
5541   }
5542
5543   private List<HRegionLocation> getRegionsInRange(TableName tableName, byte[] startKey,
5544       byte[] endKey) throws IOException {
5545     List<HRegionLocation> regionsInRange = new ArrayList<>();
5546     byte[] currentKey = startKey;
5547     final boolean endKeyIsEndOfTable = Bytes.equals(endKey, HConstants.EMPTY_END_ROW);
5548     try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
5549       do {
5550         HRegionLocation regionLocation = r.getRegionLocation(currentKey);
5551         regionsInRange.add(regionLocation);
5552         currentKey = regionLocation.getRegion().getEndKey();
5553       } while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW)
5554           && (endKeyIsEndOfTable || Bytes.compareTo(currentKey, endKey) < 0));
5555       return regionsInRange;
5556     }
5557   }
5558
5559   @Test
5560   public void testJira6912() throws Exception {
5561     final TableName tableName = name.getTableName();
5562     try (Table foo = TEST_UTIL.createTable(tableName, new byte[][] {FAMILY}, 10)) {
5563
5564       List<Put> puts = new ArrayList<Put>();
5565       for (int i = 0; i != 100; i++) {
5566         Put put = new Put(Bytes.toBytes(i));
5567         put.addColumn(FAMILY, FAMILY, Bytes.toBytes(i));
5568         puts.add(put);
5569       }
5570       foo.put(puts);
5571       // If i comment this out it works
5572       TEST_UTIL.flush();
5573
5574       Scan scan = new Scan();
5575       scan.setStartRow(Bytes.toBytes(1));
5576       scan.setStopRow(Bytes.toBytes(3));
5577       scan.addColumn(FAMILY, FAMILY);
5578       scan.setFilter(new RowFilter(CompareOperator.NOT_EQUAL,
5579               new BinaryComparator(Bytes.toBytes(1))));
5580
5581       try (ResultScanner scanner = foo.getScanner(scan)) {
5582         Result[] bar = scanner.next(100);
5583         assertEquals(1, bar.length);
5584       }
5585     }
5586   }
5587
5588   @Test
5589   public void testScan_NullQualifier() throws IOException {
5590     try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
5591       Put put = new Put(ROW);
5592       put.addColumn(FAMILY, QUALIFIER, VALUE);
5593       table.put(put);
5594
5595       put = new Put(ROW);
5596       put.addColumn(FAMILY, null, VALUE);
5597       table.put(put);
5598       LOG.info("Row put");
5599
5600       Scan scan = new Scan();
5601       scan.addColumn(FAMILY, null);
5602
5603       ResultScanner scanner = table.getScanner(scan);
5604       Result[] bar = scanner.next(100);
5605       assertEquals(1, bar.length);
5606       assertEquals(1, bar[0].size());
5607
5608       scan = new Scan();
5609       scan.addFamily(FAMILY);
5610
5611       scanner = table.getScanner(scan);
5612       bar = scanner.next(100);
5613       assertEquals(1, bar.length);
5614       assertEquals(2, bar[0].size());
5615     }
5616   }
5617
5618   @Test
5619   public void testNegativeTimestamp() throws IOException {
5620     try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
5621
5622       try {
5623         Put put = new Put(ROW, -1);
5624         put.addColumn(FAMILY, QUALIFIER, VALUE);
5625         table.put(put);
5626         fail("Negative timestamps should not have been allowed");
5627       } catch (IllegalArgumentException ex) {
5628         assertTrue(ex.getMessage().contains("negative"));
5629       }
5630
5631       try {
5632         Put put = new Put(ROW);
5633         long ts = -1;
5634         put.addColumn(FAMILY, QUALIFIER, ts, VALUE);
5635         table.put(put);
5636         fail("Negative timestamps should not have been allowed");
5637       } catch (IllegalArgumentException ex) {
5638         assertTrue(ex.getMessage().contains("negative"));
5639       }
5640
5641       try {
5642         Delete delete = new Delete(ROW, -1);
5643         table.delete(delete);
5644         fail("Negative timestamps should not have been allowed");
5645       } catch (IllegalArgumentException ex) {
5646         assertTrue(ex.getMessage().contains("negative"));
5647       }
5648
5649       try {
5650         Delete delete = new Delete(ROW);
5651         delete.addFamily(FAMILY, -1);
5652         table.delete(delete);
5653         fail("Negative timestamps should not have been allowed");
5654       } catch (IllegalArgumentException ex) {
5655         assertTrue(ex.getMessage().contains("negative"));
5656       }
5657
5658       try {
5659         Scan scan = new Scan();
5660         scan.setTimeRange(-1, 1);
5661         table.getScanner(scan);
5662         fail("Negative timestamps should not have been allowed");
5663       } catch (IllegalArgumentException ex) {
5664         assertTrue(ex.getMessage().contains("negative"));
5665       }
5666
5667       // KeyValue should allow negative timestamps for backwards compat. Otherwise, if the user
5668       // already has negative timestamps in cluster data, HBase won't be able to handle that
5669       try {
5670         new KeyValue(Bytes.toBytes(42), Bytes.toBytes(42), Bytes.toBytes(42), -1,
5671                 Bytes.toBytes(42));
5672       } catch (IllegalArgumentException ex) {
5673         fail("KeyValue SHOULD allow negative timestamps");
5674       }
5675
5676     }
5677   }
5678
5679   @Test
5680   public void testRawScanRespectsVersions() throws Exception {
5681     final TableName tableName = name.getTableName();
5682     try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) {
5683       byte[] row = Bytes.toBytes("row");
5684
5685       // put the same row 4 times, with different values
5686       Put p = new Put(row);
5687       p.addColumn(FAMILY, QUALIFIER, 10, VALUE);
5688       table.put(p);
5689       p = new Put(row);
5690       p.addColumn(FAMILY, QUALIFIER, 11, ArrayUtils.add(VALUE, (byte) 2));
5691       table.put(p);
5692
5693       p = new Put(row);
5694       p.addColumn(FAMILY, QUALIFIER, 12, ArrayUtils.add(VALUE, (byte) 3));
5695       table.put(p);
5696
5697       p = new Put(row);
5698       p.addColumn(FAMILY, QUALIFIER, 13, ArrayUtils.add(VALUE, (byte) 4));
5699       table.put(p);
5700
5701       int versions = 4;
5702       Scan s = new Scan(row);
5703       // get all the possible versions
5704       s.setMaxVersions();
5705       s.setRaw(true);
5706
5707       try (ResultScanner scanner = table.getScanner(s)) {
5708         int count = 0;
5709         for (Result r : scanner) {
5710           assertEquals("Found an unexpected number of results for the row!", versions,
5711                   r.listCells().size());
5712           count++;
5713         }
5714         assertEquals("Found more than a single row when raw scanning the table with a single row!",
5715                 1, count);
5716       }
5717
5718       // then if we decrease the number of versions, but keep the scan raw, we should see exactly
5719       // that number of versions
5720       versions = 2;
5721       s.setMaxVersions(versions);
5722       try (ResultScanner scanner = table.getScanner(s)) {
5723         int count = 0;
5724         for (Result r : scanner) {
5725           assertEquals("Found an unexpected number of results for the row!", versions,
5726                   r.listCells().size());
5727           count++;
5728         }
5729         assertEquals("Found more than a single row when raw scanning the table with a single row!",
5730                 1, count);
5731       }
5732
5733       // finally, if we turn off raw scanning, but max out the number of versions, we should go back
5734       // to seeing just three
5735       versions = 3;
5736       s.setMaxVersions(versions);
5737       try (ResultScanner scanner = table.getScanner(s)) {
5738         int count = 0;
5739         for (Result r : scanner) {
5740           assertEquals("Found an unexpected number of results for the row!", versions,
5741                   r.listCells().size());
5742           count++;
5743         }
5744         assertEquals("Found more than a single row when raw scanning the table with a single row!",
5745                 1, count);
5746       }
5747
5748     }
5749     TEST_UTIL.deleteTable(tableName);
5750   }
5751
5752   @Test
5753   public void testEmptyFilterList() throws Exception {
5754     // Test Initialization.
5755     final TableName tableName = name.getTableName();
5756     try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) {
5757
5758       // Insert one row each region
5759       Put put = new Put(Bytes.toBytes("row"));
5760       put.addColumn(FAMILY, QUALIFIER, VALUE);
5761       table.put(put);
5762
5763       List<Result> scanResults = new LinkedList<>();
5764       Scan scan = new Scan();
5765       scan.setFilter(new FilterList());
5766       try (ResultScanner scanner = table.getScanner(scan)) {
5767         for (Result r : scanner) {
5768           scanResults.add(r);
5769         }
5770       }
5771       assertEquals(1, scanResults.size());
5772       Get g = new Get(Bytes.toBytes("row"));
5773       g.setFilter(new FilterList());
5774       Result getResult = table.get(g);
5775       Result scanResult = scanResults.get(0);
5776       assertEquals(scanResult.rawCells().length, getResult.rawCells().length);
5777       for (int i = 0; i != scanResult.rawCells().length; ++i) {
5778         Cell scanCell = scanResult.rawCells()[i];
5779         Cell getCell = getResult.rawCells()[i];
5780         assertEquals(0, Bytes.compareTo(CellUtil.cloneRow(scanCell),
5781                 CellUtil.cloneRow(getCell)));
5782         assertEquals(0, Bytes.compareTo(CellUtil.cloneFamily(scanCell),
5783                 CellUtil.cloneFamily(getCell)));
5784         assertEquals(0, Bytes.compareTo(CellUtil.cloneQualifier(scanCell),
5785                 CellUtil.cloneQualifier(getCell)));
5786         assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(scanCell),
5787                 CellUtil.cloneValue(getCell)));
5788       }
5789     }
5790   }
5791
5792   @Test
5793   public void testSmallScan() throws Exception {
5794     // Test Initialization.
5795     final TableName tableName = name.getTableName();
5796     try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) {
5797
5798       // Insert one row each region
5799       int insertNum = 10;
5800       for (int i = 0; i < 10; i++) {
5801         Put put = new Put(Bytes.toBytes("row" + String.format("%03d", i)));
5802         put.addColumn(FAMILY, QUALIFIER, VALUE);
5803         table.put(put);
5804       }
5805
5806       // normal scan
5807       try (ResultScanner scanner = table.getScanner(new Scan())) {
5808         int count = 0;
5809         for (Result r : scanner) {
5810           assertTrue(!r.isEmpty());
5811           count++;
5812         }
5813         assertEquals(insertNum, count);
5814       }
5815
5816       // small scan
5817       Scan scan = new Scan(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
5818       scan.setSmall(true);
5819       scan.setCaching(2);
5820       try (ResultScanner scanner = table.getScanner(scan)) {
5821         int count = 0;
5822         for (Result r : scanner) {
5823           assertTrue(!r.isEmpty());
5824           count++;
5825         }
5826         assertEquals(insertNum, count);
5827       }
5828     }
5829   }
5830
5831   @Test
5832   public void testSuperSimpleWithReverseScan() throws Exception {
5833     final TableName tableName = name.getTableName();
5834     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
5835       Put put = new Put(Bytes.toBytes("0-b11111-0000000000000000000"));
5836       put.addColumn(FAMILY, QUALIFIER, VALUE);
5837       ht.put(put);
5838       put = new Put(Bytes.toBytes("0-b11111-0000000000000000002"));
5839       put.addColumn(FAMILY, QUALIFIER, VALUE);
5840       ht.put(put);
5841       put = new Put(Bytes.toBytes("0-b11111-0000000000000000004"));
5842       put.addColumn(FAMILY, QUALIFIER, VALUE);
5843       ht.put(put);
5844       put = new Put(Bytes.toBytes("0-b11111-0000000000000000006"));
5845       put.addColumn(FAMILY, QUALIFIER, VALUE);
5846       ht.put(put);
5847       put = new Put(Bytes.toBytes("0-b11111-0000000000000000008"));
5848       put.addColumn(FAMILY, QUALIFIER, VALUE);
5849       ht.put(put);
5850       put = new Put(Bytes.toBytes("0-b22222-0000000000000000001"));
5851       put.addColumn(FAMILY, QUALIFIER, VALUE);
5852       ht.put(put);
5853       put = new Put(Bytes.toBytes("0-b22222-0000000000000000003"));
5854       put.addColumn(FAMILY, QUALIFIER, VALUE);
5855       ht.put(put);
5856       put = new Put(Bytes.toBytes("0-b22222-0000000000000000005"));
5857       put.addColumn(FAMILY, QUALIFIER, VALUE);
5858       ht.put(put);
5859       put = new Put(Bytes.toBytes("0-b22222-0000000000000000007"));
5860       put.addColumn(FAMILY, QUALIFIER, VALUE);
5861       ht.put(put);
5862       put = new Put(Bytes.toBytes("0-b22222-0000000000000000009"));
5863       put.addColumn(FAMILY, QUALIFIER, VALUE);
5864       ht.put(put);
5865       Scan scan = new Scan(Bytes.toBytes("0-b11111-9223372036854775807"),
5866               Bytes.toBytes("0-b11111-0000000000000000000"));
5867       scan.setReversed(true);
5868       try (ResultScanner scanner = ht.getScanner(scan)) {
5869         Result result = scanner.next();
5870         assertTrue(Bytes.equals(result.getRow(),
5871                 Bytes.toBytes("0-b11111-0000000000000000008")));
5872       }
5873     }
5874   }
5875
5876   @Test
5877   public void testFiltersWithReverseScan() throws Exception {
5878     final TableName tableName = name.getTableName();
5879     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
5880       byte[][] ROWS = makeN(ROW, 10);
5881       byte[][] QUALIFIERS = {Bytes.toBytes("col0-<d2v1>-<d3v2>"),
5882               Bytes.toBytes("col1-<d2v1>-<d3v2>"),
5883               Bytes.toBytes("col2-<d2v1>-<d3v2>"),
5884               Bytes.toBytes("col3-<d2v1>-<d3v2>"),
5885               Bytes.toBytes("col4-<d2v1>-<d3v2>"),
5886               Bytes.toBytes("col5-<d2v1>-<d3v2>"),
5887               Bytes.toBytes("col6-<d2v1>-<d3v2>"),
5888               Bytes.toBytes("col7-<d2v1>-<d3v2>"),
5889               Bytes.toBytes("col8-<d2v1>-<d3v2>"),
5890               Bytes.toBytes("col9-<d2v1>-<d3v2>")};
5891       for (int i = 0; i < 10; i++) {
5892         Put put = new Put(ROWS[i]);
5893         put.addColumn(FAMILY, QUALIFIERS[i], VALUE);
5894         ht.put(put);
5895       }
5896       Scan scan = new Scan();
5897       scan.setReversed(true);
5898       scan.addFamily(FAMILY);
5899       Filter filter = new QualifierFilter(CompareOperator.EQUAL,
5900               new RegexStringComparator("col[1-5]"));
5901       scan.setFilter(filter);
5902       try (ResultScanner scanner = ht.getScanner(scan)) {
5903         int expectedIndex = 5;
5904         for (Result result : scanner) {
5905           assertEquals(1, result.size());
5906           Cell c = result.rawCells()[0];
5907           assertTrue(Bytes.equals(c.getRowArray(), c.getRowOffset(), c.getRowLength(),
5908                   ROWS[expectedIndex], 0, ROWS[expectedIndex].length));
5909           assertTrue(Bytes.equals(c.getQualifierArray(), c.getQualifierOffset(),
5910                   c.getQualifierLength(), QUALIFIERS[expectedIndex], 0,
5911                   QUALIFIERS[expectedIndex].length));
5912           expectedIndex--;
5913         }
5914         assertEquals(0, expectedIndex);
5915       }
5916     }
5917   }
5918
5919   @Test
5920   public void testKeyOnlyFilterWithReverseScan() throws Exception {
5921     final TableName tableName = name.getTableName();
5922     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
5923       byte[][] ROWS = makeN(ROW, 10);
5924       byte[][] QUALIFIERS = {Bytes.toBytes("col0-<d2v1>-<d3v2>"),
5925               Bytes.toBytes("col1-<d2v1>-<d3v2>"),
5926               Bytes.toBytes("col2-<d2v1>-<d3v2>"),
5927               Bytes.toBytes("col3-<d2v1>-<d3v2>"),
5928               Bytes.toBytes("col4-<d2v1>-<d3v2>"),
5929               Bytes.toBytes("col5-<d2v1>-<d3v2>"),
5930               Bytes.toBytes("col6-<d2v1>-<d3v2>"),
5931               Bytes.toBytes("col7-<d2v1>-<d3v2>"),
5932               Bytes.toBytes("col8-<d2v1>-<d3v2>"),
5933               Bytes.toBytes("col9-<d2v1>-<d3v2>")};
5934       for (int i = 0; i < 10; i++) {
5935         Put put = new Put(ROWS[i]);
5936         put.addColumn(FAMILY, QUALIFIERS[i], VALUE);
5937         ht.put(put);
5938       }
5939       Scan scan = new Scan();
5940       scan.setReversed(true);
5941       scan.addFamily(FAMILY);
5942       Filter filter = new KeyOnlyFilter(true);
5943       scan.setFilter(filter);
5944       try (ResultScanner scanner = ht.getScanner(scan)) {
5945         int count = 0;
5946         for (Result result : ht.getScanner(scan)) {
5947           assertEquals(1, result.size());
5948           assertEquals(Bytes.SIZEOF_INT, result.rawCells()[0].getValueLength());
5949           assertEquals(VALUE.length, Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0])));
5950           count++;
5951         }
5952         assertEquals(10, count);
5953       }
5954     }
5955   }
5956
5957   /**
5958    * Test simple table and non-existent row cases.
5959    */
5960   @Test
5961   public void testSimpleMissingWithReverseScan() throws Exception {
5962     final TableName tableName = name.getTableName();
5963     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
5964       byte[][] ROWS = makeN(ROW, 4);
5965
5966       // Try to get a row on an empty table
5967       Scan scan = new Scan();
5968       scan.setReversed(true);
5969       Result result = getSingleScanResult(ht, scan);
5970       assertNullResult(result);
5971
5972       scan = new Scan(ROWS[0]);
5973       scan.setReversed(true);
5974       result = getSingleScanResult(ht, scan);
5975       assertNullResult(result);
5976
5977       scan = new Scan(ROWS[0], ROWS[1]);
5978       scan.setReversed(true);
5979       result = getSingleScanResult(ht, scan);
5980       assertNullResult(result);
5981
5982       scan = new Scan();
5983       scan.setReversed(true);
5984       scan.addFamily(FAMILY);
5985       result = getSingleScanResult(ht, scan);
5986       assertNullResult(result);
5987
5988       scan = new Scan();
5989       scan.setReversed(true);
5990       scan.addColumn(FAMILY, QUALIFIER);
5991       result = getSingleScanResult(ht, scan);
5992       assertNullResult(result);
5993
5994       // Insert a row
5995
5996       Put put = new Put(ROWS[2]);
5997       put.addColumn(FAMILY, QUALIFIER, VALUE);
5998       ht.put(put);
5999
6000       // Make sure we can scan the row
6001       scan = new Scan();
6002       scan.setReversed(true);
6003       result = getSingleScanResult(ht, scan);
6004       assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
6005
6006       scan = new Scan(ROWS[3], ROWS[0]);
6007       scan.setReversed(true);
6008       result = getSingleScanResult(ht, scan);
6009       assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
6010
6011       scan = new Scan(ROWS[2], ROWS[1]);
6012       scan.setReversed(true);
6013       result = getSingleScanResult(ht, scan);
6014       assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
6015
6016       // Try to scan empty rows around it
6017       // Introduced MemStore#shouldSeekForReverseScan to fix the following
6018       scan = new Scan(ROWS[1]);
6019       scan.setReversed(true);
6020       result = getSingleScanResult(ht, scan);
6021       assertNullResult(result);
6022     }
6023   }
6024
6025   @Test
6026   public void testNullWithReverseScan() throws Exception {
6027     final TableName tableName = name.getTableName();
6028     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
6029       // Null qualifier (should work)
6030       Put put = new Put(ROW);
6031       put.addColumn(FAMILY, null, VALUE);
6032       ht.put(put);
6033       scanTestNull(ht, ROW, FAMILY, VALUE, true);
6034       Delete delete = new Delete(ROW);
6035       delete.addColumns(FAMILY, null);
6036       ht.delete(delete);
6037     }
6038
6039     // Use a new table
6040     try (Table ht =
6041        TEST_UTIL.createTable(TableName.valueOf(name.getTableName().toString() + "2"), FAMILY)) {
6042       // Empty qualifier, byte[0] instead of null (should work)
6043       Put put = new Put(ROW);
6044       put.addColumn(FAMILY, HConstants.EMPTY_BYTE_ARRAY, VALUE);
6045       ht.put(put);
6046       scanTestNull(ht, ROW, FAMILY, VALUE, true);
6047       TEST_UTIL.flush();
6048       scanTestNull(ht, ROW, FAMILY, VALUE, true);
6049       Delete delete = new Delete(ROW);
6050       delete.addColumns(FAMILY, HConstants.EMPTY_BYTE_ARRAY);
6051       ht.delete(delete);
6052       // Null value
6053       put = new Put(ROW);
6054       put.addColumn(FAMILY, QUALIFIER, null);
6055       ht.put(put);
6056       Scan scan = new Scan();
6057       scan.setReversed(true);
6058       scan.addColumn(FAMILY, QUALIFIER);
6059       Result result = getSingleScanResult(ht, scan);
6060       assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
6061     }
6062   }
6063
6064   @Test
6065   @SuppressWarnings("checkstyle:MethodLength")
6066   public void testDeletesWithReverseScan() throws Exception {
6067     final TableName tableName = name.getTableName();
6068     byte[][] ROWS = makeNAscii(ROW, 6);
6069     byte[][] FAMILIES = makeNAscii(FAMILY, 3);
6070     byte[][] VALUES = makeN(VALUE, 5);
6071     long[] ts = { 1000, 2000, 3000, 4000, 5000 };
6072     try (Table ht = TEST_UTIL.createTable(tableName, FAMILIES, 3)) {
6073
6074       Put put = new Put(ROW);
6075       put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
6076       put.addColumn(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]);
6077       ht.put(put);
6078
6079       Delete delete = new Delete(ROW);
6080       delete.addFamily(FAMILIES[0], ts[0]);
6081       ht.delete(delete);
6082
6083       Scan scan = new Scan(ROW);
6084       scan.setReversed(true);
6085       scan.addFamily(FAMILIES[0]);
6086       scan.setMaxVersions(Integer.MAX_VALUE);
6087       Result result = getSingleScanResult(ht, scan);
6088       assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[]{ts[1]},
6089               new byte[][]{VALUES[1]}, 0, 0);
6090
6091       // Test delete latest version
6092       put = new Put(ROW);
6093       put.addColumn(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
6094       put.addColumn(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]);
6095       put.addColumn(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]);
6096       put.addColumn(FAMILIES[0], null, ts[4], VALUES[4]);
6097       put.addColumn(FAMILIES[0], null, ts[2], VALUES[2]);
6098       put.addColumn(FAMILIES[0], null, ts[3], VALUES[3]);
6099       ht.put(put);
6100
6101       delete = new Delete(ROW);
6102       delete.addColumn(FAMILIES[0], QUALIFIER); // ts[4]
6103       ht.delete(delete);
6104
6105       scan = new Scan(ROW);
6106       scan.setReversed(true);
6107       scan.addColumn(FAMILIES[0], QUALIFIER);
6108       scan.setMaxVersions(Integer.MAX_VALUE);
6109       result = getSingleScanResult(ht, scan);
6110       assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[]{ts[1],
6111               ts[2], ts[3]}, new byte[][]{VALUES[1], VALUES[2], VALUES[3]}, 0, 2);
6112
6113       // Test for HBASE-1847
6114       delete = new Delete(ROW);
6115       delete.addColumn(FAMILIES[0], null);
6116       ht.delete(delete);
6117
6118       // Cleanup null qualifier
6119       delete = new Delete(ROW);
6120       delete.addColumns(FAMILIES[0], null);
6121       ht.delete(delete);
6122
6123       // Expected client behavior might be that you can re-put deleted values
6124       // But alas, this is not to be. We can't put them back in either case.
6125
6126       put = new Put(ROW);
6127       put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
6128       put.addColumn(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
6129       ht.put(put);
6130
6131       // The Scanner returns the previous values, the expected-naive-unexpected
6132       // behavior
6133
6134       scan = new Scan(ROW);
6135       scan.setReversed(true);
6136       scan.addFamily(FAMILIES[0]);
6137       scan.setMaxVersions(Integer.MAX_VALUE);
6138       result = getSingleScanResult(ht, scan);
6139       assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[]{ts[1],
6140               ts[2], ts[3]}, new byte[][]{VALUES[1], VALUES[2], VALUES[3]}, 0, 2);
6141
6142       // Test deleting an entire family from one row but not the other various
6143       // ways
6144
6145       put = new Put(ROWS[0]);
6146       put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
6147       put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
6148       put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
6149       put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
6150       ht.put(put);
6151
6152       put = new Put(ROWS[1]);
6153       put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
6154       put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
6155       put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
6156       put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
6157       ht.put(put);
6158
6159       put = new Put(ROWS[2]);
6160       put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
6161       put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
6162       put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
6163       put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
6164       ht.put(put);
6165
6166       delete = new Delete(ROWS[0]);
6167       delete.addFamily(FAMILIES[2]);
6168       ht.delete(delete);
6169
6170       delete = new Delete(ROWS[1]);
6171       delete.addColumns(FAMILIES[1], QUALIFIER);
6172       ht.delete(delete);
6173
6174       delete = new Delete(ROWS[2]);
6175       delete.addColumn(FAMILIES[1], QUALIFIER);
6176       delete.addColumn(FAMILIES[1], QUALIFIER);
6177       delete.addColumn(FAMILIES[2], QUALIFIER);
6178       ht.delete(delete);
6179
6180       scan = new Scan(ROWS[0]);
6181       scan.setReversed(true);
6182       scan.addFamily(FAMILIES[1]);
6183       scan.addFamily(FAMILIES[2]);
6184       scan.setMaxVersions(Integer.MAX_VALUE);
6185       result = getSingleScanResult(ht, scan);
6186       assertTrue("Expected 2 keys but received " + result.size(),
6187               result.size() == 2);
6188       assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER, new long[]{ts[0],
6189               ts[1]}, new byte[][]{VALUES[0], VALUES[1]}, 0, 1);
6190
6191       scan = new Scan(ROWS[1]);
6192       scan.setReversed(true);
6193       scan.addFamily(FAMILIES[1]);
6194       scan.addFamily(FAMILIES[2]);
6195       scan.setMaxVersions(Integer.MAX_VALUE);
6196       result = getSingleScanResult(ht, scan);
6197       assertTrue("Expected 2 keys but received " + result.size(),
6198               result.size() == 2);
6199
6200       scan = new Scan(ROWS[2]);
6201       scan.setReversed(true);
6202       scan.addFamily(FAMILIES[1]);
6203       scan.addFamily(FAMILIES[2]);
6204       scan.setMaxVersions(Integer.MAX_VALUE);
6205       result = getSingleScanResult(ht, scan);
6206       assertEquals(1, result.size());
6207       assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
6208               new long[]{ts[2]}, new byte[][]{VALUES[2]}, 0, 0);
6209
6210       // Test if we delete the family first in one row (HBASE-1541)
6211
6212       delete = new Delete(ROWS[3]);
6213       delete.addFamily(FAMILIES[1]);
6214       ht.delete(delete);
6215
6216       put = new Put(ROWS[3]);
6217       put.addColumn(FAMILIES[2], QUALIFIER, VALUES[0]);
6218       ht.put(put);
6219
6220       put = new Put(ROWS[4]);
6221       put.addColumn(FAMILIES[1], QUALIFIER, VALUES[1]);
6222       put.addColumn(FAMILIES[2], QUALIFIER, VALUES[2]);
6223       ht.put(put);
6224
6225       scan = new Scan(ROWS[4]);
6226       scan.setReversed(true);
6227       scan.addFamily(FAMILIES[1]);
6228       scan.addFamily(FAMILIES[2]);
6229       scan.setMaxVersions(Integer.MAX_VALUE);
6230       ResultScanner scanner = ht.getScanner(scan);
6231       result = scanner.next();
6232       assertTrue("Expected 2 keys but received " + result.size(),
6233               result.size() == 2);
6234       assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[4]));
6235       assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[1]), ROWS[4]));
6236       assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[1]));
6237       assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[1]), VALUES[2]));
6238       result = scanner.next();
6239       assertTrue("Expected 1 key but received " + result.size(),
6240               result.size() == 1);
6241       assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[3]));
6242       assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[0]));
6243       scanner.close();
6244     }
6245   }
6246
6247   /**
6248    * Tests reversed scan under multi regions
6249    */
6250   @Test
6251   public void testReversedScanUnderMultiRegions() throws Exception {
6252     // Test Initialization.
6253     final TableName tableName = name.getTableName();
6254     byte[] maxByteArray = ConnectionUtils.MAX_BYTE_ARRAY;
6255     byte[][] splitRows = new byte[][] { Bytes.toBytes("005"),
6256         Bytes.add(Bytes.toBytes("005"), Bytes.multiple(maxByteArray, 16)),
6257         Bytes.toBytes("006"),
6258         Bytes.add(Bytes.toBytes("006"), Bytes.multiple(maxByteArray, 8)),
6259         Bytes.toBytes("007"),
6260         Bytes.add(Bytes.toBytes("007"), Bytes.multiple(maxByteArray, 4)),
6261         Bytes.toBytes("008"), Bytes.multiple(maxByteArray, 2) };
6262     try (Table table = TEST_UTIL.createTable(tableName, FAMILY, splitRows)) {
6263       TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
6264
6265       try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
6266         assertEquals(splitRows.length + 1, l.getAllRegionLocations().size());
6267       }
6268       // Insert one row each region
6269       int insertNum = splitRows.length;
6270       for (int i = 0; i < insertNum; i++) {
6271         Put put = new Put(splitRows[i]);
6272         put.addColumn(FAMILY, QUALIFIER, VALUE);
6273         table.put(put);
6274       }
6275
6276       // scan forward
6277       try (ResultScanner scanner = table.getScanner(new Scan())) {
6278         int count = 0;
6279         for (Result r : scanner) {
6280           assertTrue(!r.isEmpty());
6281           count++;
6282         }
6283         assertEquals(insertNum, count);
6284       }
6285
6286       // scan backward
6287       Scan scan = new Scan();
6288       scan.setReversed(true);
6289       try (ResultScanner scanner = table.getScanner(scan)) {
6290         int count = 0;
6291         byte[] lastRow = null;
6292         for (Result r : scanner) {
6293           assertTrue(!r.isEmpty());
6294           count++;
6295           byte[] thisRow = r.getRow();
6296           if (lastRow != null) {
6297             assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6298                             + ",this row=" + Bytes.toString(thisRow),
6299                     Bytes.compareTo(thisRow, lastRow) < 0);
6300           }
6301           lastRow = thisRow;
6302         }
6303         assertEquals(insertNum, count);
6304       }
6305     }
6306   }
6307
6308   /**
6309    * Tests reversed scan under multi regions
6310    */
6311   @Test
6312   public void testSmallReversedScanUnderMultiRegions() throws Exception {
6313     // Test Initialization.
6314     final TableName tableName = name.getTableName();
6315     byte[][] splitRows = new byte[][]{
6316         Bytes.toBytes("000"), Bytes.toBytes("002"), Bytes.toBytes("004"),
6317         Bytes.toBytes("006"), Bytes.toBytes("008"), Bytes.toBytes("010")};
6318     try (Table table = TEST_UTIL.createTable(tableName, FAMILY, splitRows)) {
6319       TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
6320
6321       try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
6322         assertEquals(splitRows.length + 1, l.getAllRegionLocations().size());
6323       }
6324       for (byte[] splitRow : splitRows) {
6325         Put put = new Put(splitRow);
6326         put.addColumn(FAMILY, QUALIFIER, VALUE);
6327         table.put(put);
6328
6329         byte[] nextRow = Bytes.copy(splitRow);
6330         nextRow[nextRow.length - 1]++;
6331
6332         put = new Put(nextRow);
6333         put.addColumn(FAMILY, QUALIFIER, VALUE);
6334         table.put(put);
6335       }
6336
6337       // scan forward
6338       try (ResultScanner scanner = table.getScanner(new Scan())) {
6339         int count = 0;
6340         for (Result r : scanner) {
6341           assertTrue(!r.isEmpty());
6342           count++;
6343         }
6344         assertEquals(12, count);
6345       }
6346
6347       reverseScanTest(table, false);
6348       reverseScanTest(table, true);
6349     }
6350   }
6351
6352   private void reverseScanTest(Table table, boolean small) throws IOException {
6353     // scan backward
6354     Scan scan = new Scan();
6355     scan.setReversed(true);
6356     try (ResultScanner scanner = table.getScanner(scan)) {
6357       int count = 0;
6358       byte[] lastRow = null;
6359       for (Result r : scanner) {
6360         assertTrue(!r.isEmpty());
6361         count++;
6362         byte[] thisRow = r.getRow();
6363         if (lastRow != null) {
6364           assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6365                           + ",this row=" + Bytes.toString(thisRow),
6366                   Bytes.compareTo(thisRow, lastRow) < 0);
6367         }
6368         lastRow = thisRow;
6369       }
6370       assertEquals(12, count);
6371     }
6372
6373     scan = new Scan();
6374     scan.setSmall(small);
6375     scan.setReversed(true);
6376     scan.setStartRow(Bytes.toBytes("002"));
6377     try (ResultScanner scanner = table.getScanner(scan)) {
6378       int count = 0;
6379       byte[] lastRow = null;
6380       for (Result r : scanner) {
6381         assertTrue(!r.isEmpty());
6382         count++;
6383         byte[] thisRow = r.getRow();
6384         if (lastRow != null) {
6385           assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6386                           + ",this row=" + Bytes.toString(thisRow),
6387                   Bytes.compareTo(thisRow, lastRow) < 0);
6388         }
6389         lastRow = thisRow;
6390       }
6391       assertEquals(3, count); // 000 001 002
6392     }
6393
6394     scan = new Scan();
6395     scan.setSmall(small);
6396     scan.setReversed(true);
6397     scan.setStartRow(Bytes.toBytes("002"));
6398     scan.setStopRow(Bytes.toBytes("000"));
6399     try (ResultScanner scanner = table.getScanner(scan)) {
6400       int count = 0;
6401       byte[] lastRow = null;
6402       for (Result r : scanner) {
6403         assertTrue(!r.isEmpty());
6404         count++;
6405         byte[] thisRow = r.getRow();
6406         if (lastRow != null) {
6407           assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6408                           + ",this row=" + Bytes.toString(thisRow),
6409                   Bytes.compareTo(thisRow, lastRow) < 0);
6410         }
6411         lastRow = thisRow;
6412       }
6413       assertEquals(2, count); // 001 002
6414     }
6415
6416     scan = new Scan();
6417     scan.setSmall(small);
6418     scan.setReversed(true);
6419     scan.setStartRow(Bytes.toBytes("001"));
6420     try (ResultScanner scanner = table.getScanner(scan)) {
6421       int count = 0;
6422       byte[] lastRow = null;
6423       for (Result r : scanner) {
6424         assertTrue(!r.isEmpty());
6425         count++;
6426         byte[] thisRow = r.getRow();
6427         if (lastRow != null) {
6428           assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6429                           + ",this row=" + Bytes.toString(thisRow),
6430                   Bytes.compareTo(thisRow, lastRow) < 0);
6431         }
6432         lastRow = thisRow;
6433       }
6434       assertEquals(2, count); // 000 001
6435     }
6436
6437     scan = new Scan();
6438     scan.setSmall(small);
6439     scan.setReversed(true);
6440     scan.setStartRow(Bytes.toBytes("000"));
6441     try (ResultScanner scanner = table.getScanner(scan)) {
6442       int count = 0;
6443       byte[] lastRow = null;
6444       for (Result r : scanner) {
6445         assertTrue(!r.isEmpty());
6446         count++;
6447         byte[] thisRow = r.getRow();
6448         if (lastRow != null) {
6449           assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6450                           + ",this row=" + Bytes.toString(thisRow),
6451                   Bytes.compareTo(thisRow, lastRow) < 0);
6452         }
6453         lastRow = thisRow;
6454       }
6455       assertEquals(1, count); // 000
6456     }
6457
6458     scan = new Scan();
6459     scan.setSmall(small);
6460     scan.setReversed(true);
6461     scan.setStartRow(Bytes.toBytes("006"));
6462     scan.setStopRow(Bytes.toBytes("002"));
6463     try (ResultScanner scanner = table.getScanner(scan)) {
6464       int count = 0;
6465       byte[] lastRow = null;
6466       for (Result r : scanner) {
6467         assertTrue(!r.isEmpty());
6468         count++;
6469         byte[] thisRow = r.getRow();
6470         if (lastRow != null) {
6471           assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6472                           + ",this row=" + Bytes.toString(thisRow),
6473                   Bytes.compareTo(thisRow, lastRow) < 0);
6474         }
6475         lastRow = thisRow;
6476       }
6477       assertEquals(4, count); // 003 004 005 006
6478     }
6479   }
6480
6481   @Test
6482   public void testFilterAllRecords() throws IOException {
6483     Scan scan = new Scan();
6484     scan.setBatch(1);
6485     scan.setCaching(1);
6486     // Filter out any records
6487     scan.setFilter(new FilterList(new FirstKeyOnlyFilter(), new InclusiveStopFilter(new byte[0])));
6488     try (Table table = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME)) {
6489       try (ResultScanner s = table.getScanner(scan)) {
6490         assertNull(s.next());
6491       }
6492     }
6493   }
6494
6495   @Test
6496   public void testCellSizeLimit() throws IOException {
6497     final TableName tableName = TableName.valueOf("testCellSizeLimit");
6498     TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
6499       new TableDescriptorBuilder.ModifyableTableDescriptor(tableName)
6500         .setValue(HRegion.HBASE_MAX_CELL_SIZE_KEY, Integer.toString(10 * 1024));
6501     ColumnFamilyDescriptor familyDescriptor =
6502       new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(FAMILY);
6503
6504     tableDescriptor.setColumnFamily(familyDescriptor);
6505     try (Admin admin = TEST_UTIL.getAdmin()) {
6506       admin.createTable(tableDescriptor);
6507     }
6508     // Will succeed
6509     try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
6510       t.put(new Put(ROW).addColumn(FAMILY, QUALIFIER, Bytes.toBytes(0L)));
6511       t.increment(new Increment(ROW).addColumn(FAMILY, QUALIFIER, 1L));
6512     }
6513     // Will succeed
6514     try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
6515       t.put(new Put(ROW).addColumn(FAMILY, QUALIFIER, new byte[9*1024]));
6516     }
6517     // Will fail
6518     try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
6519       try {
6520         t.put(new Put(ROW).addColumn(FAMILY, QUALIFIER, new byte[10 * 1024]));
6521         fail("Oversize cell failed to trigger exception");
6522       } catch (IOException e) {
6523         // expected
6524       }
6525       try {
6526         t.append(new Append(ROW).addColumn(FAMILY, QUALIFIER, new byte[2 * 1024]));
6527         fail("Oversize cell failed to trigger exception");
6528       } catch (IOException e) {
6529         // expected
6530       }
6531     }
6532   }
6533
6534   @Test
6535   public void testDeleteSpecifiedVersionOfSpecifiedColumn() throws Exception {
6536     try (Admin admin = TEST_UTIL.getAdmin()) {
6537       final TableName tableName = name.getTableName();
6538
6539       byte[][] VALUES = makeN(VALUE, 5);
6540       long[] ts = {1000, 2000, 3000, 4000, 5000};
6541
6542       try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 5)) {
6543
6544         Put put = new Put(ROW);
6545         // Put version 1000,2000,3000,4000 of column FAMILY:QUALIFIER
6546         for (int t = 0; t < 4; t++) {
6547           put.addColumn(FAMILY, QUALIFIER, ts[t], VALUES[t]);
6548         }
6549         ht.put(put);
6550
6551         Delete delete = new Delete(ROW);
6552         // Delete version 3000 of column FAMILY:QUALIFIER
6553         delete.addColumn(FAMILY, QUALIFIER, ts[2]);
6554         ht.delete(delete);
6555
6556         Get get = new Get(ROW);
6557         get.addColumn(FAMILY, QUALIFIER);
6558         get.readVersions(Integer.MAX_VALUE);
6559         Result result = ht.get(get);
6560         // verify version 1000,2000,4000 remains for column FAMILY:QUALIFIER
6561         assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[0], ts[1], ts[3]}, new byte[][]{
6562                 VALUES[0], VALUES[1], VALUES[3]}, 0, 2);
6563
6564         delete = new Delete(ROW);
6565         // Delete a version 5000 of column FAMILY:QUALIFIER which didn't exist
6566         delete.addColumn(FAMILY, QUALIFIER, ts[4]);
6567         ht.delete(delete);
6568
6569         get = new Get(ROW);
6570         get.addColumn(FAMILY, QUALIFIER);
6571         get.readVersions(Integer.MAX_VALUE);
6572         result = ht.get(get);
6573         // verify version 1000,2000,4000 remains for column FAMILY:QUALIFIER
6574         assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[0], ts[1], ts[3]}, new byte[][]{
6575                 VALUES[0], VALUES[1], VALUES[3]}, 0, 2);
6576       }
6577     }
6578   }
6579
6580   @Test
6581   public void testDeleteLatestVersionOfSpecifiedColumn() throws Exception {
6582     try (Admin admin = TEST_UTIL.getAdmin()) {
6583       final TableName tableName = name.getTableName();
6584
6585       byte[][] VALUES = makeN(VALUE, 5);
6586       long[] ts = {1000, 2000, 3000, 4000, 5000};
6587
6588       try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 5)) {
6589
6590         Put put = new Put(ROW);
6591         // Put version 1000,2000,3000,4000 of column FAMILY:QUALIFIER
6592         for (int t = 0; t < 4; t++) {
6593           put.addColumn(FAMILY, QUALIFIER, ts[t], VALUES[t]);
6594         }
6595         ht.put(put);
6596
6597         Delete delete = new Delete(ROW);
6598         // Delete latest version of column FAMILY:QUALIFIER
6599         delete.addColumn(FAMILY, QUALIFIER);
6600         ht.delete(delete);
6601
6602         Get get = new Get(ROW);
6603         get.addColumn(FAMILY, QUALIFIER);
6604         get.readVersions(Integer.MAX_VALUE);
6605         Result result = ht.get(get);
6606         // verify version 1000,2000,3000 remains for column FAMILY:QUALIFIER
6607         assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[0], ts[1], ts[2]}, new byte[][]{
6608                 VALUES[0], VALUES[1], VALUES[2]}, 0, 2);
6609
6610         delete = new Delete(ROW);
6611         // Delete two latest version of column FAMILY:QUALIFIER
6612         delete.addColumn(FAMILY, QUALIFIER);
6613         delete.addColumn(FAMILY, QUALIFIER);
6614         ht.delete(delete);
6615
6616         get = new Get(ROW);
6617         get.addColumn(FAMILY, QUALIFIER);
6618         get.readVersions(Integer.MAX_VALUE);
6619         result = ht.get(get);
6620         // verify version 1000 remains for column FAMILY:QUALIFIER
6621         assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[0]}, new byte[][]{VALUES[0]},
6622                 0, 0);
6623
6624         put = new Put(ROW);
6625         // Put a version 5000 of column FAMILY:QUALIFIER
6626         put.addColumn(FAMILY, QUALIFIER, ts[4], VALUES[4]);
6627         ht.put(put);
6628
6629         get = new Get(ROW);
6630         get.addColumn(FAMILY, QUALIFIER);
6631         get.readVersions(Integer.MAX_VALUE);
6632         result = ht.get(get);
6633         // verify version 1000,5000 remains for column FAMILY:QUALIFIER
6634         assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[0], ts[4]}, new byte[][]{
6635                 VALUES[0], VALUES[4]}, 0, 1);
6636       }
6637     }
6638   }
6639
6640   /**
6641    * Test for HBASE-17125
6642    */
6643   @Test
6644   public void testReadWithFilter() throws Exception {
6645     try (Admin admin = TEST_UTIL.getAdmin()) {
6646       final TableName tableName = name.getTableName();
6647       try (Table table = TEST_UTIL.createTable(tableName, FAMILY, 3)) {
6648
6649         byte[] VALUEA = Bytes.toBytes("value-a");
6650         byte[] VALUEB = Bytes.toBytes("value-b");
6651         long[] ts = {1000, 2000, 3000, 4000};
6652
6653         Put put = new Put(ROW);
6654         // Put version 1000,2000,3000,4000 of column FAMILY:QUALIFIER
6655         for (int t = 0; t <= 3; t++) {
6656           if (t <= 1) {
6657             put.addColumn(FAMILY, QUALIFIER, ts[t], VALUEA);
6658           } else {
6659             put.addColumn(FAMILY, QUALIFIER, ts[t], VALUEB);
6660           }
6661         }
6662         table.put(put);
6663
6664         Scan scan =
6665                 new Scan().setFilter(new ValueFilter(CompareOperator.EQUAL,
6666                         new SubstringComparator("value-a")))
6667                         .setMaxVersions(3);
6668         ResultScanner scanner = table.getScanner(scan);
6669         Result result = scanner.next();
6670         // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
6671         assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0,
6672                 0);
6673
6674         Get get =
6675                 new Get(ROW)
6676                         .setFilter(new ValueFilter(CompareOperator.EQUAL,
6677                                 new SubstringComparator("value-a")))
6678                         .readVersions(3);
6679         result = table.get(get);
6680         // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
6681         assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0,
6682                 0);
6683
6684         // Test with max versions 1, it should still read ts[1]
6685         scan =
6686                 new Scan().setFilter(new ValueFilter(CompareOperator.EQUAL,
6687                         new SubstringComparator("value-a")))
6688                         .setMaxVersions(1);
6689         scanner = table.getScanner(scan);
6690         result = scanner.next();
6691         // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
6692         assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0,
6693                 0);
6694
6695         // Test with max versions 1, it should still read ts[1]
6696         get =
6697                 new Get(ROW)
6698                         .setFilter(new ValueFilter(CompareOperator.EQUAL,
6699                                 new SubstringComparator("value-a")))
6700                         .readVersions(1);
6701         result = table.get(get);
6702         // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
6703         assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0,
6704                 0);
6705
6706         // Test with max versions 5, it should still read ts[1]
6707         scan =
6708                 new Scan().setFilter(new ValueFilter(CompareOperator.EQUAL,
6709                         new SubstringComparator("value-a")))
6710                         .setMaxVersions(5);
6711         scanner = table.getScanner(scan);
6712         result = scanner.next();
6713         // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
6714         assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0,
6715                 0);
6716
6717         // Test with max versions 5, it should still read ts[1]
6718         get =
6719                 new Get(ROW)
6720                         .setFilter(new ValueFilter(CompareOperator.EQUAL,
6721                                 new SubstringComparator("value-a")))
6722                         .readVersions(5);
6723         result = table.get(get);
6724         // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
6725         assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0,
6726                 0);
6727       }
6728     }
6729   }
6730
6731   @Test
6732   public void testCellUtilTypeMethods() throws IOException {
6733     final TableName tableName = name.getTableName();
6734     try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) {
6735
6736       final byte[] row = Bytes.toBytes("p");
6737       Put p = new Put(row);
6738       p.addColumn(FAMILY, QUALIFIER, VALUE);
6739       table.put(p);
6740
6741       try (ResultScanner scanner = table.getScanner(new Scan())) {
6742         Result result = scanner.next();
6743         assertNotNull(result);
6744         CellScanner cs = result.cellScanner();
6745         assertTrue(cs.advance());
6746         Cell c = cs.current();
6747         assertTrue(CellUtil.isPut(c));
6748         assertFalse(CellUtil.isDelete(c));
6749         assertFalse(cs.advance());
6750         assertNull(scanner.next());
6751       }
6752
6753       Delete d = new Delete(row);
6754       d.addColumn(FAMILY, QUALIFIER);
6755       table.delete(d);
6756
6757       Scan scan = new Scan();
6758       scan.setRaw(true);
6759       try (ResultScanner scanner = table.getScanner(scan)) {
6760         Result result = scanner.next();
6761         assertNotNull(result);
6762         CellScanner cs = result.cellScanner();
6763         assertTrue(cs.advance());
6764
6765         // First cell should be the delete (masking the Put)
6766         Cell c = cs.current();
6767         assertTrue("Cell should be a Delete: " + c, CellUtil.isDelete(c));
6768         assertFalse("Cell should not be a Put: " + c, CellUtil.isPut(c));
6769
6770         // Second cell should be the original Put
6771         assertTrue(cs.advance());
6772         c = cs.current();
6773         assertFalse("Cell should not be a Delete: " + c, CellUtil.isDelete(c));
6774         assertTrue("Cell should be a Put: " + c, CellUtil.isPut(c));
6775
6776         // No more cells in this row
6777         assertFalse(cs.advance());
6778
6779         // No more results in this scan
6780         assertNull(scanner.next());
6781       }
6782     }
6783   }
6784
6785   @Test(expected = DoNotRetryIOException.class)
6786   public void testCreateTableWithZeroRegionReplicas() throws Exception {
6787     TableName tableName = name.getTableName();
6788     TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
6789         .setColumnFamily(ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("cf")))
6790         .setRegionReplication(0)
6791         .build();
6792
6793     TEST_UTIL.getAdmin().createTable(desc);
6794   }
6795
6796   @Test(expected = DoNotRetryIOException.class)
6797   public void testModifyTableWithZeroRegionReplicas() throws Exception {
6798     TableName tableName = name.getTableName();
6799     TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
6800         .setColumnFamily(ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("cf")))
6801         .build();
6802
6803     TEST_UTIL.getAdmin().createTable(desc);
6804     TableDescriptor newDesc = TableDescriptorBuilder.newBuilder(desc)
6805         .setRegionReplication(0)
6806         .build();
6807
6808     TEST_UTIL.getAdmin().modifyTable(newDesc);
6809   }
6810
6811   @Test(timeout = 60000)
6812   public void testModifyTableWithMemstoreData() throws Exception {
6813     TableName tableName = name.getTableName();
6814     createTableAndValidateTableSchemaModification(tableName, true);
6815   }
6816
6817   @Test(timeout = 60000)
6818   public void testDeleteCFWithMemstoreData() throws Exception {
6819     TableName tableName = name.getTableName();
6820     createTableAndValidateTableSchemaModification(tableName, false);
6821   }
6822
6823   /**
6824    * Create table and validate online schema modification
6825    * @param tableName Table name
6826    * @param modifyTable Modify table if true otherwise delete column family
6827    * @throws IOException in case of failures
6828    */
6829   private void createTableAndValidateTableSchemaModification(TableName tableName,
6830       boolean modifyTable) throws Exception {
6831     Admin admin = TEST_UTIL.getAdmin();
6832     // Create table with two Cfs
6833     byte[] cf1 = Bytes.toBytes("cf1");
6834     byte[] cf2 = Bytes.toBytes("cf2");
6835     TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName)
6836         .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf1))
6837         .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf2)).build();
6838     admin.createTable(tableDesc);
6839
6840     Table t = TEST_UTIL.getConnection().getTable(tableName);
6841     // Insert few records and flush the table
6842     t.put(new Put(ROW).addColumn(cf1, QUALIFIER, Bytes.toBytes("val1")));
6843     t.put(new Put(ROW).addColumn(cf2, QUALIFIER, Bytes.toBytes("val2")));
6844     admin.flush(tableName);
6845     Path tableDir = FSUtils.getTableDir(TEST_UTIL.getDefaultRootDirPath(), tableName);
6846     List<Path> regionDirs = FSUtils.getRegionDirs(TEST_UTIL.getTestFileSystem(), tableDir);
6847     assertTrue(regionDirs.size() == 1);
6848     List<Path> familyDirs = FSUtils.getFamilyDirs(TEST_UTIL.getTestFileSystem(), regionDirs.get(0));
6849     assertTrue(familyDirs.size() == 2);
6850
6851     // Insert record but dont flush the table
6852     t.put(new Put(ROW).addColumn(cf1, QUALIFIER, Bytes.toBytes("val2")));
6853     t.put(new Put(ROW).addColumn(cf2, QUALIFIER, Bytes.toBytes("val2")));
6854
6855     if (modifyTable) {
6856       tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).removeColumnFamily(cf2).build();
6857       admin.modifyTable(tableDesc);
6858     } else {
6859       admin.deleteColumnFamily(tableName, cf2);
6860     }
6861     // After table modification or delete family there should be only one CF in FS
6862     familyDirs = FSUtils.getFamilyDirs(TEST_UTIL.getTestFileSystem(), regionDirs.get(0));
6863     assertTrue("CF dir count should be 1, but was " + familyDirs.size(), familyDirs.size() == 1);
6864   }
6865 }