hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.client;
  19
  20 import static org.junit.Assert.assertArrayEquals;
  21 import static org.junit.Assert.assertEquals;
  22 import static org.junit.Assert.assertFalse;
  23 import static org.junit.Assert.assertNotNull;
  24 import static org.junit.Assert.assertNull;
  25 import static org.junit.Assert.assertSame;
  26 import static org.junit.Assert.assertTrue;
  27 import static org.junit.Assert.fail;
  28
  29 import java.io.IOException;
  30 import java.util.ArrayList;
  31 import java.util.Arrays;
  32 import java.util.EnumSet;
  33 import java.util.HashMap;
  34 import java.util.HashSet;
  35 import java.util.Iterator;
  36 import java.util.LinkedList;
  37 import java.util.List;
  38 import java.util.Map;
  39 import java.util.NavigableMap;
  40 import java.util.concurrent.Callable;
  41 import java.util.concurrent.ExecutorService;
  42 import java.util.concurrent.Executors;
  43 import java.util.concurrent.atomic.AtomicReference;
  44 import org.apache.commons.lang3.ArrayUtils;
  45 import org.apache.hadoop.conf.Configuration;
  46 import org.apache.hadoop.hbase.Cell;
  47 import org.apache.hadoop.hbase.CellScanner;
  48 import org.apache.hadoop.hbase.CellUtil;
  49 import org.apache.hadoop.hbase.ClusterMetrics.Option;
  50 import org.apache.hadoop.hbase.CompareOperator;
  51 import org.apache.hadoop.hbase.DoNotRetryIOException;
  52 import org.apache.hadoop.hbase.HBaseClassTestRule;
  53 import org.apache.hadoop.hbase.HBaseTestingUtility;
  54 import org.apache.hadoop.hbase.HColumnDescriptor;
  55 import org.apache.hadoop.hbase.HConstants;
  56 import org.apache.hadoop.hbase.HRegionInfo;
  57 import org.apache.hadoop.hbase.HRegionLocation;
  58 import org.apache.hadoop.hbase.HTableDescriptor;
  59 import org.apache.hadoop.hbase.KeepDeletedCells;
  60 import org.apache.hadoop.hbase.KeyValue;
  61 import org.apache.hadoop.hbase.MiniHBaseCluster;
  62 import org.apache.hadoop.hbase.PrivateCellUtil;
  63 import org.apache.hadoop.hbase.RegionLocations;
  64 import org.apache.hadoop.hbase.ServerName;
  65 import org.apache.hadoop.hbase.TableName;
  66 import org.apache.hadoop.hbase.Waiter;
  67 import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
  68 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
  69 import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
  70 import org.apache.hadoop.hbase.filter.BinaryComparator;
  71 import org.apache.hadoop.hbase.filter.Filter;
  72 import org.apache.hadoop.hbase.filter.FilterList;
  73 import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
  74 import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
  75 import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
  76 import org.apache.hadoop.hbase.filter.LongComparator;
  77 import org.apache.hadoop.hbase.filter.PrefixFilter;
  78 import org.apache.hadoop.hbase.filter.QualifierFilter;
  79 import org.apache.hadoop.hbase.filter.RegexStringComparator;
  80 import org.apache.hadoop.hbase.filter.RowFilter;
  81 import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
  82 import org.apache.hadoop.hbase.filter.SubstringComparator;
  83 import org.apache.hadoop.hbase.filter.ValueFilter;
  84 import org.apache.hadoop.hbase.filter.WhileMatchFilter;
  85 import org.apache.hadoop.hbase.io.TimeRange;
  86 import org.apache.hadoop.hbase.io.hfile.BlockCache;
  87 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
  88 import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
  89 import org.apache.hadoop.hbase.master.LoadBalancer;
  90 import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
  91 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto;
  92 import org.apache.hadoop.hbase.protobuf.generated.ClientProtos.MutationProto.MutationType;
  93 import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService;
  94 import org.apache.hadoop.hbase.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest;
  95 import org.apache.hadoop.hbase.regionserver.HRegion;
  96 import org.apache.hadoop.hbase.regionserver.HRegionServer;
  97 import org.apache.hadoop.hbase.regionserver.HStore;
  98 import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
  99 import org.apache.hadoop.hbase.testclassification.ClientTests;
 100 import org.apache.hadoop.hbase.testclassification.LargeTests;
 101 import org.apache.hadoop.hbase.util.Bytes;
 102 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 103 import org.apache.hadoop.hbase.util.NonRepeatedEnvironmentEdge;
 104 import org.apache.hadoop.hbase.util.Pair;
 105 import org.junit.AfterClass;
 106 import org.junit.BeforeClass;
 107 import org.junit.ClassRule;
 108 import org.junit.Ignore;
 109 import org.junit.Rule;
 110 import org.junit.Test;
 111 import org.junit.experimental.categories.Category;
 112 import org.junit.rules.TestName;
 113 import org.slf4j.Logger;
 114 import org.slf4j.LoggerFactory;
 115
 116 /**
 117  * Run tests that use the HBase clients; {@link Table}.
 118  * Sets up the HBase mini cluster once at start and runs through all client tests.
 119  * Each creates a table named for the method and does its stuff against that.
 120  */
 121 @Category({LargeTests.class, ClientTests.class})
 122 @SuppressWarnings ("deprecation")
 123 public class TestFromClientSide {
 124
 125   @ClassRule
 126   public static final HBaseClassTestRule CLASS_RULE =
 127       HBaseClassTestRule.forClass(TestFromClientSide.class);
 128
 129   // NOTE: Increment tests were moved to their own class, TestIncrementsFromClientSide.
 130   private static final Logger LOG = LoggerFactory.getLogger(TestFromClientSide.class);
 131   protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
 132   private static byte [] ROW = Bytes.toBytes("testRow");
 133   private static byte [] FAMILY = Bytes.toBytes("testFamily");
 134   private static final byte[] INVALID_FAMILY = Bytes.toBytes("invalidTestFamily");
 135   private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
 136   private static byte [] VALUE = Bytes.toBytes("testValue");
 137   protected static int SLAVES = 3;
 138
 139   @Rule
 140   public TestName name = new TestName();
 141
 142   protected static final void initialize(Class<?>... cps) throws Exception {
 143     // Uncomment the following lines if more verbosity is needed for
 144     // debugging (see HBASE-12285 for details).
 145     // ((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
 146     // ((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
 147     // ((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
 148     // make sure that we do not get the same ts twice, see HBASE-19731 for more details.
 149     EnvironmentEdgeManager.injectEdge(new NonRepeatedEnvironmentEdge());
 150     Configuration conf = TEST_UTIL.getConfiguration();
 151     conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
 152       Arrays.stream(cps).map(Class::getName).toArray(String[]::new));
 153     conf.setBoolean("hbase.table.sanity.checks", true); // enable for below tests
 154     // We need more than one region server in this test
 155     TEST_UTIL.startMiniCluster(SLAVES);
 156   }
 157
 158   @BeforeClass
 159   public static void setUpBeforeClass() throws Exception {
 160     initialize(MultiRowMutationEndpoint.class);
 161   }
 162
 163   @AfterClass
 164   public static void tearDownAfterClass() throws Exception {
 165     TEST_UTIL.shutdownMiniCluster();
 166   }
 167
 168   /**
 169    * Test append result when there are duplicate rpc request.
 170    */
 171   @Test
 172   public void testDuplicateAppend() throws Exception {
 173     HTableDescriptor hdt = TEST_UTIL.createTableDescriptor(name.getMethodName());
 174     Map<String, String> kvs = new HashMap<>();
 175     kvs.put(HConnectionTestingUtility.SleepAtFirstRpcCall.SLEEP_TIME_CONF_KEY, "2000");
 176     hdt.addCoprocessor(HConnectionTestingUtility.SleepAtFirstRpcCall.class.getName(), null, 1, kvs);
 177     TEST_UTIL.createTable(hdt, new byte[][] { ROW }).close();
 178
 179     Configuration c = new Configuration(TEST_UTIL.getConfiguration());
 180     c.setInt(HConstants.HBASE_CLIENT_PAUSE, 50);
 181     // Client will retry beacuse rpc timeout is small than the sleep time of first rpc call
 182     c.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 1500);
 183
 184     Connection connection = ConnectionFactory.createConnection(c);
 185     Table t = connection.getTable(TableName.valueOf(name.getMethodName()));
 186     if (t instanceof HTable) {
 187       HTable table = (HTable) t;
 188       table.setOperationTimeout(3 * 1000);
 189
 190       try {
 191         Append append = new Append(ROW);
 192         append.addColumn(HBaseTestingUtility.fam1, QUALIFIER, VALUE);
 193         Result result = table.append(append);
 194
 195         // Verify expected result
 196         Cell[] cells = result.rawCells();
 197         assertEquals(1, cells.length);
 198         assertKey(cells[0], ROW, HBaseTestingUtility.fam1, QUALIFIER, VALUE);
 199
 200         // Verify expected result again
 201         Result readResult = table.get(new Get(ROW));
 202         cells = readResult.rawCells();
 203         assertEquals(1, cells.length);
 204         assertKey(cells[0], ROW, HBaseTestingUtility.fam1, QUALIFIER, VALUE);
 205       } finally {
 206         table.close();
 207         connection.close();
 208       }
 209     }
 210   }
 211
 212   /**
 213    * Basic client side validation of HBASE-4536
 214    */
 215   @Test
 216   public void testKeepDeletedCells() throws Exception {
 217     final TableName tableName = TableName.valueOf(name.getMethodName());
 218     final byte[] FAMILY = Bytes.toBytes("family");
 219     final byte[] C0 = Bytes.toBytes("c0");
 220
 221     final byte[] T1 = Bytes.toBytes("T1");
 222     final byte[] T2 = Bytes.toBytes("T2");
 223     final byte[] T3 = Bytes.toBytes("T3");
 224     HColumnDescriptor hcd =
 225         new HColumnDescriptor(FAMILY).setKeepDeletedCells(KeepDeletedCells.TRUE).setMaxVersions(3);
 226
 227     HTableDescriptor desc = new HTableDescriptor(tableName);
 228     desc.addFamily(hcd);
 229     TEST_UTIL.getAdmin().createTable(desc);
 230     Table h = TEST_UTIL.getConnection().getTable(tableName);
 231
 232     long ts = System.currentTimeMillis();
 233     Put p = new Put(T1, ts);
 234     p.addColumn(FAMILY, C0, T1);
 235     h.put(p);
 236     p = new Put(T1, ts + 2);
 237     p.addColumn(FAMILY, C0, T2);
 238     h.put(p);
 239     p = new Put(T1, ts + 4);
 240     p.addColumn(FAMILY, C0, T3);
 241     h.put(p);
 242
 243     Delete d = new Delete(T1, ts + 3);
 244     h.delete(d);
 245
 246     d = new Delete(T1, ts + 3);
 247     d.addColumns(FAMILY, C0, ts + 3);
 248     h.delete(d);
 249
 250     Get g = new Get(T1);
 251     // does *not* include the delete
 252     g.setTimeRange(0, ts + 3);
 253     Result r = h.get(g);
 254     assertArrayEquals(T2, r.getValue(FAMILY, C0));
 255
 256     Scan s = new Scan(T1);
 257     s.setTimeRange(0, ts + 3);
 258     s.setMaxVersions();
 259     ResultScanner scanner = h.getScanner(s);
 260     Cell[] kvs = scanner.next().rawCells();
 261     assertArrayEquals(T2, CellUtil.cloneValue(kvs[0]));
 262     assertArrayEquals(T1, CellUtil.cloneValue(kvs[1]));
 263     scanner.close();
 264
 265     s = new Scan(T1);
 266     s.setRaw(true);
 267     s.setMaxVersions();
 268     scanner = h.getScanner(s);
 269     kvs = scanner.next().rawCells();
 270     assertTrue(PrivateCellUtil.isDeleteFamily(kvs[0]));
 271     assertArrayEquals(T3, CellUtil.cloneValue(kvs[1]));
 272     assertTrue(CellUtil.isDelete(kvs[2]));
 273     assertArrayEquals(T2, CellUtil.cloneValue(kvs[3]));
 274     assertArrayEquals(T1, CellUtil.cloneValue(kvs[4]));
 275     scanner.close();
 276     h.close();
 277   }
 278
 279   /**
 280    * Basic client side validation of HBASE-10118
 281    */
 282   @Test
 283   public void testPurgeFutureDeletes() throws Exception {
 284     final TableName tableName = TableName.valueOf(name.getMethodName());
 285     final byte[] ROW = Bytes.toBytes("row");
 286     final byte[] FAMILY = Bytes.toBytes("family");
 287     final byte[] COLUMN = Bytes.toBytes("column");
 288     final byte[] VALUE = Bytes.toBytes("value");
 289
 290     Table table = TEST_UTIL.createTable(tableName, FAMILY);
 291
 292     // future timestamp
 293     long ts = System.currentTimeMillis() * 2;
 294     Put put = new Put(ROW, ts);
 295     put.addColumn(FAMILY, COLUMN, VALUE);
 296     table.put(put);
 297
 298     Get get = new Get(ROW);
 299     Result result = table.get(get);
 300     assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
 301
 302     Delete del = new Delete(ROW);
 303     del.addColumn(FAMILY, COLUMN, ts);
 304     table.delete(del);
 305
 306     get = new Get(ROW);
 307     result = table.get(get);
 308     assertNull(result.getValue(FAMILY, COLUMN));
 309
 310     // major compaction, purged future deletes
 311     TEST_UTIL.getAdmin().flush(tableName);
 312     TEST_UTIL.getAdmin().majorCompact(tableName);
 313
 314     // waiting for the major compaction to complete
 315     TEST_UTIL.waitFor(6000, new Waiter.Predicate<IOException>() {
 316       @Override
 317       public boolean evaluate() throws IOException {
 318         return TEST_UTIL.getAdmin().getCompactionState(tableName) == CompactionState.NONE;
 319       }
 320     });
 321
 322     put = new Put(ROW, ts);
 323     put.addColumn(FAMILY, COLUMN, VALUE);
 324     table.put(put);
 325
 326     get = new Get(ROW);
 327     result = table.get(get);
 328     assertArrayEquals(VALUE, result.getValue(FAMILY, COLUMN));
 329
 330     table.close();
 331   }
 332
 333   /**
 334    * Verifies that getConfiguration returns the same Configuration object used
 335    * to create the HTable instance.
 336    */
 337   @Test
 338   public void testGetConfiguration() throws Exception {
 339     final TableName tableName = TableName.valueOf(name.getMethodName());
 340     byte[][] FAMILIES = new byte[][] { Bytes.toBytes("foo") };
 341     Configuration conf = TEST_UTIL.getConfiguration();
 342     Table table = TEST_UTIL.createTable(tableName, FAMILIES);
 343     assertSame(conf, table.getConfiguration());
 344   }
 345
 346   /**
 347    * Test from client side of an involved filter against a multi family that
 348    * involves deletes.
 349    */
 350   @Test
 351   public void testWeirdCacheBehaviour() throws Exception {
 352     final TableName tableName = TableName.valueOf(name.getMethodName());
 353     byte [][] FAMILIES = new byte[][] { Bytes.toBytes("trans-blob"),
 354         Bytes.toBytes("trans-type"), Bytes.toBytes("trans-date"),
 355         Bytes.toBytes("trans-tags"), Bytes.toBytes("trans-group") };
 356     Table ht = TEST_UTIL.createTable(tableName, FAMILIES);
 357     String value = "this is the value";
 358     String value2 = "this is some other value";
 359     String keyPrefix1 = TEST_UTIL.getRandomUUID().toString();
 360     String keyPrefix2 = TEST_UTIL.getRandomUUID().toString();
 361     String keyPrefix3 = TEST_UTIL.getRandomUUID().toString();
 362     putRows(ht, 3, value, keyPrefix1);
 363     putRows(ht, 3, value, keyPrefix2);
 364     putRows(ht, 3, value, keyPrefix3);
 365     putRows(ht, 3, value2, keyPrefix1);
 366     putRows(ht, 3, value2, keyPrefix2);
 367     putRows(ht, 3, value2, keyPrefix3);
 368     Table table = TEST_UTIL.getConnection().getTable(tableName);
 369     System.out.println("Checking values for key: " + keyPrefix1);
 370     assertEquals("Got back incorrect number of rows from scan", 3,
 371         getNumberOfRows(keyPrefix1, value2, table));
 372     System.out.println("Checking values for key: " + keyPrefix2);
 373     assertEquals("Got back incorrect number of rows from scan", 3,
 374         getNumberOfRows(keyPrefix2, value2, table));
 375     System.out.println("Checking values for key: " + keyPrefix3);
 376     assertEquals("Got back incorrect number of rows from scan", 3,
 377         getNumberOfRows(keyPrefix3, value2, table));
 378     deleteColumns(ht, value2, keyPrefix1);
 379     deleteColumns(ht, value2, keyPrefix2);
 380     deleteColumns(ht, value2, keyPrefix3);
 381     System.out.println("Starting important checks.....");
 382     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix1,
 383       0, getNumberOfRows(keyPrefix1, value2, table));
 384     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix2,
 385       0, getNumberOfRows(keyPrefix2, value2, table));
 386     assertEquals("Got back incorrect number of rows from scan: " + keyPrefix3,
 387       0, getNumberOfRows(keyPrefix3, value2, table));
 388   }
 389
 390   private void deleteColumns(Table ht, String value, String keyPrefix)
 391   throws IOException {
 392     ResultScanner scanner = buildScanner(keyPrefix, value, ht);
 393     Iterator<Result> it = scanner.iterator();
 394     int count = 0;
 395     while (it.hasNext()) {
 396       Result result = it.next();
 397       Delete delete = new Delete(result.getRow());
 398       delete.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"));
 399       ht.delete(delete);
 400       count++;
 401     }
 402     assertEquals("Did not perform correct number of deletes", 3, count);
 403   }
 404
 405   private int getNumberOfRows(String keyPrefix, String value, Table ht)
 406       throws Exception {
 407     ResultScanner resultScanner = buildScanner(keyPrefix, value, ht);
 408     Iterator<Result> scanner = resultScanner.iterator();
 409     int numberOfResults = 0;
 410     while (scanner.hasNext()) {
 411       Result result = scanner.next();
 412       System.out.println("Got back key: " + Bytes.toString(result.getRow()));
 413       for (Cell kv : result.rawCells()) {
 414         System.out.println("kv=" + kv.toString() + ", "
 415             + Bytes.toString(CellUtil.cloneValue(kv)));
 416       }
 417       numberOfResults++;
 418     }
 419     return numberOfResults;
 420   }
 421
 422   private ResultScanner buildScanner(String keyPrefix, String value, Table ht)
 423       throws IOException {
 424     // OurFilterList allFilters = new OurFilterList();
 425     FilterList allFilters = new FilterList(/* FilterList.Operator.MUST_PASS_ALL */);
 426     allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
 427     SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes
 428         .toBytes("trans-tags"), Bytes.toBytes("qual2"), CompareOperator.EQUAL, Bytes
 429         .toBytes(value));
 430     filter.setFilterIfMissing(true);
 431     allFilters.addFilter(filter);
 432
 433     // allFilters.addFilter(new
 434     // RowExcludingSingleColumnValueFilter(Bytes.toBytes("trans-tags"),
 435     // Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value)));
 436
 437     Scan scan = new Scan();
 438     scan.addFamily(Bytes.toBytes("trans-blob"));
 439     scan.addFamily(Bytes.toBytes("trans-type"));
 440     scan.addFamily(Bytes.toBytes("trans-date"));
 441     scan.addFamily(Bytes.toBytes("trans-tags"));
 442     scan.addFamily(Bytes.toBytes("trans-group"));
 443     scan.setFilter(allFilters);
 444
 445     return ht.getScanner(scan);
 446   }
 447
 448   private void putRows(Table ht, int numRows, String value, String key)
 449       throws IOException {
 450     for (int i = 0; i < numRows; i++) {
 451       String row = key + "_" + TEST_UTIL.getRandomUUID().toString();
 452       System.out.println(String.format("Saving row: %s, with value %s", row,
 453           value));
 454       Put put = new Put(Bytes.toBytes(row));
 455       put.setDurability(Durability.SKIP_WAL);
 456       put.addColumn(Bytes.toBytes("trans-blob"), null, Bytes
 457           .toBytes("value for blob"));
 458       put.addColumn(Bytes.toBytes("trans-type"), null, Bytes.toBytes("statement"));
 459       put.addColumn(Bytes.toBytes("trans-date"), null, Bytes
 460           .toBytes("20090921010101999"));
 461       put.addColumn(Bytes.toBytes("trans-tags"), Bytes.toBytes("qual2"), Bytes
 462           .toBytes(value));
 463       put.addColumn(Bytes.toBytes("trans-group"), null, Bytes
 464           .toBytes("adhocTransactionGroupId"));
 465       ht.put(put);
 466     }
 467   }
 468
 469   /**
 470    * Test filters when multiple regions.  It does counts.  Needs eye-balling of
 471    * logs to ensure that we're not scanning more regions that we're supposed to.
 472    * Related to the TestFilterAcrossRegions over in the o.a.h.h.filter package.
 473    */
 474   @Test
 475   public void testFilterAcrossMultipleRegions()
 476   throws IOException, InterruptedException {
 477     final TableName tableName = TableName.valueOf(name.getMethodName());
 478     Table t = TEST_UTIL.createTable(tableName, FAMILY);
 479     int rowCount = TEST_UTIL.loadTable(t, FAMILY, false);
 480     assertRowCount(t, rowCount);
 481     // Split the table.  Should split on a reasonable key; 'lqj'
 482     List<HRegionLocation> regions  = splitTable(t);
 483     assertRowCount(t, rowCount);
 484     // Get end key of first region.
 485     byte [] endKey = regions.get(0).getRegionInfo().getEndKey();
 486     // Count rows with a filter that stops us before passed 'endKey'.
 487     // Should be count of rows in first region.
 488     int endKeyCount = TEST_UTIL.countRows(t, createScanWithRowFilter(endKey));
 489     assertTrue(endKeyCount < rowCount);
 490
 491     // How do I know I did not got to second region?  Thats tough.  Can't really
 492     // do that in client-side region test.  I verified by tracing in debugger.
 493     // I changed the messages that come out when set to DEBUG so should see
 494     // when scanner is done. Says "Finished with scanning..." with region name.
 495     // Check that its finished in right region.
 496
 497     // New test.  Make it so scan goes into next region by one and then two.
 498     // Make sure count comes out right.
 499     byte [] key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 1)};
 500     int plusOneCount = TEST_UTIL.countRows(t, createScanWithRowFilter(key));
 501     assertEquals(endKeyCount + 1, plusOneCount);
 502     key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] + 2)};
 503     int plusTwoCount = TEST_UTIL.countRows(t, createScanWithRowFilter(key));
 504     assertEquals(endKeyCount + 2, plusTwoCount);
 505
 506     // New test.  Make it so I scan one less than endkey.
 507     key = new byte [] {endKey[0], endKey[1], (byte)(endKey[2] - 1)};
 508     int minusOneCount = TEST_UTIL.countRows(t, createScanWithRowFilter(key));
 509     assertEquals(endKeyCount - 1, minusOneCount);
 510     // For above test... study logs.  Make sure we do "Finished with scanning.."
 511     // in first region and that we do not fall into the next region.
 512
 513     key = new byte [] {'a', 'a', 'a'};
 514     int countBBB = TEST_UTIL.countRows(t,
 515       createScanWithRowFilter(key, null, CompareOperator.EQUAL));
 516     assertEquals(1, countBBB);
 517
 518     int countGreater = TEST_UTIL.countRows(t, createScanWithRowFilter(endKey, null,
 519       CompareOperator.GREATER_OR_EQUAL));
 520     // Because started at start of table.
 521     assertEquals(0, countGreater);
 522     countGreater = TEST_UTIL.countRows(t, createScanWithRowFilter(endKey, endKey,
 523       CompareOperator.GREATER_OR_EQUAL));
 524     assertEquals(rowCount - endKeyCount, countGreater);
 525   }
 526
 527   /*
 528    * @param key
 529    * @return Scan with RowFilter that does LESS than passed key.
 530    */
 531   private Scan createScanWithRowFilter(final byte [] key) {
 532     return createScanWithRowFilter(key, null, CompareOperator.LESS);
 533   }
 534
 535   /*
 536    * @param key
 537    * @param op
 538    * @param startRow
 539    * @return Scan with RowFilter that does CompareOp op on passed key.
 540    */
 541   private Scan createScanWithRowFilter(final byte [] key,
 542       final byte [] startRow, CompareOperator op) {
 543     // Make sure key is of some substance... non-null and > than first key.
 544     assertTrue(key != null && key.length > 0 &&
 545       Bytes.BYTES_COMPARATOR.compare(key, new byte [] {'a', 'a', 'a'}) >= 0);
 546     LOG.info("Key=" + Bytes.toString(key));
 547     Scan s = startRow == null? new Scan(): new Scan(startRow);
 548     Filter f = new RowFilter(op, new BinaryComparator(key));
 549     f = new WhileMatchFilter(f);
 550     s.setFilter(f);
 551     return s;
 552   }
 553
 554   private void assertRowCount(final Table t, final int expected)
 555   throws IOException {
 556     assertEquals(expected, TEST_UTIL.countRows(t, new Scan()));
 557   }
 558
 559   /*
 560    * Split table into multiple regions.
 561    * @param t Table to split.
 562    * @return Map of regions to servers.
 563    * @throws IOException
 564    */
 565   private List<HRegionLocation> splitTable(final Table t)
 566   throws IOException, InterruptedException {
 567     // Split this table in two.
 568     Admin admin = TEST_UTIL.getAdmin();
 569     admin.split(t.getName());
 570     admin.close();
 571     List<HRegionLocation> regions = waitOnSplit(t);
 572     assertTrue(regions.size() > 1);
 573     return regions;
 574   }
 575
 576   /*
 577    * Wait on table split.  May return because we waited long enough on the split
 578    * and it didn't happen.  Caller should check.
 579    * @param t
 580    * @return Map of table regions; caller needs to check table actually split.
 581    */
 582   private List<HRegionLocation> waitOnSplit(final Table t)
 583   throws IOException {
 584     try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(t.getName())) {
 585       List<HRegionLocation> regions = locator.getAllRegionLocations();
 586       int originalCount = regions.size();
 587       for (int i = 0; i < TEST_UTIL.getConfiguration().getInt("hbase.test.retries", 30); i++) {
 588         Thread.currentThread();
 589         try {
 590           Thread.sleep(1000);
 591         } catch (InterruptedException e) {
 592           e.printStackTrace();
 593         }
 594         regions = locator.getAllRegionLocations();
 595         if (regions.size() > originalCount)
 596           break;
 597       }
 598       return regions;
 599     }
 600   }
 601
 602   @Test
 603   public void testSuperSimple() throws Exception {
 604     final TableName tableName = TableName.valueOf(name.getMethodName());
 605     Table ht = TEST_UTIL.createTable(tableName, FAMILY);
 606     Put put = new Put(ROW);
 607     put.addColumn(FAMILY, QUALIFIER, VALUE);
 608     ht.put(put);
 609     Scan scan = new Scan();
 610     scan.addColumn(FAMILY, tableName.toBytes());
 611     ResultScanner scanner = ht.getScanner(scan);
 612     Result result = scanner.next();
 613     assertTrue("Expected null result", result == null);
 614     scanner.close();
 615   }
 616
 617   @Test
 618   public void testMaxKeyValueSize() throws Exception {
 619     final TableName tableName = TableName.valueOf(name.getMethodName());
 620     Configuration conf = TEST_UTIL.getConfiguration();
 621     String oldMaxSize = conf.get(ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY);
 622     Table ht = TEST_UTIL.createTable(tableName, FAMILY);
 623     byte[] value = new byte[4 * 1024 * 1024];
 624     Put put = new Put(ROW);
 625     put.addColumn(FAMILY, QUALIFIER, value);
 626     ht.put(put);
 627     try {
 628       TEST_UTIL.getConfiguration().setInt(
 629           ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY, 2 * 1024 * 1024);
 630       // Create new table so we pick up the change in Configuration.
 631       try (Connection connection =
 632           ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) {
 633         try (Table t = connection.getTable(TableName.valueOf(FAMILY))) {
 634           put = new Put(ROW);
 635           put.addColumn(FAMILY, QUALIFIER, value);
 636           t.put(put);
 637         }
 638       }
 639       fail("Inserting a too large KeyValue worked, should throw exception");
 640     } catch(Exception e) {}
 641     conf.set(ConnectionConfiguration.MAX_KEYVALUE_SIZE_KEY, oldMaxSize);
 642   }
 643
 644   @Test
 645   public void testFilters() throws Exception {
 646     final TableName tableName = TableName.valueOf(name.getMethodName());
 647     Table ht = TEST_UTIL.createTable(tableName, FAMILY);
 648     byte [][] ROWS = makeN(ROW, 10);
 649     byte [][] QUALIFIERS = {
 650         Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
 651         Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
 652         Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
 653         Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
 654         Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>")
 655     };
 656     for(int i=0;i<10;i++) {
 657       Put put = new Put(ROWS[i]);
 658       put.setDurability(Durability.SKIP_WAL);
 659       put.addColumn(FAMILY, QUALIFIERS[i], VALUE);
 660       ht.put(put);
 661     }
 662     Scan scan = new Scan();
 663     scan.addFamily(FAMILY);
 664     Filter filter = new QualifierFilter(CompareOperator.EQUAL,
 665       new RegexStringComparator("col[1-5]"));
 666     scan.setFilter(filter);
 667     ResultScanner scanner = ht.getScanner(scan);
 668     int expectedIndex = 1;
 669     for(Result result : ht.getScanner(scan)) {
 670       assertEquals(1, result.size());
 671       assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[expectedIndex]));
 672       assertTrue(Bytes.equals(CellUtil.cloneQualifier(result.rawCells()[0]),
 673           QUALIFIERS[expectedIndex]));
 674       expectedIndex++;
 675     }
 676     assertEquals(6, expectedIndex);
 677     scanner.close();
 678   }
 679
 680   @Test
 681   public void testFilterWithLongCompartor() throws Exception {
 682     final TableName tableName = TableName.valueOf(name.getMethodName());
 683     Table ht = TEST_UTIL.createTable(tableName, FAMILY);
 684     byte [][] ROWS = makeN(ROW, 10);
 685     byte [][] values = new byte[10][];
 686     for (int i = 0; i < 10; i ++) {
 687         values[i] = Bytes.toBytes(100L * i);
 688     }
 689     for(int i = 0; i < 10; i ++) {
 690       Put put = new Put(ROWS[i]);
 691       put.setDurability(Durability.SKIP_WAL);
 692       put.addColumn(FAMILY, QUALIFIER, values[i]);
 693       ht.put(put);
 694     }
 695     Scan scan = new Scan();
 696     scan.addFamily(FAMILY);
 697     Filter filter = new SingleColumnValueFilter(FAMILY, QUALIFIER, CompareOperator.GREATER,
 698       new LongComparator(500));
 699     scan.setFilter(filter);
 700     ResultScanner scanner = ht.getScanner(scan);
 701     int expectedIndex = 0;
 702     for(Result result : ht.getScanner(scan)) {
 703       assertEquals(1, result.size());
 704       assertTrue(Bytes.toLong(result.getValue(FAMILY, QUALIFIER)) > 500);
 705       expectedIndex++;
 706     }
 707     assertEquals(4, expectedIndex);
 708     scanner.close();
 709 }
 710
 711   @Test
 712   public void testKeyOnlyFilter() throws Exception {
 713     final TableName tableName = TableName.valueOf(name.getMethodName());
 714     Table ht = TEST_UTIL.createTable(tableName, FAMILY);
 715     byte [][] ROWS = makeN(ROW, 10);
 716     byte [][] QUALIFIERS = {
 717         Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
 718         Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
 719         Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
 720         Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
 721         Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>")
 722     };
 723     for(int i=0;i<10;i++) {
 724       Put put = new Put(ROWS[i]);
 725       put.setDurability(Durability.SKIP_WAL);
 726       put.addColumn(FAMILY, QUALIFIERS[i], VALUE);
 727       ht.put(put);
 728     }
 729     Scan scan = new Scan();
 730     scan.addFamily(FAMILY);
 731     Filter filter = new KeyOnlyFilter(true);
 732     scan.setFilter(filter);
 733     ResultScanner scanner = ht.getScanner(scan);
 734     int count = 0;
 735     for(Result result : ht.getScanner(scan)) {
 736       assertEquals(1, result.size());
 737       assertEquals(Bytes.SIZEOF_INT, result.rawCells()[0].getValueLength());
 738       assertEquals(VALUE.length, Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0])));
 739       count++;
 740     }
 741     assertEquals(10, count);
 742     scanner.close();
 743   }
 744
 745   /**
 746    * Test simple table and non-existent row cases.
 747    */
 748   @Test
 749   public void testSimpleMissing() throws Exception {
 750     final TableName tableName = TableName.valueOf(name.getMethodName());
 751     Table ht = TEST_UTIL.createTable(tableName, FAMILY);
 752     byte [][] ROWS = makeN(ROW, 4);
 753
 754     // Try to get a row on an empty table
 755     Get get = new Get(ROWS[0]);
 756     Result result = ht.get(get);
 757     assertEmptyResult(result);
 758
 759     get = new Get(ROWS[0]);
 760     get.addFamily(FAMILY);
 761     result = ht.get(get);
 762     assertEmptyResult(result);
 763
 764     get = new Get(ROWS[0]);
 765     get.addColumn(FAMILY, QUALIFIER);
 766     result = ht.get(get);
 767     assertEmptyResult(result);
 768
 769     Scan scan = new Scan();
 770     result = getSingleScanResult(ht, scan);
 771     assertNullResult(result);
 772
 773
 774     scan = new Scan(ROWS[0]);
 775     result = getSingleScanResult(ht, scan);
 776     assertNullResult(result);
 777
 778     scan = new Scan(ROWS[0],ROWS[1]);
 779     result = getSingleScanResult(ht, scan);
 780     assertNullResult(result);
 781
 782     scan = new Scan();
 783     scan.addFamily(FAMILY);
 784     result = getSingleScanResult(ht, scan);
 785     assertNullResult(result);
 786
 787     scan = new Scan();
 788     scan.addColumn(FAMILY, QUALIFIER);
 789     result = getSingleScanResult(ht, scan);
 790     assertNullResult(result);
 791
 792     // Insert a row
 793
 794     Put put = new Put(ROWS[2]);
 795     put.addColumn(FAMILY, QUALIFIER, VALUE);
 796     ht.put(put);
 797
 798     // Try to get empty rows around it
 799
 800     get = new Get(ROWS[1]);
 801     result = ht.get(get);
 802     assertEmptyResult(result);
 803
 804     get = new Get(ROWS[0]);
 805     get.addFamily(FAMILY);
 806     result = ht.get(get);
 807     assertEmptyResult(result);
 808
 809     get = new Get(ROWS[3]);
 810     get.addColumn(FAMILY, QUALIFIER);
 811     result = ht.get(get);
 812     assertEmptyResult(result);
 813
 814     // Try to scan empty rows around it
 815
 816     scan = new Scan(ROWS[3]);
 817     result = getSingleScanResult(ht, scan);
 818     assertNullResult(result);
 819
 820     scan = new Scan(ROWS[0],ROWS[2]);
 821     result = getSingleScanResult(ht, scan);
 822     assertNullResult(result);
 823
 824     // Make sure we can actually get the row
 825
 826     get = new Get(ROWS[2]);
 827     result = ht.get(get);
 828     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
 829
 830     get = new Get(ROWS[2]);
 831     get.addFamily(FAMILY);
 832     result = ht.get(get);
 833     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
 834
 835     get = new Get(ROWS[2]);
 836     get.addColumn(FAMILY, QUALIFIER);
 837     result = ht.get(get);
 838     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
 839
 840     // Make sure we can scan the row
 841
 842     scan = new Scan();
 843     result = getSingleScanResult(ht, scan);
 844     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
 845
 846     scan = new Scan(ROWS[0],ROWS[3]);
 847     result = getSingleScanResult(ht, scan);
 848     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
 849
 850     scan = new Scan(ROWS[2],ROWS[3]);
 851     result = getSingleScanResult(ht, scan);
 852     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
 853   }
 854
 855   /**
 856    * Test basic puts, gets, scans, and deletes for a single row
 857    * in a multiple family table.
 858    */
 859   @Test
 860   public void testSingleRowMultipleFamily() throws Exception {
 861     final TableName tableName = TableName.valueOf(name.getMethodName());
 862     byte [][] ROWS = makeN(ROW, 3);
 863     byte [][] FAMILIES = makeNAscii(FAMILY, 10);
 864     byte [][] QUALIFIERS = makeN(QUALIFIER, 10);
 865     byte [][] VALUES = makeN(VALUE, 10);
 866
 867     Table ht = TEST_UTIL.createTable(tableName, FAMILIES);
 868
 869     Get get;
 870     Scan scan;
 871     Delete delete;
 872     Put put;
 873     Result result;
 874
 875     ////////////////////////////////////////////////////////////////////////////
 876     // Insert one column to one family
 877     ////////////////////////////////////////////////////////////////////////////
 878
 879     put = new Put(ROWS[0]);
 880     put.addColumn(FAMILIES[4], QUALIFIERS[0], VALUES[0]);
 881     ht.put(put);
 882
 883     // Get the single column
 884     getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
 885
 886     // Scan the single column
 887     scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
 888
 889     // Get empty results around inserted column
 890     getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
 891
 892     // Scan empty results around inserted column
 893     scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
 894
 895     ////////////////////////////////////////////////////////////////////////////
 896     // Flush memstore and run same tests from storefiles
 897     ////////////////////////////////////////////////////////////////////////////
 898
 899     TEST_UTIL.flush();
 900
 901     // Redo get and scan tests from storefile
 902     getVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
 903     scanVerifySingleColumn(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0, VALUES, 0);
 904     getVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
 905     scanVerifySingleEmpty(ht, ROWS, 0, FAMILIES, 4, QUALIFIERS, 0);
 906
 907     ////////////////////////////////////////////////////////////////////////////
 908     // Now, Test reading from memstore and storefiles at once
 909     ////////////////////////////////////////////////////////////////////////////
 910
 911     // Insert multiple columns to two other families
 912     put = new Put(ROWS[0]);
 913     put.addColumn(FAMILIES[2], QUALIFIERS[2], VALUES[2]);
 914     put.addColumn(FAMILIES[2], QUALIFIERS[4], VALUES[4]);
 915     put.addColumn(FAMILIES[4], QUALIFIERS[4], VALUES[4]);
 916     put.addColumn(FAMILIES[6], QUALIFIERS[6], VALUES[6]);
 917     put.addColumn(FAMILIES[6], QUALIFIERS[7], VALUES[7]);
 918     put.addColumn(FAMILIES[7], QUALIFIERS[7], VALUES[7]);
 919     put.addColumn(FAMILIES[9], QUALIFIERS[0], VALUES[0]);
 920     ht.put(put);
 921
 922     // Get multiple columns across multiple families and get empties around it
 923     singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
 924
 925     // Scan multiple columns across multiple families and scan empties around it
 926     singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
 927
 928     ////////////////////////////////////////////////////////////////////////////
 929     // Flush the table again
 930     ////////////////////////////////////////////////////////////////////////////
 931
 932     TEST_UTIL.flush();
 933
 934     // Redo tests again
 935     singleRowGetTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
 936     singleRowScanTest(ht, ROWS, FAMILIES, QUALIFIERS, VALUES);
 937
 938     // Insert more data to memstore
 939     put = new Put(ROWS[0]);
 940     put.addColumn(FAMILIES[6], QUALIFIERS[5], VALUES[5]);
 941     put.addColumn(FAMILIES[6], QUALIFIERS[8], VALUES[8]);
 942     put.addColumn(FAMILIES[6], QUALIFIERS[9], VALUES[9]);
 943     put.addColumn(FAMILIES[4], QUALIFIERS[3], VALUES[3]);
 944     ht.put(put);
 945
 946     ////////////////////////////////////////////////////////////////////////////
 947     // Delete a storefile column
 948     ////////////////////////////////////////////////////////////////////////////
 949     delete = new Delete(ROWS[0]);
 950     delete.addColumns(FAMILIES[6], QUALIFIERS[7]);
 951     ht.delete(delete);
 952
 953     // Try to get deleted column
 954     get = new Get(ROWS[0]);
 955     get.addColumn(FAMILIES[6], QUALIFIERS[7]);
 956     result = ht.get(get);
 957     assertEmptyResult(result);
 958
 959     // Try to scan deleted column
 960     scan = new Scan();
 961     scan.addColumn(FAMILIES[6], QUALIFIERS[7]);
 962     result = getSingleScanResult(ht, scan);
 963     assertNullResult(result);
 964
 965     // Make sure we can still get a column before it and after it
 966     get = new Get(ROWS[0]);
 967     get.addColumn(FAMILIES[6], QUALIFIERS[6]);
 968     result = ht.get(get);
 969     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
 970
 971     get = new Get(ROWS[0]);
 972     get.addColumn(FAMILIES[6], QUALIFIERS[8]);
 973     result = ht.get(get);
 974     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]);
 975
 976     // Make sure we can still scan a column before it and after it
 977     scan = new Scan();
 978     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
 979     result = getSingleScanResult(ht, scan);
 980     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
 981
 982     scan = new Scan();
 983     scan.addColumn(FAMILIES[6], QUALIFIERS[8]);
 984     result = getSingleScanResult(ht, scan);
 985     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[8], VALUES[8]);
 986
 987     ////////////////////////////////////////////////////////////////////////////
 988     // Delete a memstore column
 989     ////////////////////////////////////////////////////////////////////////////
 990     delete = new Delete(ROWS[0]);
 991     delete.addColumns(FAMILIES[6], QUALIFIERS[8]);
 992     ht.delete(delete);
 993
 994     // Try to get deleted column
 995     get = new Get(ROWS[0]);
 996     get.addColumn(FAMILIES[6], QUALIFIERS[8]);
 997     result = ht.get(get);
 998     assertEmptyResult(result);
 999
1000     // Try to scan deleted column
1001     scan = new Scan();
1002     scan.addColumn(FAMILIES[6], QUALIFIERS[8]);
1003     result = getSingleScanResult(ht, scan);
1004     assertNullResult(result);
1005
1006     // Make sure we can still get a column before it and after it
1007     get = new Get(ROWS[0]);
1008     get.addColumn(FAMILIES[6], QUALIFIERS[6]);
1009     result = ht.get(get);
1010     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1011
1012     get = new Get(ROWS[0]);
1013     get.addColumn(FAMILIES[6], QUALIFIERS[9]);
1014     result = ht.get(get);
1015     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1016
1017     // Make sure we can still scan a column before it and after it
1018     scan = new Scan();
1019     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1020     result = getSingleScanResult(ht, scan);
1021     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1022
1023     scan = new Scan();
1024     scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
1025     result = getSingleScanResult(ht, scan);
1026     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1027
1028     ////////////////////////////////////////////////////////////////////////////
1029     // Delete joint storefile/memstore family
1030     ////////////////////////////////////////////////////////////////////////////
1031
1032     delete = new Delete(ROWS[0]);
1033     delete.addFamily(FAMILIES[4]);
1034     ht.delete(delete);
1035
1036     // Try to get storefile column in deleted family
1037     get = new Get(ROWS[0]);
1038     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
1039     result = ht.get(get);
1040     assertEmptyResult(result);
1041
1042     // Try to get memstore column in deleted family
1043     get = new Get(ROWS[0]);
1044     get.addColumn(FAMILIES[4], QUALIFIERS[3]);
1045     result = ht.get(get);
1046     assertEmptyResult(result);
1047
1048     // Try to get deleted family
1049     get = new Get(ROWS[0]);
1050     get.addFamily(FAMILIES[4]);
1051     result = ht.get(get);
1052     assertEmptyResult(result);
1053
1054     // Try to scan storefile column in deleted family
1055     scan = new Scan();
1056     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
1057     result = getSingleScanResult(ht, scan);
1058     assertNullResult(result);
1059
1060     // Try to scan memstore column in deleted family
1061     scan = new Scan();
1062     scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
1063     result = getSingleScanResult(ht, scan);
1064     assertNullResult(result);
1065
1066     // Try to scan deleted family
1067     scan = new Scan();
1068     scan.addFamily(FAMILIES[4]);
1069     result = getSingleScanResult(ht, scan);
1070     assertNullResult(result);
1071
1072     // Make sure we can still get another family
1073     get = new Get(ROWS[0]);
1074     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
1075     result = ht.get(get);
1076     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
1077
1078     get = new Get(ROWS[0]);
1079     get.addColumn(FAMILIES[6], QUALIFIERS[9]);
1080     result = ht.get(get);
1081     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1082
1083     // Make sure we can still scan another family
1084     scan = new Scan();
1085     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1086     result = getSingleScanResult(ht, scan);
1087     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1088
1089     scan = new Scan();
1090     scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
1091     result = getSingleScanResult(ht, scan);
1092     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1093
1094     ////////////////////////////////////////////////////////////////////////////
1095     // Flush everything and rerun delete tests
1096     ////////////////////////////////////////////////////////////////////////////
1097
1098     TEST_UTIL.flush();
1099
1100     // Try to get storefile column in deleted family
1101     get = new Get(ROWS[0]);
1102     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
1103     result = ht.get(get);
1104     assertEmptyResult(result);
1105
1106     // Try to get memstore column in deleted family
1107     get = new Get(ROWS[0]);
1108     get.addColumn(FAMILIES[4], QUALIFIERS[3]);
1109     result = ht.get(get);
1110     assertEmptyResult(result);
1111
1112     // Try to get deleted family
1113     get = new Get(ROWS[0]);
1114     get.addFamily(FAMILIES[4]);
1115     result = ht.get(get);
1116     assertEmptyResult(result);
1117
1118     // Try to scan storefile column in deleted family
1119     scan = new Scan();
1120     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
1121     result = getSingleScanResult(ht, scan);
1122     assertNullResult(result);
1123
1124     // Try to scan memstore column in deleted family
1125     scan = new Scan();
1126     scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
1127     result = getSingleScanResult(ht, scan);
1128     assertNullResult(result);
1129
1130     // Try to scan deleted family
1131     scan = new Scan();
1132     scan.addFamily(FAMILIES[4]);
1133     result = getSingleScanResult(ht, scan);
1134     assertNullResult(result);
1135
1136     // Make sure we can still get another family
1137     get = new Get(ROWS[0]);
1138     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
1139     result = ht.get(get);
1140     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
1141
1142     get = new Get(ROWS[0]);
1143     get.addColumn(FAMILIES[6], QUALIFIERS[9]);
1144     result = ht.get(get);
1145     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1146
1147     // Make sure we can still scan another family
1148     scan = new Scan();
1149     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
1150     result = getSingleScanResult(ht, scan);
1151     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[6], VALUES[6]);
1152
1153     scan = new Scan();
1154     scan.addColumn(FAMILIES[6], QUALIFIERS[9]);
1155     result = getSingleScanResult(ht, scan);
1156     assertSingleResult(result, ROWS[0], FAMILIES[6], QUALIFIERS[9], VALUES[9]);
1157
1158   }
1159
1160   @Test
1161   public void testNull() throws Exception {
1162     final TableName tableName = TableName.valueOf(name.getMethodName());
1163
1164     // Null table name (should NOT work)
1165     try {
1166       TEST_UTIL.createTable((TableName)null, FAMILY);
1167       fail("Creating a table with null name passed, should have failed");
1168     } catch(Exception e) {}
1169
1170     // Null family (should NOT work)
1171     try {
1172       TEST_UTIL.createTable(tableName, new byte[][]{null});
1173       fail("Creating a table with a null family passed, should fail");
1174     } catch(Exception e) {}
1175
1176     Table ht = TEST_UTIL.createTable(tableName, FAMILY);
1177
1178     // Null row (should NOT work)
1179     try {
1180       Put put = new Put((byte[])null);
1181       put.addColumn(FAMILY, QUALIFIER, VALUE);
1182       ht.put(put);
1183       fail("Inserting a null row worked, should throw exception");
1184     } catch(Exception e) {}
1185
1186     // Null qualifier (should work)
1187     {
1188       Put put = new Put(ROW);
1189       put.addColumn(FAMILY, null, VALUE);
1190       ht.put(put);
1191
1192       getTestNull(ht, ROW, FAMILY, VALUE);
1193
1194       scanTestNull(ht, ROW, FAMILY, VALUE);
1195
1196       Delete delete = new Delete(ROW);
1197       delete.addColumns(FAMILY, null);
1198       ht.delete(delete);
1199
1200       Get get = new Get(ROW);
1201       Result result = ht.get(get);
1202       assertEmptyResult(result);
1203     }
1204
1205     // Use a new table
1206     ht = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName() + "2"), FAMILY);
1207
1208     // Empty qualifier, byte[0] instead of null (should work)
1209     try {
1210       Put put = new Put(ROW);
1211       put.addColumn(FAMILY, HConstants.EMPTY_BYTE_ARRAY, VALUE);
1212       ht.put(put);
1213
1214       getTestNull(ht, ROW, FAMILY, VALUE);
1215
1216       scanTestNull(ht, ROW, FAMILY, VALUE);
1217
1218       // Flush and try again
1219
1220       TEST_UTIL.flush();
1221
1222       getTestNull(ht, ROW, FAMILY, VALUE);
1223
1224       scanTestNull(ht, ROW, FAMILY, VALUE);
1225
1226       Delete delete = new Delete(ROW);
1227       delete.addColumns(FAMILY, HConstants.EMPTY_BYTE_ARRAY);
1228       ht.delete(delete);
1229
1230       Get get = new Get(ROW);
1231       Result result = ht.get(get);
1232       assertEmptyResult(result);
1233
1234     } catch(Exception e) {
1235       throw new IOException("Using a row with null qualifier threw exception, should ");
1236     }
1237
1238     // Null value
1239     try {
1240       Put put = new Put(ROW);
1241       put.addColumn(FAMILY, QUALIFIER, null);
1242       ht.put(put);
1243
1244       Get get = new Get(ROW);
1245       get.addColumn(FAMILY, QUALIFIER);
1246       Result result = ht.get(get);
1247       assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
1248
1249       Scan scan = new Scan();
1250       scan.addColumn(FAMILY, QUALIFIER);
1251       result = getSingleScanResult(ht, scan);
1252       assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
1253
1254       Delete delete = new Delete(ROW);
1255       delete.addColumns(FAMILY, QUALIFIER);
1256       ht.delete(delete);
1257
1258       get = new Get(ROW);
1259       result = ht.get(get);
1260       assertEmptyResult(result);
1261
1262     } catch(Exception e) {
1263       throw new IOException("Null values should be allowed, but threw exception");
1264     }
1265   }
1266
1267   @Test
1268   public void testNullQualifier() throws Exception {
1269     final TableName tableName = TableName.valueOf(name.getMethodName());
1270     Table table = TEST_UTIL.createTable(tableName, FAMILY);
1271
1272     // Work for Put
1273     Put put = new Put(ROW);
1274     put.addColumn(FAMILY, null, VALUE);
1275     table.put(put);
1276
1277     // Work for Get, Scan
1278     getTestNull(table, ROW, FAMILY, VALUE);
1279     scanTestNull(table, ROW, FAMILY, VALUE);
1280
1281     // Work for Delete
1282     Delete delete = new Delete(ROW);
1283     delete.addColumns(FAMILY, null);
1284     table.delete(delete);
1285
1286     Get get = new Get(ROW);
1287     Result result = table.get(get);
1288     assertEmptyResult(result);
1289
1290     // Work for Increment/Append
1291     Increment increment = new Increment(ROW);
1292     increment.addColumn(FAMILY, null, 1L);
1293     table.increment(increment);
1294     getTestNull(table, ROW, FAMILY, 1L);
1295
1296     table.incrementColumnValue(ROW, FAMILY, null, 1L);
1297     getTestNull(table, ROW, FAMILY, 2L);
1298
1299     delete = new Delete(ROW);
1300     delete.addColumns(FAMILY, null);
1301     table.delete(delete);
1302
1303     Append append = new Append(ROW);
1304     append.addColumn(FAMILY, null, VALUE);
1305     table.append(append);
1306     getTestNull(table, ROW, FAMILY, VALUE);
1307
1308     // Work for checkAndMutate using thenPut, thenMutate and thenDelete
1309     put = new Put(ROW);
1310     put.addColumn(FAMILY, null, Bytes.toBytes("checkAndPut"));
1311     table.put(put);
1312     table.checkAndMutate(ROW, FAMILY).ifEquals(VALUE).thenPut(put);
1313
1314     RowMutations mutate = new RowMutations(ROW);
1315     mutate.add(new Put(ROW).addColumn(FAMILY, null, Bytes.toBytes("checkAndMutate")));
1316     table.checkAndMutate(ROW, FAMILY).ifEquals(Bytes.toBytes("checkAndPut")).thenMutate(mutate);
1317
1318     delete = new Delete(ROW);
1319     delete.addColumns(FAMILY, null);
1320     table.checkAndMutate(ROW, FAMILY).ifEquals(Bytes.toBytes("checkAndMutate")).thenDelete(delete);
1321   }
1322
1323   @Test
1324   public void testVersions() throws Exception {
1325     final TableName tableName = TableName.valueOf(name.getMethodName());
1326
1327     long [] STAMPS = makeStamps(20);
1328     byte [][] VALUES = makeNAscii(VALUE, 20);
1329
1330     Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10);
1331
1332     // Insert 4 versions of same column
1333     Put put = new Put(ROW);
1334     put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1335     put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1336     put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1337     put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1338     ht.put(put);
1339
1340     // Verify we can get each one properly
1341     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1342     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1343     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1344     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1345     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1346     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1347     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1348     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1349
1350     // Verify we don't accidentally get others
1351     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1352     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1353     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1354     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1355     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1356     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1357
1358     // Ensure maxVersions in query is respected
1359     Get get = new Get(ROW);
1360     get.addColumn(FAMILY, QUALIFIER);
1361     get.setMaxVersions(2);
1362     Result result = ht.get(get);
1363     assertNResult(result, ROW, FAMILY, QUALIFIER,
1364         new long [] {STAMPS[4], STAMPS[5]},
1365         new byte[][] {VALUES[4], VALUES[5]},
1366         0, 1);
1367
1368     Scan scan = new Scan(ROW);
1369     scan.addColumn(FAMILY, QUALIFIER);
1370     scan.setMaxVersions(2);
1371     result = getSingleScanResult(ht, scan);
1372     assertNResult(result, ROW, FAMILY, QUALIFIER,
1373         new long [] {STAMPS[4], STAMPS[5]},
1374         new byte[][] {VALUES[4], VALUES[5]},
1375         0, 1);
1376
1377     // Flush and redo
1378
1379     TEST_UTIL.flush();
1380
1381     // Verify we can get each one properly
1382     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1383     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1384     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1385     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1386     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1387     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1388     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1389     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
1390
1391     // Verify we don't accidentally get others
1392     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1393     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1394     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1395     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1396     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
1397     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
1398
1399     // Ensure maxVersions in query is respected
1400     get = new Get(ROW);
1401     get.addColumn(FAMILY, QUALIFIER);
1402     get.setMaxVersions(2);
1403     result = ht.get(get);
1404     assertNResult(result, ROW, FAMILY, QUALIFIER,
1405         new long [] {STAMPS[4], STAMPS[5]},
1406         new byte[][] {VALUES[4], VALUES[5]},
1407         0, 1);
1408
1409     scan = new Scan(ROW);
1410     scan.addColumn(FAMILY, QUALIFIER);
1411     scan.setMaxVersions(2);
1412     result = getSingleScanResult(ht, scan);
1413     assertNResult(result, ROW, FAMILY, QUALIFIER,
1414         new long [] {STAMPS[4], STAMPS[5]},
1415         new byte[][] {VALUES[4], VALUES[5]},
1416         0, 1);
1417
1418
1419     // Add some memstore and retest
1420
1421     // Insert 4 more versions of same column and a dupe
1422     put = new Put(ROW);
1423     put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
1424     put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
1425     put.addColumn(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1426     put.addColumn(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
1427     ht.put(put);
1428
1429     // Ensure maxVersions in query is respected
1430     get = new Get(ROW);
1431     get.addColumn(FAMILY, QUALIFIER);
1432     get.setMaxVersions();
1433     result = ht.get(get);
1434     assertNResult(result, ROW, FAMILY, QUALIFIER,
1435         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1436         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1437         0, 7);
1438
1439     scan = new Scan(ROW);
1440     scan.addColumn(FAMILY, QUALIFIER);
1441     scan.setMaxVersions();
1442     result = getSingleScanResult(ht, scan);
1443     assertNResult(result, ROW, FAMILY, QUALIFIER,
1444         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1445         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1446         0, 7);
1447
1448     get = new Get(ROW);
1449     get.setMaxVersions();
1450     result = ht.get(get);
1451     assertNResult(result, ROW, FAMILY, QUALIFIER,
1452         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1453         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1454         0, 7);
1455
1456     scan = new Scan(ROW);
1457     scan.setMaxVersions();
1458     result = getSingleScanResult(ht, scan);
1459     assertNResult(result, ROW, FAMILY, QUALIFIER,
1460         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
1461         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
1462         0, 7);
1463
1464     // Verify we can get each one properly
1465     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1466     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1467     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1468     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1469     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
1470     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
1471     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
1472     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
1473
1474     // Verify we don't accidentally get others
1475     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1476     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
1477     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
1478     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
1479
1480     // Ensure maxVersions of table is respected
1481
1482     TEST_UTIL.flush();
1483
1484     // Insert 4 more versions of same column and a dupe
1485     put = new Put(ROW);
1486     put.addColumn(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
1487     put.addColumn(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
1488     put.addColumn(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
1489     put.addColumn(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
1490     ht.put(put);
1491
1492     get = new Get(ROW);
1493     get.addColumn(FAMILY, QUALIFIER);
1494     get.setMaxVersions(Integer.MAX_VALUE);
1495     result = ht.get(get);
1496     assertNResult(result, ROW, FAMILY, QUALIFIER,
1497         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
1498         new byte[][] {VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
1499         0, 9);
1500
1501     scan = new Scan(ROW);
1502     scan.addColumn(FAMILY, QUALIFIER);
1503     scan.setMaxVersions(Integer.MAX_VALUE);
1504     result = getSingleScanResult(ht, scan);
1505     assertNResult(result, ROW, FAMILY, QUALIFIER,
1506         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
1507         new byte[][] {VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
1508         0, 9);
1509
1510     // Delete a version in the memstore and a version in a storefile
1511     Delete delete = new Delete(ROW);
1512     delete.addColumn(FAMILY, QUALIFIER, STAMPS[11]);
1513     delete.addColumn(FAMILY, QUALIFIER, STAMPS[7]);
1514     ht.delete(delete);
1515
1516     // Test that it's gone
1517     get = new Get(ROW);
1518     get.addColumn(FAMILY, QUALIFIER);
1519     get.setMaxVersions(Integer.MAX_VALUE);
1520     result = ht.get(get);
1521     assertNResult(result, ROW, FAMILY, QUALIFIER,
1522         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
1523         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
1524         0, 9);
1525
1526     scan = new Scan(ROW);
1527     scan.addColumn(FAMILY, QUALIFIER);
1528     scan.setMaxVersions(Integer.MAX_VALUE);
1529     result = getSingleScanResult(ht, scan);
1530     assertNResult(result, ROW, FAMILY, QUALIFIER,
1531         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
1532         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
1533         0, 9);
1534
1535   }
1536
1537   @Test
1538   public void testVersionLimits() throws Exception {
1539     final TableName tableName = TableName.valueOf(name.getMethodName());
1540     byte [][] FAMILIES = makeNAscii(FAMILY, 3);
1541     int [] LIMITS = {1,3,5};
1542     long [] STAMPS = makeStamps(10);
1543     byte [][] VALUES = makeNAscii(VALUE, 10);
1544     Table ht = TEST_UTIL.createTable(tableName, FAMILIES, LIMITS);
1545
1546     // Insert limit + 1 on each family
1547     Put put = new Put(ROW);
1548     put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[0], VALUES[0]);
1549     put.addColumn(FAMILIES[0], QUALIFIER, STAMPS[1], VALUES[1]);
1550     put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[0], VALUES[0]);
1551     put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[1], VALUES[1]);
1552     put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[2], VALUES[2]);
1553     put.addColumn(FAMILIES[1], QUALIFIER, STAMPS[3], VALUES[3]);
1554     put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[0], VALUES[0]);
1555     put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[1], VALUES[1]);
1556     put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[2], VALUES[2]);
1557     put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[3], VALUES[3]);
1558     put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[4], VALUES[4]);
1559     put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[5], VALUES[5]);
1560     put.addColumn(FAMILIES[2], QUALIFIER, STAMPS[6], VALUES[6]);
1561     ht.put(put);
1562
1563     // Verify we only get the right number out of each
1564
1565     // Family0
1566
1567     Get get = new Get(ROW);
1568     get.addColumn(FAMILIES[0], QUALIFIER);
1569     get.setMaxVersions(Integer.MAX_VALUE);
1570     Result result = ht.get(get);
1571     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1572         new long [] {STAMPS[1]},
1573         new byte[][] {VALUES[1]},
1574         0, 0);
1575
1576     get = new Get(ROW);
1577     get.addFamily(FAMILIES[0]);
1578     get.setMaxVersions(Integer.MAX_VALUE);
1579     result = ht.get(get);
1580     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1581         new long [] {STAMPS[1]},
1582         new byte[][] {VALUES[1]},
1583         0, 0);
1584
1585     Scan scan = new Scan(ROW);
1586     scan.addColumn(FAMILIES[0], QUALIFIER);
1587     scan.setMaxVersions(Integer.MAX_VALUE);
1588     result = getSingleScanResult(ht, scan);
1589     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1590         new long [] {STAMPS[1]},
1591         new byte[][] {VALUES[1]},
1592         0, 0);
1593
1594     scan = new Scan(ROW);
1595     scan.addFamily(FAMILIES[0]);
1596     scan.setMaxVersions(Integer.MAX_VALUE);
1597     result = getSingleScanResult(ht, scan);
1598     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1599         new long [] {STAMPS[1]},
1600         new byte[][] {VALUES[1]},
1601         0, 0);
1602
1603     // Family1
1604
1605     get = new Get(ROW);
1606     get.addColumn(FAMILIES[1], QUALIFIER);
1607     get.setMaxVersions(Integer.MAX_VALUE);
1608     result = ht.get(get);
1609     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1610         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1611         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1612         0, 2);
1613
1614     get = new Get(ROW);
1615     get.addFamily(FAMILIES[1]);
1616     get.setMaxVersions(Integer.MAX_VALUE);
1617     result = ht.get(get);
1618     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1619         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1620         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1621         0, 2);
1622
1623     scan = new Scan(ROW);
1624     scan.addColumn(FAMILIES[1], QUALIFIER);
1625     scan.setMaxVersions(Integer.MAX_VALUE);
1626     result = getSingleScanResult(ht, scan);
1627     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1628         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1629         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1630         0, 2);
1631
1632     scan = new Scan(ROW);
1633     scan.addFamily(FAMILIES[1]);
1634     scan.setMaxVersions(Integer.MAX_VALUE);
1635     result = getSingleScanResult(ht, scan);
1636     assertNResult(result, ROW, FAMILIES[1], QUALIFIER,
1637         new long [] {STAMPS[1], STAMPS[2], STAMPS[3]},
1638         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1639         0, 2);
1640
1641     // Family2
1642
1643     get = new Get(ROW);
1644     get.addColumn(FAMILIES[2], QUALIFIER);
1645     get.setMaxVersions(Integer.MAX_VALUE);
1646     result = ht.get(get);
1647     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1648         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1649         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1650         0, 4);
1651
1652     get = new Get(ROW);
1653     get.addFamily(FAMILIES[2]);
1654     get.setMaxVersions(Integer.MAX_VALUE);
1655     result = ht.get(get);
1656     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1657         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1658         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1659         0, 4);
1660
1661     scan = new Scan(ROW);
1662     scan.addColumn(FAMILIES[2], QUALIFIER);
1663     scan.setMaxVersions(Integer.MAX_VALUE);
1664     result = getSingleScanResult(ht, scan);
1665     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1666         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1667         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1668         0, 4);
1669
1670     scan = new Scan(ROW);
1671     scan.addFamily(FAMILIES[2]);
1672     scan.setMaxVersions(Integer.MAX_VALUE);
1673     result = getSingleScanResult(ht, scan);
1674     assertNResult(result, ROW, FAMILIES[2], QUALIFIER,
1675         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6]},
1676         new byte[][] {VALUES[2], VALUES[3], VALUES[4], VALUES[5], VALUES[6]},
1677         0, 4);
1678
1679     // Try all families
1680
1681     get = new Get(ROW);
1682     get.setMaxVersions(Integer.MAX_VALUE);
1683     result = ht.get(get);
1684     assertTrue("Expected 9 keys but received " + result.size(),
1685         result.size() == 9);
1686
1687     get = new Get(ROW);
1688     get.addFamily(FAMILIES[0]);
1689     get.addFamily(FAMILIES[1]);
1690     get.addFamily(FAMILIES[2]);
1691     get.setMaxVersions(Integer.MAX_VALUE);
1692     result = ht.get(get);
1693     assertTrue("Expected 9 keys but received " + result.size(),
1694         result.size() == 9);
1695
1696     get = new Get(ROW);
1697     get.addColumn(FAMILIES[0], QUALIFIER);
1698     get.addColumn(FAMILIES[1], QUALIFIER);
1699     get.addColumn(FAMILIES[2], QUALIFIER);
1700     get.setMaxVersions(Integer.MAX_VALUE);
1701     result = ht.get(get);
1702     assertTrue("Expected 9 keys but received " + result.size(),
1703         result.size() == 9);
1704
1705     scan = new Scan(ROW);
1706     scan.setMaxVersions(Integer.MAX_VALUE);
1707     result = getSingleScanResult(ht, scan);
1708     assertTrue("Expected 9 keys but received " + result.size(),
1709         result.size() == 9);
1710
1711     scan = new Scan(ROW);
1712     scan.setMaxVersions(Integer.MAX_VALUE);
1713     scan.addFamily(FAMILIES[0]);
1714     scan.addFamily(FAMILIES[1]);
1715     scan.addFamily(FAMILIES[2]);
1716     result = getSingleScanResult(ht, scan);
1717     assertTrue("Expected 9 keys but received " + result.size(),
1718         result.size() == 9);
1719
1720     scan = new Scan(ROW);
1721     scan.setMaxVersions(Integer.MAX_VALUE);
1722     scan.addColumn(FAMILIES[0], QUALIFIER);
1723     scan.addColumn(FAMILIES[1], QUALIFIER);
1724     scan.addColumn(FAMILIES[2], QUALIFIER);
1725     result = getSingleScanResult(ht, scan);
1726     assertTrue("Expected 9 keys but received " + result.size(),
1727         result.size() == 9);
1728
1729   }
1730
1731   @Test
1732   public void testDeleteFamilyVersion() throws Exception {
1733     Admin admin = TEST_UTIL.getAdmin();
1734     final TableName tableName = TableName.valueOf(name.getMethodName());
1735
1736     byte [][] QUALIFIERS = makeNAscii(QUALIFIER, 1);
1737     byte [][] VALUES = makeN(VALUE, 5);
1738     long [] ts = {1000, 2000, 3000, 4000, 5000};
1739
1740     Table ht = TEST_UTIL.createTable(tableName, FAMILY, 5);
1741
1742     Put put = new Put(ROW);
1743     for (int q = 0; q < 1; q++)
1744       for (int t = 0; t < 5; t++)
1745         put.addColumn(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
1746     ht.put(put);
1747     admin.flush(tableName);
1748
1749     Delete delete = new Delete(ROW);
1750     delete.addFamilyVersion(FAMILY, ts[1]);  // delete version '2000'
1751     delete.addFamilyVersion(FAMILY, ts[3]);  // delete version '4000'
1752     ht.delete(delete);
1753     admin.flush(tableName);
1754
1755     for (int i = 0; i < 1; i++) {
1756       Get get = new Get(ROW);
1757       get.addColumn(FAMILY, QUALIFIERS[i]);
1758       get.setMaxVersions(Integer.MAX_VALUE);
1759       Result result = ht.get(get);
1760       // verify version '1000'/'3000'/'5000' remains for all columns
1761       assertNResult(result, ROW, FAMILY, QUALIFIERS[i],
1762           new long [] {ts[0], ts[2], ts[4]},
1763           new byte[][] {VALUES[0], VALUES[2], VALUES[4]},
1764           0, 2);
1765     }
1766     ht.close();
1767     admin.close();
1768   }
1769
1770   @Test
1771   public void testDeleteFamilyVersionWithOtherDeletes() throws Exception {
1772     final TableName tableName = TableName.valueOf(name.getMethodName());
1773
1774     byte [][] QUALIFIERS = makeNAscii(QUALIFIER, 5);
1775     byte [][] VALUES = makeN(VALUE, 5);
1776     long [] ts = {1000, 2000, 3000, 4000, 5000};
1777
1778     Admin admin = TEST_UTIL.getAdmin();
1779     Table ht = TEST_UTIL.createTable(tableName, FAMILY, 5);
1780     Put put = null;
1781     Result result = null;
1782     Get get = null;
1783     Delete delete = null;
1784
1785     // 1. put on ROW
1786     put = new Put(ROW);
1787     for (int q = 0; q < 5; q++)
1788       for (int t = 0; t < 5; t++)
1789         put.addColumn(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
1790     ht.put(put);
1791     admin.flush(tableName);
1792
1793     // 2. put on ROWS[0]
1794     byte [] ROW2 = Bytes.toBytes("myRowForTest");
1795     put = new Put(ROW2);
1796     for (int q = 0; q < 5; q++)
1797       for (int t = 0; t < 5; t++)
1798         put.addColumn(FAMILY, QUALIFIERS[q], ts[t], VALUES[t]);
1799     ht.put(put);
1800     admin.flush(tableName);
1801
1802     // 3. delete on ROW
1803     delete = new Delete(ROW);
1804     // delete version <= 2000 of all columns
1805     // note: addFamily must be the first since it will mask
1806     // the subsequent other type deletes!
1807     delete.addFamily(FAMILY, ts[1]);
1808     // delete version '4000' of all columns
1809     delete.addFamilyVersion(FAMILY, ts[3]);
1810    // delete version <= 3000 of column 0
1811     delete.addColumns(FAMILY, QUALIFIERS[0], ts[2]);
1812     // delete version <= 5000 of column 2
1813     delete.addColumns(FAMILY, QUALIFIERS[2], ts[4]);
1814     // delete version 5000 of column 4
1815     delete.addColumn(FAMILY, QUALIFIERS[4], ts[4]);
1816     ht.delete(delete);
1817     admin.flush(tableName);
1818
1819      // 4. delete on ROWS[0]
1820     delete = new Delete(ROW2);
1821     delete.addFamilyVersion(FAMILY, ts[1]);  // delete version '2000'
1822     delete.addFamilyVersion(FAMILY, ts[3]);  // delete version '4000'
1823     ht.delete(delete);
1824     admin.flush(tableName);
1825
1826     // 5. check ROW
1827     get = new Get(ROW);
1828     get.addColumn(FAMILY, QUALIFIERS[0]);
1829     get.setMaxVersions(Integer.MAX_VALUE);
1830     result = ht.get(get);
1831     assertNResult(result, ROW, FAMILY, QUALIFIERS[0],
1832         new long [] {ts[4]},
1833         new byte[][] {VALUES[4]},
1834         0, 0);
1835
1836     get = new Get(ROW);
1837     get.addColumn(FAMILY, QUALIFIERS[1]);
1838     get.setMaxVersions(Integer.MAX_VALUE);
1839     result = ht.get(get);
1840     assertNResult(result, ROW, FAMILY, QUALIFIERS[1],
1841         new long [] {ts[2], ts[4]},
1842         new byte[][] {VALUES[2], VALUES[4]},
1843         0, 1);
1844
1845     get = new Get(ROW);
1846     get.addColumn(FAMILY, QUALIFIERS[2]);
1847     get.setMaxVersions(Integer.MAX_VALUE);
1848     result = ht.get(get);
1849     assertEquals(0, result.size());
1850
1851     get = new Get(ROW);
1852     get.addColumn(FAMILY, QUALIFIERS[3]);
1853     get.setMaxVersions(Integer.MAX_VALUE);
1854     result = ht.get(get);
1855     assertNResult(result, ROW, FAMILY, QUALIFIERS[3],
1856         new long [] {ts[2], ts[4]},
1857         new byte[][] {VALUES[2], VALUES[4]},
1858         0, 1);
1859
1860     get = new Get(ROW);
1861     get.addColumn(FAMILY, QUALIFIERS[4]);
1862     get.setMaxVersions(Integer.MAX_VALUE);
1863     result = ht.get(get);
1864     assertNResult(result, ROW, FAMILY, QUALIFIERS[4],
1865         new long [] {ts[2]},
1866         new byte[][] {VALUES[2]},
1867         0, 0);
1868
1869     // 6. check ROWS[0]
1870     for (int i = 0; i < 5; i++) {
1871       get = new Get(ROW2);
1872       get.addColumn(FAMILY, QUALIFIERS[i]);
1873       get.setMaxVersions(Integer.MAX_VALUE);
1874       result = ht.get(get);
1875       // verify version '1000'/'3000'/'5000' remains for all columns
1876       assertNResult(result, ROW2, FAMILY, QUALIFIERS[i],
1877           new long [] {ts[0], ts[2], ts[4]},
1878           new byte[][] {VALUES[0], VALUES[2], VALUES[4]},
1879           0, 2);
1880     }
1881     ht.close();
1882     admin.close();
1883   }
1884
1885   @Test
1886   public void testDeleteWithFailed() throws Exception {
1887     final TableName tableName = TableName.valueOf(name.getMethodName());
1888
1889     byte [][] FAMILIES = makeNAscii(FAMILY, 3);
1890     byte [][] VALUES = makeN(VALUE, 5);
1891     long [] ts = {1000, 2000, 3000, 4000, 5000};
1892
1893     Table ht = TEST_UTIL.createTable(tableName, FAMILIES, 3);
1894
1895     Put put = new Put(ROW);
1896     put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
1897     ht.put(put);
1898
1899     // delete wrong family
1900     Delete delete = new Delete(ROW);
1901     delete.addFamily(FAMILIES[1], ts[0]);
1902     ht.delete(delete);
1903
1904     Get get = new Get(ROW);
1905     get.addFamily(FAMILIES[0]);
1906     get.readAllVersions();
1907     Result result = ht.get(get);
1908     assertTrue(Bytes.equals(result.getValue(FAMILIES[0], QUALIFIER), VALUES[0]));
1909   }
1910
1911   @Test
1912   public void testDeletes() throws Exception {
1913     final TableName tableName = TableName.valueOf(name.getMethodName());
1914
1915     byte [][] ROWS = makeNAscii(ROW, 6);
1916     byte [][] FAMILIES = makeNAscii(FAMILY, 3);
1917     byte [][] VALUES = makeN(VALUE, 5);
1918     long [] ts = {1000, 2000, 3000, 4000, 5000};
1919
1920     Table ht = TEST_UTIL.createTable(tableName, FAMILIES, 3);
1921
1922     Put put = new Put(ROW);
1923     put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
1924     put.addColumn(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]);
1925     ht.put(put);
1926
1927     Delete delete = new Delete(ROW);
1928     delete.addFamily(FAMILIES[0], ts[0]);
1929     ht.delete(delete);
1930
1931     Get get = new Get(ROW);
1932     get.addFamily(FAMILIES[0]);
1933     get.setMaxVersions(Integer.MAX_VALUE);
1934     Result result = ht.get(get);
1935     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1936         new long [] {ts[1]},
1937         new byte[][] {VALUES[1]},
1938         0, 0);
1939
1940     Scan scan = new Scan(ROW);
1941     scan.addFamily(FAMILIES[0]);
1942     scan.setMaxVersions(Integer.MAX_VALUE);
1943     result = getSingleScanResult(ht, scan);
1944     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1945         new long [] {ts[1]},
1946         new byte[][] {VALUES[1]},
1947         0, 0);
1948
1949     // Test delete latest version
1950     put = new Put(ROW);
1951     put.addColumn(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
1952     put.addColumn(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]);
1953     put.addColumn(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]);
1954     put.addColumn(FAMILIES[0], null, ts[4], VALUES[4]);
1955     put.addColumn(FAMILIES[0], null, ts[2], VALUES[2]);
1956     put.addColumn(FAMILIES[0], null, ts[3], VALUES[3]);
1957     ht.put(put);
1958
1959     delete = new Delete(ROW);
1960     delete.addColumn(FAMILIES[0], QUALIFIER); // ts[4]
1961     ht.delete(delete);
1962
1963     get = new Get(ROW);
1964     get.addColumn(FAMILIES[0], QUALIFIER);
1965     get.setMaxVersions(Integer.MAX_VALUE);
1966     result = ht.get(get);
1967     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1968         new long [] {ts[1], ts[2], ts[3]},
1969         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1970         0, 2);
1971
1972     scan = new Scan(ROW);
1973     scan.addColumn(FAMILIES[0], QUALIFIER);
1974     scan.setMaxVersions(Integer.MAX_VALUE);
1975     result = getSingleScanResult(ht, scan);
1976     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
1977         new long [] {ts[1], ts[2], ts[3]},
1978         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
1979         0, 2);
1980
1981     // Test for HBASE-1847
1982     delete = new Delete(ROW);
1983     delete.addColumn(FAMILIES[0], null);
1984     ht.delete(delete);
1985
1986     // Cleanup null qualifier
1987     delete = new Delete(ROW);
1988     delete.addColumns(FAMILIES[0], null);
1989     ht.delete(delete);
1990
1991     // Expected client behavior might be that you can re-put deleted values
1992     // But alas, this is not to be.  We can't put them back in either case.
1993
1994     put = new Put(ROW);
1995     put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]); // 1000
1996     put.addColumn(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]); // 5000
1997     ht.put(put);
1998
1999
2000     // It used to be due to the internal implementation of Get, that
2001     // the Get() call would return ts[4] UNLIKE the Scan below. With
2002     // the switch to using Scan for Get this is no longer the case.
2003     get = new Get(ROW);
2004     get.addFamily(FAMILIES[0]);
2005     get.setMaxVersions(Integer.MAX_VALUE);
2006     result = ht.get(get);
2007     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2008         new long [] {ts[1], ts[2], ts[3]},
2009         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
2010         0, 2);
2011
2012     // The Scanner returns the previous values, the expected-naive-unexpected behavior
2013
2014     scan = new Scan(ROW);
2015     scan.addFamily(FAMILIES[0]);
2016     scan.setMaxVersions(Integer.MAX_VALUE);
2017     result = getSingleScanResult(ht, scan);
2018     assertNResult(result, ROW, FAMILIES[0], QUALIFIER,
2019         new long [] {ts[1], ts[2], ts[3]},
2020         new byte[][] {VALUES[1], VALUES[2], VALUES[3]},
2021         0, 2);
2022
2023     // Test deleting an entire family from one row but not the other various ways
2024
2025     put = new Put(ROWS[0]);
2026     put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
2027     put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
2028     put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
2029     put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
2030     ht.put(put);
2031
2032     put = new Put(ROWS[1]);
2033     put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
2034     put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
2035     put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
2036     put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
2037     ht.put(put);
2038
2039     put = new Put(ROWS[2]);
2040     put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
2041     put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
2042     put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
2043     put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
2044     ht.put(put);
2045
2046     // Assert that above went in.
2047     get = new Get(ROWS[2]);
2048     get.addFamily(FAMILIES[1]);
2049     get.addFamily(FAMILIES[2]);
2050     get.setMaxVersions(Integer.MAX_VALUE);
2051     result = ht.get(get);
2052     assertTrue("Expected 4 key but received " + result.size() + ": " + result,
2053         result.size() == 4);
2054
2055     delete = new Delete(ROWS[0]);
2056     delete.addFamily(FAMILIES[2]);
2057     ht.delete(delete);
2058
2059     delete = new Delete(ROWS[1]);
2060     delete.addColumns(FAMILIES[1], QUALIFIER);
2061     ht.delete(delete);
2062
2063     delete = new Delete(ROWS[2]);
2064     delete.addColumn(FAMILIES[1], QUALIFIER);
2065     delete.addColumn(FAMILIES[1], QUALIFIER);
2066     delete.addColumn(FAMILIES[2], QUALIFIER);
2067     ht.delete(delete);
2068
2069     get = new Get(ROWS[0]);
2070     get.addFamily(FAMILIES[1]);
2071     get.addFamily(FAMILIES[2]);
2072     get.setMaxVersions(Integer.MAX_VALUE);
2073     result = ht.get(get);
2074     assertTrue("Expected 2 keys but received " + result.size(),
2075         result.size() == 2);
2076     assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER,
2077         new long [] {ts[0], ts[1]},
2078         new byte[][] {VALUES[0], VALUES[1]},
2079         0, 1);
2080
2081     scan = new Scan(ROWS[0]);
2082     scan.addFamily(FAMILIES[1]);
2083     scan.addFamily(FAMILIES[2]);
2084     scan.setMaxVersions(Integer.MAX_VALUE);
2085     result = getSingleScanResult(ht, scan);
2086     assertTrue("Expected 2 keys but received " + result.size(),
2087         result.size() == 2);
2088     assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER,
2089         new long [] {ts[0], ts[1]},
2090         new byte[][] {VALUES[0], VALUES[1]},
2091         0, 1);
2092
2093     get = new Get(ROWS[1]);
2094     get.addFamily(FAMILIES[1]);
2095     get.addFamily(FAMILIES[2]);
2096     get.setMaxVersions(Integer.MAX_VALUE);
2097     result = ht.get(get);
2098     assertTrue("Expected 2 keys but received " + result.size(),
2099         result.size() == 2);
2100
2101     scan = new Scan(ROWS[1]);
2102     scan.addFamily(FAMILIES[1]);
2103     scan.addFamily(FAMILIES[2]);
2104     scan.setMaxVersions(Integer.MAX_VALUE);
2105     result = getSingleScanResult(ht, scan);
2106     assertTrue("Expected 2 keys but received " + result.size(),
2107         result.size() == 2);
2108
2109     get = new Get(ROWS[2]);
2110     get.addFamily(FAMILIES[1]);
2111     get.addFamily(FAMILIES[2]);
2112     get.setMaxVersions(Integer.MAX_VALUE);
2113     result = ht.get(get);
2114     assertEquals(1, result.size());
2115     assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
2116         new long [] {ts[2]},
2117         new byte[][] {VALUES[2]},
2118         0, 0);
2119
2120     scan = new Scan(ROWS[2]);
2121     scan.addFamily(FAMILIES[1]);
2122     scan.addFamily(FAMILIES[2]);
2123     scan.setMaxVersions(Integer.MAX_VALUE);
2124     result = getSingleScanResult(ht, scan);
2125     assertEquals(1, result.size());
2126     assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
2127         new long [] {ts[2]},
2128         new byte[][] {VALUES[2]},
2129         0, 0);
2130
2131     // Test if we delete the family first in one row (HBASE-1541)
2132
2133     delete = new Delete(ROWS[3]);
2134     delete.addFamily(FAMILIES[1]);
2135     ht.delete(delete);
2136
2137     put = new Put(ROWS[3]);
2138     put.addColumn(FAMILIES[2], QUALIFIER, VALUES[0]);
2139     ht.put(put);
2140
2141     put = new Put(ROWS[4]);
2142     put.addColumn(FAMILIES[1], QUALIFIER, VALUES[1]);
2143     put.addColumn(FAMILIES[2], QUALIFIER, VALUES[2]);
2144     ht.put(put);
2145
2146     get = new Get(ROWS[3]);
2147     get.addFamily(FAMILIES[1]);
2148     get.addFamily(FAMILIES[2]);
2149     get.setMaxVersions(Integer.MAX_VALUE);
2150     result = ht.get(get);
2151     assertTrue("Expected 1 key but received " + result.size(),
2152         result.size() == 1);
2153
2154     get = new Get(ROWS[4]);
2155     get.addFamily(FAMILIES[1]);
2156     get.addFamily(FAMILIES[2]);
2157     get.setMaxVersions(Integer.MAX_VALUE);
2158     result = ht.get(get);
2159     assertTrue("Expected 2 keys but received " + result.size(),
2160         result.size() == 2);
2161
2162     scan = new Scan(ROWS[3]);
2163     scan.addFamily(FAMILIES[1]);
2164     scan.addFamily(FAMILIES[2]);
2165     scan.setMaxVersions(Integer.MAX_VALUE);
2166     ResultScanner scanner = ht.getScanner(scan);
2167     result = scanner.next();
2168     assertTrue("Expected 1 key but received " + result.size(),
2169         result.size() == 1);
2170     assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[3]));
2171     assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[0]));
2172     result = scanner.next();
2173     assertTrue("Expected 2 keys but received " + result.size(),
2174         result.size() == 2);
2175     assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[4]));
2176     assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[1]), ROWS[4]));
2177     assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[1]));
2178     assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[1]), VALUES[2]));
2179     scanner.close();
2180
2181     // Add test of bulk deleting.
2182     for (int i = 0; i < 10; i++) {
2183       byte [] bytes = Bytes.toBytes(i);
2184       put = new Put(bytes);
2185       put.setDurability(Durability.SKIP_WAL);
2186       put.addColumn(FAMILIES[0], QUALIFIER, bytes);
2187       ht.put(put);
2188     }
2189     for (int i = 0; i < 10; i++) {
2190       byte [] bytes = Bytes.toBytes(i);
2191       get = new Get(bytes);
2192       get.addFamily(FAMILIES[0]);
2193       result = ht.get(get);
2194       assertTrue(result.size() == 1);
2195     }
2196     ArrayList<Delete> deletes = new ArrayList<>();
2197     for (int i = 0; i < 10; i++) {
2198       byte [] bytes = Bytes.toBytes(i);
2199       delete = new Delete(bytes);
2200       delete.addFamily(FAMILIES[0]);
2201       deletes.add(delete);
2202     }
2203     ht.delete(deletes);
2204     for (int i = 0; i < 10; i++) {
2205       byte [] bytes = Bytes.toBytes(i);
2206       get = new Get(bytes);
2207       get.addFamily(FAMILIES[0]);
2208       result = ht.get(get);
2209       assertTrue(result.isEmpty());
2210     }
2211   }
2212
2213   /**
2214    * Test batch operations with combination of valid and invalid args
2215    */
2216   @Test
2217   public void testBatchOperationsWithErrors() throws Exception {
2218     final TableName tableName = TableName.valueOf(name.getMethodName());
2219     Table foo = TEST_UTIL.createTable(tableName, new byte[][] {FAMILY}, 10);
2220
2221     int NUM_OPS = 100;
2222     int FAILED_OPS = 50;
2223
2224     RetriesExhaustedWithDetailsException expectedException = null;
2225     IllegalArgumentException iae = null;
2226
2227     // 1.1 Put with no column families (local validation, runtime exception)
2228     List<Put> puts = new ArrayList<Put>(NUM_OPS);
2229     for (int i = 0; i != NUM_OPS; i++) {
2230       Put put = new Put(Bytes.toBytes(i));
2231       puts.add(put);
2232     }
2233
2234     try {
2235       foo.put(puts);
2236     } catch (IllegalArgumentException e) {
2237       iae = e;
2238     }
2239     assertNotNull(iae);
2240     assertEquals(NUM_OPS, puts.size());
2241
2242     // 1.2 Put with invalid column family
2243     iae = null;
2244     puts.clear();
2245     for (int i = 0; i != NUM_OPS; i++) {
2246       Put put = new Put(Bytes.toBytes(i));
2247       put.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY, Bytes.toBytes(i));
2248       puts.add(put);
2249     }
2250
2251     try {
2252       foo.put(puts);
2253     } catch (RetriesExhaustedWithDetailsException e) {
2254       expectedException = e;
2255     }
2256     assertNotNull(expectedException);
2257     assertEquals(FAILED_OPS, expectedException.exceptions.size());
2258     assertTrue(expectedException.actions.contains(puts.get(1)));
2259
2260     // 2.1 Get non-existent rows
2261     List<Get> gets = new ArrayList<>(NUM_OPS);
2262     for (int i = 0; i < NUM_OPS; i++) {
2263       Get get = new Get(Bytes.toBytes(i));
2264       // get.addColumn(FAMILY, FAMILY);
2265       gets.add(get);
2266     }
2267     Result[] getsResult = foo.get(gets);
2268
2269     assertNotNull(getsResult);
2270     assertEquals(NUM_OPS, getsResult.length);
2271     assertNull(getsResult[1].getRow());
2272
2273     // 2.2 Get with invalid column family
2274     gets.clear();
2275     getsResult = null;
2276     expectedException = null;
2277     for (int i = 0; i < NUM_OPS; i++) {
2278       Get get = new Get(Bytes.toBytes(i));
2279       get.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY);
2280       gets.add(get);
2281     }
2282     try {
2283       getsResult = foo.get(gets);
2284     } catch (RetriesExhaustedWithDetailsException e) {
2285       expectedException = e;
2286     }
2287     assertNull(getsResult);
2288     assertNotNull(expectedException);
2289     assertEquals(FAILED_OPS, expectedException.exceptions.size());
2290     assertTrue(expectedException.actions.contains(gets.get(1)));
2291
2292     // 3.1 Delete with invalid column family
2293     expectedException = null;
2294     List<Delete> deletes = new ArrayList<>(NUM_OPS);
2295     for (int i = 0; i < NUM_OPS; i++) {
2296       Delete delete = new Delete(Bytes.toBytes(i));
2297       delete.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY);
2298       deletes.add(delete);
2299     }
2300     try {
2301       foo.delete(deletes);
2302     } catch (RetriesExhaustedWithDetailsException e) {
2303       expectedException = e;
2304     }
2305     assertEquals((NUM_OPS - FAILED_OPS), deletes.size());
2306     assertNotNull(expectedException);
2307     assertEquals(FAILED_OPS, expectedException.exceptions.size());
2308     assertTrue(expectedException.actions.contains(deletes.get(1)));
2309
2310
2311     // 3.2 Delete non-existent rows
2312     deletes.clear();
2313     for (int i = 0; i < NUM_OPS; i++) {
2314       Delete delete = new Delete(Bytes.toBytes(i));
2315       deletes.add(delete);
2316     }
2317     foo.delete(deletes);
2318
2319     assertTrue(deletes.isEmpty());
2320   }
2321
2322   /*
2323    * Baseline "scalability" test.
2324    *
2325    * Tests one hundred families, one million columns, one million versions
2326    */
2327   @Ignore @Test
2328   public void testMillions() throws Exception {
2329
2330     // 100 families
2331
2332     // millions of columns
2333
2334     // millions of versions
2335
2336   }
2337
2338   @Ignore @Test
2339   public void testMultipleRegionsAndBatchPuts() throws Exception {
2340     // Two family table
2341
2342     // Insert lots of rows
2343
2344     // Insert to the same row with batched puts
2345
2346     // Insert to multiple rows with batched puts
2347
2348     // Split the table
2349
2350     // Get row from first region
2351
2352     // Get row from second region
2353
2354     // Scan all rows
2355
2356     // Insert to multiple regions with batched puts
2357
2358     // Get row from first region
2359
2360     // Get row from second region
2361
2362     // Scan all rows
2363
2364
2365   }
2366
2367   @Ignore @Test
2368   public void testMultipleRowMultipleFamily() throws Exception {
2369
2370   }
2371
2372   //
2373   // JIRA Testers
2374   //
2375
2376   /**
2377    * HBASE-867
2378    *    If millions of columns in a column family, hbase scanner won't come up
2379    *
2380    *    Test will create numRows rows, each with numColsPerRow columns
2381    *    (1 version each), and attempt to scan them all.
2382    *
2383    *    To test at scale, up numColsPerRow to the millions
2384    *    (have not gotten that to work running as junit though)
2385    */
2386   @Test
2387   public void testJiraTest867() throws Exception {
2388     int numRows = 10;
2389     int numColsPerRow = 2000;
2390
2391     final TableName tableName = TableName.valueOf(name.getMethodName());
2392
2393     byte [][] ROWS = makeN(ROW, numRows);
2394     byte [][] QUALIFIERS = makeN(QUALIFIER, numColsPerRow);
2395
2396     Table ht = TEST_UTIL.createTable(tableName, FAMILY);
2397
2398     // Insert rows
2399
2400     for(int i=0;i<numRows;i++) {
2401       Put put = new Put(ROWS[i]);
2402       put.setDurability(Durability.SKIP_WAL);
2403       for(int j=0;j<numColsPerRow;j++) {
2404         put.addColumn(FAMILY, QUALIFIERS[j], QUALIFIERS[j]);
2405       }
2406       assertTrue("Put expected to contain " + numColsPerRow + " columns but " +
2407           "only contains " + put.size(), put.size() == numColsPerRow);
2408       ht.put(put);
2409     }
2410
2411     // Get a row
2412     Get get = new Get(ROWS[numRows-1]);
2413     Result result = ht.get(get);
2414     assertNumKeys(result, numColsPerRow);
2415     Cell [] keys = result.rawCells();
2416     for(int i=0;i<result.size();i++) {
2417       assertKey(keys[i], ROWS[numRows-1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2418     }
2419
2420     // Scan the rows
2421     Scan scan = new Scan();
2422     ResultScanner scanner = ht.getScanner(scan);
2423     int rowCount = 0;
2424     while((result = scanner.next()) != null) {
2425       assertNumKeys(result, numColsPerRow);
2426       Cell [] kvs = result.rawCells();
2427       for(int i=0;i<numColsPerRow;i++) {
2428         assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2429       }
2430       rowCount++;
2431     }
2432     scanner.close();
2433     assertTrue("Expected to scan " + numRows + " rows but actually scanned "
2434         + rowCount + " rows", rowCount == numRows);
2435
2436     // flush and try again
2437
2438     TEST_UTIL.flush();
2439
2440     // Get a row
2441     get = new Get(ROWS[numRows-1]);
2442     result = ht.get(get);
2443     assertNumKeys(result, numColsPerRow);
2444     keys = result.rawCells();
2445     for(int i=0;i<result.size();i++) {
2446       assertKey(keys[i], ROWS[numRows-1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2447     }
2448
2449     // Scan the rows
2450     scan = new Scan();
2451     scanner = ht.getScanner(scan);
2452     rowCount = 0;
2453     while((result = scanner.next()) != null) {
2454       assertNumKeys(result, numColsPerRow);
2455       Cell [] kvs = result.rawCells();
2456       for(int i=0;i<numColsPerRow;i++) {
2457         assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
2458       }
2459       rowCount++;
2460     }
2461     scanner.close();
2462     assertTrue("Expected to scan " + numRows + " rows but actually scanned "
2463         + rowCount + " rows", rowCount == numRows);
2464
2465   }
2466
2467   /**
2468    * HBASE-861
2469    *    get with timestamp will return a value if there is a version with an
2470    *    earlier timestamp
2471    */
2472   @Test
2473   public void testJiraTest861() throws Exception {
2474     final TableName tableName = TableName.valueOf(name.getMethodName());
2475     byte [][] VALUES = makeNAscii(VALUE, 7);
2476     long [] STAMPS = makeStamps(7);
2477
2478     Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10);
2479
2480     // Insert three versions
2481
2482     Put put = new Put(ROW);
2483     put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2484     put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2485     put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2486     ht.put(put);
2487
2488     // Get the middle value
2489     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2490
2491     // Try to get one version before (expect fail)
2492     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2493
2494     // Try to get one version after (expect fail)
2495     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2496
2497     // Try same from storefile
2498     TEST_UTIL.flush();
2499     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2500     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2501     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2502
2503     // Insert two more versions surrounding others, into memstore
2504     put = new Put(ROW);
2505     put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2506     put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
2507     ht.put(put);
2508
2509     // Check we can get everything we should and can't get what we shouldn't
2510     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2511     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2512     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2513     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2514     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2515     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2516     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
2517
2518     // Try same from two storefiles
2519     TEST_UTIL.flush();
2520     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2521     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
2522     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2523     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2524     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2525     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
2526     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
2527
2528   }
2529
2530   /**
2531    * HBASE-33
2532    *    Add a HTable get/obtainScanner method that retrieves all versions of a
2533    *    particular column and row between two timestamps
2534    */
2535   @Test
2536   public void testJiraTest33() throws Exception {
2537     final TableName tableName = TableName.valueOf(name.getMethodName());
2538     byte [][] VALUES = makeNAscii(VALUE, 7);
2539     long [] STAMPS = makeStamps(7);
2540
2541     Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10);
2542
2543     // Insert lots versions
2544
2545     Put put = new Put(ROW);
2546     put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2547     put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2548     put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2549     put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2550     put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2551     put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2552     ht.put(put);
2553
2554     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2555     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2556     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2557     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2558
2559     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2560     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2561     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2562     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2563
2564     // Try same from storefile
2565     TEST_UTIL.flush();
2566
2567     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2568     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2569     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2570     getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2571
2572     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2573     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
2574     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2575     scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
2576
2577   }
2578
2579   /**
2580    * HBASE-1014
2581    *    commit(BatchUpdate) method should return timestamp
2582    */
2583   @Test
2584   public void testJiraTest1014() throws Exception {
2585     final TableName tableName = TableName.valueOf(name.getMethodName());
2586
2587     Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10);
2588
2589     long manualStamp = 12345;
2590
2591     // Insert lots versions
2592
2593     Put put = new Put(ROW);
2594     put.addColumn(FAMILY, QUALIFIER, manualStamp, VALUE);
2595     ht.put(put);
2596
2597     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, manualStamp, VALUE);
2598     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp-1);
2599     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp+1);
2600
2601   }
2602
2603   /**
2604    * HBASE-1182
2605    *    Scan for columns > some timestamp
2606    */
2607   @Test
2608   public void testJiraTest1182() throws Exception {
2609     final TableName tableName = TableName.valueOf(name.getMethodName());
2610     byte [][] VALUES = makeNAscii(VALUE, 7);
2611     long [] STAMPS = makeStamps(7);
2612
2613     Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10);
2614
2615     // Insert lots versions
2616
2617     Put put = new Put(ROW);
2618     put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2619     put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2620     put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2621     put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2622     put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2623     put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2624     ht.put(put);
2625
2626     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2627     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2628     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2629
2630     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2631     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2632     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2633
2634     // Try same from storefile
2635     TEST_UTIL.flush();
2636
2637     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2638     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2639     getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2640
2641     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2642     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
2643     scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
2644   }
2645
2646   /**
2647    * HBASE-52
2648    *    Add a means of scanning over all versions
2649    */
2650   @Test
2651   public void testJiraTest52() throws Exception {
2652     final TableName tableName = TableName.valueOf(name.getMethodName());
2653     byte [][] VALUES = makeNAscii(VALUE, 7);
2654     long [] STAMPS = makeStamps(7);
2655
2656     Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10);
2657
2658     // Insert lots versions
2659
2660     Put put = new Put(ROW);
2661     put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
2662     put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
2663     put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
2664     put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
2665     put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
2666     put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
2667     ht.put(put);
2668
2669     getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2670
2671     scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2672
2673     // Try same from storefile
2674     TEST_UTIL.flush();
2675
2676     getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2677
2678     scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
2679   }
2680
2681   //
2682   // Bulk Testers
2683   //
2684
2685   private void getVersionRangeAndVerifyGreaterThan(Table ht, byte [] row,
2686       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
2687       int start, int end)
2688   throws IOException {
2689     Get get = new Get(row);
2690     get.addColumn(family, qualifier);
2691     get.setMaxVersions(Integer.MAX_VALUE);
2692     get.setTimeRange(stamps[start+1], Long.MAX_VALUE);
2693     Result result = ht.get(get);
2694     assertNResult(result, row, family, qualifier, stamps, values, start+1, end);
2695   }
2696
2697   private void getVersionRangeAndVerify(Table ht, byte [] row, byte [] family,
2698       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2699   throws IOException {
2700     Get get = new Get(row);
2701     get.addColumn(family, qualifier);
2702     get.setMaxVersions(Integer.MAX_VALUE);
2703     get.setTimeRange(stamps[start], stamps[end]+1);
2704     Result result = ht.get(get);
2705     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2706   }
2707
2708   private void getAllVersionsAndVerify(Table ht, byte [] row, byte [] family,
2709       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2710   throws IOException {
2711     Get get = new Get(row);
2712     get.addColumn(family, qualifier);
2713     get.setMaxVersions(Integer.MAX_VALUE);
2714     Result result = ht.get(get);
2715     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2716   }
2717
2718   private void scanVersionRangeAndVerifyGreaterThan(Table ht, byte [] row,
2719       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
2720       int start, int end)
2721   throws IOException {
2722     Scan scan = new Scan(row);
2723     scan.addColumn(family, qualifier);
2724     scan.setMaxVersions(Integer.MAX_VALUE);
2725     scan.setTimeRange(stamps[start+1], Long.MAX_VALUE);
2726     Result result = getSingleScanResult(ht, scan);
2727     assertNResult(result, row, family, qualifier, stamps, values, start+1, end);
2728   }
2729
2730   private void scanVersionRangeAndVerify(Table ht, byte [] row, byte [] family,
2731       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2732   throws IOException {
2733     Scan scan = new Scan(row);
2734     scan.addColumn(family, qualifier);
2735     scan.setMaxVersions(Integer.MAX_VALUE);
2736     scan.setTimeRange(stamps[start], stamps[end]+1);
2737     Result result = getSingleScanResult(ht, scan);
2738     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2739   }
2740
2741   private void scanAllVersionsAndVerify(Table ht, byte [] row, byte [] family,
2742       byte [] qualifier, long [] stamps, byte [][] values, int start, int end)
2743   throws IOException {
2744     Scan scan = new Scan(row);
2745     scan.addColumn(family, qualifier);
2746     scan.setMaxVersions(Integer.MAX_VALUE);
2747     Result result = getSingleScanResult(ht, scan);
2748     assertNResult(result, row, family, qualifier, stamps, values, start, end);
2749   }
2750
2751   private void getVersionAndVerify(Table ht, byte [] row, byte [] family,
2752       byte [] qualifier, long stamp, byte [] value)
2753   throws Exception {
2754     Get get = new Get(row);
2755     get.addColumn(family, qualifier);
2756     get.setTimestamp(stamp);
2757     get.setMaxVersions(Integer.MAX_VALUE);
2758     Result result = ht.get(get);
2759     assertSingleResult(result, row, family, qualifier, stamp, value);
2760   }
2761
2762   private void getVersionAndVerifyMissing(Table ht, byte [] row, byte [] family,
2763       byte [] qualifier, long stamp)
2764   throws Exception {
2765     Get get = new Get(row);
2766     get.addColumn(family, qualifier);
2767     get.setTimestamp(stamp);
2768     get.setMaxVersions(Integer.MAX_VALUE);
2769     Result result = ht.get(get);
2770     assertEmptyResult(result);
2771   }
2772
2773   private void scanVersionAndVerify(Table ht, byte [] row, byte [] family,
2774       byte [] qualifier, long stamp, byte [] value)
2775   throws Exception {
2776     Scan scan = new Scan(row);
2777     scan.addColumn(family, qualifier);
2778     scan.setTimestamp(stamp);
2779     scan.setMaxVersions(Integer.MAX_VALUE);
2780     Result result = getSingleScanResult(ht, scan);
2781     assertSingleResult(result, row, family, qualifier, stamp, value);
2782   }
2783
2784   private void scanVersionAndVerifyMissing(Table ht, byte [] row,
2785       byte [] family, byte [] qualifier, long stamp)
2786   throws Exception {
2787     Scan scan = new Scan(row);
2788     scan.addColumn(family, qualifier);
2789     scan.setTimestamp(stamp);
2790     scan.setMaxVersions(Integer.MAX_VALUE);
2791     Result result = getSingleScanResult(ht, scan);
2792     assertNullResult(result);
2793   }
2794
2795   private void getTestNull(Table ht, byte [] row, byte [] family,
2796       byte [] value)
2797   throws Exception {
2798
2799     Get get = new Get(row);
2800     get.addColumn(family, null);
2801     Result result = ht.get(get);
2802     assertSingleResult(result, row, family, null, value);
2803
2804     get = new Get(row);
2805     get.addColumn(family, HConstants.EMPTY_BYTE_ARRAY);
2806     result = ht.get(get);
2807     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2808
2809     get = new Get(row);
2810     get.addFamily(family);
2811     result = ht.get(get);
2812     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2813
2814     get = new Get(row);
2815     result = ht.get(get);
2816     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2817
2818   }
2819
2820   private void getTestNull(Table ht, byte[] row, byte[] family, long value) throws Exception {
2821     Get get = new Get(row);
2822     get.addColumn(family, null);
2823     Result result = ht.get(get);
2824     assertSingleResult(result, row, family, null, value);
2825
2826     get = new Get(row);
2827     get.addColumn(family, HConstants.EMPTY_BYTE_ARRAY);
2828     result = ht.get(get);
2829     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2830
2831     get = new Get(row);
2832     get.addFamily(family);
2833     result = ht.get(get);
2834     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2835
2836     get = new Get(row);
2837     result = ht.get(get);
2838     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2839   }
2840
2841   private void scanTestNull(Table ht, byte[] row, byte[] family, byte[] value)
2842       throws Exception {
2843     scanTestNull(ht, row, family, value, false);
2844   }
2845
2846   private void scanTestNull(Table ht, byte[] row, byte[] family, byte[] value,
2847       boolean isReversedScan) throws Exception {
2848
2849     Scan scan = new Scan();
2850     scan.setReversed(isReversedScan);
2851     scan.addColumn(family, null);
2852     Result result = getSingleScanResult(ht, scan);
2853     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2854
2855     scan = new Scan();
2856     scan.setReversed(isReversedScan);
2857     scan.addColumn(family, HConstants.EMPTY_BYTE_ARRAY);
2858     result = getSingleScanResult(ht, scan);
2859     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2860
2861     scan = new Scan();
2862     scan.setReversed(isReversedScan);
2863     scan.addFamily(family);
2864     result = getSingleScanResult(ht, scan);
2865     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2866
2867     scan = new Scan();
2868     scan.setReversed(isReversedScan);
2869     result = getSingleScanResult(ht, scan);
2870     assertSingleResult(result, row, family, HConstants.EMPTY_BYTE_ARRAY, value);
2871
2872   }
2873
2874   private void singleRowGetTest(Table ht, byte [][] ROWS, byte [][] FAMILIES,
2875       byte [][] QUALIFIERS, byte [][] VALUES)
2876   throws Exception {
2877
2878     // Single column from memstore
2879     Get get = new Get(ROWS[0]);
2880     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
2881     Result result = ht.get(get);
2882     assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]);
2883
2884     // Single column from storefile
2885     get = new Get(ROWS[0]);
2886     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
2887     result = ht.get(get);
2888     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
2889
2890     // Single column from storefile, family match
2891     get = new Get(ROWS[0]);
2892     get.addFamily(FAMILIES[7]);
2893     result = ht.get(get);
2894     assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]);
2895
2896     // Two columns, one from memstore one from storefile, same family,
2897     // wildcard match
2898     get = new Get(ROWS[0]);
2899     get.addFamily(FAMILIES[4]);
2900     result = ht.get(get);
2901     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2902         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2903
2904     // Two columns, one from memstore one from storefile, same family,
2905     // explicit match
2906     get = new Get(ROWS[0]);
2907     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
2908     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
2909     result = ht.get(get);
2910     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
2911         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
2912
2913     // Three column, one from memstore two from storefile, different families,
2914     // wildcard match
2915     get = new Get(ROWS[0]);
2916     get.addFamily(FAMILIES[4]);
2917     get.addFamily(FAMILIES[7]);
2918     result = ht.get(get);
2919     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2920         new int [][] { {4, 0, 0}, {4, 4, 4}, {7, 7, 7} });
2921
2922     // Multiple columns from everywhere storefile, many family, wildcard
2923     get = new Get(ROWS[0]);
2924     get.addFamily(FAMILIES[2]);
2925     get.addFamily(FAMILIES[4]);
2926     get.addFamily(FAMILIES[6]);
2927     get.addFamily(FAMILIES[7]);
2928     result = ht.get(get);
2929     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2930         new int [][] {
2931           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
2932     });
2933
2934     // Multiple columns from everywhere storefile, many family, wildcard
2935     get = new Get(ROWS[0]);
2936     get.addColumn(FAMILIES[2], QUALIFIERS[2]);
2937     get.addColumn(FAMILIES[2], QUALIFIERS[4]);
2938     get.addColumn(FAMILIES[4], QUALIFIERS[0]);
2939     get.addColumn(FAMILIES[4], QUALIFIERS[4]);
2940     get.addColumn(FAMILIES[6], QUALIFIERS[6]);
2941     get.addColumn(FAMILIES[6], QUALIFIERS[7]);
2942     get.addColumn(FAMILIES[7], QUALIFIERS[7]);
2943     get.addColumn(FAMILIES[7], QUALIFIERS[8]);
2944     result = ht.get(get);
2945     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2946         new int [][] {
2947           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
2948     });
2949
2950     // Everything
2951     get = new Get(ROWS[0]);
2952     result = ht.get(get);
2953     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
2954         new int [][] {
2955           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}, {9, 0, 0}
2956     });
2957
2958     // Get around inserted columns
2959
2960     get = new Get(ROWS[1]);
2961     result = ht.get(get);
2962     assertEmptyResult(result);
2963
2964     get = new Get(ROWS[0]);
2965     get.addColumn(FAMILIES[4], QUALIFIERS[3]);
2966     get.addColumn(FAMILIES[2], QUALIFIERS[3]);
2967     result = ht.get(get);
2968     assertEmptyResult(result);
2969
2970   }
2971
2972   private void singleRowScanTest(Table ht, byte [][] ROWS, byte [][] FAMILIES,
2973       byte [][] QUALIFIERS, byte [][] VALUES)
2974   throws Exception {
2975
2976     // Single column from memstore
2977     Scan scan = new Scan();
2978     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
2979     Result result = getSingleScanResult(ht, scan);
2980     assertSingleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0]);
2981
2982     // Single column from storefile
2983     scan = new Scan();
2984     scan.addColumn(FAMILIES[2], QUALIFIERS[2]);
2985     result = getSingleScanResult(ht, scan);
2986     assertSingleResult(result, ROWS[0], FAMILIES[2], QUALIFIERS[2], VALUES[2]);
2987
2988     // Single column from storefile, family match
2989     scan = new Scan();
2990     scan.addFamily(FAMILIES[7]);
2991     result = getSingleScanResult(ht, scan);
2992     assertSingleResult(result, ROWS[0], FAMILIES[7], QUALIFIERS[7], VALUES[7]);
2993
2994     // Two columns, one from memstore one from storefile, same family,
2995     // wildcard match
2996     scan = new Scan();
2997     scan.addFamily(FAMILIES[4]);
2998     result = getSingleScanResult(ht, scan);
2999     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
3000         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
3001
3002     // Two columns, one from memstore one from storefile, same family,
3003     // explicit match
3004     scan = new Scan();
3005     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
3006     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
3007     result = getSingleScanResult(ht, scan);
3008     assertDoubleResult(result, ROWS[0], FAMILIES[4], QUALIFIERS[0], VALUES[0],
3009         FAMILIES[4], QUALIFIERS[4], VALUES[4]);
3010
3011     // Three column, one from memstore two from storefile, different families,
3012     // wildcard match
3013     scan = new Scan();
3014     scan.addFamily(FAMILIES[4]);
3015     scan.addFamily(FAMILIES[7]);
3016     result = getSingleScanResult(ht, scan);
3017     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
3018         new int [][] { {4, 0, 0}, {4, 4, 4}, {7, 7, 7} });
3019
3020     // Multiple columns from everywhere storefile, many family, wildcard
3021     scan = new Scan();
3022     scan.addFamily(FAMILIES[2]);
3023     scan.addFamily(FAMILIES[4]);
3024     scan.addFamily(FAMILIES[6]);
3025     scan.addFamily(FAMILIES[7]);
3026     result = getSingleScanResult(ht, scan);
3027     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
3028         new int [][] {
3029           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
3030     });
3031
3032     // Multiple columns from everywhere storefile, many family, wildcard
3033     scan = new Scan();
3034     scan.addColumn(FAMILIES[2], QUALIFIERS[2]);
3035     scan.addColumn(FAMILIES[2], QUALIFIERS[4]);
3036     scan.addColumn(FAMILIES[4], QUALIFIERS[0]);
3037     scan.addColumn(FAMILIES[4], QUALIFIERS[4]);
3038     scan.addColumn(FAMILIES[6], QUALIFIERS[6]);
3039     scan.addColumn(FAMILIES[6], QUALIFIERS[7]);
3040     scan.addColumn(FAMILIES[7], QUALIFIERS[7]);
3041     scan.addColumn(FAMILIES[7], QUALIFIERS[8]);
3042     result = getSingleScanResult(ht, scan);
3043     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
3044         new int [][] {
3045           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}
3046     });
3047
3048     // Everything
3049     scan = new Scan();
3050     result = getSingleScanResult(ht, scan);
3051     assertNResult(result, ROWS[0], FAMILIES, QUALIFIERS, VALUES,
3052         new int [][] {
3053           {2, 2, 2}, {2, 4, 4}, {4, 0, 0}, {4, 4, 4}, {6, 6, 6}, {6, 7, 7}, {7, 7, 7}, {9, 0, 0}
3054     });
3055
3056     // Scan around inserted columns
3057
3058     scan = new Scan(ROWS[1]);
3059     result = getSingleScanResult(ht, scan);
3060     assertNullResult(result);
3061
3062     scan = new Scan();
3063     scan.addColumn(FAMILIES[4], QUALIFIERS[3]);
3064     scan.addColumn(FAMILIES[2], QUALIFIERS[3]);
3065     result = getSingleScanResult(ht, scan);
3066     assertNullResult(result);
3067   }
3068
3069   /**
3070    * Verify a single column using gets.
3071    * Expects family and qualifier arrays to be valid for at least
3072    * the range:  idx-2 < idx < idx+2
3073    */
3074   private void getVerifySingleColumn(Table ht,
3075       byte [][] ROWS, int ROWIDX,
3076       byte [][] FAMILIES, int FAMILYIDX,
3077       byte [][] QUALIFIERS, int QUALIFIERIDX,
3078       byte [][] VALUES, int VALUEIDX)
3079   throws Exception {
3080
3081     Get get = new Get(ROWS[ROWIDX]);
3082     Result result = ht.get(get);
3083     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3084         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3085
3086     get = new Get(ROWS[ROWIDX]);
3087     get.addFamily(FAMILIES[FAMILYIDX]);
3088     result = ht.get(get);
3089     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3090         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3091
3092     get = new Get(ROWS[ROWIDX]);
3093     get.addFamily(FAMILIES[FAMILYIDX-2]);
3094     get.addFamily(FAMILIES[FAMILYIDX]);
3095     get.addFamily(FAMILIES[FAMILYIDX+2]);
3096     result = ht.get(get);
3097     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3098         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3099
3100     get = new Get(ROWS[ROWIDX]);
3101     get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[0]);
3102     result = ht.get(get);
3103     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3104         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3105
3106     get = new Get(ROWS[ROWIDX]);
3107     get.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[1]);
3108     get.addFamily(FAMILIES[FAMILYIDX]);
3109     result = ht.get(get);
3110     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3111         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3112
3113     get = new Get(ROWS[ROWIDX]);
3114     get.addFamily(FAMILIES[FAMILYIDX]);
3115     get.addColumn(FAMILIES[FAMILYIDX+1], QUALIFIERS[1]);
3116     get.addColumn(FAMILIES[FAMILYIDX-2], QUALIFIERS[1]);
3117     get.addFamily(FAMILIES[FAMILYIDX-1]);
3118     get.addFamily(FAMILIES[FAMILYIDX+2]);
3119     result = ht.get(get);
3120     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3121         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3122
3123   }
3124
3125
3126   /**
3127    * Verify a single column using scanners.
3128    * Expects family and qualifier arrays to be valid for at least
3129    * the range:  idx-2 to idx+2
3130    * Expects row array to be valid for at least idx to idx+2
3131    */
3132   private void scanVerifySingleColumn(Table ht,
3133       byte [][] ROWS, int ROWIDX,
3134       byte [][] FAMILIES, int FAMILYIDX,
3135       byte [][] QUALIFIERS, int QUALIFIERIDX,
3136       byte [][] VALUES, int VALUEIDX)
3137   throws Exception {
3138
3139     Scan scan = new Scan();
3140     Result result = getSingleScanResult(ht, scan);
3141     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3142         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3143
3144     scan = new Scan(ROWS[ROWIDX]);
3145     result = getSingleScanResult(ht, scan);
3146     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3147         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3148
3149     scan = new Scan(ROWS[ROWIDX], ROWS[ROWIDX+1]);
3150     result = getSingleScanResult(ht, scan);
3151     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3152         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3153
3154     scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX+1]);
3155     result = getSingleScanResult(ht, scan);
3156     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3157         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3158
3159     scan = new Scan();
3160     scan.addFamily(FAMILIES[FAMILYIDX]);
3161     result = getSingleScanResult(ht, scan);
3162     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3163         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3164
3165     scan = new Scan();
3166     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]);
3167     result = getSingleScanResult(ht, scan);
3168     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3169         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3170
3171     scan = new Scan();
3172     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX+1]);
3173     scan.addFamily(FAMILIES[FAMILYIDX]);
3174     result = getSingleScanResult(ht, scan);
3175     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3176         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3177
3178     scan = new Scan();
3179     scan.addColumn(FAMILIES[FAMILYIDX-1], QUALIFIERS[QUALIFIERIDX+1]);
3180     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX]);
3181     scan.addFamily(FAMILIES[FAMILYIDX+1]);
3182     result = getSingleScanResult(ht, scan);
3183     assertSingleResult(result, ROWS[ROWIDX], FAMILIES[FAMILYIDX],
3184         QUALIFIERS[QUALIFIERIDX], VALUES[VALUEIDX]);
3185
3186   }
3187
3188   /**
3189    * Verify we do not read any values by accident around a single column
3190    * Same requirements as getVerifySingleColumn
3191    */
3192   private void getVerifySingleEmpty(Table ht,
3193       byte [][] ROWS, int ROWIDX,
3194       byte [][] FAMILIES, int FAMILYIDX,
3195       byte [][] QUALIFIERS, int QUALIFIERIDX)
3196   throws Exception {
3197
3198     Get get = new Get(ROWS[ROWIDX]);
3199     get.addFamily(FAMILIES[4]);
3200     get.addColumn(FAMILIES[4], QUALIFIERS[1]);
3201     Result result = ht.get(get);
3202     assertEmptyResult(result);
3203
3204     get = new Get(ROWS[ROWIDX]);
3205     get.addFamily(FAMILIES[4]);
3206     get.addColumn(FAMILIES[4], QUALIFIERS[2]);
3207     result = ht.get(get);
3208     assertEmptyResult(result);
3209
3210     get = new Get(ROWS[ROWIDX]);
3211     get.addFamily(FAMILIES[3]);
3212     get.addColumn(FAMILIES[4], QUALIFIERS[2]);
3213     get.addFamily(FAMILIES[5]);
3214     result = ht.get(get);
3215     assertEmptyResult(result);
3216
3217     get = new Get(ROWS[ROWIDX+1]);
3218     result = ht.get(get);
3219     assertEmptyResult(result);
3220
3221   }
3222
3223   private void scanVerifySingleEmpty(Table ht,
3224       byte [][] ROWS, int ROWIDX,
3225       byte [][] FAMILIES, int FAMILYIDX,
3226       byte [][] QUALIFIERS, int QUALIFIERIDX)
3227   throws Exception {
3228
3229     Scan scan = new Scan(ROWS[ROWIDX+1]);
3230     Result result = getSingleScanResult(ht, scan);
3231     assertNullResult(result);
3232
3233     scan = new Scan(ROWS[ROWIDX+1],ROWS[ROWIDX+2]);
3234     result = getSingleScanResult(ht, scan);
3235     assertNullResult(result);
3236
3237     scan = new Scan(HConstants.EMPTY_START_ROW, ROWS[ROWIDX]);
3238     result = getSingleScanResult(ht, scan);
3239     assertNullResult(result);
3240
3241     scan = new Scan();
3242     scan.addColumn(FAMILIES[FAMILYIDX], QUALIFIERS[QUALIFIERIDX+1]);
3243     scan.addFamily(FAMILIES[FAMILYIDX-1]);
3244     result = getSingleScanResult(ht, scan);
3245     assertNullResult(result);
3246
3247   }
3248
3249   //
3250   // Verifiers
3251   //
3252
3253   private void assertKey(Cell key, byte [] row, byte [] family,
3254       byte [] qualifier, byte [] value)
3255   throws Exception {
3256     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3257         "Got row [" + Bytes.toString(CellUtil.cloneRow(key)) +"]",
3258         equals(row, CellUtil.cloneRow(key)));
3259     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3260         "Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
3261         equals(family, CellUtil.cloneFamily(key)));
3262     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3263         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key)) + "]",
3264         equals(qualifier, CellUtil.cloneQualifier(key)));
3265     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
3266         "Got value [" + Bytes.toString(CellUtil.cloneValue(key)) + "]",
3267         equals(value, CellUtil.cloneValue(key)));
3268   }
3269
3270   static void assertIncrementKey(Cell key, byte [] row, byte [] family,
3271       byte [] qualifier, long value)
3272   throws Exception {
3273     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3274         "Got row [" + Bytes.toString(CellUtil.cloneRow(key)) +"]",
3275         equals(row, CellUtil.cloneRow(key)));
3276     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3277         "Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
3278         equals(family, CellUtil.cloneFamily(key)));
3279     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3280         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key)) + "]",
3281         equals(qualifier, CellUtil.cloneQualifier(key)));
3282     assertTrue("Expected value [" + value + "] " +
3283         "Got value [" + Bytes.toLong(CellUtil.cloneValue(key)) + "]",
3284         Bytes.toLong(CellUtil.cloneValue(key)) == value);
3285   }
3286
3287   private void assertNumKeys(Result result, int n) throws Exception {
3288     assertTrue("Expected " + n + " keys but got " + result.size(),
3289         result.size() == n);
3290   }
3291
3292   private void assertNResult(Result result, byte [] row,
3293       byte [][] families, byte [][] qualifiers, byte [][] values,
3294       int [][] idxs)
3295   throws Exception {
3296     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3297         "Got row [" + Bytes.toString(result.getRow()) +"]",
3298         equals(row, result.getRow()));
3299     assertTrue("Expected " + idxs.length + " keys but result contains "
3300         + result.size(), result.size() == idxs.length);
3301
3302     Cell [] keys = result.rawCells();
3303
3304     for(int i=0;i<keys.length;i++) {
3305       byte [] family = families[idxs[i][0]];
3306       byte [] qualifier = qualifiers[idxs[i][1]];
3307       byte [] value = values[idxs[i][2]];
3308       Cell key = keys[i];
3309
3310       byte[] famb = CellUtil.cloneFamily(key);
3311       byte[] qualb = CellUtil.cloneQualifier(key);
3312       byte[] valb = CellUtil.cloneValue(key);
3313       assertTrue("(" + i + ") Expected family [" + Bytes.toString(family)
3314           + "] " + "Got family [" + Bytes.toString(famb) + "]",
3315           equals(family, famb));
3316       assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier)
3317           + "] " + "Got qualifier [" + Bytes.toString(qualb) + "]",
3318           equals(qualifier, qualb));
3319       assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] "
3320           + "Got value [" + Bytes.toString(valb) + "]",
3321           equals(value, valb));
3322     }
3323   }
3324
3325   private void assertNResult(Result result, byte [] row,
3326       byte [] family, byte [] qualifier, long [] stamps, byte [][] values,
3327       int start, int end)
3328   throws IOException {
3329     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3330         "Got row [" + Bytes.toString(result.getRow()) +"]",
3331         equals(row, result.getRow()));
3332     int expectedResults = end - start + 1;
3333     assertEquals(expectedResults, result.size());
3334
3335     Cell[] keys = result.rawCells();
3336
3337     for (int i=0; i<keys.length; i++) {
3338       byte [] value = values[end-i];
3339       long ts = stamps[end-i];
3340       Cell key = keys[i];
3341
3342       assertTrue("(" + i + ") Expected family [" + Bytes.toString(family)
3343           + "] " + "Got family [" + Bytes.toString(CellUtil.cloneFamily(key)) + "]",
3344           CellUtil.matchingFamily(key, family));
3345       assertTrue("(" + i + ") Expected qualifier [" + Bytes.toString(qualifier)
3346           + "] " + "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(key))+ "]",
3347           CellUtil.matchingQualifier(key, qualifier));
3348       assertTrue("Expected ts [" + ts + "] " +
3349           "Got ts [" + key.getTimestamp() + "]", ts == key.getTimestamp());
3350       assertTrue("(" + i + ") Expected value [" + Bytes.toString(value) + "] "
3351           + "Got value [" + Bytes.toString(CellUtil.cloneValue(key)) + "]",
3352           CellUtil.matchingValue(key,  value));
3353     }
3354   }
3355
3356   /**
3357    * Validate that result contains two specified keys, exactly.
3358    * It is assumed key A sorts before key B.
3359    */
3360   private void assertDoubleResult(Result result, byte [] row,
3361       byte [] familyA, byte [] qualifierA, byte [] valueA,
3362       byte [] familyB, byte [] qualifierB, byte [] valueB)
3363   throws Exception {
3364     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3365         "Got row [" + Bytes.toString(result.getRow()) +"]",
3366         equals(row, result.getRow()));
3367     assertTrue("Expected two keys but result contains " + result.size(),
3368         result.size() == 2);
3369     Cell [] kv = result.rawCells();
3370     Cell kvA = kv[0];
3371     assertTrue("(A) Expected family [" + Bytes.toString(familyA) + "] " +
3372         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kvA)) + "]",
3373         equals(familyA, CellUtil.cloneFamily(kvA)));
3374     assertTrue("(A) Expected qualifier [" + Bytes.toString(qualifierA) + "] " +
3375         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kvA)) + "]",
3376         equals(qualifierA, CellUtil.cloneQualifier(kvA)));
3377     assertTrue("(A) Expected value [" + Bytes.toString(valueA) + "] " +
3378         "Got value [" + Bytes.toString(CellUtil.cloneValue(kvA)) + "]",
3379         equals(valueA, CellUtil.cloneValue(kvA)));
3380     Cell kvB = kv[1];
3381     assertTrue("(B) Expected family [" + Bytes.toString(familyB) + "] " +
3382         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kvB)) + "]",
3383         equals(familyB, CellUtil.cloneFamily(kvB)));
3384     assertTrue("(B) Expected qualifier [" + Bytes.toString(qualifierB) + "] " +
3385         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kvB)) + "]",
3386         equals(qualifierB, CellUtil.cloneQualifier(kvB)));
3387     assertTrue("(B) Expected value [" + Bytes.toString(valueB) + "] " +
3388         "Got value [" + Bytes.toString(CellUtil.cloneValue(kvB)) + "]",
3389         equals(valueB, CellUtil.cloneValue(kvB)));
3390   }
3391
3392   private void assertSingleResult(Result result, byte [] row, byte [] family,
3393       byte [] qualifier, byte [] value)
3394   throws Exception {
3395     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3396         "Got row [" + Bytes.toString(result.getRow()) +"]",
3397         equals(row, result.getRow()));
3398     assertTrue("Expected a single key but result contains " + result.size(),
3399         result.size() == 1);
3400     Cell kv = result.rawCells()[0];
3401     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3402         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kv)) + "]",
3403         equals(family, CellUtil.cloneFamily(kv)));
3404     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3405         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kv)) + "]",
3406         equals(qualifier, CellUtil.cloneQualifier(kv)));
3407     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
3408         "Got value [" + Bytes.toString(CellUtil.cloneValue(kv)) + "]",
3409         equals(value, CellUtil.cloneValue(kv)));
3410   }
3411
3412   private void assertSingleResult(Result result, byte[] row, byte[] family, byte[] qualifier,
3413       long value) throws Exception {
3414     assertTrue(
3415       "Expected row [" + Bytes.toString(row) + "] " + "Got row [" + Bytes.toString(result.getRow())
3416           + "]", equals(row, result.getRow()));
3417     assertTrue("Expected a single key but result contains " + result.size(), result.size() == 1);
3418     Cell kv = result.rawCells()[0];
3419     assertTrue(
3420       "Expected family [" + Bytes.toString(family) + "] " + "Got family ["
3421           + Bytes.toString(CellUtil.cloneFamily(kv)) + "]",
3422       equals(family, CellUtil.cloneFamily(kv)));
3423     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " + "Got qualifier ["
3424         + Bytes.toString(CellUtil.cloneQualifier(kv)) + "]",
3425       equals(qualifier, CellUtil.cloneQualifier(kv)));
3426     assertTrue(
3427       "Expected value [" + value + "] " + "Got value [" + Bytes.toLong(CellUtil.cloneValue(kv))
3428           + "]", value == Bytes.toLong(CellUtil.cloneValue(kv)));
3429   }
3430
3431   private void assertSingleResult(Result result, byte [] row, byte [] family,
3432       byte [] qualifier, long ts, byte [] value)
3433   throws Exception {
3434     assertTrue("Expected row [" + Bytes.toString(row) + "] " +
3435         "Got row [" + Bytes.toString(result.getRow()) +"]",
3436         equals(row, result.getRow()));
3437     assertTrue("Expected a single key but result contains " + result.size(),
3438         result.size() == 1);
3439     Cell kv = result.rawCells()[0];
3440     assertTrue("Expected family [" + Bytes.toString(family) + "] " +
3441         "Got family [" + Bytes.toString(CellUtil.cloneFamily(kv)) + "]",
3442         equals(family, CellUtil.cloneFamily(kv)));
3443     assertTrue("Expected qualifier [" + Bytes.toString(qualifier) + "] " +
3444         "Got qualifier [" + Bytes.toString(CellUtil.cloneQualifier(kv)) + "]",
3445         equals(qualifier, CellUtil.cloneQualifier(kv)));
3446     assertTrue("Expected ts [" + ts + "] " +
3447         "Got ts [" + kv.getTimestamp() + "]", ts == kv.getTimestamp());
3448     assertTrue("Expected value [" + Bytes.toString(value) + "] " +
3449         "Got value [" + Bytes.toString(CellUtil.cloneValue(kv)) + "]",
3450         equals(value, CellUtil.cloneValue(kv)));
3451   }
3452
3453   private void assertEmptyResult(Result result) throws Exception {
3454     assertTrue("expected an empty result but result contains " +
3455         result.size() + " keys", result.isEmpty());
3456   }
3457
3458   private void assertNullResult(Result result) throws Exception {
3459     assertTrue("expected null result but received a non-null result",
3460         result == null);
3461   }
3462
3463   //
3464   // Helpers
3465   //
3466
3467   private Result getSingleScanResult(Table ht, Scan scan) throws IOException {
3468     ResultScanner scanner = ht.getScanner(scan);
3469     Result result = scanner.next();
3470     scanner.close();
3471     return result;
3472   }
3473
3474   private byte [][] makeNAscii(byte [] base, int n) {
3475     if(n > 256) {
3476       return makeNBig(base, n);
3477     }
3478     byte [][] ret = new byte[n][];
3479     for(int i=0;i<n;i++) {
3480       byte [] tail = Bytes.toBytes(Integer.toString(i));
3481       ret[i] = Bytes.add(base, tail);
3482     }
3483     return ret;
3484   }
3485
3486   private byte [][] makeN(byte [] base, int n) {
3487     if (n > 256) {
3488       return makeNBig(base, n);
3489     }
3490     byte [][] ret = new byte[n][];
3491     for(int i=0;i<n;i++) {
3492       ret[i] = Bytes.add(base, new byte[]{(byte)i});
3493     }
3494     return ret;
3495   }
3496
3497   private byte [][] makeNBig(byte [] base, int n) {
3498     byte [][] ret = new byte[n][];
3499     for(int i=0;i<n;i++) {
3500       int byteA = (i % 256);
3501       int byteB = (i >> 8);
3502       ret[i] = Bytes.add(base, new byte[]{(byte)byteB,(byte)byteA});
3503     }
3504     return ret;
3505   }
3506
3507   private long [] makeStamps(int n) {
3508     long [] stamps = new long[n];
3509     for (int i = 0; i < n; i++) {
3510       stamps[i] = i+1L;
3511     }
3512     return stamps;
3513   }
3514
3515   static boolean equals(byte [] left, byte [] right) {
3516     if (left == null && right == null) return true;
3517     if (left == null && right.length == 0) return true;
3518     if (right == null && left.length == 0) return true;
3519     return Bytes.equals(left, right);
3520   }
3521
3522   @Test
3523   public void testDuplicateVersions() throws Exception {
3524     final TableName tableName = TableName.valueOf(name.getMethodName());
3525
3526     long [] STAMPS = makeStamps(20);
3527     byte [][] VALUES = makeNAscii(VALUE, 20);
3528
3529     Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10);
3530
3531     // Insert 4 versions of same column
3532     Put put = new Put(ROW);
3533     put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3534     put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3535     put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3536     put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3537     ht.put(put);
3538
3539     // Verify we can get each one properly
3540     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3541     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3542     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3543     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3544     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3545     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3546     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3547     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3548
3549     // Verify we don't accidentally get others
3550     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3551     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3552     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3553     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3554     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3555     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3556
3557     // Ensure maxVersions in query is respected
3558     Get get = new Get(ROW);
3559     get.addColumn(FAMILY, QUALIFIER);
3560     get.setMaxVersions(2);
3561     Result result = ht.get(get);
3562     assertNResult(result, ROW, FAMILY, QUALIFIER,
3563         new long [] {STAMPS[4], STAMPS[5]},
3564         new byte[][] {VALUES[4], VALUES[5]},
3565         0, 1);
3566
3567     Scan scan = new Scan(ROW);
3568     scan.addColumn(FAMILY, QUALIFIER);
3569     scan.setMaxVersions(2);
3570     result = getSingleScanResult(ht, scan);
3571     assertNResult(result, ROW, FAMILY, QUALIFIER,
3572         new long [] {STAMPS[4], STAMPS[5]},
3573         new byte[][] {VALUES[4], VALUES[5]},
3574         0, 1);
3575
3576     // Flush and redo
3577
3578     TEST_UTIL.flush();
3579
3580     // Verify we can get each one properly
3581     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3582     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3583     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3584     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3585     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3586     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3587     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
3588     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
3589
3590     // Verify we don't accidentally get others
3591     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3592     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3593     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3594     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3595     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
3596     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
3597
3598     // Ensure maxVersions in query is respected
3599     get = new Get(ROW);
3600     get.addColumn(FAMILY, QUALIFIER);
3601     get.setMaxVersions(2);
3602     result = ht.get(get);
3603     assertNResult(result, ROW, FAMILY, QUALIFIER,
3604         new long [] {STAMPS[4], STAMPS[5]},
3605         new byte[][] {VALUES[4], VALUES[5]},
3606         0, 1);
3607
3608     scan = new Scan(ROW);
3609     scan.addColumn(FAMILY, QUALIFIER);
3610     scan.setMaxVersions(2);
3611     result = getSingleScanResult(ht, scan);
3612     assertNResult(result, ROW, FAMILY, QUALIFIER,
3613         new long [] {STAMPS[4], STAMPS[5]},
3614         new byte[][] {VALUES[4], VALUES[5]},
3615         0, 1);
3616
3617
3618     // Add some memstore and retest
3619
3620     // Insert 4 more versions of same column and a dupe
3621     put = new Put(ROW);
3622     put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
3623     put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3624     put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
3625     put.addColumn(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3626     put.addColumn(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
3627     ht.put(put);
3628
3629     // Ensure maxVersions in query is respected
3630     get = new Get(ROW);
3631     get.addColumn(FAMILY, QUALIFIER);
3632     get.setMaxVersions(7);
3633     result = ht.get(get);
3634     assertNResult(result, ROW, FAMILY, QUALIFIER,
3635         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3636         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3637         0, 6);
3638
3639     scan = new Scan(ROW);
3640     scan.addColumn(FAMILY, QUALIFIER);
3641     scan.setMaxVersions(7);
3642     result = getSingleScanResult(ht, scan);
3643     assertNResult(result, ROW, FAMILY, QUALIFIER,
3644         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3645         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3646         0, 6);
3647
3648     get = new Get(ROW);
3649     get.setMaxVersions(7);
3650     result = ht.get(get);
3651     assertNResult(result, ROW, FAMILY, QUALIFIER,
3652         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3653         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3654         0, 6);
3655
3656     scan = new Scan(ROW);
3657     scan.setMaxVersions(7);
3658     result = getSingleScanResult(ht, scan);
3659     assertNResult(result, ROW, FAMILY, QUALIFIER,
3660         new long [] {STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8]},
3661         new byte[][] {VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8]},
3662         0, 6);
3663
3664     // Verify we can get each one properly
3665     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3666     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3667     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3668     getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3669     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
3670     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
3671     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
3672     scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
3673
3674     // Verify we don't accidentally get others
3675     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3676     getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
3677     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
3678     scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
3679
3680     // Ensure maxVersions of table is respected
3681
3682     TEST_UTIL.flush();
3683
3684     // Insert 4 more versions of same column and a dupe
3685     put = new Put(ROW);
3686     put.addColumn(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
3687     put.addColumn(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
3688     put.addColumn(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
3689     put.addColumn(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
3690     ht.put(put);
3691
3692     get = new Get(ROW);
3693     get.addColumn(FAMILY, QUALIFIER);
3694     get.setMaxVersions(Integer.MAX_VALUE);
3695     result = ht.get(get);
3696     assertNResult(result, ROW, FAMILY, QUALIFIER,
3697         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
3698         new byte[][] {VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
3699         0, 9);
3700
3701     scan = new Scan(ROW);
3702     scan.addColumn(FAMILY, QUALIFIER);
3703     scan.setMaxVersions(Integer.MAX_VALUE);
3704     result = getSingleScanResult(ht, scan);
3705     assertNResult(result, ROW, FAMILY, QUALIFIER,
3706         new long [] {STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9], STAMPS[11], STAMPS[13], STAMPS[15]},
3707         new byte[][] {VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9], VALUES[11], VALUES[13], VALUES[15]},
3708         0, 9);
3709
3710     // Delete a version in the memstore and a version in a storefile
3711     Delete delete = new Delete(ROW);
3712     delete.addColumn(FAMILY, QUALIFIER, STAMPS[11]);
3713     delete.addColumn(FAMILY, QUALIFIER, STAMPS[7]);
3714     ht.delete(delete);
3715
3716     // Test that it's gone
3717     get = new Get(ROW);
3718     get.addColumn(FAMILY, QUALIFIER);
3719     get.setMaxVersions(Integer.MAX_VALUE);
3720     result = ht.get(get);
3721     assertNResult(result, ROW, FAMILY, QUALIFIER,
3722         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
3723         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
3724         0, 9);
3725
3726     scan = new Scan(ROW);
3727     scan.addColumn(FAMILY, QUALIFIER);
3728     scan.setMaxVersions(Integer.MAX_VALUE);
3729     result = getSingleScanResult(ht, scan);
3730     assertNResult(result, ROW, FAMILY, QUALIFIER,
3731         new long [] {STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8], STAMPS[9], STAMPS[13], STAMPS[15]},
3732         new byte[][] {VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8], VALUES[9], VALUES[13], VALUES[15]},
3733         0, 9);
3734   }
3735
3736   @Test
3737   public void testUpdates() throws Exception {
3738     final TableName tableName = TableName.valueOf(name.getMethodName());
3739     Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10);
3740
3741     // Write a column with values at timestamp 1, 2 and 3
3742     byte[] row = Bytes.toBytes("row1");
3743     byte[] qualifier = Bytes.toBytes("myCol");
3744     Put put = new Put(row);
3745     put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3746     hTable.put(put);
3747
3748     put = new Put(row);
3749     put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3750     hTable.put(put);
3751
3752     put = new Put(row);
3753     put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3754     hTable.put(put);
3755
3756     Get get = new Get(row);
3757     get.addColumn(FAMILY, qualifier);
3758     get.setMaxVersions();
3759
3760     // Check that the column indeed has the right values at timestamps 1 and
3761     // 2
3762     Result result = hTable.get(get);
3763     NavigableMap<Long, byte[]> navigableMap =
3764         result.getMap().get(FAMILY).get(qualifier);
3765     assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3766     assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3767
3768     // Update the value at timestamp 1
3769     put = new Put(row);
3770     put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3771     hTable.put(put);
3772
3773     // Update the value at timestamp 2
3774     put = new Put(row);
3775     put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3776     hTable.put(put);
3777
3778     // Check that the values at timestamp 2 and 1 got updated
3779     result = hTable.get(get);
3780     navigableMap = result.getMap().get(FAMILY).get(qualifier);
3781     assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3782     assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3783   }
3784
3785   @Test
3786   public void testUpdatesWithMajorCompaction() throws Exception {
3787     final TableName tableName = TableName.valueOf(name.getMethodName());
3788     Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10);
3789     Admin admin = TEST_UTIL.getAdmin();
3790
3791     // Write a column with values at timestamp 1, 2 and 3
3792     byte[] row = Bytes.toBytes("row2");
3793     byte[] qualifier = Bytes.toBytes("myCol");
3794     Put put = new Put(row);
3795     put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3796     hTable.put(put);
3797
3798     put = new Put(row);
3799     put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3800     hTable.put(put);
3801
3802     put = new Put(row);
3803     put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3804     hTable.put(put);
3805
3806     Get get = new Get(row);
3807     get.addColumn(FAMILY, qualifier);
3808     get.setMaxVersions();
3809
3810     // Check that the column indeed has the right values at timestamps 1 and
3811     // 2
3812     Result result = hTable.get(get);
3813     NavigableMap<Long, byte[]> navigableMap =
3814         result.getMap().get(FAMILY).get(qualifier);
3815     assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3816     assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3817
3818     // Trigger a major compaction
3819     admin.flush(tableName);
3820     admin.majorCompact(tableName);
3821     Thread.sleep(6000);
3822
3823     // Update the value at timestamp 1
3824     put = new Put(row);
3825     put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3826     hTable.put(put);
3827
3828     // Update the value at timestamp 2
3829     put = new Put(row);
3830     put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3831     hTable.put(put);
3832
3833     // Trigger a major compaction
3834     admin.flush(tableName);
3835     admin.majorCompact(tableName);
3836     Thread.sleep(6000);
3837
3838     // Check that the values at timestamp 2 and 1 got updated
3839     result = hTable.get(get);
3840     navigableMap = result.getMap().get(FAMILY).get(qualifier);
3841     assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3842     assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3843   }
3844
3845   @Test
3846   public void testMajorCompactionBetweenTwoUpdates() throws Exception {
3847     final TableName tableName = TableName.valueOf(name.getMethodName());
3848     Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10);
3849     Admin admin = TEST_UTIL.getAdmin();
3850
3851     // Write a column with values at timestamp 1, 2 and 3
3852     byte[] row = Bytes.toBytes("row3");
3853     byte[] qualifier = Bytes.toBytes("myCol");
3854     Put put = new Put(row);
3855     put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
3856     hTable.put(put);
3857
3858     put = new Put(row);
3859     put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
3860     hTable.put(put);
3861
3862     put = new Put(row);
3863     put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
3864     hTable.put(put);
3865
3866     Get get = new Get(row);
3867     get.addColumn(FAMILY, qualifier);
3868     get.setMaxVersions();
3869
3870     // Check that the column indeed has the right values at timestamps 1 and
3871     // 2
3872     Result result = hTable.get(get);
3873     NavigableMap<Long, byte[]> navigableMap =
3874         result.getMap().get(FAMILY).get(qualifier);
3875     assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
3876     assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
3877
3878     // Trigger a major compaction
3879     admin.flush(tableName);
3880     admin.majorCompact(tableName);
3881     Thread.sleep(6000);
3882
3883     // Update the value at timestamp 1
3884     put = new Put(row);
3885     put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
3886     hTable.put(put);
3887
3888     // Trigger a major compaction
3889     admin.flush(tableName);
3890     admin.majorCompact(tableName);
3891     Thread.sleep(6000);
3892
3893     // Update the value at timestamp 2
3894     put = new Put(row);
3895     put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
3896     hTable.put(put);
3897
3898     // Trigger a major compaction
3899     admin.flush(tableName);
3900     admin.majorCompact(tableName);
3901     Thread.sleep(6000);
3902
3903     // Check that the values at timestamp 2 and 1 got updated
3904     result = hTable.get(get);
3905     navigableMap = result.getMap().get(FAMILY).get(qualifier);
3906
3907     assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
3908     assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
3909   }
3910
3911   @Test
3912   public void testGet_EmptyTable() throws IOException {
3913     Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY);
3914     Get get = new Get(ROW);
3915     get.addFamily(FAMILY);
3916     Result r = table.get(get);
3917     assertTrue(r.isEmpty());
3918   }
3919
3920   @Test
3921   public void testGet_NullQualifier() throws IOException {
3922     Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY);
3923     Put put = new Put(ROW);
3924     put.addColumn(FAMILY, QUALIFIER, VALUE);
3925     table.put(put);
3926
3927     put = new Put(ROW);
3928     put.addColumn(FAMILY, null, VALUE);
3929     table.put(put);
3930     LOG.info("Row put");
3931
3932     Get get = new Get(ROW);
3933     get.addColumn(FAMILY, null);
3934     Result r = table.get(get);
3935     assertEquals(1, r.size());
3936
3937     get = new Get(ROW);
3938     get.addFamily(FAMILY);
3939     r = table.get(get);
3940     assertEquals(2, r.size());
3941   }
3942
3943   @Test
3944   public void testGet_NonExistentRow() throws IOException {
3945     Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY);
3946     Put put = new Put(ROW);
3947     put.addColumn(FAMILY, QUALIFIER, VALUE);
3948     table.put(put);
3949     LOG.info("Row put");
3950
3951     Get get = new Get(ROW);
3952     get.addFamily(FAMILY);
3953     Result r = table.get(get);
3954     assertFalse(r.isEmpty());
3955     System.out.println("Row retrieved successfully");
3956
3957     byte [] missingrow = Bytes.toBytes("missingrow");
3958     get = new Get(missingrow);
3959     get.addFamily(FAMILY);
3960     r = table.get(get);
3961     assertTrue(r.isEmpty());
3962     LOG.info("Row missing as it should be");
3963   }
3964
3965   @Test
3966   public void testPut() throws IOException {
3967     final byte [] CONTENTS_FAMILY = Bytes.toBytes("contents");
3968     final byte [] SMALL_FAMILY = Bytes.toBytes("smallfam");
3969     final byte [] row1 = Bytes.toBytes("row1");
3970     final byte [] row2 = Bytes.toBytes("row2");
3971     final byte [] value = Bytes.toBytes("abcd");
3972     Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()),
3973         new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY });
3974     Put put = new Put(row1);
3975     put.addColumn(CONTENTS_FAMILY, null, value);
3976     table.put(put);
3977
3978     put = new Put(row2);
3979     put.addColumn(CONTENTS_FAMILY, null, value);
3980
3981     assertEquals(1, put.size());
3982     assertEquals(1, put.getFamilyCellMap().get(CONTENTS_FAMILY).size());
3983
3984     // KeyValue v1 expectation.  Cast for now until we go all Cell all the time. TODO
3985     KeyValue kv = (KeyValue)put.getFamilyCellMap().get(CONTENTS_FAMILY).get(0);
3986
3987     assertTrue(Bytes.equals(CellUtil.cloneFamily(kv), CONTENTS_FAMILY));
3988     // will it return null or an empty byte array?
3989     assertTrue(Bytes.equals(CellUtil.cloneQualifier(kv), new byte[0]));
3990
3991     assertTrue(Bytes.equals(CellUtil.cloneValue(kv), value));
3992
3993     table.put(put);
3994
3995     Scan scan = new Scan();
3996     scan.addColumn(CONTENTS_FAMILY, null);
3997     ResultScanner scanner = table.getScanner(scan);
3998     for (Result r : scanner) {
3999       for(Cell key : r.rawCells()) {
4000         System.out.println(Bytes.toString(r.getRow()) + ": " + key.toString());
4001       }
4002     }
4003   }
4004
4005   @Test
4006   public void testPutNoCF() throws IOException {
4007     final byte[] BAD_FAM = Bytes.toBytes("BAD_CF");
4008     final byte[] VAL = Bytes.toBytes(100);
4009     Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY);
4010
4011     boolean caughtNSCFE = false;
4012
4013     try {
4014       Put p = new Put(ROW);
4015       p.addColumn(BAD_FAM, QUALIFIER, VAL);
4016       table.put(p);
4017     } catch (Exception e) {
4018       caughtNSCFE = e instanceof NoSuchColumnFamilyException;
4019     }
4020     assertTrue("Should throw NoSuchColumnFamilyException", caughtNSCFE);
4021
4022   }
4023
4024   @Test
4025   public void testRowsPut() throws IOException {
4026     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
4027     final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
4028     final int NB_BATCH_ROWS = 10;
4029     final byte[] value = Bytes.toBytes("abcd");
4030     Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()),
4031       new byte[][] {CONTENTS_FAMILY, SMALL_FAMILY });
4032     ArrayList<Put> rowsUpdate = new ArrayList<Put>();
4033     for (int i = 0; i < NB_BATCH_ROWS; i++) {
4034       byte[] row = Bytes.toBytes("row" + i);
4035       Put put = new Put(row);
4036       put.setDurability(Durability.SKIP_WAL);
4037       put.addColumn(CONTENTS_FAMILY, null, value);
4038       rowsUpdate.add(put);
4039     }
4040     table.put(rowsUpdate);
4041     Scan scan = new Scan();
4042     scan.addFamily(CONTENTS_FAMILY);
4043     ResultScanner scanner = table.getScanner(scan);
4044     int nbRows = 0;
4045     for (@SuppressWarnings("unused")
4046     Result row : scanner)
4047       nbRows++;
4048     assertEquals(NB_BATCH_ROWS, nbRows);
4049   }
4050
4051   @Test
4052   public void testRowsPutBufferedManyManyFlushes() throws IOException {
4053     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
4054     final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
4055     final byte[] value = Bytes.toBytes("abcd");
4056     final int NB_BATCH_ROWS = 10;
4057     Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()),
4058         new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY });
4059     ArrayList<Put> rowsUpdate = new ArrayList<Put>();
4060     for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
4061       byte[] row = Bytes.toBytes("row" + i);
4062       Put put = new Put(row);
4063       put.setDurability(Durability.SKIP_WAL);
4064       put.addColumn(CONTENTS_FAMILY, null, value);
4065       rowsUpdate.add(put);
4066     }
4067     table.put(rowsUpdate);
4068
4069     Scan scan = new Scan();
4070     scan.addFamily(CONTENTS_FAMILY);
4071     ResultScanner scanner = table.getScanner(scan);
4072     int nbRows = 0;
4073     for (@SuppressWarnings("unused")
4074     Result row : scanner)
4075       nbRows++;
4076     assertEquals(NB_BATCH_ROWS * 10, nbRows);
4077   }
4078
4079   @Test
4080   public void testAddKeyValue() throws IOException {
4081     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
4082     final byte[] value = Bytes.toBytes("abcd");
4083     final byte[] row1 = Bytes.toBytes("row1");
4084     final byte[] row2 = Bytes.toBytes("row2");
4085     byte[] qualifier = Bytes.toBytes("qf1");
4086     Put put = new Put(row1);
4087
4088     // Adding KeyValue with the same row
4089     KeyValue kv = new KeyValue(row1, CONTENTS_FAMILY, qualifier, value);
4090     boolean ok = true;
4091     try {
4092       put.add(kv);
4093     } catch (IOException e) {
4094       ok = false;
4095     }
4096     assertEquals(true, ok);
4097
4098     // Adding KeyValue with the different row
4099     kv = new KeyValue(row2, CONTENTS_FAMILY, qualifier, value);
4100     ok = false;
4101     try {
4102       put.add(kv);
4103     } catch (IOException e) {
4104       ok = true;
4105     }
4106     assertEquals(true, ok);
4107   }
4108
4109   /**
4110    * test for HBASE-737
4111    */
4112   @Test
4113   public void testHBase737 () throws IOException {
4114     final byte [] FAM1 = Bytes.toBytes("fam1");
4115     final byte [] FAM2 = Bytes.toBytes("fam2");
4116     // Open table
4117     Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()),
4118       new byte [][] {FAM1, FAM2});
4119     // Insert some values
4120     Put put = new Put(ROW);
4121     put.addColumn(FAM1, Bytes.toBytes("letters"), Bytes.toBytes("abcdefg"));
4122     table.put(put);
4123     try {
4124       Thread.sleep(1000);
4125     } catch (InterruptedException i) {
4126       //ignore
4127     }
4128
4129     put = new Put(ROW);
4130     put.addColumn(FAM1, Bytes.toBytes("numbers"), Bytes.toBytes("123456"));
4131     table.put(put);
4132
4133     try {
4134       Thread.sleep(1000);
4135     } catch (InterruptedException i) {
4136       //ignore
4137     }
4138
4139     put = new Put(ROW);
4140     put.addColumn(FAM2, Bytes.toBytes("letters"), Bytes.toBytes("hijklmnop"));
4141     table.put(put);
4142
4143     long times[] = new long[3];
4144
4145     // First scan the memstore
4146
4147     Scan scan = new Scan();
4148     scan.addFamily(FAM1);
4149     scan.addFamily(FAM2);
4150     ResultScanner s = table.getScanner(scan);
4151     try {
4152       int index = 0;
4153       Result r = null;
4154       while ((r = s.next()) != null) {
4155         for(Cell key : r.rawCells()) {
4156           times[index++] = key.getTimestamp();
4157         }
4158       }
4159     } finally {
4160       s.close();
4161     }
4162     for (int i = 0; i < times.length - 1; i++) {
4163       for (int j = i + 1; j < times.length; j++) {
4164         assertTrue(times[j] > times[i]);
4165       }
4166     }
4167
4168     // Flush data to disk and try again
4169     TEST_UTIL.flush();
4170
4171     // Reset times
4172     for(int i=0;i<times.length;i++) {
4173       times[i] = 0;
4174     }
4175
4176     try {
4177       Thread.sleep(1000);
4178     } catch (InterruptedException i) {
4179       //ignore
4180     }
4181     scan = new Scan();
4182     scan.addFamily(FAM1);
4183     scan.addFamily(FAM2);
4184     s = table.getScanner(scan);
4185     try {
4186       int index = 0;
4187       Result r = null;
4188       while ((r = s.next()) != null) {
4189         for(Cell key : r.rawCells()) {
4190           times[index++] = key.getTimestamp();
4191         }
4192       }
4193     } finally {
4194       s.close();
4195     }
4196     for (int i = 0; i < times.length - 1; i++) {
4197       for (int j = i + 1; j < times.length; j++) {
4198         assertTrue(times[j] > times[i]);
4199       }
4200     }
4201   }
4202
4203   @Test
4204   public void testListTables() throws IOException, InterruptedException {
4205     final TableName tableName1 = TableName.valueOf(name.getMethodName() + "1");
4206     final TableName tableName2 = TableName.valueOf(name.getMethodName() + "2");
4207     final TableName tableName3 = TableName.valueOf(name.getMethodName() + "3");
4208     TableName [] tables = new TableName[] { tableName1, tableName2, tableName3 };
4209     for (int i = 0; i < tables.length; i++) {
4210       TEST_UTIL.createTable(tables[i], FAMILY);
4211     }
4212     Admin admin = TEST_UTIL.getAdmin();
4213     List<TableDescriptor> ts = admin.listTableDescriptors();
4214     HashSet<TableDescriptor> result = new HashSet<>(ts);
4215     int size = result.size();
4216     assertTrue(size >= tables.length);
4217     for (int i = 0; i < tables.length && i < size; i++) {
4218       boolean found = false;
4219       for (int j = 0; j < ts.size(); j++) {
4220         if (ts.get(j).getTableName().equals(tables[i])) {
4221           found = true;
4222           break;
4223         }
4224       }
4225       assertTrue("Not found: " + tables[i], found);
4226     }
4227   }
4228
4229   /**
4230    * simple test that just executes parts of the client
4231    * API that accept a pre-created Connection instance
4232    */
4233   @Test
4234   public void testUnmanagedHConnection() throws IOException {
4235     final TableName tableName = TableName.valueOf(name.getMethodName());
4236     TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
4237     Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
4238     Table t = conn.getTable(tableName);
4239     Admin admin = conn.getAdmin();
4240     assertTrue(admin.tableExists(tableName));
4241     assertTrue(t.get(new Get(ROW)).isEmpty());
4242     admin.close();
4243   }
4244
4245   /**
4246    * test of that unmanaged HConnections are able to reconnect
4247    * properly (see HBASE-5058)
4248    */
4249   @Test
4250   public void testUnmanagedHConnectionReconnect() throws Exception {
4251     final TableName tableName = TableName.valueOf(name.getMethodName());
4252     TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
4253     Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
4254     Table t = conn.getTable(tableName);
4255     try (Admin admin = conn.getAdmin()) {
4256       assertTrue(admin.tableExists(tableName));
4257       assertTrue(t.get(new Get(ROW)).isEmpty());
4258     }
4259
4260     // stop the master
4261     MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
4262     cluster.stopMaster(0, false);
4263     cluster.waitOnMaster(0);
4264
4265     // start up a new master
4266     cluster.startMaster();
4267     assertTrue(cluster.waitForActiveAndReadyMaster());
4268
4269     // test that the same unmanaged connection works with a new
4270     // Admin and can connect to the new master;
4271     boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration());
4272     try (Admin admin = conn.getAdmin()) {
4273       assertTrue(admin.tableExists(tableName));
4274       assertTrue(admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
4275           .getLiveServerMetrics().size() == SLAVES + (tablesOnMaster ? 1 : 0));
4276     }
4277   }
4278
4279   @Test
4280   public void testMiscHTableStuff() throws IOException {
4281     final TableName tableAname = TableName.valueOf(name.getMethodName() + "A");
4282     final TableName tableBname = TableName.valueOf(name.getMethodName() + "B");
4283     final byte[] attrName = Bytes.toBytes("TESTATTR");
4284     final byte[] attrValue = Bytes.toBytes("somevalue");
4285     byte[] value = Bytes.toBytes("value");
4286
4287     Table a = TEST_UTIL.createTable(tableAname, HConstants.CATALOG_FAMILY);
4288     Table b = TEST_UTIL.createTable(tableBname, HConstants.CATALOG_FAMILY);
4289     Put put = new Put(ROW);
4290     put.addColumn(HConstants.CATALOG_FAMILY, null, value);
4291     a.put(put);
4292
4293     // open a new connection to A and a connection to b
4294     Table newA = TEST_UTIL.getConnection().getTable(tableAname);
4295
4296     // copy data from A to B
4297     Scan scan = new Scan();
4298     scan.addFamily(HConstants.CATALOG_FAMILY);
4299     ResultScanner s = newA.getScanner(scan);
4300     try {
4301       for (Result r : s) {
4302         put = new Put(r.getRow());
4303         put.setDurability(Durability.SKIP_WAL);
4304         for (Cell kv : r.rawCells()) {
4305           put.add(kv);
4306         }
4307         b.put(put);
4308       }
4309     } finally {
4310       s.close();
4311     }
4312
4313     // Opening a new connection to A will cause the tables to be reloaded
4314     Table anotherA = TEST_UTIL.getConnection().getTable(tableAname);
4315     Get get = new Get(ROW);
4316     get.addFamily(HConstants.CATALOG_FAMILY);
4317     anotherA.get(get);
4318
4319     // We can still access A through newA because it has the table information
4320     // cached. And if it needs to recalibrate, that will cause the information
4321     // to be reloaded.
4322
4323     // Test user metadata
4324     Admin admin = TEST_UTIL.getAdmin();
4325     // make a modifiable descriptor
4326     HTableDescriptor desc = new HTableDescriptor(a.getTableDescriptor());
4327     // offline the table
4328     admin.disableTable(tableAname);
4329     // add a user attribute to HTD
4330     desc.setValue(attrName, attrValue);
4331     // add a user attribute to HCD
4332     for (HColumnDescriptor c : desc.getFamilies())
4333       c.setValue(attrName, attrValue);
4334     // update metadata for all regions of this table
4335     admin.modifyTable(desc);
4336     // enable the table
4337     admin.enableTable(tableAname);
4338
4339     // Test that attribute changes were applied
4340     desc = a.getTableDescriptor();
4341     assertEquals("wrong table descriptor returned", desc.getTableName(), tableAname);
4342     // check HTD attribute
4343     value = desc.getValue(attrName);
4344     assertFalse("missing HTD attribute value", value == null);
4345     assertFalse("HTD attribute value is incorrect",
4346       Bytes.compareTo(value, attrValue) != 0);
4347     // check HCD attribute
4348     for (HColumnDescriptor c : desc.getFamilies()) {
4349       value = c.getValue(attrName);
4350       assertFalse("missing HCD attribute value", value == null);
4351       assertFalse("HCD attribute value is incorrect",
4352         Bytes.compareTo(value, attrValue) != 0);
4353     }
4354   }
4355
4356   @Test
4357   public void testGetClosestRowBefore() throws IOException, InterruptedException {
4358     final TableName tableName = TableName.valueOf(name.getMethodName());
4359     final byte[] firstRow = Bytes.toBytes("row111");
4360     final byte[] secondRow = Bytes.toBytes("row222");
4361     final byte[] thirdRow = Bytes.toBytes("row333");
4362     final byte[] forthRow = Bytes.toBytes("row444");
4363     final byte[] beforeFirstRow = Bytes.toBytes("row");
4364     final byte[] beforeSecondRow = Bytes.toBytes("row22");
4365     final byte[] beforeThirdRow = Bytes.toBytes("row33");
4366     final byte[] beforeForthRow = Bytes.toBytes("row44");
4367
4368     try (Table table =
4369         TEST_UTIL.createTable(tableName,
4370           new byte[][] { HConstants.CATALOG_FAMILY, Bytes.toBytes("info2") }, 1, 1024);
4371       RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
4372
4373       // set block size to 64 to making 2 kvs into one block, bypassing the walkForwardInSingleRow
4374       // in Store.rowAtOrBeforeFromStoreFile
4375       String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName();
4376       HRegion region = TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
4377       Put put1 = new Put(firstRow);
4378       Put put2 = new Put(secondRow);
4379       Put put3 = new Put(thirdRow);
4380       Put put4 = new Put(forthRow);
4381       byte[] one = new byte[] { 1 };
4382       byte[] two = new byte[] { 2 };
4383       byte[] three = new byte[] { 3 };
4384       byte[] four = new byte[] { 4 };
4385
4386       put1.addColumn(HConstants.CATALOG_FAMILY, null, one);
4387       put2.addColumn(HConstants.CATALOG_FAMILY, null, two);
4388       put3.addColumn(HConstants.CATALOG_FAMILY, null, three);
4389       put4.addColumn(HConstants.CATALOG_FAMILY, null, four);
4390       table.put(put1);
4391       table.put(put2);
4392       table.put(put3);
4393       table.put(put4);
4394       region.flush(true);
4395
4396       Result result;
4397
4398       // Test before first that null is returned
4399       result = getReverseScanResult(table, beforeFirstRow,
4400         HConstants.CATALOG_FAMILY);
4401       assertNull(result);
4402
4403       // Test at first that first is returned
4404       result = getReverseScanResult(table, firstRow, HConstants.CATALOG_FAMILY);
4405       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4406       assertTrue(Bytes.equals(result.getRow(), firstRow));
4407       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));
4408
4409       // Test in between first and second that first is returned
4410       result = getReverseScanResult(table, beforeSecondRow, HConstants.CATALOG_FAMILY);
4411       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4412       assertTrue(Bytes.equals(result.getRow(), firstRow));
4413       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));
4414
4415       // Test at second make sure second is returned
4416       result = getReverseScanResult(table, secondRow, HConstants.CATALOG_FAMILY);
4417       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4418       assertTrue(Bytes.equals(result.getRow(), secondRow));
4419       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), two));
4420
4421       // Test in second and third, make sure second is returned
4422       result = getReverseScanResult(table, beforeThirdRow, HConstants.CATALOG_FAMILY);
4423       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4424       assertTrue(Bytes.equals(result.getRow(), secondRow));
4425       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), two));
4426
4427       // Test at third make sure third is returned
4428       result = getReverseScanResult(table, thirdRow, HConstants.CATALOG_FAMILY);
4429       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4430       assertTrue(Bytes.equals(result.getRow(), thirdRow));
4431       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), three));
4432
4433       // Test in third and forth, make sure third is returned
4434       result = getReverseScanResult(table, beforeForthRow, HConstants.CATALOG_FAMILY);
4435       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4436       assertTrue(Bytes.equals(result.getRow(), thirdRow));
4437       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), three));
4438
4439       // Test at forth make sure forth is returned
4440       result = getReverseScanResult(table, forthRow, HConstants.CATALOG_FAMILY);
4441       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4442       assertTrue(Bytes.equals(result.getRow(), forthRow));
4443       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), four));
4444
4445       // Test after forth make sure forth is returned
4446       result = getReverseScanResult(table, Bytes.add(forthRow, one), HConstants.CATALOG_FAMILY);
4447       assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
4448       assertTrue(Bytes.equals(result.getRow(), forthRow));
4449       assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), four));
4450     }
4451   }
4452
4453   private Result getReverseScanResult(Table table, byte[] row, byte[] fam) throws IOException {
4454     Scan scan = new Scan(row);
4455     scan.setSmall(true);
4456     scan.setReversed(true);
4457     scan.setCaching(1);
4458     scan.addFamily(fam);
4459     try (ResultScanner scanner = table.getScanner(scan)) {
4460       return scanner.next();
4461     }
4462   }
4463
4464   /**
4465    * For HBASE-2156
4466    */
4467   @Test
4468   public void testScanVariableReuse() throws Exception {
4469     Scan scan = new Scan();
4470     scan.addFamily(FAMILY);
4471     scan.addColumn(FAMILY, ROW);
4472
4473     assertTrue(scan.getFamilyMap().get(FAMILY).size() == 1);
4474
4475     scan = new Scan();
4476     scan.addFamily(FAMILY);
4477
4478     assertTrue(scan.getFamilyMap().get(FAMILY) == null);
4479     assertTrue(scan.getFamilyMap().containsKey(FAMILY));
4480   }
4481
4482   @Test
4483   public void testMultiRowMutation() throws Exception {
4484     LOG.info("Starting testMultiRowMutation");
4485     final TableName tableName = TableName.valueOf(name.getMethodName());
4486     final byte [] ROW1 = Bytes.toBytes("testRow1");
4487
4488     Table t = TEST_UTIL.createTable(tableName, FAMILY);
4489     Put p = new Put(ROW);
4490     p.addColumn(FAMILY, QUALIFIER, VALUE);
4491     MutationProto m1 = ProtobufUtil.toMutation(MutationType.PUT, p);
4492
4493     p = new Put(ROW1);
4494     p.addColumn(FAMILY, QUALIFIER, VALUE);
4495     MutationProto m2 = ProtobufUtil.toMutation(MutationType.PUT, p);
4496
4497     MutateRowsRequest.Builder mrmBuilder = MutateRowsRequest.newBuilder();
4498     mrmBuilder.addMutationRequest(m1);
4499     mrmBuilder.addMutationRequest(m2);
4500     MutateRowsRequest mrm = mrmBuilder.build();
4501     CoprocessorRpcChannel channel = t.coprocessorService(ROW);
4502     MultiRowMutationService.BlockingInterface service =
4503        MultiRowMutationService.newBlockingStub(channel);
4504     service.mutateRows(null, mrm);
4505     Get g = new Get(ROW);
4506     Result r = t.get(g);
4507     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)));
4508     g = new Get(ROW1);
4509     r = t.get(g);
4510     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)));
4511   }
4512
4513   @Test
4514   public void testRowMutation() throws Exception {
4515     LOG.info("Starting testRowMutation");
4516     final TableName tableName = TableName.valueOf(name.getMethodName());
4517     Table t = TEST_UTIL.createTable(tableName, FAMILY);
4518     byte [][] QUALIFIERS = new byte [][] {
4519         Bytes.toBytes("a"), Bytes.toBytes("b")
4520     };
4521     RowMutations arm = new RowMutations(ROW);
4522     Put p = new Put(ROW);
4523     p.addColumn(FAMILY, QUALIFIERS[0], VALUE);
4524     arm.add(p);
4525     t.mutateRow(arm);
4526
4527     Get g = new Get(ROW);
4528     Result r = t.get(g);
4529     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[0])));
4530
4531     arm = new RowMutations(ROW);
4532     p = new Put(ROW);
4533     p.addColumn(FAMILY, QUALIFIERS[1], VALUE);
4534     arm.add(p);
4535     Delete d = new Delete(ROW);
4536     d.addColumns(FAMILY, QUALIFIERS[0]);
4537     arm.add(d);
4538     // TODO: Trying mutateRow again.  The batch was failing with a one try only.
4539     t.mutateRow(arm);
4540     r = t.get(g);
4541     assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[1])));
4542     assertNull(r.getValue(FAMILY, QUALIFIERS[0]));
4543
4544     //Test that we get a region level exception
4545     try {
4546       arm = new RowMutations(ROW);
4547       p = new Put(ROW);
4548       p.addColumn(new byte[]{'b', 'o', 'g', 'u', 's'}, QUALIFIERS[0], VALUE);
4549       arm.add(p);
4550       t.mutateRow(arm);
4551       fail("Expected NoSuchColumnFamilyException");
4552     } catch(RetriesExhaustedWithDetailsException e) {
4553       for(Throwable rootCause: e.getCauses()){
4554         if(rootCause instanceof NoSuchColumnFamilyException){
4555           return;
4556         }
4557       }
4558       throw e;
4559     }
4560   }
4561
4562   @Test
4563   public void testBatchAppendWithReturnResultFalse() throws Exception {
4564     LOG.info("Starting testBatchAppendWithReturnResultFalse");
4565     final TableName tableName = TableName.valueOf(name.getMethodName());
4566     Table table = TEST_UTIL.createTable(tableName, FAMILY);
4567     Append append1 = new Append(Bytes.toBytes("row1"));
4568     append1.setReturnResults(false);
4569     append1.addColumn(FAMILY, Bytes.toBytes("f1"), Bytes.toBytes("value1"));
4570     Append append2 = new Append(Bytes.toBytes("row1"));
4571     append2.setReturnResults(false);
4572     append2.addColumn(FAMILY, Bytes.toBytes("f1"), Bytes.toBytes("value2"));
4573     List<Append> appends = new ArrayList<>();
4574     appends.add(append1);
4575     appends.add(append2);
4576     Object[] results = new Object[2];
4577     table.batch(appends, results);
4578     assertTrue(results.length == 2);
4579     for(Object r : results) {
4580       Result result = (Result)r;
4581       assertTrue(result.isEmpty());
4582     }
4583     table.close();
4584   }
4585
4586   @Test
4587   public void testAppend() throws Exception {
4588     LOG.info("Starting testAppend");
4589     final TableName tableName = TableName.valueOf(name.getMethodName());
4590     Table t = TEST_UTIL.createTable(tableName, FAMILY);
4591     byte[] v1 = Bytes.toBytes("42");
4592     byte[] v2 = Bytes.toBytes("23");
4593     byte [][] QUALIFIERS = new byte [][] {
4594         Bytes.toBytes("b"), Bytes.toBytes("a"), Bytes.toBytes("c")
4595     };
4596     Append a = new Append(ROW);
4597     a.addColumn(FAMILY, QUALIFIERS[0], v1);
4598     a.addColumn(FAMILY, QUALIFIERS[1], v2);
4599     a.setReturnResults(false);
4600     assertEmptyResult(t.append(a));
4601
4602     a = new Append(ROW);
4603     a.addColumn(FAMILY, QUALIFIERS[0], v2);
4604     a.addColumn(FAMILY, QUALIFIERS[1], v1);
4605     a.addColumn(FAMILY, QUALIFIERS[2], v2);
4606     Result r = t.append(a);
4607     assertEquals(0, Bytes.compareTo(Bytes.add(v1, v2), r.getValue(FAMILY, QUALIFIERS[0])));
4608     assertEquals(0, Bytes.compareTo(Bytes.add(v2, v1), r.getValue(FAMILY, QUALIFIERS[1])));
4609     // QUALIFIERS[2] previously not exist, verify both value and timestamp are correct
4610     assertEquals(0, Bytes.compareTo(v2, r.getValue(FAMILY, QUALIFIERS[2])));
4611     assertEquals(r.getColumnLatestCell(FAMILY, QUALIFIERS[0]).getTimestamp(),
4612         r.getColumnLatestCell(FAMILY, QUALIFIERS[2]).getTimestamp());
4613   }
4614   private List<Result> doAppend(final boolean walUsed) throws IOException {
4615     LOG.info("Starting testAppend, walUsed is " + walUsed);
4616     final TableName TABLENAME = TableName.valueOf(walUsed ? "testAppendWithWAL" : "testAppendWithoutWAL");
4617     Table t = TEST_UTIL.createTable(TABLENAME, FAMILY);
4618     final byte[] row1 = Bytes.toBytes("c");
4619     final byte[] row2 = Bytes.toBytes("b");
4620     final byte[] row3 = Bytes.toBytes("a");
4621     final byte[] qual = Bytes.toBytes("qual");
4622     Put put_0 = new Put(row2);
4623     put_0.addColumn(FAMILY, qual, Bytes.toBytes("put"));
4624     Put put_1 = new Put(row3);
4625     put_1.addColumn(FAMILY, qual, Bytes.toBytes("put"));
4626     Append append_0 = new Append(row1);
4627     append_0.addColumn(FAMILY, qual, Bytes.toBytes("i"));
4628     Append append_1 = new Append(row1);
4629     append_1.addColumn(FAMILY, qual, Bytes.toBytes("k"));
4630     Append append_2 = new Append(row1);
4631     append_2.addColumn(FAMILY, qual, Bytes.toBytes("e"));
4632     if (!walUsed) {
4633       append_2.setDurability(Durability.SKIP_WAL);
4634     }
4635     Append append_3 = new Append(row1);
4636     append_3.addColumn(FAMILY, qual, Bytes.toBytes("a"));
4637     Scan s = new Scan();
4638     s.setCaching(1);
4639     t.append(append_0);
4640     t.put(put_0);
4641     t.put(put_1);
4642     List<Result> results = new LinkedList<>();
4643     try (ResultScanner scanner = t.getScanner(s)) {
4644       t.append(append_1);
4645       t.append(append_2);
4646       t.append(append_3);
4647       for (Result r : scanner) {
4648         results.add(r);
4649       }
4650     }
4651     TEST_UTIL.deleteTable(TABLENAME);
4652     return results;
4653   }
4654
4655   @Test
4656   public void testAppendWithoutWAL() throws Exception {
4657     List<Result> resultsWithWal = doAppend(true);
4658     List<Result> resultsWithoutWal = doAppend(false);
4659     assertEquals(resultsWithWal.size(), resultsWithoutWal.size());
4660     for (int i = 0; i != resultsWithWal.size(); ++i) {
4661       Result resultWithWal = resultsWithWal.get(i);
4662       Result resultWithoutWal = resultsWithoutWal.get(i);
4663       assertEquals(resultWithWal.rawCells().length, resultWithoutWal.rawCells().length);
4664       for (int j = 0; j != resultWithWal.rawCells().length; ++j) {
4665         Cell cellWithWal = resultWithWal.rawCells()[j];
4666         Cell cellWithoutWal = resultWithoutWal.rawCells()[j];
4667         assertTrue(Bytes.equals(CellUtil.cloneRow(cellWithWal), CellUtil.cloneRow(cellWithoutWal)));
4668         assertTrue(Bytes.equals(CellUtil.cloneFamily(cellWithWal), CellUtil.cloneFamily(cellWithoutWal)));
4669         assertTrue(Bytes.equals(CellUtil.cloneQualifier(cellWithWal), CellUtil.cloneQualifier(cellWithoutWal)));
4670         assertTrue(Bytes.equals(CellUtil.cloneValue(cellWithWal), CellUtil.cloneValue(cellWithoutWal)));
4671       }
4672     }
4673   }
4674
4675   @Test
4676   public void testClientPoolRoundRobin() throws IOException {
4677     final TableName tableName = TableName.valueOf(name.getMethodName());
4678
4679     int poolSize = 3;
4680     int numVersions = poolSize * 2;
4681     Configuration conf = TEST_UTIL.getConfiguration();
4682     conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "round-robin");
4683     conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize);
4684
4685     Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, Integer.MAX_VALUE);
4686
4687     final long ts = EnvironmentEdgeManager.currentTime();
4688     Get get = new Get(ROW);
4689     get.addColumn(FAMILY, QUALIFIER);
4690     get.setMaxVersions();
4691
4692     for (int versions = 1; versions <= numVersions; versions++) {
4693       Put put = new Put(ROW);
4694       put.addColumn(FAMILY, QUALIFIER, ts + versions, VALUE);
4695       table.put(put);
4696
4697       Result result = table.get(get);
4698       NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY)
4699           .get(QUALIFIER);
4700
4701       assertEquals("The number of versions of '" + Bytes.toString(FAMILY) + ":"
4702           + Bytes.toString(QUALIFIER) + " did not match", versions, navigableMap.size());
4703       for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
4704         assertTrue("The value at time " + entry.getKey()
4705             + " did not match what was put",
4706             Bytes.equals(VALUE, entry.getValue()));
4707       }
4708     }
4709   }
4710
4711   @Ignore ("Flakey: HBASE-8989") @Test
4712   public void testClientPoolThreadLocal() throws IOException {
4713     final TableName tableName = TableName.valueOf(name.getMethodName());
4714
4715     int poolSize = Integer.MAX_VALUE;
4716     int numVersions = 3;
4717     Configuration conf = TEST_UTIL.getConfiguration();
4718     conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "thread-local");
4719     conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize);
4720
4721     final Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY },  3);
4722
4723     final long ts = EnvironmentEdgeManager.currentTime();
4724     final Get get = new Get(ROW);
4725     get.addColumn(FAMILY, QUALIFIER);
4726     get.setMaxVersions();
4727
4728     for (int versions = 1; versions <= numVersions; versions++) {
4729       Put put = new Put(ROW);
4730       put.addColumn(FAMILY, QUALIFIER, ts + versions, VALUE);
4731       table.put(put);
4732
4733       Result result = table.get(get);
4734       NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY)
4735           .get(QUALIFIER);
4736
4737       assertEquals("The number of versions of '" + Bytes.toString(FAMILY) + ":"
4738           + Bytes.toString(QUALIFIER) + " did not match", versions, navigableMap.size());
4739       for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
4740         assertTrue("The value at time " + entry.getKey()
4741             + " did not match what was put",
4742             Bytes.equals(VALUE, entry.getValue()));
4743       }
4744     }
4745
4746     final Object waitLock = new Object();
4747     ExecutorService executorService = Executors.newFixedThreadPool(numVersions);
4748     final AtomicReference<AssertionError> error = new AtomicReference<>(null);
4749     for (int versions = numVersions; versions < numVersions * 2; versions++) {
4750       final int versionsCopy = versions;
4751       executorService.submit(new Callable<Void>() {
4752         @Override
4753         public Void call() {
4754           try {
4755             Put put = new Put(ROW);
4756             put.addColumn(FAMILY, QUALIFIER, ts + versionsCopy, VALUE);
4757             table.put(put);
4758
4759             Result result = table.get(get);
4760             NavigableMap<Long, byte[]> navigableMap = result.getMap()
4761                 .get(FAMILY).get(QUALIFIER);
4762
4763             assertEquals("The number of versions of '" + Bytes.toString(FAMILY) + ":"
4764                 + Bytes.toString(QUALIFIER) + " did not match " + versionsCopy, versionsCopy,
4765                 navigableMap.size());
4766             for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
4767               assertTrue("The value at time " + entry.getKey()
4768                   + " did not match what was put",
4769                   Bytes.equals(VALUE, entry.getValue()));
4770             }
4771             synchronized (waitLock) {
4772               waitLock.wait();
4773             }
4774           } catch (Exception e) {
4775           } catch (AssertionError e) {
4776             // the error happens in a thread, it won't fail the test,
4777             // need to pass it to the caller for proper handling.
4778             error.set(e);
4779             LOG.error(e.toString(), e);
4780           }
4781
4782           return null;
4783         }
4784       });
4785     }
4786     synchronized (waitLock) {
4787       waitLock.notifyAll();
4788     }
4789     executorService.shutdownNow();
4790     assertNull(error.get());
4791   }
4792
4793   @Test
4794   public void testCheckAndPut() throws IOException {
4795     final byte [] anotherrow = Bytes.toBytes("anotherrow");
4796     final byte [] value2 = Bytes.toBytes("abcd");
4797
4798     Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY);
4799     Put put1 = new Put(ROW);
4800     put1.addColumn(FAMILY, QUALIFIER, VALUE);
4801
4802     // row doesn't exist, so using non-null value should be considered "not match".
4803     boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4804         .ifEquals(VALUE).thenPut(put1);
4805     assertFalse(ok);
4806
4807     // row doesn't exist, so using "ifNotExists" should be considered "match".
4808     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifNotExists().thenPut(put1);
4809     assertTrue(ok);
4810
4811     // row now exists, so using "ifNotExists" should be considered "not match".
4812     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifNotExists().thenPut(put1);
4813     assertFalse(ok);
4814
4815     Put put2 = new Put(ROW);
4816     put2.addColumn(FAMILY, QUALIFIER, value2);
4817
4818     // row now exists, use the matching value to check
4819     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifEquals(VALUE).thenPut(put2);
4820     assertTrue(ok);
4821
4822     Put put3 = new Put(anotherrow);
4823     put3.addColumn(FAMILY, QUALIFIER, VALUE);
4824
4825     // try to do CheckAndPut on different rows
4826     try {
4827       table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifEquals(value2).thenPut(put3);
4828       fail("trying to check and modify different rows should have failed.");
4829     } catch(Exception e) {}
4830
4831   }
4832
4833   @Test
4834   public void testCheckAndMutateWithTimeRange() throws IOException {
4835     Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY);
4836     final long ts = System.currentTimeMillis() / 2;
4837     Put put = new Put(ROW);
4838     put.addColumn(FAMILY, QUALIFIER, ts, VALUE);
4839
4840     boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4841       .ifNotExists()
4842       .thenPut(put);
4843     assertTrue(ok);
4844
4845     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4846       .timeRange(TimeRange.at(ts + 10000))
4847       .ifEquals(VALUE)
4848       .thenPut(put);
4849     assertFalse(ok);
4850
4851     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4852       .timeRange(TimeRange.at(ts))
4853       .ifEquals(VALUE)
4854       .thenPut(put);
4855     assertTrue(ok);
4856
4857     RowMutations rm = new RowMutations(ROW)
4858       .add((Mutation) put);
4859     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4860       .timeRange(TimeRange.at(ts + 10000))
4861       .ifEquals(VALUE)
4862       .thenMutate(rm);
4863     assertFalse(ok);
4864
4865     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4866       .timeRange(TimeRange.at(ts))
4867       .ifEquals(VALUE)
4868       .thenMutate(rm);
4869     assertTrue(ok);
4870
4871     Delete delete = new Delete(ROW)
4872       .addColumn(FAMILY, QUALIFIER);
4873
4874     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4875       .timeRange(TimeRange.at(ts + 10000))
4876       .ifEquals(VALUE)
4877       .thenDelete(delete);
4878     assertFalse(ok);
4879
4880     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4881       .timeRange(TimeRange.at(ts))
4882       .ifEquals(VALUE)
4883       .thenDelete(delete);
4884     assertTrue(ok);
4885   }
4886
4887   @Test
4888   public void testCheckAndPutWithCompareOp() throws IOException {
4889     final byte [] value1 = Bytes.toBytes("aaaa");
4890     final byte [] value2 = Bytes.toBytes("bbbb");
4891     final byte [] value3 = Bytes.toBytes("cccc");
4892     final byte [] value4 = Bytes.toBytes("dddd");
4893
4894     Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY);
4895
4896     Put put2 = new Put(ROW);
4897     put2.addColumn(FAMILY, QUALIFIER, value2);
4898
4899     Put put3 = new Put(ROW);
4900     put3.addColumn(FAMILY, QUALIFIER, value3);
4901
4902     // row doesn't exist, so using "ifNotExists" should be considered "match".
4903     boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifNotExists().thenPut(put2);
4904     assertTrue(ok);
4905
4906     // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL
4907     // turns out "match"
4908     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4909         .ifMatches(CompareOperator.GREATER, value1).thenPut(put2);
4910     assertFalse(ok);
4911     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4912         .ifMatches(CompareOperator.EQUAL, value1).thenPut(put2);
4913     assertFalse(ok);
4914     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4915         .ifMatches(CompareOperator.GREATER_OR_EQUAL, value1).thenPut(put2);
4916     assertFalse(ok);
4917     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4918         .ifMatches(CompareOperator.LESS, value1).thenPut(put2);
4919     assertTrue(ok);
4920     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4921         .ifMatches(CompareOperator.LESS_OR_EQUAL, value1).thenPut(put2);
4922     assertTrue(ok);
4923     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4924         .ifMatches(CompareOperator.NOT_EQUAL, value1).thenPut(put3);
4925     assertTrue(ok);
4926
4927     // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL
4928     // turns out "match"
4929     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4930         .ifMatches(CompareOperator.LESS, value4).thenPut(put3);
4931     assertFalse(ok);
4932     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4933         .ifMatches(CompareOperator.LESS_OR_EQUAL, value4).thenPut(put3);
4934     assertFalse(ok);
4935     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4936         .ifMatches(CompareOperator.EQUAL, value4).thenPut(put3);
4937     assertFalse(ok);
4938     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4939         .ifMatches(CompareOperator.GREATER, value4).thenPut(put3);
4940     assertTrue(ok);
4941     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4942         .ifMatches(CompareOperator.GREATER_OR_EQUAL, value4).thenPut(put3);
4943     assertTrue(ok);
4944     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4945         .ifMatches(CompareOperator.NOT_EQUAL, value4).thenPut(put2);
4946     assertTrue(ok);
4947
4948     // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL
4949     // turns out "match"
4950     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4951         .ifMatches(CompareOperator.GREATER, value2).thenPut(put2);
4952     assertFalse(ok);
4953     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4954         .ifMatches(CompareOperator.NOT_EQUAL, value2).thenPut(put2);
4955     assertFalse(ok);
4956     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4957         .ifMatches(CompareOperator.LESS, value2).thenPut(put2);
4958     assertFalse(ok);
4959     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4960         .ifMatches(CompareOperator.GREATER_OR_EQUAL, value2).thenPut(put2);
4961     assertTrue(ok);
4962     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4963         .ifMatches(CompareOperator.LESS_OR_EQUAL, value2).thenPut(put2);
4964     assertTrue(ok);
4965     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4966         .ifMatches(CompareOperator.EQUAL, value2).thenPut(put3);
4967     assertTrue(ok);
4968   }
4969
4970   @Test
4971   public void testCheckAndDelete() throws IOException {
4972     final byte [] value1 = Bytes.toBytes("aaaa");
4973
4974     Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()),
4975         FAMILY);
4976
4977     Put put = new Put(ROW);
4978     put.addColumn(FAMILY, QUALIFIER, value1);
4979     table.put(put);
4980
4981     Delete delete = new Delete(ROW);
4982     delete.addColumns(FAMILY, QUALIFIER);
4983
4984     boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
4985         .ifEquals(value1).thenDelete(delete);
4986     assertTrue(ok);
4987   }
4988
4989   @Test
4990   public void testCheckAndDeleteWithCompareOp() throws IOException {
4991     final byte [] value1 = Bytes.toBytes("aaaa");
4992     final byte [] value2 = Bytes.toBytes("bbbb");
4993     final byte [] value3 = Bytes.toBytes("cccc");
4994     final byte [] value4 = Bytes.toBytes("dddd");
4995
4996     Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()),
4997         FAMILY);
4998
4999     Put put2 = new Put(ROW);
5000     put2.addColumn(FAMILY, QUALIFIER, value2);
5001     table.put(put2);
5002
5003     Put put3 = new Put(ROW);
5004     put3.addColumn(FAMILY, QUALIFIER, value3);
5005
5006     Delete delete = new Delete(ROW);
5007     delete.addColumns(FAMILY, QUALIFIER);
5008
5009     // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL
5010     // turns out "match"
5011     boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5012         .ifMatches(CompareOperator.GREATER, value1).thenDelete(delete);
5013     assertFalse(ok);
5014     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5015         .ifMatches(CompareOperator.EQUAL, value1).thenDelete(delete);
5016     assertFalse(ok);
5017     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5018         .ifMatches(CompareOperator.GREATER_OR_EQUAL, value1).thenDelete(delete);
5019     assertFalse(ok);
5020     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5021         .ifMatches(CompareOperator.LESS, value1).thenDelete(delete);
5022     assertTrue(ok);
5023     table.put(put2);
5024     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5025         .ifMatches(CompareOperator.LESS_OR_EQUAL, value1).thenDelete(delete);
5026     assertTrue(ok);
5027     table.put(put2);
5028     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5029         .ifMatches(CompareOperator.NOT_EQUAL, value1).thenDelete(delete);
5030     assertTrue(ok);
5031
5032     // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL
5033     // turns out "match"
5034     table.put(put3);
5035     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5036         .ifMatches(CompareOperator.LESS, value4).thenDelete(delete);
5037     assertFalse(ok);
5038     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5039         .ifMatches(CompareOperator.LESS_OR_EQUAL, value4).thenDelete(delete);
5040     assertFalse(ok);
5041     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5042         .ifMatches(CompareOperator.EQUAL, value4).thenDelete(delete);
5043     assertFalse(ok);
5044     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5045         .ifMatches(CompareOperator.GREATER, value4).thenDelete(delete);
5046     assertTrue(ok);
5047     table.put(put3);
5048     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5049         .ifMatches(CompareOperator.GREATER_OR_EQUAL, value4).thenDelete(delete);
5050     assertTrue(ok);
5051     table.put(put3);
5052     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5053         .ifMatches(CompareOperator.NOT_EQUAL, value4).thenDelete(delete);
5054     assertTrue(ok);
5055
5056     // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL
5057     // turns out "match"
5058     table.put(put2);
5059     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5060         .ifMatches(CompareOperator.GREATER, value2).thenDelete(delete);
5061     assertFalse(ok);
5062     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5063         .ifMatches(CompareOperator.NOT_EQUAL, value2).thenDelete(delete);
5064     assertFalse(ok);
5065     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5066         .ifMatches(CompareOperator.LESS, value2).thenDelete(delete);
5067     assertFalse(ok);
5068     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5069         .ifMatches(CompareOperator.GREATER_OR_EQUAL, value2).thenDelete(delete);
5070     assertTrue(ok);
5071     table.put(put2);
5072     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5073         .ifMatches(CompareOperator.LESS_OR_EQUAL, value2).thenDelete(delete);
5074     assertTrue(ok);
5075     table.put(put2);
5076     ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
5077         .ifMatches(CompareOperator.EQUAL, value2).thenDelete(delete);
5078     assertTrue(ok);
5079   }
5080
5081   /**
5082   * Test ScanMetrics
5083   */
5084   @Test
5085   @SuppressWarnings ("unused")
5086   public void testScanMetrics() throws Exception {
5087     final TableName tableName = TableName.valueOf(name.getMethodName());
5088
5089     // Set up test table:
5090     // Create table:
5091     Table ht = TEST_UTIL.createMultiRegionTable(tableName, FAMILY);
5092     int numOfRegions = -1;
5093     try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
5094       numOfRegions = r.getStartKeys().length;
5095     }
5096     // Create 3 rows in the table, with rowkeys starting with "zzz*" so that
5097     // scan are forced to hit all the regions.
5098     Put put1 = new Put(Bytes.toBytes("zzz1"));
5099     put1.addColumn(FAMILY, QUALIFIER, VALUE);
5100     Put put2 = new Put(Bytes.toBytes("zzz2"));
5101     put2.addColumn(FAMILY, QUALIFIER, VALUE);
5102     Put put3 = new Put(Bytes.toBytes("zzz3"));
5103     put3.addColumn(FAMILY, QUALIFIER, VALUE);
5104     ht.put(Arrays.asList(put1, put2, put3));
5105
5106     Scan scan1 = new Scan();
5107     int numRecords = 0;
5108     ResultScanner scanner = ht.getScanner(scan1);
5109     for(Result result : scanner) {
5110       numRecords++;
5111     }
5112     scanner.close();
5113     LOG.info("test data has " + numRecords + " records.");
5114
5115     // by default, scan metrics collection is turned off
5116     assertEquals(null, scanner.getScanMetrics());
5117
5118     // turn on scan metrics
5119     Scan scan2 = new Scan();
5120     scan2.setScanMetricsEnabled(true);
5121     scan2.setCaching(numRecords+1);
5122     scanner = ht.getScanner(scan2);
5123     for (Result result : scanner.next(numRecords - 1)) {
5124     }
5125     scanner.close();
5126     // closing the scanner will set the metrics.
5127     assertNotNull(scanner.getScanMetrics());
5128
5129     // set caching to 1, because metrics are collected in each roundtrip only
5130     scan2 = new Scan();
5131     scan2.setScanMetricsEnabled(true);
5132     scan2.setCaching(1);
5133     scanner = ht.getScanner(scan2);
5134     // per HBASE-5717, this should still collect even if you don't run all the way to
5135     // the end of the scanner. So this is asking for 2 of the 3 rows we inserted.
5136     for (Result result : scanner.next(numRecords - 1)) {
5137     }
5138     scanner.close();
5139
5140     ScanMetrics scanMetrics = scanner.getScanMetrics();
5141     assertEquals("Did not access all the regions in the table", numOfRegions,
5142         scanMetrics.countOfRegions.get());
5143
5144     // check byte counters
5145     scan2 = new Scan();
5146     scan2.setScanMetricsEnabled(true);
5147     scan2.setCaching(1);
5148     scanner = ht.getScanner(scan2);
5149     int numBytes = 0;
5150     for (Result result : scanner.next(1)) {
5151       for (Cell cell: result.listCells()) {
5152         numBytes += PrivateCellUtil.estimatedSerializedSizeOf(cell);
5153       }
5154     }
5155     scanner.close();
5156     scanMetrics = scanner.getScanMetrics();
5157     assertEquals("Did not count the result bytes", numBytes,
5158       scanMetrics.countOfBytesInResults.get());
5159
5160     // check byte counters on a small scan
5161     scan2 = new Scan();
5162     scan2.setScanMetricsEnabled(true);
5163     scan2.setCaching(1);
5164     scan2.setSmall(true);
5165     scanner = ht.getScanner(scan2);
5166     numBytes = 0;
5167     for (Result result : scanner.next(1)) {
5168       for (Cell cell: result.listCells()) {
5169         numBytes += PrivateCellUtil.estimatedSerializedSizeOf(cell);
5170       }
5171     }
5172     scanner.close();
5173     scanMetrics = scanner.getScanMetrics();
5174     assertEquals("Did not count the result bytes", numBytes,
5175       scanMetrics.countOfBytesInResults.get());
5176
5177     // now, test that the metrics are still collected even if you don't call close, but do
5178     // run past the end of all the records
5179     /** There seems to be a timing issue here.  Comment out for now. Fix when time.
5180     Scan scanWithoutClose = new Scan();
5181     scanWithoutClose.setCaching(1);
5182     scanWithoutClose.setScanMetricsEnabled(true);
5183     ResultScanner scannerWithoutClose = ht.getScanner(scanWithoutClose);
5184     for (Result result : scannerWithoutClose.next(numRecords + 1)) {
5185     }
5186     ScanMetrics scanMetricsWithoutClose = getScanMetrics(scanWithoutClose);
5187     assertEquals("Did not access all the regions in the table", numOfRegions,
5188         scanMetricsWithoutClose.countOfRegions.get());
5189     */
5190
5191     // finally, test that the metrics are collected correctly if you both run past all the records,
5192     // AND close the scanner
5193     Scan scanWithClose = new Scan();
5194     // make sure we can set caching up to the number of a scanned values
5195     scanWithClose.setCaching(numRecords);
5196     scanWithClose.setScanMetricsEnabled(true);
5197     ResultScanner scannerWithClose = ht.getScanner(scanWithClose);
5198     for (Result result : scannerWithClose.next(numRecords + 1)) {
5199     }
5200     scannerWithClose.close();
5201     ScanMetrics scanMetricsWithClose = scannerWithClose.getScanMetrics();
5202     assertEquals("Did not access all the regions in the table", numOfRegions,
5203         scanMetricsWithClose.countOfRegions.get());
5204   }
5205
5206   /**
5207    * Tests that cache on write works all the way up from the client-side.
5208    *
5209    * Performs inserts, flushes, and compactions, verifying changes in the block
5210    * cache along the way.
5211    */
5212   @Test
5213   public void testCacheOnWriteEvictOnClose() throws Exception {
5214     final TableName tableName = TableName.valueOf(name.getMethodName());
5215     byte [] data = Bytes.toBytes("data");
5216     Table table = TEST_UTIL.createTable(tableName, FAMILY);
5217     try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
5218       // get the block cache and region
5219       String regionName = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName();
5220
5221       HRegion region = TEST_UTIL.getRSForFirstRegionInTable(tableName)
5222           .getRegion(regionName);
5223       HStore store = region.getStores().iterator().next();
5224       CacheConfig cacheConf = store.getCacheConfig();
5225       cacheConf.setCacheDataOnWrite(true);
5226       cacheConf.setEvictOnClose(true);
5227       BlockCache cache = cacheConf.getBlockCache().get();
5228
5229       // establish baseline stats
5230       long startBlockCount = cache.getBlockCount();
5231       long startBlockHits = cache.getStats().getHitCount();
5232       long startBlockMiss = cache.getStats().getMissCount();
5233
5234       // wait till baseline is stable, (minimal 500 ms)
5235       for (int i = 0; i < 5; i++) {
5236         Thread.sleep(100);
5237         if (startBlockCount != cache.getBlockCount()
5238             || startBlockHits != cache.getStats().getHitCount()
5239             || startBlockMiss != cache.getStats().getMissCount()) {
5240           startBlockCount = cache.getBlockCount();
5241           startBlockHits = cache.getStats().getHitCount();
5242           startBlockMiss = cache.getStats().getMissCount();
5243           i = -1;
5244         }
5245       }
5246
5247       // insert data
5248       Put put = new Put(ROW);
5249       put.addColumn(FAMILY, QUALIFIER, data);
5250       table.put(put);
5251       assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data));
5252       // data was in memstore so don't expect any changes
5253       assertEquals(startBlockCount, cache.getBlockCount());
5254       assertEquals(startBlockHits, cache.getStats().getHitCount());
5255       assertEquals(startBlockMiss, cache.getStats().getMissCount());
5256       // flush the data
5257       System.out.println("Flushing cache");
5258       region.flush(true);
5259       // expect one more block in cache, no change in hits/misses
5260       long expectedBlockCount = startBlockCount + 1;
5261       long expectedBlockHits = startBlockHits;
5262       long expectedBlockMiss = startBlockMiss;
5263       assertEquals(expectedBlockCount, cache.getBlockCount());
5264       assertEquals(expectedBlockHits, cache.getStats().getHitCount());
5265       assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5266       // read the data and expect same blocks, one new hit, no misses
5267       assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data));
5268       assertEquals(expectedBlockCount, cache.getBlockCount());
5269       assertEquals(++expectedBlockHits, cache.getStats().getHitCount());
5270       assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5271       // insert a second column, read the row, no new blocks, one new hit
5272       byte [] QUALIFIER2 = Bytes.add(QUALIFIER, QUALIFIER);
5273       byte [] data2 = Bytes.add(data, data);
5274       put = new Put(ROW);
5275       put.addColumn(FAMILY, QUALIFIER2, data2);
5276       table.put(put);
5277       Result r = table.get(new Get(ROW));
5278       assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data));
5279       assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2));
5280       assertEquals(expectedBlockCount, cache.getBlockCount());
5281       assertEquals(++expectedBlockHits, cache.getStats().getHitCount());
5282       assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5283       // flush, one new block
5284       System.out.println("Flushing cache");
5285       region.flush(true);
5286       assertEquals(++expectedBlockCount, cache.getBlockCount());
5287       assertEquals(expectedBlockHits, cache.getStats().getHitCount());
5288       assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5289       // compact, net minus two blocks, two hits, no misses
5290       System.out.println("Compacting");
5291       assertEquals(2, store.getStorefilesCount());
5292       store.triggerMajorCompaction();
5293       region.compact(true);
5294       store.closeAndArchiveCompactedFiles();
5295       waitForStoreFileCount(store, 1, 10000); // wait 10 seconds max
5296       assertEquals(1, store.getStorefilesCount());
5297       expectedBlockCount -= 2; // evicted two blocks, cached none
5298       assertEquals(expectedBlockCount, cache.getBlockCount());
5299       expectedBlockHits += 2;
5300       assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
5301       assertEquals(expectedBlockHits, cache.getStats().getHitCount());
5302       // read the row, this should be a cache miss because we don't cache data
5303       // blocks on compaction
5304       r = table.get(new Get(ROW));
5305       assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data));
5306       assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2));
5307       expectedBlockCount += 1; // cached one data block
5308       assertEquals(expectedBlockCount, cache.getBlockCount());
5309       assertEquals(expectedBlockHits, cache.getStats().getHitCount());
5310       assertEquals(++expectedBlockMiss, cache.getStats().getMissCount());
5311     }
5312   }
5313
5314   private void waitForStoreFileCount(HStore store, int count, int timeout)
5315       throws InterruptedException {
5316     long start = System.currentTimeMillis();
5317     while (start + timeout > System.currentTimeMillis() && store.getStorefilesCount() != count) {
5318       Thread.sleep(100);
5319     }
5320     System.out.println("start=" + start + ", now=" + System.currentTimeMillis() + ", cur=" +
5321         store.getStorefilesCount());
5322     assertEquals(count, store.getStorefilesCount());
5323   }
5324
5325   @Test
5326   /**
5327    * Tests the non cached version of getRegionLocator by moving a region.
5328    */
5329   public void testNonCachedGetRegionLocation() throws Exception {
5330     // Test Initialization.
5331     final TableName tableName = TableName.valueOf(name.getMethodName());
5332     byte [] family1 = Bytes.toBytes("f1");
5333     byte [] family2 = Bytes.toBytes("f2");
5334     try (Table table = TEST_UTIL.createTable(tableName, new byte[][] {family1, family2}, 10);
5335         Admin admin = TEST_UTIL.getAdmin();
5336         RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
5337       List<HRegionLocation> allRegionLocations = locator.getAllRegionLocations();
5338       assertEquals(1, allRegionLocations.size());
5339       HRegionInfo regionInfo = allRegionLocations.get(0).getRegionInfo();
5340       ServerName addrBefore = allRegionLocations.get(0).getServerName();
5341       // Verify region location before move.
5342       HRegionLocation addrCache = locator.getRegionLocation(regionInfo.getStartKey(), false);
5343       HRegionLocation addrNoCache = locator.getRegionLocation(regionInfo.getStartKey(),  true);
5344
5345       assertEquals(addrBefore.getPort(), addrCache.getPort());
5346       assertEquals(addrBefore.getPort(), addrNoCache.getPort());
5347
5348       ServerName addrAfter = null;
5349       // Now move the region to a different server.
5350       for (int i = 0; i < SLAVES; i++) {
5351         HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(i);
5352         ServerName addr = regionServer.getServerName();
5353         if (addr.getPort() != addrBefore.getPort()) {
5354           admin.move(regionInfo.getEncodedNameAsBytes(),
5355               Bytes.toBytes(addr.toString()));
5356           // Wait for the region to move.
5357           Thread.sleep(5000);
5358           addrAfter = addr;
5359           break;
5360         }
5361       }
5362
5363       // Verify the region was moved.
5364       addrCache = locator.getRegionLocation(regionInfo.getStartKey(), false);
5365       addrNoCache = locator.getRegionLocation(regionInfo.getStartKey(), true);
5366       assertNotNull(addrAfter);
5367       assertTrue(addrAfter.getPort() != addrCache.getPort());
5368       assertEquals(addrAfter.getPort(), addrNoCache.getPort());
5369     }
5370   }
5371
5372   @Test
5373   /**
5374    * Tests getRegionsInRange by creating some regions over which a range of
5375    * keys spans; then changing the key range.
5376    */
5377   public void testGetRegionsInRange() throws Exception {
5378     // Test Initialization.
5379     byte [] startKey = Bytes.toBytes("ddc");
5380     byte [] endKey = Bytes.toBytes("mmm");
5381     TableName tableName = TableName.valueOf(name.getMethodName());
5382     TEST_UTIL.createMultiRegionTable(tableName, new byte[][] { FAMILY }, 10);
5383
5384     int numOfRegions = -1;
5385     try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
5386       numOfRegions = r.getStartKeys().length;
5387     }
5388     assertEquals(26, numOfRegions);
5389
5390     // Get the regions in this range
5391     List<HRegionLocation> regionsList = getRegionsInRange(tableName, startKey, endKey);
5392     assertEquals(10, regionsList.size());
5393
5394     // Change the start key
5395     startKey = Bytes.toBytes("fff");
5396     regionsList = getRegionsInRange(tableName, startKey, endKey);
5397     assertEquals(7, regionsList.size());
5398
5399     // Change the end key
5400     endKey = Bytes.toBytes("nnn");
5401     regionsList = getRegionsInRange(tableName, startKey, endKey);
5402     assertEquals(8, regionsList.size());
5403
5404     // Empty start key
5405     regionsList = getRegionsInRange(tableName, HConstants.EMPTY_START_ROW, endKey);
5406     assertEquals(13, regionsList.size());
5407
5408     // Empty end key
5409     regionsList = getRegionsInRange(tableName, startKey, HConstants.EMPTY_END_ROW);
5410     assertEquals(21, regionsList.size());
5411
5412     // Both start and end keys empty
5413     regionsList = getRegionsInRange(tableName, HConstants.EMPTY_START_ROW,
5414         HConstants.EMPTY_END_ROW);
5415     assertEquals(26, regionsList.size());
5416
5417     // Change the end key to somewhere in the last block
5418     endKey = Bytes.toBytes("zzz1");
5419     regionsList = getRegionsInRange(tableName, startKey, endKey);
5420     assertEquals(21, regionsList.size());
5421
5422     // Change the start key to somewhere in the first block
5423     startKey = Bytes.toBytes("aac");
5424     regionsList = getRegionsInRange(tableName, startKey, endKey);
5425     assertEquals(26, regionsList.size());
5426
5427     // Make start and end key the same
5428     startKey = endKey = Bytes.toBytes("ccc");
5429     regionsList = getRegionsInRange(tableName, startKey, endKey);
5430     assertEquals(1, regionsList.size());
5431   }
5432
5433   private List<HRegionLocation> getRegionsInRange(TableName tableName, byte[] startKey,
5434       byte[] endKey) throws IOException {
5435     List<HRegionLocation> regionsInRange = new ArrayList<>();
5436     byte[] currentKey = startKey;
5437     final boolean endKeyIsEndOfTable = Bytes.equals(endKey, HConstants.EMPTY_END_ROW);
5438     try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
5439       do {
5440         HRegionLocation regionLocation = r.getRegionLocation(currentKey);
5441         regionsInRange.add(regionLocation);
5442         currentKey = regionLocation.getRegionInfo().getEndKey();
5443       } while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW)
5444           && (endKeyIsEndOfTable || Bytes.compareTo(currentKey, endKey) < 0));
5445       return regionsInRange;
5446     }
5447   }
5448
5449   @Test
5450   public void testJira6912() throws Exception {
5451     final TableName tableName = TableName.valueOf(name.getMethodName());
5452     Table foo = TEST_UTIL.createTable(tableName, new byte[][] {FAMILY}, 10);
5453
5454     List<Put> puts = new ArrayList<Put>();
5455     for (int i=0;i !=100; i++){
5456       Put put = new Put(Bytes.toBytes(i));
5457       put.addColumn(FAMILY, FAMILY, Bytes.toBytes(i));
5458       puts.add(put);
5459     }
5460     foo.put(puts);
5461     // If i comment this out it works
5462     TEST_UTIL.flush();
5463
5464     Scan scan = new Scan();
5465     scan.setStartRow(Bytes.toBytes(1));
5466     scan.setStopRow(Bytes.toBytes(3));
5467     scan.addColumn(FAMILY, FAMILY);
5468     scan.setFilter(new RowFilter(CompareOperator.NOT_EQUAL,
5469         new BinaryComparator(Bytes.toBytes(1))));
5470
5471     ResultScanner scanner = foo.getScanner(scan);
5472     Result[] bar = scanner.next(100);
5473     assertEquals(1, bar.length);
5474   }
5475
5476   @Test
5477   public void testScan_NullQualifier() throws IOException {
5478     Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY);
5479     Put put = new Put(ROW);
5480     put.addColumn(FAMILY, QUALIFIER, VALUE);
5481     table.put(put);
5482
5483     put = new Put(ROW);
5484     put.addColumn(FAMILY, null, VALUE);
5485     table.put(put);
5486     LOG.info("Row put");
5487
5488     Scan scan = new Scan();
5489     scan.addColumn(FAMILY, null);
5490
5491     ResultScanner scanner = table.getScanner(scan);
5492     Result[] bar = scanner.next(100);
5493     assertEquals(1, bar.length);
5494     assertEquals(1, bar[0].size());
5495
5496     scan = new Scan();
5497     scan.addFamily(FAMILY);
5498
5499     scanner = table.getScanner(scan);
5500     bar = scanner.next(100);
5501     assertEquals(1, bar.length);
5502     assertEquals(2, bar[0].size());
5503   }
5504
5505   @Test
5506   public void testNegativeTimestamp() throws IOException {
5507     Table table = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), FAMILY);
5508
5509     try {
5510       Put put = new Put(ROW, -1);
5511       put.addColumn(FAMILY, QUALIFIER, VALUE);
5512       table.put(put);
5513       fail("Negative timestamps should not have been allowed");
5514     } catch (IllegalArgumentException ex) {
5515       assertTrue(ex.getMessage().contains("negative"));
5516     }
5517
5518     try {
5519       Put put = new Put(ROW);
5520       long ts = -1;
5521       put.addColumn(FAMILY, QUALIFIER, ts, VALUE);
5522       table.put(put);
5523       fail("Negative timestamps should not have been allowed");
5524     } catch (IllegalArgumentException ex) {
5525       assertTrue(ex.getMessage().contains("negative"));
5526     }
5527
5528     try {
5529       Delete delete = new Delete(ROW, -1);
5530       table.delete(delete);
5531       fail("Negative timestamps should not have been allowed");
5532     } catch (IllegalArgumentException ex) {
5533       assertTrue(ex.getMessage().contains("negative"));
5534     }
5535
5536     try {
5537       Delete delete = new Delete(ROW);
5538       delete.addFamily(FAMILY, -1);
5539       table.delete(delete);
5540       fail("Negative timestamps should not have been allowed");
5541     } catch (IllegalArgumentException ex) {
5542       assertTrue(ex.getMessage().contains("negative"));
5543     }
5544
5545     try {
5546       Scan scan = new Scan();
5547       scan.setTimeRange(-1, 1);
5548       table.getScanner(scan);
5549       fail("Negative timestamps should not have been allowed");
5550     } catch (IllegalArgumentException ex) {
5551       assertTrue(ex.getMessage().contains("negative"));
5552     }
5553
5554     // KeyValue should allow negative timestamps for backwards compat. Otherwise, if the user
5555     // already has negative timestamps in cluster data, HBase won't be able to handle that
5556     try {
5557       new KeyValue(Bytes.toBytes(42), Bytes.toBytes(42), Bytes.toBytes(42), -1, Bytes.toBytes(42));
5558     } catch (IllegalArgumentException ex) {
5559       fail("KeyValue SHOULD allow negative timestamps");
5560     }
5561
5562     table.close();
5563   }
5564
5565   @Test
5566   public void testRawScanRespectsVersions() throws Exception {
5567     final TableName tableName = TableName.valueOf(name.getMethodName());
5568     Table table = TEST_UTIL.createTable(tableName, FAMILY);
5569     byte[] row = Bytes.toBytes("row");
5570
5571     // put the same row 4 times, with different values
5572     Put p = new Put(row);
5573     p.addColumn(FAMILY, QUALIFIER, 10, VALUE);
5574     table.put(p);
5575     p = new Put(row);
5576     p.addColumn(FAMILY, QUALIFIER, 11, ArrayUtils.add(VALUE, (byte) 2));
5577     table.put(p);
5578
5579     p = new Put(row);
5580     p.addColumn(FAMILY, QUALIFIER, 12, ArrayUtils.add(VALUE, (byte) 3));
5581     table.put(p);
5582
5583     p = new Put(row);
5584     p.addColumn(FAMILY, QUALIFIER, 13, ArrayUtils.add(VALUE, (byte) 4));
5585     table.put(p);
5586
5587     int versions = 4;
5588     Scan s = new Scan(row);
5589     // get all the possible versions
5590     s.setMaxVersions();
5591     s.setRaw(true);
5592
5593     ResultScanner scanner = table.getScanner(s);
5594     int count = 0;
5595     for (Result r : scanner) {
5596       assertEquals("Found an unexpected number of results for the row!", versions, r.listCells().size());
5597       count++;
5598     }
5599     assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
5600       count);
5601     scanner.close();
5602
5603     // then if we decrease the number of versions, but keep the scan raw, we should see exactly that
5604     // number of versions
5605     versions = 2;
5606     s.setMaxVersions(versions);
5607     scanner = table.getScanner(s);
5608     count = 0;
5609     for (Result r : scanner) {
5610       assertEquals("Found an unexpected number of results for the row!", versions, r.listCells().size());
5611       count++;
5612     }
5613     assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
5614       count);
5615     scanner.close();
5616
5617     // finally, if we turn off raw scanning, but max out the number of versions, we should go back
5618     // to seeing just three
5619     versions = 3;
5620     s.setMaxVersions(versions);
5621     scanner = table.getScanner(s);
5622     count = 0;
5623     for (Result r : scanner) {
5624       assertEquals("Found an unexpected number of results for the row!", versions, r.listCells().size());
5625       count++;
5626     }
5627     assertEquals("Found more than a single row when raw scanning the table with a single row!", 1,
5628       count);
5629     scanner.close();
5630
5631     table.close();
5632     TEST_UTIL.deleteTable(tableName);
5633   }
5634
5635   @Test
5636   public void testEmptyFilterList() throws Exception {
5637     // Test Initialization.
5638     final TableName tableName = TableName.valueOf(name.getMethodName());
5639     Table table = TEST_UTIL.createTable(tableName, FAMILY);
5640
5641     // Insert one row each region
5642     Put put = new Put(Bytes.toBytes("row"));
5643     put.addColumn(FAMILY, QUALIFIER, VALUE);
5644     table.put(put);
5645
5646     List<Result> scanResults = new LinkedList<>();
5647     Scan scan = new Scan();
5648     scan.setFilter(new FilterList());
5649     try (ResultScanner scanner = table.getScanner(scan)) {
5650       for (Result r : scanner) {
5651         scanResults.add(r);
5652       }
5653     }
5654     assertEquals(1, scanResults.size());
5655     Get g = new Get(Bytes.toBytes("row"));
5656     g.setFilter(new FilterList());
5657     Result getResult = table.get(g);
5658     Result scanResult = scanResults.get(0);
5659     assertEquals(scanResult.rawCells().length, getResult.rawCells().length);
5660     for (int i = 0; i != scanResult.rawCells().length; ++i) {
5661       Cell scanCell = scanResult.rawCells()[i];
5662       Cell getCell = getResult.rawCells()[i];
5663       assertEquals(0, Bytes.compareTo(CellUtil.cloneRow(scanCell), CellUtil.cloneRow(getCell)));
5664       assertEquals(0, Bytes.compareTo(CellUtil.cloneFamily(scanCell), CellUtil.cloneFamily(getCell)));
5665       assertEquals(0, Bytes.compareTo(CellUtil.cloneQualifier(scanCell), CellUtil.cloneQualifier(getCell)));
5666       assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(scanCell), CellUtil.cloneValue(getCell)));
5667     }
5668   }
5669
5670   @Test
5671   public void testSmallScan() throws Exception {
5672     // Test Initialization.
5673     final TableName tableName = TableName.valueOf(name.getMethodName());
5674     Table table = TEST_UTIL.createTable(tableName, FAMILY);
5675
5676     // Insert one row each region
5677     int insertNum = 10;
5678     for (int i = 0; i < 10; i++) {
5679       Put put = new Put(Bytes.toBytes("row" + String.format("%03d", i)));
5680       put.addColumn(FAMILY, QUALIFIER, VALUE);
5681       table.put(put);
5682     }
5683
5684     // normal scan
5685     ResultScanner scanner = table.getScanner(new Scan());
5686     int count = 0;
5687     for (Result r : scanner) {
5688       assertTrue(!r.isEmpty());
5689       count++;
5690     }
5691     assertEquals(insertNum, count);
5692
5693     // small scan
5694     Scan scan = new Scan(HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
5695     scan.setSmall(true);
5696     scan.setCaching(2);
5697     scanner = table.getScanner(scan);
5698     count = 0;
5699     for (Result r : scanner) {
5700       assertTrue(!r.isEmpty());
5701       count++;
5702     }
5703     assertEquals(insertNum, count);
5704
5705   }
5706
5707   @Test
5708   public void testSuperSimpleWithReverseScan() throws Exception {
5709     final TableName tableName = TableName.valueOf(name.getMethodName());
5710     Table ht = TEST_UTIL.createTable(tableName, FAMILY);
5711     Put put = new Put(Bytes.toBytes("0-b11111-0000000000000000000"));
5712     put.addColumn(FAMILY, QUALIFIER, VALUE);
5713     ht.put(put);
5714     put = new Put(Bytes.toBytes("0-b11111-0000000000000000002"));
5715     put.addColumn(FAMILY, QUALIFIER, VALUE);
5716     ht.put(put);
5717     put = new Put(Bytes.toBytes("0-b11111-0000000000000000004"));
5718     put.addColumn(FAMILY, QUALIFIER, VALUE);
5719     ht.put(put);
5720     put = new Put(Bytes.toBytes("0-b11111-0000000000000000006"));
5721     put.addColumn(FAMILY, QUALIFIER, VALUE);
5722     ht.put(put);
5723     put = new Put(Bytes.toBytes("0-b11111-0000000000000000008"));
5724     put.addColumn(FAMILY, QUALIFIER, VALUE);
5725     ht.put(put);
5726     put = new Put(Bytes.toBytes("0-b22222-0000000000000000001"));
5727     put.addColumn(FAMILY, QUALIFIER, VALUE);
5728     ht.put(put);
5729     put = new Put(Bytes.toBytes("0-b22222-0000000000000000003"));
5730     put.addColumn(FAMILY, QUALIFIER, VALUE);
5731     ht.put(put);
5732     put = new Put(Bytes.toBytes("0-b22222-0000000000000000005"));
5733     put.addColumn(FAMILY, QUALIFIER, VALUE);
5734     ht.put(put);
5735     put = new Put(Bytes.toBytes("0-b22222-0000000000000000007"));
5736     put.addColumn(FAMILY, QUALIFIER, VALUE);
5737     ht.put(put);
5738     put = new Put(Bytes.toBytes("0-b22222-0000000000000000009"));
5739     put.addColumn(FAMILY, QUALIFIER, VALUE);
5740     ht.put(put);
5741     Scan scan = new Scan(Bytes.toBytes("0-b11111-9223372036854775807"),
5742         Bytes.toBytes("0-b11111-0000000000000000000"));
5743     scan.setReversed(true);
5744     ResultScanner scanner = ht.getScanner(scan);
5745     Result result = scanner.next();
5746     assertTrue(Bytes.equals(result.getRow(),
5747         Bytes.toBytes("0-b11111-0000000000000000008")));
5748     scanner.close();
5749     ht.close();
5750   }
5751
5752   @Test
5753   public void testFiltersWithReverseScan() throws Exception {
5754     final TableName tableName = TableName.valueOf(name.getMethodName());
5755     Table ht = TEST_UTIL.createTable(tableName, FAMILY);
5756     byte[][] ROWS = makeN(ROW, 10);
5757     byte[][] QUALIFIERS = { Bytes.toBytes("col0-<d2v1>-<d3v2>"),
5758         Bytes.toBytes("col1-<d2v1>-<d3v2>"),
5759         Bytes.toBytes("col2-<d2v1>-<d3v2>"),
5760         Bytes.toBytes("col3-<d2v1>-<d3v2>"),
5761         Bytes.toBytes("col4-<d2v1>-<d3v2>"),
5762         Bytes.toBytes("col5-<d2v1>-<d3v2>"),
5763         Bytes.toBytes("col6-<d2v1>-<d3v2>"),
5764         Bytes.toBytes("col7-<d2v1>-<d3v2>"),
5765         Bytes.toBytes("col8-<d2v1>-<d3v2>"),
5766         Bytes.toBytes("col9-<d2v1>-<d3v2>") };
5767     for (int i = 0; i < 10; i++) {
5768       Put put = new Put(ROWS[i]);
5769       put.addColumn(FAMILY, QUALIFIERS[i], VALUE);
5770       ht.put(put);
5771     }
5772     Scan scan = new Scan();
5773     scan.setReversed(true);
5774     scan.addFamily(FAMILY);
5775     Filter filter = new QualifierFilter(CompareOperator.EQUAL,
5776         new RegexStringComparator("col[1-5]"));
5777     scan.setFilter(filter);
5778     ResultScanner scanner = ht.getScanner(scan);
5779     int expectedIndex = 5;
5780     for (Result result : scanner) {
5781       assertEquals(1, result.size());
5782       Cell c = result.rawCells()[0];
5783       assertTrue(Bytes.equals(c.getRowArray(), c.getRowOffset(), c.getRowLength(),
5784         ROWS[expectedIndex], 0, ROWS[expectedIndex].length));
5785       assertTrue(Bytes.equals(c.getQualifierArray(), c.getQualifierOffset(),
5786         c.getQualifierLength(), QUALIFIERS[expectedIndex], 0, QUALIFIERS[expectedIndex].length));
5787       expectedIndex--;
5788     }
5789     assertEquals(0, expectedIndex);
5790     scanner.close();
5791     ht.close();
5792   }
5793
5794   @Test
5795   public void testKeyOnlyFilterWithReverseScan() throws Exception {
5796     final TableName tableName = TableName.valueOf(name.getMethodName());
5797     Table ht = TEST_UTIL.createTable(tableName, FAMILY);
5798     byte[][] ROWS = makeN(ROW, 10);
5799     byte[][] QUALIFIERS = { Bytes.toBytes("col0-<d2v1>-<d3v2>"),
5800         Bytes.toBytes("col1-<d2v1>-<d3v2>"),
5801         Bytes.toBytes("col2-<d2v1>-<d3v2>"),
5802         Bytes.toBytes("col3-<d2v1>-<d3v2>"),
5803         Bytes.toBytes("col4-<d2v1>-<d3v2>"),
5804         Bytes.toBytes("col5-<d2v1>-<d3v2>"),
5805         Bytes.toBytes("col6-<d2v1>-<d3v2>"),
5806         Bytes.toBytes("col7-<d2v1>-<d3v2>"),
5807         Bytes.toBytes("col8-<d2v1>-<d3v2>"),
5808         Bytes.toBytes("col9-<d2v1>-<d3v2>") };
5809     for (int i = 0; i < 10; i++) {
5810       Put put = new Put(ROWS[i]);
5811       put.addColumn(FAMILY, QUALIFIERS[i], VALUE);
5812       ht.put(put);
5813     }
5814     Scan scan = new Scan();
5815     scan.setReversed(true);
5816     scan.addFamily(FAMILY);
5817     Filter filter = new KeyOnlyFilter(true);
5818     scan.setFilter(filter);
5819     ResultScanner scanner = ht.getScanner(scan);
5820     int count = 0;
5821     for (Result result : ht.getScanner(scan)) {
5822       assertEquals(1, result.size());
5823       assertEquals(Bytes.SIZEOF_INT, result.rawCells()[0].getValueLength());
5824       assertEquals(VALUE.length, Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0])));
5825       count++;
5826     }
5827     assertEquals(10, count);
5828     scanner.close();
5829     ht.close();
5830   }
5831
5832   /**
5833    * Test simple table and non-existent row cases.
5834    */
5835   @Test
5836   public void testSimpleMissingWithReverseScan() throws Exception {
5837     final TableName tableName = TableName.valueOf(name.getMethodName());
5838     Table ht = TEST_UTIL.createTable(tableName, FAMILY);
5839     byte[][] ROWS = makeN(ROW, 4);
5840
5841     // Try to get a row on an empty table
5842     Scan scan = new Scan();
5843     scan.setReversed(true);
5844     Result result = getSingleScanResult(ht, scan);
5845     assertNullResult(result);
5846
5847     scan = new Scan(ROWS[0]);
5848     scan.setReversed(true);
5849     result = getSingleScanResult(ht, scan);
5850     assertNullResult(result);
5851
5852     scan = new Scan(ROWS[0], ROWS[1]);
5853     scan.setReversed(true);
5854     result = getSingleScanResult(ht, scan);
5855     assertNullResult(result);
5856
5857     scan = new Scan();
5858     scan.setReversed(true);
5859     scan.addFamily(FAMILY);
5860     result = getSingleScanResult(ht, scan);
5861     assertNullResult(result);
5862
5863     scan = new Scan();
5864     scan.setReversed(true);
5865     scan.addColumn(FAMILY, QUALIFIER);
5866     result = getSingleScanResult(ht, scan);
5867     assertNullResult(result);
5868
5869     // Insert a row
5870
5871     Put put = new Put(ROWS[2]);
5872     put.addColumn(FAMILY, QUALIFIER, VALUE);
5873     ht.put(put);
5874
5875     // Make sure we can scan the row
5876     scan = new Scan();
5877     scan.setReversed(true);
5878     result = getSingleScanResult(ht, scan);
5879     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
5880
5881     scan = new Scan(ROWS[3], ROWS[0]);
5882     scan.setReversed(true);
5883     result = getSingleScanResult(ht, scan);
5884     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
5885
5886     scan = new Scan(ROWS[2], ROWS[1]);
5887     scan.setReversed(true);
5888     result = getSingleScanResult(ht, scan);
5889     assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
5890
5891     // Try to scan empty rows around it
5892     // Introduced MemStore#shouldSeekForReverseScan to fix the following
5893     scan = new Scan(ROWS[1]);
5894     scan.setReversed(true);
5895     result = getSingleScanResult(ht, scan);
5896     assertNullResult(result);
5897     ht.close();
5898   }
5899
5900   @Test
5901   public void testNullWithReverseScan() throws Exception {
5902     final TableName tableName = TableName.valueOf(name.getMethodName());
5903     Table ht = TEST_UTIL.createTable(tableName, FAMILY);
5904     // Null qualifier (should work)
5905     Put put = new Put(ROW);
5906     put.addColumn(FAMILY, null, VALUE);
5907     ht.put(put);
5908     scanTestNull(ht, ROW, FAMILY, VALUE, true);
5909     Delete delete = new Delete(ROW);
5910     delete.addColumns(FAMILY, null);
5911     ht.delete(delete);
5912     // Use a new table
5913     ht = TEST_UTIL.createTable(TableName.valueOf(name.getMethodName() + "2"), FAMILY);
5914     // Empty qualifier, byte[0] instead of null (should work)
5915     put = new Put(ROW);
5916     put.addColumn(FAMILY, HConstants.EMPTY_BYTE_ARRAY, VALUE);
5917     ht.put(put);
5918     scanTestNull(ht, ROW, FAMILY, VALUE, true);
5919     TEST_UTIL.flush();
5920     scanTestNull(ht, ROW, FAMILY, VALUE, true);
5921     delete = new Delete(ROW);
5922     delete.addColumns(FAMILY, HConstants.EMPTY_BYTE_ARRAY);
5923     ht.delete(delete);
5924     // Null value
5925     put = new Put(ROW);
5926     put.addColumn(FAMILY, QUALIFIER, null);
5927     ht.put(put);
5928     Scan scan = new Scan();
5929     scan.setReversed(true);
5930     scan.addColumn(FAMILY, QUALIFIER);
5931     Result result = getSingleScanResult(ht, scan);
5932     assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
5933     ht.close();
5934   }
5935
5936   @Test
5937   public void testDeletesWithReverseScan() throws Exception {
5938     final TableName tableName = TableName.valueOf(name.getMethodName());
5939     byte[][] ROWS = makeNAscii(ROW, 6);
5940     byte[][] FAMILIES = makeNAscii(FAMILY, 3);
5941     byte[][] VALUES = makeN(VALUE, 5);
5942     long[] ts = { 1000, 2000, 3000, 4000, 5000 };
5943     Table ht = TEST_UTIL.createTable(tableName, FAMILIES, 3);
5944
5945     Put put = new Put(ROW);
5946     put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
5947     put.addColumn(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]);
5948     ht.put(put);
5949
5950     Delete delete = new Delete(ROW);
5951     delete.addFamily(FAMILIES[0], ts[0]);
5952     ht.delete(delete);
5953
5954     Scan scan = new Scan(ROW);
5955     scan.setReversed(true);
5956     scan.addFamily(FAMILIES[0]);
5957     scan.setMaxVersions(Integer.MAX_VALUE);
5958     Result result = getSingleScanResult(ht, scan);
5959     assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1] },
5960         new byte[][] { VALUES[1] }, 0, 0);
5961
5962     // Test delete latest version
5963     put = new Put(ROW);
5964     put.addColumn(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
5965     put.addColumn(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]);
5966     put.addColumn(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]);
5967     put.addColumn(FAMILIES[0], null, ts[4], VALUES[4]);
5968     put.addColumn(FAMILIES[0], null, ts[2], VALUES[2]);
5969     put.addColumn(FAMILIES[0], null, ts[3], VALUES[3]);
5970     ht.put(put);
5971
5972     delete = new Delete(ROW);
5973     delete.addColumn(FAMILIES[0], QUALIFIER); // ts[4]
5974     ht.delete(delete);
5975
5976     scan = new Scan(ROW);
5977     scan.setReversed(true);
5978     scan.addColumn(FAMILIES[0], QUALIFIER);
5979     scan.setMaxVersions(Integer.MAX_VALUE);
5980     result = getSingleScanResult(ht, scan);
5981     assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1],
5982         ts[2], ts[3] }, new byte[][] { VALUES[1], VALUES[2], VALUES[3] }, 0, 2);
5983
5984     // Test for HBASE-1847
5985     delete = new Delete(ROW);
5986     delete.addColumn(FAMILIES[0], null);
5987     ht.delete(delete);
5988
5989     // Cleanup null qualifier
5990     delete = new Delete(ROW);
5991     delete.addColumns(FAMILIES[0], null);
5992     ht.delete(delete);
5993
5994     // Expected client behavior might be that you can re-put deleted values
5995     // But alas, this is not to be. We can't put them back in either case.
5996
5997     put = new Put(ROW);
5998     put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
5999     put.addColumn(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
6000     ht.put(put);
6001
6002     // The Scanner returns the previous values, the expected-naive-unexpected
6003     // behavior
6004
6005     scan = new Scan(ROW);
6006     scan.setReversed(true);
6007     scan.addFamily(FAMILIES[0]);
6008     scan.setMaxVersions(Integer.MAX_VALUE);
6009     result = getSingleScanResult(ht, scan);
6010     assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[] { ts[1],
6011         ts[2], ts[3] }, new byte[][] { VALUES[1], VALUES[2], VALUES[3] }, 0, 2);
6012
6013     // Test deleting an entire family from one row but not the other various
6014     // ways
6015
6016     put = new Put(ROWS[0]);
6017     put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
6018     put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
6019     put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
6020     put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
6021     ht.put(put);
6022
6023     put = new Put(ROWS[1]);
6024     put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
6025     put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
6026     put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
6027     put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
6028     ht.put(put);
6029
6030     put = new Put(ROWS[2]);
6031     put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
6032     put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
6033     put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
6034     put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
6035     ht.put(put);
6036
6037     delete = new Delete(ROWS[0]);
6038     delete.addFamily(FAMILIES[2]);
6039     ht.delete(delete);
6040
6041     delete = new Delete(ROWS[1]);
6042     delete.addColumns(FAMILIES[1], QUALIFIER);
6043     ht.delete(delete);
6044
6045     delete = new Delete(ROWS[2]);
6046     delete.addColumn(FAMILIES[1], QUALIFIER);
6047     delete.addColumn(FAMILIES[1], QUALIFIER);
6048     delete.addColumn(FAMILIES[2], QUALIFIER);
6049     ht.delete(delete);
6050
6051     scan = new Scan(ROWS[0]);
6052     scan.setReversed(true);
6053     scan.addFamily(FAMILIES[1]);
6054     scan.addFamily(FAMILIES[2]);
6055     scan.setMaxVersions(Integer.MAX_VALUE);
6056     result = getSingleScanResult(ht, scan);
6057     assertTrue("Expected 2 keys but received " + result.size(),
6058         result.size() == 2);
6059     assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER, new long[] { ts[0],
6060         ts[1] }, new byte[][] { VALUES[0], VALUES[1] }, 0, 1);
6061
6062     scan = new Scan(ROWS[1]);
6063     scan.setReversed(true);
6064     scan.addFamily(FAMILIES[1]);
6065     scan.addFamily(FAMILIES[2]);
6066     scan.setMaxVersions(Integer.MAX_VALUE);
6067     result = getSingleScanResult(ht, scan);
6068     assertTrue("Expected 2 keys but received " + result.size(),
6069         result.size() == 2);
6070
6071     scan = new Scan(ROWS[2]);
6072     scan.setReversed(true);
6073     scan.addFamily(FAMILIES[1]);
6074     scan.addFamily(FAMILIES[2]);
6075     scan.setMaxVersions(Integer.MAX_VALUE);
6076     result = getSingleScanResult(ht, scan);
6077     assertEquals(1, result.size());
6078     assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
6079         new long[] { ts[2] }, new byte[][] { VALUES[2] }, 0, 0);
6080
6081     // Test if we delete the family first in one row (HBASE-1541)
6082
6083     delete = new Delete(ROWS[3]);
6084     delete.addFamily(FAMILIES[1]);
6085     ht.delete(delete);
6086
6087     put = new Put(ROWS[3]);
6088     put.addColumn(FAMILIES[2], QUALIFIER, VALUES[0]);
6089     ht.put(put);
6090
6091     put = new Put(ROWS[4]);
6092     put.addColumn(FAMILIES[1], QUALIFIER, VALUES[1]);
6093     put.addColumn(FAMILIES[2], QUALIFIER, VALUES[2]);
6094     ht.put(put);
6095
6096     scan = new Scan(ROWS[4]);
6097     scan.setReversed(true);
6098     scan.addFamily(FAMILIES[1]);
6099     scan.addFamily(FAMILIES[2]);
6100     scan.setMaxVersions(Integer.MAX_VALUE);
6101     ResultScanner scanner = ht.getScanner(scan);
6102     result = scanner.next();
6103     assertTrue("Expected 2 keys but received " + result.size(),
6104         result.size() == 2);
6105     assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[4]));
6106     assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[1]), ROWS[4]));
6107     assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[1]));
6108     assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[1]), VALUES[2]));
6109     result = scanner.next();
6110     assertTrue("Expected 1 key but received " + result.size(),
6111         result.size() == 1);
6112     assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[3]));
6113     assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[0]));
6114     scanner.close();
6115     ht.close();
6116   }
6117
6118   /**
6119    * Tests reversed scan under multi regions
6120    */
6121   @Test
6122   public void testReversedScanUnderMultiRegions() throws Exception {
6123     // Test Initialization.
6124     final TableName tableName = TableName.valueOf(name.getMethodName());
6125     byte[] maxByteArray = ConnectionUtils.MAX_BYTE_ARRAY;
6126     byte[][] splitRows = new byte[][] { Bytes.toBytes("005"),
6127         Bytes.add(Bytes.toBytes("005"), Bytes.multiple(maxByteArray, 16)),
6128         Bytes.toBytes("006"),
6129         Bytes.add(Bytes.toBytes("006"), Bytes.multiple(maxByteArray, 8)),
6130         Bytes.toBytes("007"),
6131         Bytes.add(Bytes.toBytes("007"), Bytes.multiple(maxByteArray, 4)),
6132         Bytes.toBytes("008"), Bytes.multiple(maxByteArray, 2) };
6133     Table table = TEST_UTIL.createTable(tableName, FAMILY, splitRows);
6134     TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
6135
6136     try(RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
6137       assertEquals(splitRows.length + 1, l.getAllRegionLocations().size());
6138     }
6139     // Insert one row each region
6140     int insertNum = splitRows.length;
6141     for (int i = 0; i < insertNum; i++) {
6142       Put put = new Put(splitRows[i]);
6143       put.addColumn(FAMILY, QUALIFIER, VALUE);
6144       table.put(put);
6145     }
6146
6147     // scan forward
6148     ResultScanner scanner = table.getScanner(new Scan());
6149     int count = 0;
6150     for (Result r : scanner) {
6151       assertTrue(!r.isEmpty());
6152       count++;
6153     }
6154     assertEquals(insertNum, count);
6155
6156     // scan backward
6157     Scan scan = new Scan();
6158     scan.setReversed(true);
6159     scanner = table.getScanner(scan);
6160     count = 0;
6161     byte[] lastRow = null;
6162     for (Result r : scanner) {
6163       assertTrue(!r.isEmpty());
6164       count++;
6165       byte[] thisRow = r.getRow();
6166       if (lastRow != null) {
6167         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6168             + ",this row=" + Bytes.toString(thisRow),
6169             Bytes.compareTo(thisRow, lastRow) < 0);
6170       }
6171       lastRow = thisRow;
6172     }
6173     assertEquals(insertNum, count);
6174     table.close();
6175   }
6176
6177   /**
6178    * Tests reversed scan under multi regions
6179    */
6180   @Test
6181   public void testSmallReversedScanUnderMultiRegions() throws Exception {
6182     // Test Initialization.
6183     final TableName tableName = TableName.valueOf(name.getMethodName());
6184     byte[][] splitRows = new byte[][]{
6185         Bytes.toBytes("000"), Bytes.toBytes("002"), Bytes.toBytes("004"),
6186         Bytes.toBytes("006"), Bytes.toBytes("008"), Bytes.toBytes("010")};
6187     Table table = TEST_UTIL.createTable(tableName, FAMILY, splitRows);
6188     TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
6189
6190     try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
6191       assertEquals(splitRows.length + 1, l.getAllRegionLocations().size());
6192     }
6193     for (byte[] splitRow : splitRows) {
6194       Put put = new Put(splitRow);
6195       put.addColumn(FAMILY, QUALIFIER, VALUE);
6196       table.put(put);
6197
6198       byte[] nextRow = Bytes.copy(splitRow);
6199       nextRow[nextRow.length - 1]++;
6200
6201       put = new Put(nextRow);
6202       put.addColumn(FAMILY, QUALIFIER, VALUE);
6203       table.put(put);
6204     }
6205
6206     // scan forward
6207     ResultScanner scanner = table.getScanner(new Scan());
6208     int count = 0;
6209     for (Result r : scanner) {
6210       assertTrue(!r.isEmpty());
6211       count++;
6212     }
6213     assertEquals(12, count);
6214
6215     reverseScanTest(table, false);
6216     reverseScanTest(table, true);
6217
6218     table.close();
6219   }
6220
6221   private void reverseScanTest(Table table, boolean small) throws IOException {
6222     // scan backward
6223     Scan scan = new Scan();
6224     scan.setReversed(true);
6225     ResultScanner scanner = table.getScanner(scan);
6226     int count = 0;
6227     byte[] lastRow = null;
6228     for (Result r : scanner) {
6229       assertTrue(!r.isEmpty());
6230       count++;
6231       byte[] thisRow = r.getRow();
6232       if (lastRow != null) {
6233         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6234             + ",this row=" + Bytes.toString(thisRow),
6235             Bytes.compareTo(thisRow, lastRow) < 0);
6236       }
6237       lastRow = thisRow;
6238     }
6239     assertEquals(12, count);
6240
6241     scan = new Scan();
6242     scan.setSmall(small);
6243     scan.setReversed(true);
6244     scan.setStartRow(Bytes.toBytes("002"));
6245     scanner = table.getScanner(scan);
6246     count = 0;
6247     lastRow = null;
6248     for (Result r : scanner) {
6249       assertTrue(!r.isEmpty());
6250       count++;
6251       byte[] thisRow = r.getRow();
6252       if (lastRow != null) {
6253         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6254             + ",this row=" + Bytes.toString(thisRow),
6255             Bytes.compareTo(thisRow, lastRow) < 0);
6256       }
6257       lastRow = thisRow;
6258     }
6259     assertEquals(3, count); // 000 001 002
6260
6261     scan = new Scan();
6262     scan.setSmall(small);
6263     scan.setReversed(true);
6264     scan.setStartRow(Bytes.toBytes("002"));
6265     scan.setStopRow(Bytes.toBytes("000"));
6266     scanner = table.getScanner(scan);
6267     count = 0;
6268     lastRow = null;
6269     for (Result r : scanner) {
6270       assertTrue(!r.isEmpty());
6271       count++;
6272       byte[] thisRow = r.getRow();
6273       if (lastRow != null) {
6274         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6275             + ",this row=" + Bytes.toString(thisRow),
6276             Bytes.compareTo(thisRow, lastRow) < 0);
6277       }
6278       lastRow = thisRow;
6279     }
6280     assertEquals(2, count); // 001 002
6281
6282     scan = new Scan();
6283     scan.setSmall(small);
6284     scan.setReversed(true);
6285     scan.setStartRow(Bytes.toBytes("001"));
6286     scanner = table.getScanner(scan);
6287     count = 0;
6288     lastRow = null;
6289     for (Result r : scanner) {
6290       assertTrue(!r.isEmpty());
6291       count++;
6292       byte[] thisRow = r.getRow();
6293       if (lastRow != null) {
6294         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6295             + ",this row=" + Bytes.toString(thisRow),
6296             Bytes.compareTo(thisRow, lastRow) < 0);
6297       }
6298       lastRow = thisRow;
6299     }
6300     assertEquals(2, count); // 000 001
6301
6302     scan = new Scan();
6303     scan.setSmall(small);
6304     scan.setReversed(true);
6305     scan.setStartRow(Bytes.toBytes("000"));
6306     scanner = table.getScanner(scan);
6307     count = 0;
6308     lastRow = null;
6309     for (Result r : scanner) {
6310       assertTrue(!r.isEmpty());
6311       count++;
6312       byte[] thisRow = r.getRow();
6313       if (lastRow != null) {
6314         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6315             + ",this row=" + Bytes.toString(thisRow),
6316             Bytes.compareTo(thisRow, lastRow) < 0);
6317       }
6318       lastRow = thisRow;
6319     }
6320     assertEquals(1, count); // 000
6321
6322     scan = new Scan();
6323     scan.setSmall(small);
6324     scan.setReversed(true);
6325     scan.setStartRow(Bytes.toBytes("006"));
6326     scan.setStopRow(Bytes.toBytes("002"));
6327     scanner = table.getScanner(scan);
6328     count = 0;
6329     lastRow = null;
6330     for (Result r : scanner) {
6331       assertTrue(!r.isEmpty());
6332       count++;
6333       byte[] thisRow = r.getRow();
6334       if (lastRow != null) {
6335         assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
6336             + ",this row=" + Bytes.toString(thisRow),
6337             Bytes.compareTo(thisRow, lastRow) < 0);
6338       }
6339       lastRow = thisRow;
6340     }
6341     assertEquals(4, count); // 003 004 005 006
6342   }
6343
6344   private static Pair<byte[][], byte[][]> getStartEndKeys(List<RegionLocations> regions) {
6345     final byte[][] startKeyList = new byte[regions.size()][];
6346     final byte[][] endKeyList = new byte[regions.size()][];
6347
6348     for (int i = 0; i < regions.size(); i++) {
6349       RegionInfo region = regions.get(i).getRegionLocation().getRegion();
6350       startKeyList[i] = region.getStartKey();
6351       endKeyList[i] = region.getEndKey();
6352     }
6353
6354     return new Pair<>(startKeyList, endKeyList);
6355   }
6356
6357   @Test
6358   public void testGetStartEndKeysWithRegionReplicas() throws IOException {
6359     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
6360     HColumnDescriptor fam = new HColumnDescriptor(FAMILY);
6361     htd.addFamily(fam);
6362     byte[][] KEYS = HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE;
6363     Admin admin = TEST_UTIL.getAdmin();
6364     admin.createTable(htd, KEYS);
6365     List<RegionInfo> regions = admin.getRegions(htd.getTableName());
6366
6367     HRegionLocator locator =
6368         (HRegionLocator) admin.getConnection().getRegionLocator(htd.getTableName());
6369     for (int regionReplication = 1; regionReplication < 4; regionReplication++) {
6370       List<RegionLocations> regionLocations = new ArrayList<>();
6371
6372       // mock region locations coming from meta with multiple replicas
6373       for (RegionInfo region : regions) {
6374         HRegionLocation[] arr = new HRegionLocation[regionReplication];
6375         for (int i = 0; i < arr.length; i++) {
6376           arr[i] = new HRegionLocation(RegionReplicaUtil.getRegionInfoForReplica(region, i), null);
6377         }
6378         regionLocations.add(new RegionLocations(arr));
6379       }
6380
6381       Pair<byte[][], byte[][]> startEndKeys = getStartEndKeys(regionLocations);
6382
6383       assertEquals(KEYS.length + 1, startEndKeys.getFirst().length);
6384
6385       for (int i = 0; i < KEYS.length + 1; i++) {
6386         byte[] startKey = i == 0 ? HConstants.EMPTY_START_ROW : KEYS[i - 1];
6387         byte[] endKey = i == KEYS.length ? HConstants.EMPTY_END_ROW : KEYS[i];
6388         assertArrayEquals(startKey, startEndKeys.getFirst()[i]);
6389         assertArrayEquals(endKey, startEndKeys.getSecond()[i]);
6390       }
6391     }
6392   }
6393
6394   @Test
6395   public void testFilterAllRecords() throws IOException {
6396     Scan scan = new Scan();
6397     scan.setBatch(1);
6398     scan.setCaching(1);
6399     // Filter out any records
6400     scan.setFilter(new FilterList(new FirstKeyOnlyFilter(), new InclusiveStopFilter(new byte[0])));
6401     try (Table table = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME)) {
6402       try (ResultScanner s = table.getScanner(scan)) {
6403         assertNull(s.next());
6404       }
6405     }
6406   }
6407
6408   @Test
6409   public void testRegionCache() throws IOException {
6410     HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName()));
6411     HColumnDescriptor fam = new HColumnDescriptor(FAMILY);
6412     htd.addFamily(fam);
6413     byte[][] KEYS = HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE;
6414     Admin admin = TEST_UTIL.getAdmin();
6415     admin.createTable(htd, KEYS);
6416     HRegionLocator locator =
6417       (HRegionLocator) admin.getConnection().getRegionLocator(htd.getTableName());
6418     List<HRegionLocation> results = locator.getAllRegionLocations();
6419     int number = ((ConnectionImplementation)admin.getConnection())
6420       .getNumberOfCachedRegionLocations(htd.getTableName());
6421     assertEquals(results.size(), number);
6422     ConnectionImplementation conn = ((ConnectionImplementation)admin.getConnection());
6423     assertNotNull("Can't get cached location for row aaa",
6424         conn.getCachedLocation(htd.getTableName(),Bytes.toBytes("aaa")));
6425     for(byte[] startKey:HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE){
6426       assertNotNull("Can't get cached location for row "+
6427         Bytes.toString(startKey),(conn.getCachedLocation(htd.getTableName(),startKey)));
6428     }
6429   }
6430
6431   @Test
6432   public void testCellSizeLimit() throws IOException {
6433     final TableName tableName = TableName.valueOf("testCellSizeLimit");
6434     HTableDescriptor htd = new HTableDescriptor(tableName);
6435     htd.setConfiguration(HRegion.HBASE_MAX_CELL_SIZE_KEY, Integer.toString(10 * 1024)); // 10K
6436     HColumnDescriptor fam = new HColumnDescriptor(FAMILY);
6437     htd.addFamily(fam);
6438     Admin admin = TEST_UTIL.getAdmin();
6439     admin.createTable(htd);
6440     // Will succeed
6441     try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
6442       t.put(new Put(ROW).addColumn(FAMILY, QUALIFIER, Bytes.toBytes(0L)));
6443       t.increment(new Increment(ROW).addColumn(FAMILY, QUALIFIER, 1L));
6444     }
6445     // Will succeed
6446     try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
6447       t.put(new Put(ROW).addColumn(FAMILY, QUALIFIER, new byte[9*1024]));
6448     }
6449     // Will fail
6450     try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
6451       try {
6452         t.put(new Put(ROW).addColumn(FAMILY, QUALIFIER, new byte[10 * 1024]));
6453         fail("Oversize cell failed to trigger exception");
6454       } catch (IOException e) {
6455         // expected
6456       }
6457       try {
6458         t.append(new Append(ROW).addColumn(FAMILY, QUALIFIER, new byte[10 * 1024]));
6459         fail("Oversize cell failed to trigger exception");
6460       } catch (IOException e) {
6461         // expected
6462       }
6463     }
6464   }
6465
6466   @Test
6467   public void testDeleteSpecifiedVersionOfSpecifiedColumn() throws Exception {
6468     Admin admin = TEST_UTIL.getAdmin();
6469     final TableName tableName = TableName.valueOf(name.getMethodName());
6470
6471     byte[][] VALUES = makeN(VALUE, 5);
6472     long[] ts = { 1000, 2000, 3000, 4000, 5000 };
6473
6474     Table ht = TEST_UTIL.createTable(tableName, FAMILY, 5);
6475
6476     Put put = new Put(ROW);
6477     // Put version 1000,2000,3000,4000 of column FAMILY:QUALIFIER
6478     for (int t = 0; t < 4; t++) {
6479       put.addColumn(FAMILY, QUALIFIER, ts[t], VALUES[t]);
6480     }
6481     ht.put(put);
6482
6483     Delete delete = new Delete(ROW);
6484     // Delete version 3000 of column FAMILY:QUALIFIER
6485     delete.addColumn(FAMILY, QUALIFIER, ts[2]);
6486     ht.delete(delete);
6487
6488     Get get = new Get(ROW);
6489     get.addColumn(FAMILY, QUALIFIER);
6490     get.setMaxVersions(Integer.MAX_VALUE);
6491     Result result = ht.get(get);
6492     // verify version 1000,2000,4000 remains for column FAMILY:QUALIFIER
6493     assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { ts[0], ts[1], ts[3] }, new byte[][] {
6494         VALUES[0], VALUES[1], VALUES[3] }, 0, 2);
6495
6496     delete = new Delete(ROW);
6497     // Delete a version 5000 of column FAMILY:QUALIFIER which didn't exist
6498     delete.addColumn(FAMILY, QUALIFIER, ts[4]);
6499     ht.delete(delete);
6500
6501     get = new Get(ROW);
6502     get.addColumn(FAMILY, QUALIFIER);
6503     get.setMaxVersions(Integer.MAX_VALUE);
6504     result = ht.get(get);
6505     // verify version 1000,2000,4000 remains for column FAMILY:QUALIFIER
6506     assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { ts[0], ts[1], ts[3] }, new byte[][] {
6507         VALUES[0], VALUES[1], VALUES[3] }, 0, 2);
6508
6509     ht.close();
6510     admin.close();
6511   }
6512
6513   @Test
6514   public void testDeleteLatestVersionOfSpecifiedColumn() throws Exception {
6515     Admin admin = TEST_UTIL.getAdmin();
6516     final TableName tableName = TableName.valueOf(name.getMethodName());
6517
6518     byte[][] VALUES = makeN(VALUE, 5);
6519     long[] ts = { 1000, 2000, 3000, 4000, 5000 };
6520
6521     Table ht = TEST_UTIL.createTable(tableName, FAMILY, 5);
6522
6523     Put put = new Put(ROW);
6524     // Put version 1000,2000,3000,4000 of column FAMILY:QUALIFIER
6525     for (int t = 0; t < 4; t++) {
6526       put.addColumn(FAMILY, QUALIFIER, ts[t], VALUES[t]);
6527     }
6528     ht.put(put);
6529
6530     Delete delete = new Delete(ROW);
6531     // Delete latest version of column FAMILY:QUALIFIER
6532     delete.addColumn(FAMILY, QUALIFIER);
6533     ht.delete(delete);
6534
6535     Get get = new Get(ROW);
6536     get.addColumn(FAMILY, QUALIFIER);
6537     get.setMaxVersions(Integer.MAX_VALUE);
6538     Result result = ht.get(get);
6539     // verify version 1000,2000,3000 remains for column FAMILY:QUALIFIER
6540     assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { ts[0], ts[1], ts[2] }, new byte[][] {
6541         VALUES[0], VALUES[1], VALUES[2] }, 0, 2);
6542
6543     delete = new Delete(ROW);
6544     // Delete two latest version of column FAMILY:QUALIFIER
6545     delete.addColumn(FAMILY, QUALIFIER);
6546     delete.addColumn(FAMILY, QUALIFIER);
6547     ht.delete(delete);
6548
6549     get = new Get(ROW);
6550     get.addColumn(FAMILY, QUALIFIER);
6551     get.setMaxVersions(Integer.MAX_VALUE);
6552     result = ht.get(get);
6553     // verify version 1000 remains for column FAMILY:QUALIFIER
6554     assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { ts[0] }, new byte[][] { VALUES[0] },
6555       0, 0);
6556
6557     put = new Put(ROW);
6558     // Put a version 5000 of column FAMILY:QUALIFIER
6559     put.addColumn(FAMILY, QUALIFIER, ts[4], VALUES[4]);
6560     ht.put(put);
6561
6562     get = new Get(ROW);
6563     get.addColumn(FAMILY, QUALIFIER);
6564     get.setMaxVersions(Integer.MAX_VALUE);
6565     result = ht.get(get);
6566     // verify version 1000,5000 remains for column FAMILY:QUALIFIER
6567     assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { ts[0], ts[4] }, new byte[][] {
6568         VALUES[0], VALUES[4] }, 0, 1);
6569
6570     ht.close();
6571     admin.close();
6572   }
6573
6574   /**
6575    * Test for HBASE-17125
6576    */
6577   @Test
6578   public void testReadWithFilter() throws Exception {
6579     Admin admin = TEST_UTIL.getAdmin();
6580     final TableName tableName = TableName.valueOf(name.getMethodName());
6581     Table table = TEST_UTIL.createTable(tableName, FAMILY, 3);
6582
6583     byte[] VALUEA = Bytes.toBytes("value-a");
6584     byte[] VALUEB = Bytes.toBytes("value-b");
6585     long[] ts = { 1000, 2000, 3000, 4000 };
6586
6587     Put put = new Put(ROW);
6588     // Put version 1000,2000,3000,4000 of column FAMILY:QUALIFIER
6589     for (int t = 0; t <= 3; t++) {
6590       if (t <= 1) {
6591         put.addColumn(FAMILY, QUALIFIER, ts[t], VALUEA);
6592       } else {
6593         put.addColumn(FAMILY, QUALIFIER, ts[t], VALUEB);
6594       }
6595     }
6596     table.put(put);
6597
6598     Scan scan =
6599         new Scan().setFilter(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("value-a")))
6600             .setMaxVersions(3);
6601     ResultScanner scanner = table.getScanner(scan);
6602     Result result = scanner.next();
6603     // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
6604     assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { ts[1] }, new byte[][] { VALUEA }, 0,
6605       0);
6606
6607     Get get =
6608         new Get(ROW)
6609             .setFilter(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("value-a")))
6610             .setMaxVersions(3);
6611     result = table.get(get);
6612     // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
6613     assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { ts[1] }, new byte[][] { VALUEA }, 0,
6614       0);
6615
6616     // Test with max versions 1, it should still read ts[1]
6617     scan =
6618         new Scan().setFilter(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("value-a")))
6619             .setMaxVersions(1);
6620     scanner = table.getScanner(scan);
6621     result = scanner.next();
6622     // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
6623     assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { ts[1] }, new byte[][] { VALUEA }, 0,
6624       0);
6625
6626     // Test with max versions 1, it should still read ts[1]
6627     get =
6628         new Get(ROW)
6629             .setFilter(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("value-a")))
6630             .setMaxVersions(1);
6631     result = table.get(get);
6632     // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
6633     assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { ts[1] }, new byte[][] { VALUEA }, 0,
6634       0);
6635
6636     // Test with max versions 5, it should still read ts[1]
6637     scan =
6638         new Scan().setFilter(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("value-a")))
6639             .setMaxVersions(5);
6640     scanner = table.getScanner(scan);
6641     result = scanner.next();
6642     // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
6643     assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { ts[1] }, new byte[][] { VALUEA }, 0,
6644       0);
6645
6646     // Test with max versions 5, it should still read ts[1]
6647     get =
6648         new Get(ROW)
6649             .setFilter(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("value-a")))
6650             .setMaxVersions(5);
6651     result = table.get(get);
6652     // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
6653     assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { ts[1] }, new byte[][] { VALUEA }, 0,
6654       0);
6655
6656     table.close();
6657     admin.close();
6658   }
6659
6660   @Test
6661   public void testCellUtilTypeMethods() throws IOException {
6662     final TableName tableName = TableName.valueOf(name.getMethodName());
6663     Table table = TEST_UTIL.createTable(tableName, FAMILY);
6664
6665     final byte[] row = Bytes.toBytes("p");
6666     Put p = new Put(row);
6667     p.addColumn(FAMILY, QUALIFIER, VALUE);
6668     table.put(p);
6669
6670     try (ResultScanner scanner = table.getScanner(new Scan())) {
6671       Result result = scanner.next();
6672       assertNotNull(result);
6673       CellScanner cs = result.cellScanner();
6674       assertTrue(cs.advance());
6675       Cell c = cs.current();
6676       assertTrue(CellUtil.isPut(c));
6677       assertFalse(CellUtil.isDelete(c));
6678       assertFalse(cs.advance());
6679       assertNull(scanner.next());
6680     }
6681
6682     Delete d = new Delete(row);
6683     d.addColumn(FAMILY, QUALIFIER);
6684     table.delete(d);
6685
6686     Scan scan = new Scan();
6687     scan.setRaw(true);
6688     try (ResultScanner scanner = table.getScanner(scan)) {
6689       Result result = scanner.next();
6690       assertNotNull(result);
6691       CellScanner cs = result.cellScanner();
6692       assertTrue(cs.advance());
6693
6694       // First cell should be the delete (masking the Put)
6695       Cell c = cs.current();
6696       assertTrue("Cell should be a Delete: " + c, CellUtil.isDelete(c));
6697       assertFalse("Cell should not be a Put: " + c, CellUtil.isPut(c));
6698
6699       // Second cell should be the original Put
6700       assertTrue(cs.advance());
6701       c = cs.current();
6702       assertFalse("Cell should not be a Delete: " + c, CellUtil.isDelete(c));
6703       assertTrue("Cell should be a Put: " + c, CellUtil.isPut(c));
6704
6705       // No more cells in this row
6706       assertFalse(cs.advance());
6707
6708       // No more results in this scan
6709       assertNull(scanner.next());
6710     }
6711   }
6712
6713   @Test(expected = DoNotRetryIOException.class)
6714   public void testCreateTableWithZeroRegionReplicas() throws Exception {
6715     TableName tableName = TableName.valueOf(name.getMethodName());
6716     TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
6717         .setColumnFamily(ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("cf")))
6718         .setRegionReplication(0)
6719         .build();
6720
6721     TEST_UTIL.getAdmin().createTable(desc);
6722   }
6723
6724   @Test(expected = DoNotRetryIOException.class)
6725   public void testModifyTableWithZeroRegionReplicas() throws Exception {
6726     TableName tableName = TableName.valueOf(name.getMethodName());
6727     TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
6728         .setColumnFamily(ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("cf")))
6729         .build();
6730
6731     TEST_UTIL.getAdmin().createTable(desc);
6732     TableDescriptor newDesc = TableDescriptorBuilder.newBuilder(desc)
6733         .setRegionReplication(0)
6734         .build();
6735
6736     TEST_UTIL.getAdmin().modifyTable(newDesc);
6737   }
6738 }