hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide4.java

   1 /*
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.client;
  19
  20 import static org.hamcrest.CoreMatchers.instanceOf;
  21 import static org.hamcrest.MatcherAssert.assertThat;
  22 import static org.junit.Assert.assertEquals;
  23 import static org.junit.Assert.assertFalse;
  24 import static org.junit.Assert.assertNotNull;
  25 import static org.junit.Assert.assertTrue;
  26 import static org.junit.Assert.fail;
  27
  28 import java.io.IOException;
  29 import java.util.ArrayList;
  30 import java.util.Arrays;
  31 import java.util.Collection;
  32 import java.util.EnumSet;
  33 import java.util.HashSet;
  34 import java.util.List;
  35 import java.util.NavigableMap;
  36 import org.apache.hadoop.conf.Configuration;
  37 import org.apache.hadoop.hbase.Cell;
  38 import org.apache.hadoop.hbase.CellUtil;
  39 import org.apache.hadoop.hbase.ClusterMetrics.Option;
  40 import org.apache.hadoop.hbase.HBaseClassTestRule;
  41 import org.apache.hadoop.hbase.HConstants;
  42 import org.apache.hadoop.hbase.KeyValue;
  43 import org.apache.hadoop.hbase.SingleProcessHBaseCluster;
  44 import org.apache.hadoop.hbase.TableName;
  45 import org.apache.hadoop.hbase.TableNameTestRule;
  46 import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
  47 import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
  48 import org.apache.hadoop.hbase.testclassification.ClientTests;
  49 import org.apache.hadoop.hbase.testclassification.LargeTests;
  50 import org.apache.hadoop.hbase.util.Bytes;
  51 import org.junit.AfterClass;
  52 import org.junit.Assume;
  53 import org.junit.ClassRule;
  54 import org.junit.Rule;
  55 import org.junit.Test;
  56 import org.junit.experimental.categories.Category;
  57 import org.junit.runner.RunWith;
  58 import org.junit.runners.Parameterized;
  59 import org.slf4j.Logger;
  60 import org.slf4j.LoggerFactory;
  61
  62 /**
  63  * Run tests that use the HBase clients; {@link Table}.
  64  * Sets up the HBase mini cluster once at start and runs through all client tests.
  65  * Each creates a table named for the method and does its stuff against that.
  66  *
  67  * Parameterized to run with different registry implementations.
  68  */
  69 @Category({LargeTests.class, ClientTests.class})
  70 @SuppressWarnings ("deprecation")
  71 @RunWith(Parameterized.class)
  72 public class TestFromClientSide4 extends FromClientSideBase {
  73   private static final Logger LOG = LoggerFactory.getLogger(TestFromClientSide4.class);
  74   @ClassRule
  75   public static final HBaseClassTestRule CLASS_RULE =
  76     HBaseClassTestRule.forClass(TestFromClientSide4.class);
  77   @Rule
  78   public TableNameTestRule name = new TableNameTestRule();
  79
  80   // To keep the child classes happy.
  81   TestFromClientSide4() {
  82   }
  83
  84   public TestFromClientSide4(Class registry, int numHedgedReqs) throws Exception {
  85     initialize(registry, numHedgedReqs, MultiRowMutationEndpoint.class);
  86   }
  87
  88   @Parameterized.Parameters
  89   public static Collection parameters() {
  90     return Arrays.asList(new Object[][] { { MasterRegistry.class, 1 }, { MasterRegistry.class, 2 },
  91       { ZKConnectionRegistry.class, 1 } });
  92   }
  93
  94   @AfterClass
  95   public static void tearDownAfterClass() throws Exception {
  96     afterClass();
  97   }
  98
  99   /**
 100    * Test batch operations with combination of valid and invalid args
 101    */
 102   @Test public void testBatchOperationsWithErrors() throws Exception {
 103     final TableName tableName = name.getTableName();
 104     try (Table foo = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, 10)) {
 105
 106       int NUM_OPS = 100;
 107
 108       // 1.1 Put with no column families (local validation, runtime exception)
 109       List<Put> puts = new ArrayList<>(NUM_OPS);
 110       for (int i = 0; i != NUM_OPS; i++) {
 111         Put put = new Put(Bytes.toBytes(i));
 112         puts.add(put);
 113       }
 114
 115       try {
 116         foo.put(puts);
 117         fail();
 118       } catch (IllegalArgumentException e) {
 119         // expected
 120         assertEquals(NUM_OPS, puts.size());
 121       }
 122
 123       // 1.2 Put with invalid column family
 124       puts.clear();
 125       for (int i = 0; i < NUM_OPS; i++) {
 126         Put put = new Put(Bytes.toBytes(i));
 127         put.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY, Bytes.toBytes(i));
 128         puts.add(put);
 129       }
 130
 131       try {
 132         foo.put(puts);
 133         fail();
 134       } catch (RetriesExhaustedException e) {
 135         // expected
 136         assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class));
 137       }
 138
 139       // 2.1 Get non-existent rows
 140       List<Get> gets = new ArrayList<>(NUM_OPS);
 141       for (int i = 0; i < NUM_OPS; i++) {
 142         Get get = new Get(Bytes.toBytes(i));
 143         gets.add(get);
 144       }
 145       Result[] getsResult = foo.get(gets);
 146       assertNotNull(getsResult);
 147       assertEquals(NUM_OPS, getsResult.length);
 148       for (int i = 0; i < NUM_OPS; i++) {
 149         Result getResult = getsResult[i];
 150         if (i % 2 == 0) {
 151           assertFalse(getResult.isEmpty());
 152         } else {
 153           assertTrue(getResult.isEmpty());
 154         }
 155       }
 156
 157       // 2.2 Get with invalid column family
 158       gets.clear();
 159       for (int i = 0; i < NUM_OPS; i++) {
 160         Get get = new Get(Bytes.toBytes(i));
 161         get.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY);
 162         gets.add(get);
 163       }
 164       try {
 165         foo.get(gets);
 166         fail();
 167       } catch (RetriesExhaustedException e) {
 168         // expected
 169         assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class));
 170       }
 171
 172       // 3.1 Delete with invalid column family
 173       List<Delete> deletes = new ArrayList<>(NUM_OPS);
 174       for (int i = 0; i < NUM_OPS; i++) {
 175         Delete delete = new Delete(Bytes.toBytes(i));
 176         delete.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY);
 177         deletes.add(delete);
 178       }
 179       try {
 180         foo.delete(deletes);
 181         fail();
 182       } catch (RetriesExhaustedException e) {
 183         // expected
 184         assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class));
 185       }
 186
 187       // all valid rows should have been deleted
 188       gets.clear();
 189       for (int i = 0; i < NUM_OPS; i++) {
 190         Get get = new Get(Bytes.toBytes(i));
 191         gets.add(get);
 192       }
 193       getsResult = foo.get(gets);
 194       assertNotNull(getsResult);
 195       assertEquals(NUM_OPS, getsResult.length);
 196       for (Result getResult : getsResult) {
 197         assertTrue(getResult.isEmpty());
 198       }
 199
 200       // 3.2 Delete non-existent rows
 201       deletes.clear();
 202       for (int i = 0; i < NUM_OPS; i++) {
 203         Delete delete = new Delete(Bytes.toBytes(i));
 204         deletes.add(delete);
 205       }
 206       foo.delete(deletes);
 207     }
 208   }
 209
 210   //
 211   // JIRA Testers
 212   //
 213
 214   /**
 215    * HBASE-867
 216    * If millions of columns in a column family, hbase scanner won't come up
 217    * Test will create numRows rows, each with numColsPerRow columns
 218    * (1 version each), and attempt to scan them all.
 219    * To test at scale, up numColsPerRow to the millions
 220    * (have not gotten that to work running as junit though)
 221    */
 222   @Test public void testJiraTest867() throws Exception {
 223     int numRows = 10;
 224     int numColsPerRow = 2000;
 225
 226     final TableName tableName = name.getTableName();
 227
 228     byte[][] ROWS = makeN(ROW, numRows);
 229     byte[][] QUALIFIERS = makeN(QUALIFIER, numColsPerRow);
 230
 231     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
 232
 233       // Insert rows
 234
 235       for (int i = 0; i < numRows; i++) {
 236         Put put = new Put(ROWS[i]);
 237         put.setDurability(Durability.SKIP_WAL);
 238         for (int j = 0; j < numColsPerRow; j++) {
 239           put.addColumn(FAMILY, QUALIFIERS[j], QUALIFIERS[j]);
 240         }
 241         assertEquals(
 242           "Put expected to contain " + numColsPerRow + " columns but " + "only contains " + put
 243             .size(), put.size(), numColsPerRow);
 244         ht.put(put);
 245       }
 246
 247       // Get a row
 248       Get get = new Get(ROWS[numRows - 1]);
 249       Result result = ht.get(get);
 250       assertNumKeys(result, numColsPerRow);
 251       Cell[] keys = result.rawCells();
 252       for (int i = 0; i < result.size(); i++) {
 253         assertKey(keys[i], ROWS[numRows - 1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
 254       }
 255
 256       // Scan the rows
 257       Scan scan = new Scan();
 258       try (ResultScanner scanner = ht.getScanner(scan)) {
 259         int rowCount = 0;
 260         while ((result = scanner.next()) != null) {
 261           assertNumKeys(result, numColsPerRow);
 262           Cell[] kvs = result.rawCells();
 263           for (int i = 0; i < numColsPerRow; i++) {
 264             assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
 265           }
 266           rowCount++;
 267         }
 268         assertEquals(
 269           "Expected to scan " + numRows + " rows but actually scanned " + rowCount + " rows",
 270           rowCount, numRows);
 271       }
 272
 273       // flush and try again
 274
 275       TEST_UTIL.flush();
 276
 277       // Get a row
 278       get = new Get(ROWS[numRows - 1]);
 279       result = ht.get(get);
 280       assertNumKeys(result, numColsPerRow);
 281       keys = result.rawCells();
 282       for (int i = 0; i < result.size(); i++) {
 283         assertKey(keys[i], ROWS[numRows - 1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
 284       }
 285
 286       // Scan the rows
 287       scan = new Scan();
 288       try (ResultScanner scanner = ht.getScanner(scan)) {
 289         int rowCount = 0;
 290         while ((result = scanner.next()) != null) {
 291           assertNumKeys(result, numColsPerRow);
 292           Cell[] kvs = result.rawCells();
 293           for (int i = 0; i < numColsPerRow; i++) {
 294             assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
 295           }
 296           rowCount++;
 297         }
 298         assertEquals(
 299           "Expected to scan " + numRows + " rows but actually scanned " + rowCount + " rows",
 300           rowCount, numRows);
 301       }
 302     }
 303   }
 304
 305   /**
 306    * HBASE-861
 307    * get with timestamp will return a value if there is a version with an
 308    * earlier timestamp
 309    */
 310   @Test public void testJiraTest861() throws Exception {
 311     final TableName tableName = name.getTableName();
 312     byte[][] VALUES = makeNAscii(VALUE, 7);
 313     long[] STAMPS = makeStamps(7);
 314
 315     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
 316
 317       // Insert three versions
 318
 319       Put put = new Put(ROW);
 320       put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
 321       put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 322       put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
 323       ht.put(put);
 324
 325       // Get the middle value
 326       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 327
 328       // Try to get one version before (expect fail)
 329       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
 330
 331       // Try to get one version after (expect fail)
 332       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
 333
 334       // Try same from storefile
 335       TEST_UTIL.flush();
 336       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 337       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
 338       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
 339
 340       // Insert two more versions surrounding others, into memstore
 341       put = new Put(ROW);
 342       put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
 343       put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
 344       ht.put(put);
 345
 346       // Check we can get everything we should and can't get what we shouldn't
 347       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
 348       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
 349       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 350       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
 351       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
 352       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
 353       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
 354
 355       // Try same from two storefiles
 356       TEST_UTIL.flush();
 357       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
 358       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
 359       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 360       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
 361       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
 362       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
 363       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
 364     }
 365   }
 366
 367   /**
 368    * HBASE-33
 369    * Add a HTable get/obtainScanner method that retrieves all versions of a
 370    * particular column and row between two timestamps
 371    */
 372   @Test public void testJiraTest33() throws Exception {
 373     final TableName tableName = name.getTableName();
 374     byte[][] VALUES = makeNAscii(VALUE, 7);
 375     long[] STAMPS = makeStamps(7);
 376
 377     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
 378
 379       // Insert lots versions
 380
 381       Put put = new Put(ROW);
 382       put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
 383       put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
 384       put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 385       put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
 386       put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
 387       put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
 388       ht.put(put);
 389
 390       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 391       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
 392       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
 393       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
 394
 395       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 396       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
 397       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
 398       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
 399
 400       // Try same from storefile
 401       TEST_UTIL.flush();
 402
 403       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 404       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
 405       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
 406       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
 407
 408       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 409       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
 410       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
 411       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
 412     }
 413   }
 414
 415   /**
 416    * HBASE-1014
 417    * commit(BatchUpdate) method should return timestamp
 418    */
 419   @Test public void testJiraTest1014() throws Exception {
 420     final TableName tableName = name.getTableName();
 421
 422     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
 423
 424       long manualStamp = 12345;
 425
 426       // Insert lots versions
 427
 428       Put put = new Put(ROW);
 429       put.addColumn(FAMILY, QUALIFIER, manualStamp, VALUE);
 430       ht.put(put);
 431
 432       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, manualStamp, VALUE);
 433       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp - 1);
 434       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp + 1);
 435     }
 436   }
 437
 438   /**
 439    * HBASE-1182
 440    * Scan for columns > some timestamp
 441    */
 442   @Test public void testJiraTest1182() throws Exception {
 443     final TableName tableName = name.getTableName();
 444     byte[][] VALUES = makeNAscii(VALUE, 7);
 445     long[] STAMPS = makeStamps(7);
 446
 447     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
 448
 449       // Insert lots versions
 450
 451       Put put = new Put(ROW);
 452       put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
 453       put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
 454       put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 455       put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
 456       put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
 457       put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
 458       ht.put(put);
 459
 460       getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 461       getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
 462       getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
 463
 464       scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 465       scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
 466       scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
 467
 468       // Try same from storefile
 469       TEST_UTIL.flush();
 470
 471       getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 472       getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
 473       getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
 474
 475       scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 476       scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
 477       scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
 478     }
 479   }
 480
 481   /**
 482    * HBASE-52
 483    * Add a means of scanning over all versions
 484    */
 485   @Test public void testJiraTest52() throws Exception {
 486     final TableName tableName = name.getTableName();
 487     byte[][] VALUES = makeNAscii(VALUE, 7);
 488     long[] STAMPS = makeStamps(7);
 489
 490     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
 491
 492       // Insert lots versions
 493
 494       Put put = new Put(ROW);
 495       put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
 496       put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
 497       put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 498       put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
 499       put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
 500       put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
 501       ht.put(put);
 502
 503       getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 504
 505       scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 506
 507       // Try same from storefile
 508       TEST_UTIL.flush();
 509
 510       getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 511
 512       scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 513     }
 514   }
 515
 516   @Test
 517   @SuppressWarnings("checkstyle:MethodLength")
 518   public void testDuplicateVersions() throws Exception {
 519     final TableName tableName = name.getTableName();
 520
 521     long[] STAMPS = makeStamps(20);
 522     byte[][] VALUES = makeNAscii(VALUE, 20);
 523
 524     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
 525
 526       // Insert 4 versions of same column
 527       Put put = new Put(ROW);
 528       put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
 529       put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 530       put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
 531       put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
 532       ht.put(put);
 533
 534       // Verify we can get each one properly
 535       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
 536       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 537       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
 538       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
 539       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
 540       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 541       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
 542       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
 543
 544       // Verify we don't accidentally get others
 545       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
 546       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
 547       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
 548       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
 549       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
 550       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
 551
 552       // Ensure maxVersions in query is respected
 553       Get get = new Get(ROW);
 554       get.addColumn(FAMILY, QUALIFIER);
 555       get.readVersions(2);
 556       Result result = ht.get(get);
 557       assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
 558         new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
 559
 560       Scan scan = new Scan().withStartRow(ROW);
 561       scan.addColumn(FAMILY, QUALIFIER);
 562       scan.readVersions(2);
 563       result = getSingleScanResult(ht, scan);
 564       assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
 565         new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
 566
 567       // Flush and redo
 568
 569       TEST_UTIL.flush();
 570
 571       // Verify we can get each one properly
 572       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
 573       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 574       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
 575       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
 576       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
 577       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 578       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
 579       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
 580
 581       // Verify we don't accidentally get others
 582       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
 583       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
 584       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
 585       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
 586       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
 587       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
 588
 589       // Ensure maxVersions in query is respected
 590       get = new Get(ROW);
 591       get.addColumn(FAMILY, QUALIFIER);
 592       get.readVersions(2);
 593       result = ht.get(get);
 594       assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
 595         new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
 596
 597       scan = new Scan().withStartRow(ROW);
 598       scan.addColumn(FAMILY, QUALIFIER);
 599       scan.readVersions(2);
 600       result = getSingleScanResult(ht, scan);
 601       assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
 602         new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
 603
 604       // Add some memstore and retest
 605
 606       // Insert 4 more versions of same column and a dupe
 607       put = new Put(ROW);
 608       put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
 609       put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
 610       put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
 611       put.addColumn(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
 612       put.addColumn(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
 613       ht.put(put);
 614
 615       // Ensure maxVersions in query is respected
 616       get = new Get(ROW);
 617       get.addColumn(FAMILY, QUALIFIER);
 618       get.readVersions(7);
 619       result = ht.get(get);
 620       assertNResult(result, ROW, FAMILY, QUALIFIER,
 621         new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
 622         new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
 623           VALUES[8] }, 0, 6);
 624
 625       scan = new Scan().withStartRow(ROW);
 626       scan.addColumn(FAMILY, QUALIFIER);
 627       scan.readVersions(7);
 628       result = getSingleScanResult(ht, scan);
 629       assertNResult(result, ROW, FAMILY, QUALIFIER,
 630         new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
 631         new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
 632           VALUES[8] }, 0, 6);
 633
 634       get = new Get(ROW);
 635       get.readVersions(7);
 636       result = ht.get(get);
 637       assertNResult(result, ROW, FAMILY, QUALIFIER,
 638         new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
 639         new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
 640           VALUES[8] }, 0, 6);
 641
 642       scan = new Scan().withStartRow(ROW);
 643       scan.readVersions(7);
 644       result = getSingleScanResult(ht, scan);
 645       assertNResult(result, ROW, FAMILY, QUALIFIER,
 646         new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
 647         new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
 648           VALUES[8] }, 0, 6);
 649
 650       // Verify we can get each one properly
 651       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
 652       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 653       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
 654       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
 655       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
 656       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 657       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
 658       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
 659
 660       // Verify we don't accidentally get others
 661       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
 662       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
 663       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
 664       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
 665
 666       // Ensure maxVersions of table is respected
 667
 668       TEST_UTIL.flush();
 669
 670       // Insert 4 more versions of same column and a dupe
 671       put = new Put(ROW);
 672       put.addColumn(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
 673       put.addColumn(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
 674       put.addColumn(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
 675       put.addColumn(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
 676       ht.put(put);
 677
 678       get = new Get(ROW);
 679       get.addColumn(FAMILY, QUALIFIER);
 680       get.readVersions(Integer.MAX_VALUE);
 681       result = ht.get(get);
 682       assertNResult(result, ROW, FAMILY, QUALIFIER,
 683         new long[] { STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9],
 684           STAMPS[11], STAMPS[13], STAMPS[15] },
 685         new byte[][] { VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9],
 686           VALUES[11], VALUES[13], VALUES[15] }, 0, 9);
 687
 688       scan = new Scan().withStartRow(ROW);
 689       scan.addColumn(FAMILY, QUALIFIER);
 690       scan.readVersions(Integer.MAX_VALUE);
 691       result = getSingleScanResult(ht, scan);
 692       assertNResult(result, ROW, FAMILY, QUALIFIER,
 693         new long[] { STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9],
 694           STAMPS[11], STAMPS[13], STAMPS[15] },
 695         new byte[][] { VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9],
 696           VALUES[11], VALUES[13], VALUES[15] }, 0, 9);
 697
 698       // Delete a version in the memstore and a version in a storefile
 699       Delete delete = new Delete(ROW);
 700       delete.addColumn(FAMILY, QUALIFIER, STAMPS[11]);
 701       delete.addColumn(FAMILY, QUALIFIER, STAMPS[7]);
 702       ht.delete(delete);
 703
 704       // Test that it's gone
 705       get = new Get(ROW);
 706       get.addColumn(FAMILY, QUALIFIER);
 707       get.readVersions(Integer.MAX_VALUE);
 708       result = ht.get(get);
 709       assertNResult(result, ROW, FAMILY, QUALIFIER,
 710         new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8],
 711           STAMPS[9], STAMPS[13], STAMPS[15] },
 712         new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8],
 713           VALUES[9], VALUES[13], VALUES[15] }, 0, 9);
 714
 715       scan = new Scan().withStartRow(ROW);
 716       scan.addColumn(FAMILY, QUALIFIER);
 717       scan.readVersions(Integer.MAX_VALUE);
 718       result = getSingleScanResult(ht, scan);
 719       assertNResult(result, ROW, FAMILY, QUALIFIER,
 720         new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8],
 721           STAMPS[9], STAMPS[13], STAMPS[15] },
 722         new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8],
 723           VALUES[9], VALUES[13], VALUES[15] }, 0, 9);
 724     }
 725   }
 726
 727   @Test public void testUpdates() throws Exception {
 728     final TableName tableName = name.getTableName();
 729     try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
 730
 731       // Write a column with values at timestamp 1, 2 and 3
 732       byte[] row = Bytes.toBytes("row1");
 733       byte[] qualifier = Bytes.toBytes("myCol");
 734       Put put = new Put(row);
 735       put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
 736       hTable.put(put);
 737
 738       put = new Put(row);
 739       put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
 740       hTable.put(put);
 741
 742       put = new Put(row);
 743       put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
 744       hTable.put(put);
 745
 746       Get get = new Get(row);
 747       get.addColumn(FAMILY, qualifier);
 748       get.readAllVersions();
 749
 750       // Check that the column indeed has the right values at timestamps 1 and
 751       // 2
 752       Result result = hTable.get(get);
 753       NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
 754       assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
 755       assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
 756
 757       // Update the value at timestamp 1
 758       put = new Put(row);
 759       put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
 760       hTable.put(put);
 761
 762       // Update the value at timestamp 2
 763       put = new Put(row);
 764       put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
 765       hTable.put(put);
 766
 767       // Check that the values at timestamp 2 and 1 got updated
 768       result = hTable.get(get);
 769       navigableMap = result.getMap().get(FAMILY).get(qualifier);
 770       assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
 771       assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
 772     }
 773   }
 774
 775   @Test public void testUpdatesWithMajorCompaction() throws Exception {
 776     final TableName tableName = name.getTableName();
 777     try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10);
 778         Admin admin = TEST_UTIL.getAdmin()) {
 779
 780       // Write a column with values at timestamp 1, 2 and 3
 781       byte[] row = Bytes.toBytes("row2");
 782       byte[] qualifier = Bytes.toBytes("myCol");
 783       Put put = new Put(row);
 784       put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
 785       hTable.put(put);
 786
 787       put = new Put(row);
 788       put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
 789       hTable.put(put);
 790
 791       put = new Put(row);
 792       put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
 793       hTable.put(put);
 794
 795       Get get = new Get(row);
 796       get.addColumn(FAMILY, qualifier);
 797       get.readAllVersions();
 798
 799       // Check that the column indeed has the right values at timestamps 1 and
 800       // 2
 801       Result result = hTable.get(get);
 802       NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
 803       assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
 804       assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
 805
 806       // Trigger a major compaction
 807       admin.flush(tableName);
 808       admin.majorCompact(tableName);
 809       Thread.sleep(6000);
 810
 811       // Update the value at timestamp 1
 812       put = new Put(row);
 813       put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
 814       hTable.put(put);
 815
 816       // Update the value at timestamp 2
 817       put = new Put(row);
 818       put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
 819       hTable.put(put);
 820
 821       // Trigger a major compaction
 822       admin.flush(tableName);
 823       admin.majorCompact(tableName);
 824       Thread.sleep(6000);
 825
 826       // Check that the values at timestamp 2 and 1 got updated
 827       result = hTable.get(get);
 828       navigableMap = result.getMap().get(FAMILY).get(qualifier);
 829       assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
 830       assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
 831     }
 832   }
 833
 834   @Test public void testMajorCompactionBetweenTwoUpdates() throws Exception {
 835     final TableName tableName = name.getTableName();
 836     try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10);
 837         Admin admin = TEST_UTIL.getAdmin()) {
 838
 839       // Write a column with values at timestamp 1, 2 and 3
 840       byte[] row = Bytes.toBytes("row3");
 841       byte[] qualifier = Bytes.toBytes("myCol");
 842       Put put = new Put(row);
 843       put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
 844       hTable.put(put);
 845
 846       put = new Put(row);
 847       put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
 848       hTable.put(put);
 849
 850       put = new Put(row);
 851       put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
 852       hTable.put(put);
 853
 854       Get get = new Get(row);
 855       get.addColumn(FAMILY, qualifier);
 856       get.readAllVersions();
 857
 858       // Check that the column indeed has the right values at timestamps 1 and
 859       // 2
 860       Result result = hTable.get(get);
 861       NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
 862       assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
 863       assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
 864
 865       // Trigger a major compaction
 866       admin.flush(tableName);
 867       admin.majorCompact(tableName);
 868       Thread.sleep(6000);
 869
 870       // Update the value at timestamp 1
 871       put = new Put(row);
 872       put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
 873       hTable.put(put);
 874
 875       // Trigger a major compaction
 876       admin.flush(tableName);
 877       admin.majorCompact(tableName);
 878       Thread.sleep(6000);
 879
 880       // Update the value at timestamp 2
 881       put = new Put(row);
 882       put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
 883       hTable.put(put);
 884
 885       // Trigger a major compaction
 886       admin.flush(tableName);
 887       admin.majorCompact(tableName);
 888       Thread.sleep(6000);
 889
 890       // Check that the values at timestamp 2 and 1 got updated
 891       result = hTable.get(get);
 892       navigableMap = result.getMap().get(FAMILY).get(qualifier);
 893
 894       assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
 895       assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
 896     }
 897   }
 898
 899   @Test public void testGet_EmptyTable() throws IOException {
 900     try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
 901       Get get = new Get(ROW);
 902       get.addFamily(FAMILY);
 903       Result r = table.get(get);
 904       assertTrue(r.isEmpty());
 905     }
 906   }
 907
 908   @Test public void testGet_NullQualifier() throws IOException {
 909     try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
 910       Put put = new Put(ROW);
 911       put.addColumn(FAMILY, QUALIFIER, VALUE);
 912       table.put(put);
 913
 914       put = new Put(ROW);
 915       put.addColumn(FAMILY, null, VALUE);
 916       table.put(put);
 917       LOG.info("Row put");
 918
 919       Get get = new Get(ROW);
 920       get.addColumn(FAMILY, null);
 921       Result r = table.get(get);
 922       assertEquals(1, r.size());
 923
 924       get = new Get(ROW);
 925       get.addFamily(FAMILY);
 926       r = table.get(get);
 927       assertEquals(2, r.size());
 928     }
 929   }
 930
 931   @Test public void testGet_NonExistentRow() throws IOException {
 932     try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
 933       Put put = new Put(ROW);
 934       put.addColumn(FAMILY, QUALIFIER, VALUE);
 935       table.put(put);
 936       LOG.info("Row put");
 937
 938       Get get = new Get(ROW);
 939       get.addFamily(FAMILY);
 940       Result r = table.get(get);
 941       assertFalse(r.isEmpty());
 942       System.out.println("Row retrieved successfully");
 943
 944       byte[] missingrow = Bytes.toBytes("missingrow");
 945       get = new Get(missingrow);
 946       get.addFamily(FAMILY);
 947       r = table.get(get);
 948       assertTrue(r.isEmpty());
 949       LOG.info("Row missing as it should be");
 950     }
 951   }
 952
 953   @Test public void testPut() throws IOException {
 954     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
 955     final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
 956     final byte[] row1 = Bytes.toBytes("row1");
 957     final byte[] row2 = Bytes.toBytes("row2");
 958     final byte[] value = Bytes.toBytes("abcd");
 959     try (Table table = TEST_UTIL
 960       .createTable(name.getTableName(), new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) {
 961       Put put = new Put(row1);
 962       put.addColumn(CONTENTS_FAMILY, null, value);
 963       table.put(put);
 964
 965       put = new Put(row2);
 966       put.addColumn(CONTENTS_FAMILY, null, value);
 967
 968       assertEquals(1, put.size());
 969       assertEquals(1, put.getFamilyCellMap().get(CONTENTS_FAMILY).size());
 970
 971       // KeyValue v1 expectation.  Cast for now until we go all Cell all the time. TODO
 972       KeyValue kv = (KeyValue) put.getFamilyCellMap().get(CONTENTS_FAMILY).get(0);
 973
 974       assertTrue(Bytes.equals(CellUtil.cloneFamily(kv), CONTENTS_FAMILY));
 975       // will it return null or an empty byte array?
 976       assertTrue(Bytes.equals(CellUtil.cloneQualifier(kv), new byte[0]));
 977
 978       assertTrue(Bytes.equals(CellUtil.cloneValue(kv), value));
 979
 980       table.put(put);
 981
 982       Scan scan = new Scan();
 983       scan.addColumn(CONTENTS_FAMILY, null);
 984       try (ResultScanner scanner = table.getScanner(scan)) {
 985         for (Result r : scanner) {
 986           for (Cell key : r.rawCells()) {
 987             System.out.println(Bytes.toString(r.getRow()) + ": " + key.toString());
 988           }
 989         }
 990       }
 991     }
 992   }
 993
 994   @Test public void testPutNoCF() throws IOException {
 995     final byte[] BAD_FAM = Bytes.toBytes("BAD_CF");
 996     final byte[] VAL = Bytes.toBytes(100);
 997     try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
 998       boolean caughtNSCFE = false;
 999
1000       try {
1001         Put p = new Put(ROW);
1002         p.addColumn(BAD_FAM, QUALIFIER, VAL);
1003         table.put(p);
1004       } catch (Exception e) {
1005         caughtNSCFE = e instanceof NoSuchColumnFamilyException;
1006       }
1007       assertTrue("Should throw NoSuchColumnFamilyException", caughtNSCFE);
1008     }
1009   }
1010
1011   @Test public void testRowsPut() throws IOException {
1012     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
1013     final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
1014     final int NB_BATCH_ROWS = 10;
1015     final byte[] value = Bytes.toBytes("abcd");
1016     try (Table table = TEST_UTIL
1017       .createTable(name.getTableName(), new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) {
1018       ArrayList<Put> rowsUpdate = new ArrayList<>();
1019       for (int i = 0; i < NB_BATCH_ROWS; i++) {
1020         byte[] row = Bytes.toBytes("row" + i);
1021         Put put = new Put(row);
1022         put.setDurability(Durability.SKIP_WAL);
1023         put.addColumn(CONTENTS_FAMILY, null, value);
1024         rowsUpdate.add(put);
1025       }
1026       table.put(rowsUpdate);
1027       Scan scan = new Scan();
1028       scan.addFamily(CONTENTS_FAMILY);
1029       try (ResultScanner scanner = table.getScanner(scan)) {
1030         int nbRows = 0;
1031         for (@SuppressWarnings("unused") Result row : scanner) {
1032           nbRows++;
1033         }
1034         assertEquals(NB_BATCH_ROWS, nbRows);
1035       }
1036     }
1037   }
1038
1039   @Test public void testRowsPutBufferedManyManyFlushes() throws IOException {
1040     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
1041     final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
1042     final byte[] value = Bytes.toBytes("abcd");
1043     final int NB_BATCH_ROWS = 10;
1044     try (Table table = TEST_UTIL
1045       .createTable(name.getTableName(), new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) {
1046       ArrayList<Put> rowsUpdate = new ArrayList<>();
1047       for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
1048         byte[] row = Bytes.toBytes("row" + i);
1049         Put put = new Put(row);
1050         put.setDurability(Durability.SKIP_WAL);
1051         put.addColumn(CONTENTS_FAMILY, null, value);
1052         rowsUpdate.add(put);
1053       }
1054       table.put(rowsUpdate);
1055
1056       Scan scan = new Scan();
1057       scan.addFamily(CONTENTS_FAMILY);
1058       try (ResultScanner scanner = table.getScanner(scan)) {
1059         int nbRows = 0;
1060         for (@SuppressWarnings("unused") Result row : scanner) {
1061           nbRows++;
1062         }
1063         assertEquals(NB_BATCH_ROWS * 10, nbRows);
1064       }
1065     }
1066   }
1067
1068   @Test public void testAddKeyValue() {
1069     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
1070     final byte[] value = Bytes.toBytes("abcd");
1071     final byte[] row1 = Bytes.toBytes("row1");
1072     final byte[] row2 = Bytes.toBytes("row2");
1073     byte[] qualifier = Bytes.toBytes("qf1");
1074     Put put = new Put(row1);
1075
1076     // Adding KeyValue with the same row
1077     KeyValue kv = new KeyValue(row1, CONTENTS_FAMILY, qualifier, value);
1078     boolean ok = true;
1079     try {
1080       put.add(kv);
1081     } catch (IOException e) {
1082       ok = false;
1083     }
1084     assertTrue(ok);
1085
1086     // Adding KeyValue with the different row
1087     kv = new KeyValue(row2, CONTENTS_FAMILY, qualifier, value);
1088     ok = false;
1089     try {
1090       put.add(kv);
1091     } catch (IOException e) {
1092       ok = true;
1093     }
1094     assertTrue(ok);
1095   }
1096
1097   /**
1098    * test for HBASE-737
1099    */
1100   @Test public void testHBase737() throws IOException {
1101     final byte[] FAM1 = Bytes.toBytes("fam1");
1102     final byte[] FAM2 = Bytes.toBytes("fam2");
1103     // Open table
1104     try (Table table = TEST_UTIL.createTable(name.getTableName(), new byte[][] { FAM1, FAM2 })) {
1105       // Insert some values
1106       Put put = new Put(ROW);
1107       put.addColumn(FAM1, Bytes.toBytes("letters"), Bytes.toBytes("abcdefg"));
1108       table.put(put);
1109       try {
1110         Thread.sleep(1000);
1111       } catch (InterruptedException i) {
1112         //ignore
1113       }
1114
1115       put = new Put(ROW);
1116       put.addColumn(FAM1, Bytes.toBytes("numbers"), Bytes.toBytes("123456"));
1117       table.put(put);
1118
1119       try {
1120         Thread.sleep(1000);
1121       } catch (InterruptedException i) {
1122         //ignore
1123       }
1124
1125       put = new Put(ROW);
1126       put.addColumn(FAM2, Bytes.toBytes("letters"), Bytes.toBytes("hijklmnop"));
1127       table.put(put);
1128
1129       long[] times = new long[3];
1130
1131       // First scan the memstore
1132
1133       Scan scan = new Scan();
1134       scan.addFamily(FAM1);
1135       scan.addFamily(FAM2);
1136       try (ResultScanner s = table.getScanner(scan)) {
1137         int index = 0;
1138         Result r;
1139         while ((r = s.next()) != null) {
1140           for (Cell key : r.rawCells()) {
1141             times[index++] = key.getTimestamp();
1142           }
1143         }
1144       }
1145       for (int i = 0; i < times.length - 1; i++) {
1146         for (int j = i + 1; j < times.length; j++) {
1147           assertTrue(times[j] > times[i]);
1148         }
1149       }
1150
1151       // Flush data to disk and try again
1152       TEST_UTIL.flush();
1153
1154       // Reset times
1155       Arrays.fill(times, 0);
1156
1157       try {
1158         Thread.sleep(1000);
1159       } catch (InterruptedException i) {
1160         //ignore
1161       }
1162       scan = new Scan();
1163       scan.addFamily(FAM1);
1164       scan.addFamily(FAM2);
1165       try (ResultScanner s = table.getScanner(scan)) {
1166         int index = 0;
1167         Result r = null;
1168         while ((r = s.next()) != null) {
1169           for (Cell key : r.rawCells()) {
1170             times[index++] = key.getTimestamp();
1171           }
1172         }
1173         for (int i = 0; i < times.length - 1; i++) {
1174           for (int j = i + 1; j < times.length; j++) {
1175             assertTrue(times[j] > times[i]);
1176           }
1177         }
1178       }
1179     }
1180   }
1181
1182   @Test public void testListTables() throws IOException {
1183     final String testTableName = name.getTableName().toString();
1184     final TableName tableName1 = TableName.valueOf(testTableName + "1");
1185     final TableName tableName2 = TableName.valueOf(testTableName + "2");
1186     final TableName tableName3 = TableName.valueOf(testTableName + "3");
1187     TableName[] tables = new TableName[] { tableName1, tableName2, tableName3 };
1188     for (TableName table : tables) {
1189       TEST_UTIL.createTable(table, FAMILY);
1190     }
1191     try (Admin admin = TEST_UTIL.getAdmin()) {
1192       List<TableDescriptor> ts = admin.listTableDescriptors();
1193       HashSet<TableDescriptor> result = new HashSet<>(ts);
1194       int size = result.size();
1195       assertTrue(size >= tables.length);
1196       for (TableName table : tables) {
1197         boolean found = false;
1198         for (TableDescriptor t : ts) {
1199           if (t.getTableName().equals(table)) {
1200             found = true;
1201             break;
1202           }
1203         }
1204         assertTrue("Not found: " + table, found);
1205       }
1206     }
1207   }
1208
1209   /**
1210    * simple test that just executes parts of the client
1211    * API that accept a pre-created Connection instance
1212    */
1213   @Test public void testUnmanagedHConnection() throws IOException {
1214     final TableName tableName = name.getTableName();
1215     TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
1216     try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
1217       Table t = conn.getTable(tableName);
1218       Admin admin = conn.getAdmin()) {
1219       assertTrue(admin.tableExists(tableName));
1220       assertTrue(t.get(new Get(ROW)).isEmpty());
1221     }
1222   }
1223
1224   /**
1225    * test of that unmanaged HConnections are able to reconnect
1226    * properly (see HBASE-5058)
1227    */
1228   @Test public void testUnmanagedHConnectionReconnect() throws Exception {
1229     Configuration conf = TEST_UTIL.getConfiguration();
1230     Class registryImpl = conf
1231       .getClass(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY, ZKConnectionRegistry.class);
1232     // This test does not make sense for MasterRegistry since it stops the only master in the
1233     // cluster and starts a new master without populating the underlying config for the connection.
1234     Assume.assumeFalse(registryImpl.equals(MasterRegistry.class));
1235     final TableName tableName = name.getTableName();
1236     TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
1237     try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) {
1238       try (Table t = conn.getTable(tableName); Admin admin = conn.getAdmin()) {
1239         assertTrue(admin.tableExists(tableName));
1240         assertTrue(t.get(new Get(ROW)).isEmpty());
1241       }
1242
1243       // stop the master
1244       SingleProcessHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1245       cluster.stopMaster(0, false);
1246       cluster.waitOnMaster(0);
1247
1248       // start up a new master
1249       cluster.startMaster();
1250       assertTrue(cluster.waitForActiveAndReadyMaster());
1251
1252       // test that the same unmanaged connection works with a new
1253       // Admin and can connect to the new master;
1254       try (Admin admin = conn.getAdmin()) {
1255         assertTrue(admin.tableExists(tableName));
1256         assertEquals(
1257           admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().size(),
1258           SLAVES);
1259       }
1260     }
1261   }
1262
1263   @Test public void testMiscHTableStuff() throws IOException {
1264     final String testTableName = name.getTableName().toString();
1265     final TableName tableAname = TableName.valueOf(testTableName + "A");
1266     final TableName tableBname = TableName.valueOf(testTableName + "B");
1267     final byte[] attrName = Bytes.toBytes("TESTATTR");
1268     final byte[] attrValue = Bytes.toBytes("somevalue");
1269     byte[] value = Bytes.toBytes("value");
1270
1271     try (Table a = TEST_UTIL.createTable(tableAname, HConstants.CATALOG_FAMILY);
1272       Table b = TEST_UTIL.createTable(tableBname, HConstants.CATALOG_FAMILY)) {
1273       Put put = new Put(ROW);
1274       put.addColumn(HConstants.CATALOG_FAMILY, null, value);
1275       a.put(put);
1276
1277       // open a new connection to A and a connection to b
1278       try (Table newA = TEST_UTIL.getConnection().getTable(tableAname)) {
1279
1280         // copy data from A to B
1281         Scan scan = new Scan();
1282         scan.addFamily(HConstants.CATALOG_FAMILY);
1283         try (ResultScanner s = newA.getScanner(scan)) {
1284           for (Result r : s) {
1285             put = new Put(r.getRow());
1286             put.setDurability(Durability.SKIP_WAL);
1287             for (Cell kv : r.rawCells()) {
1288               put.add(kv);
1289             }
1290             b.put(put);
1291           }
1292         }
1293       }
1294
1295       // Opening a new connection to A will cause the tables to be reloaded
1296       try (Table anotherA = TEST_UTIL.getConnection().getTable(tableAname)) {
1297         Get get = new Get(ROW);
1298         get.addFamily(HConstants.CATALOG_FAMILY);
1299         anotherA.get(get);
1300       }
1301
1302       // We can still access A through newA because it has the table information
1303       // cached. And if it needs to recalibrate, that will cause the information
1304       // to be reloaded.
1305
1306       // Test user metadata
1307       Admin admin = TEST_UTIL.getAdmin();
1308       // make a modifiable descriptor
1309       TableDescriptor desc = a.getDescriptor();
1310       // offline the table
1311       admin.disableTable(tableAname);
1312       // add a user attribute to HTD
1313       TableDescriptorBuilder builder =
1314         TableDescriptorBuilder.newBuilder(desc).setValue(attrName, attrValue);
1315       // add a user attribute to HCD
1316       for (ColumnFamilyDescriptor c : desc.getColumnFamilies()) {
1317         builder.modifyColumnFamily(
1318           ColumnFamilyDescriptorBuilder.newBuilder(c).setValue(attrName, attrValue).build());
1319       }
1320       // update metadata for all regions of this table
1321       admin.modifyTable(builder.build());
1322       // enable the table
1323       admin.enableTable(tableAname);
1324
1325       // Test that attribute changes were applied
1326       desc = a.getDescriptor();
1327       assertEquals("wrong table descriptor returned", desc.getTableName(), tableAname);
1328       // check HTD attribute
1329       value = desc.getValue(attrName);
1330       assertNotNull("missing HTD attribute value", value);
1331       assertFalse("HTD attribute value is incorrect", Bytes.compareTo(value, attrValue) != 0);
1332       // check HCD attribute
1333       for (ColumnFamilyDescriptor c : desc.getColumnFamilies()) {
1334         value = c.getValue(attrName);
1335         assertNotNull("missing HCD attribute value", value);
1336         assertFalse("HCD attribute value is incorrect", Bytes.compareTo(value, attrValue) != 0);
1337       }
1338     }
1339   }
1340 }
1341