hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestFromClientSide4.java

   1 /*
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.client;
  19
  20 import static org.hamcrest.CoreMatchers.instanceOf;
  21 import static org.junit.Assert.assertEquals;
  22 import static org.junit.Assert.assertFalse;
  23 import static org.junit.Assert.assertNotNull;
  24 import static org.junit.Assert.assertThat;
  25 import static org.junit.Assert.assertTrue;
  26 import static org.junit.Assert.fail;
  27 import java.io.IOException;
  28 import java.util.ArrayList;
  29 import java.util.Arrays;
  30 import java.util.Collection;
  31 import java.util.EnumSet;
  32 import java.util.HashSet;
  33 import java.util.List;
  34 import java.util.NavigableMap;
  35 import org.apache.hadoop.conf.Configuration;
  36 import org.apache.hadoop.hbase.Cell;
  37 import org.apache.hadoop.hbase.CellUtil;
  38 import org.apache.hadoop.hbase.ClusterMetrics.Option;
  39 import org.apache.hadoop.hbase.HBaseClassTestRule;
  40 import org.apache.hadoop.hbase.HColumnDescriptor;
  41 import org.apache.hadoop.hbase.HConstants;
  42 import org.apache.hadoop.hbase.HTableDescriptor;
  43 import org.apache.hadoop.hbase.KeyValue;
  44 import org.apache.hadoop.hbase.MiniHBaseCluster;
  45 import org.apache.hadoop.hbase.TableName;
  46 import org.apache.hadoop.hbase.TableNameTestRule;
  47 import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
  48 import org.apache.hadoop.hbase.master.LoadBalancer;
  49 import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
  50 import org.apache.hadoop.hbase.testclassification.ClientTests;
  51 import org.apache.hadoop.hbase.testclassification.LargeTests;
  52 import org.apache.hadoop.hbase.util.Bytes;
  53 import org.junit.AfterClass;
  54 import org.junit.Assume;
  55 import org.junit.ClassRule;
  56 import org.junit.Rule;
  57 import org.junit.Test;
  58 import org.junit.experimental.categories.Category;
  59 import org.junit.runner.RunWith;
  60 import org.junit.runners.Parameterized;
  61 import org.slf4j.Logger;
  62 import org.slf4j.LoggerFactory;
  63
  64 /**
  65  * Run tests that use the HBase clients; {@link Table}.
  66  * Sets up the HBase mini cluster once at start and runs through all client tests.
  67  * Each creates a table named for the method and does its stuff against that.
  68  *
  69  * Parameterized to run with different registry implementations.
  70  */
  71 @Category({LargeTests.class, ClientTests.class})
  72 @SuppressWarnings ("deprecation")
  73 @RunWith(Parameterized.class)
  74 public class TestFromClientSide4 extends FromClientSideBase {
  75   private static final Logger LOG = LoggerFactory.getLogger(TestFromClientSide4.class);
  76   @ClassRule
  77   public static final HBaseClassTestRule CLASS_RULE =
  78     HBaseClassTestRule.forClass(TestFromClientSide4.class);
  79   @Rule
  80   public TableNameTestRule name = new TableNameTestRule();
  81
  82   // To keep the child classes happy.
  83   TestFromClientSide4() {
  84   }
  85
  86   public TestFromClientSide4(Class registry, int numHedgedReqs) throws Exception {
  87     initialize(registry, numHedgedReqs, MultiRowMutationEndpoint.class);
  88   }
  89
  90   @Parameterized.Parameters
  91   public static Collection parameters() {
  92     return Arrays.asList(new Object[][] { { MasterRegistry.class, 1 }, { MasterRegistry.class, 2 },
  93       { ZKConnectionRegistry.class, 1 } });
  94   }
  95
  96   @AfterClass
  97   public static void tearDownAfterClass() throws Exception {
  98     afterClass();
  99   }
 100
 101   /**
 102    * Test batch operations with combination of valid and invalid args
 103    */
 104   @Test public void testBatchOperationsWithErrors() throws Exception {
 105     final TableName tableName = name.getTableName();
 106     try (Table foo = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, 10)) {
 107
 108       int NUM_OPS = 100;
 109
 110       // 1.1 Put with no column families (local validation, runtime exception)
 111       List<Put> puts = new ArrayList<>(NUM_OPS);
 112       for (int i = 0; i != NUM_OPS; i++) {
 113         Put put = new Put(Bytes.toBytes(i));
 114         puts.add(put);
 115       }
 116
 117       try {
 118         foo.put(puts);
 119         fail();
 120       } catch (IllegalArgumentException e) {
 121         // expected
 122         assertEquals(NUM_OPS, puts.size());
 123       }
 124
 125       // 1.2 Put with invalid column family
 126       puts.clear();
 127       for (int i = 0; i < NUM_OPS; i++) {
 128         Put put = new Put(Bytes.toBytes(i));
 129         put.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY, Bytes.toBytes(i));
 130         puts.add(put);
 131       }
 132
 133       try {
 134         foo.put(puts);
 135         fail();
 136       } catch (RetriesExhaustedException e) {
 137         // expected
 138         assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class));
 139       }
 140
 141       // 2.1 Get non-existent rows
 142       List<Get> gets = new ArrayList<>(NUM_OPS);
 143       for (int i = 0; i < NUM_OPS; i++) {
 144         Get get = new Get(Bytes.toBytes(i));
 145         gets.add(get);
 146       }
 147       Result[] getsResult = foo.get(gets);
 148       assertNotNull(getsResult);
 149       assertEquals(NUM_OPS, getsResult.length);
 150       for (int i = 0; i < NUM_OPS; i++) {
 151         Result getResult = getsResult[i];
 152         if (i % 2 == 0) {
 153           assertFalse(getResult.isEmpty());
 154         } else {
 155           assertTrue(getResult.isEmpty());
 156         }
 157       }
 158
 159       // 2.2 Get with invalid column family
 160       gets.clear();
 161       for (int i = 0; i < NUM_OPS; i++) {
 162         Get get = new Get(Bytes.toBytes(i));
 163         get.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY);
 164         gets.add(get);
 165       }
 166       try {
 167         foo.get(gets);
 168         fail();
 169       } catch (RetriesExhaustedException e) {
 170         // expected
 171         assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class));
 172       }
 173
 174       // 3.1 Delete with invalid column family
 175       List<Delete> deletes = new ArrayList<>(NUM_OPS);
 176       for (int i = 0; i < NUM_OPS; i++) {
 177         Delete delete = new Delete(Bytes.toBytes(i));
 178         delete.addColumn((i % 2) == 0 ? FAMILY : INVALID_FAMILY, FAMILY);
 179         deletes.add(delete);
 180       }
 181       try {
 182         foo.delete(deletes);
 183         fail();
 184       } catch (RetriesExhaustedException e) {
 185         // expected
 186         assertThat(e.getCause(), instanceOf(NoSuchColumnFamilyException.class));
 187       }
 188
 189       // all valid rows should have been deleted
 190       gets.clear();
 191       for (int i = 0; i < NUM_OPS; i++) {
 192         Get get = new Get(Bytes.toBytes(i));
 193         gets.add(get);
 194       }
 195       getsResult = foo.get(gets);
 196       assertNotNull(getsResult);
 197       assertEquals(NUM_OPS, getsResult.length);
 198       for (Result getResult : getsResult) {
 199         assertTrue(getResult.isEmpty());
 200       }
 201
 202       // 3.2 Delete non-existent rows
 203       deletes.clear();
 204       for (int i = 0; i < NUM_OPS; i++) {
 205         Delete delete = new Delete(Bytes.toBytes(i));
 206         deletes.add(delete);
 207       }
 208       foo.delete(deletes);
 209     }
 210   }
 211
 212   //
 213   // JIRA Testers
 214   //
 215
 216   /**
 217    * HBASE-867
 218    * If millions of columns in a column family, hbase scanner won't come up
 219    * Test will create numRows rows, each with numColsPerRow columns
 220    * (1 version each), and attempt to scan them all.
 221    * To test at scale, up numColsPerRow to the millions
 222    * (have not gotten that to work running as junit though)
 223    */
 224   @Test public void testJiraTest867() throws Exception {
 225     int numRows = 10;
 226     int numColsPerRow = 2000;
 227
 228     final TableName tableName = name.getTableName();
 229
 230     byte[][] ROWS = makeN(ROW, numRows);
 231     byte[][] QUALIFIERS = makeN(QUALIFIER, numColsPerRow);
 232
 233     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
 234
 235       // Insert rows
 236
 237       for (int i = 0; i < numRows; i++) {
 238         Put put = new Put(ROWS[i]);
 239         put.setDurability(Durability.SKIP_WAL);
 240         for (int j = 0; j < numColsPerRow; j++) {
 241           put.addColumn(FAMILY, QUALIFIERS[j], QUALIFIERS[j]);
 242         }
 243         assertEquals(
 244           "Put expected to contain " + numColsPerRow + " columns but " + "only contains " + put
 245             .size(), put.size(), numColsPerRow);
 246         ht.put(put);
 247       }
 248
 249       // Get a row
 250       Get get = new Get(ROWS[numRows - 1]);
 251       Result result = ht.get(get);
 252       assertNumKeys(result, numColsPerRow);
 253       Cell[] keys = result.rawCells();
 254       for (int i = 0; i < result.size(); i++) {
 255         assertKey(keys[i], ROWS[numRows - 1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
 256       }
 257
 258       // Scan the rows
 259       Scan scan = new Scan();
 260       try (ResultScanner scanner = ht.getScanner(scan)) {
 261         int rowCount = 0;
 262         while ((result = scanner.next()) != null) {
 263           assertNumKeys(result, numColsPerRow);
 264           Cell[] kvs = result.rawCells();
 265           for (int i = 0; i < numColsPerRow; i++) {
 266             assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
 267           }
 268           rowCount++;
 269         }
 270         assertEquals(
 271           "Expected to scan " + numRows + " rows but actually scanned " + rowCount + " rows",
 272           rowCount, numRows);
 273       }
 274
 275       // flush and try again
 276
 277       TEST_UTIL.flush();
 278
 279       // Get a row
 280       get = new Get(ROWS[numRows - 1]);
 281       result = ht.get(get);
 282       assertNumKeys(result, numColsPerRow);
 283       keys = result.rawCells();
 284       for (int i = 0; i < result.size(); i++) {
 285         assertKey(keys[i], ROWS[numRows - 1], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
 286       }
 287
 288       // Scan the rows
 289       scan = new Scan();
 290       try (ResultScanner scanner = ht.getScanner(scan)) {
 291         int rowCount = 0;
 292         while ((result = scanner.next()) != null) {
 293           assertNumKeys(result, numColsPerRow);
 294           Cell[] kvs = result.rawCells();
 295           for (int i = 0; i < numColsPerRow; i++) {
 296             assertKey(kvs[i], ROWS[rowCount], FAMILY, QUALIFIERS[i], QUALIFIERS[i]);
 297           }
 298           rowCount++;
 299         }
 300         assertEquals(
 301           "Expected to scan " + numRows + " rows but actually scanned " + rowCount + " rows",
 302           rowCount, numRows);
 303       }
 304     }
 305   }
 306
 307   /**
 308    * HBASE-861
 309    * get with timestamp will return a value if there is a version with an
 310    * earlier timestamp
 311    */
 312   @Test public void testJiraTest861() throws Exception {
 313     final TableName tableName = name.getTableName();
 314     byte[][] VALUES = makeNAscii(VALUE, 7);
 315     long[] STAMPS = makeStamps(7);
 316
 317     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
 318
 319       // Insert three versions
 320
 321       Put put = new Put(ROW);
 322       put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
 323       put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 324       put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
 325       ht.put(put);
 326
 327       // Get the middle value
 328       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 329
 330       // Try to get one version before (expect fail)
 331       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
 332
 333       // Try to get one version after (expect fail)
 334       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
 335
 336       // Try same from storefile
 337       TEST_UTIL.flush();
 338       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 339       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
 340       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
 341
 342       // Insert two more versions surrounding others, into memstore
 343       put = new Put(ROW);
 344       put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
 345       put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
 346       ht.put(put);
 347
 348       // Check we can get everything we should and can't get what we shouldn't
 349       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
 350       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
 351       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 352       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
 353       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
 354       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
 355       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
 356
 357       // Try same from two storefiles
 358       TEST_UTIL.flush();
 359       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
 360       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[1]);
 361       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 362       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
 363       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
 364       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[5]);
 365       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
 366     }
 367   }
 368
 369   /**
 370    * HBASE-33
 371    * Add a HTable get/obtainScanner method that retrieves all versions of a
 372    * particular column and row between two timestamps
 373    */
 374   @Test public void testJiraTest33() throws Exception {
 375     final TableName tableName = name.getTableName();
 376     byte[][] VALUES = makeNAscii(VALUE, 7);
 377     long[] STAMPS = makeStamps(7);
 378
 379     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
 380
 381       // Insert lots versions
 382
 383       Put put = new Put(ROW);
 384       put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
 385       put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
 386       put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 387       put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
 388       put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
 389       put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
 390       ht.put(put);
 391
 392       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 393       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
 394       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
 395       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
 396
 397       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 398       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
 399       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
 400       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
 401
 402       // Try same from storefile
 403       TEST_UTIL.flush();
 404
 405       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 406       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
 407       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
 408       getVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
 409
 410       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 411       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 2);
 412       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
 413       scanVersionRangeAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 3);
 414     }
 415   }
 416
 417   /**
 418    * HBASE-1014
 419    * commit(BatchUpdate) method should return timestamp
 420    */
 421   @Test public void testJiraTest1014() throws Exception {
 422     final TableName tableName = name.getTableName();
 423
 424     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
 425
 426       long manualStamp = 12345;
 427
 428       // Insert lots versions
 429
 430       Put put = new Put(ROW);
 431       put.addColumn(FAMILY, QUALIFIER, manualStamp, VALUE);
 432       ht.put(put);
 433
 434       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, manualStamp, VALUE);
 435       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp - 1);
 436       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, manualStamp + 1);
 437     }
 438   }
 439
 440   /**
 441    * HBASE-1182
 442    * Scan for columns > some timestamp
 443    */
 444   @Test public void testJiraTest1182() throws Exception {
 445     final TableName tableName = name.getTableName();
 446     byte[][] VALUES = makeNAscii(VALUE, 7);
 447     long[] STAMPS = makeStamps(7);
 448
 449     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
 450
 451       // Insert lots versions
 452
 453       Put put = new Put(ROW);
 454       put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
 455       put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
 456       put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 457       put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
 458       put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
 459       put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
 460       ht.put(put);
 461
 462       getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 463       getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
 464       getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
 465
 466       scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 467       scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
 468       scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
 469
 470       // Try same from storefile
 471       TEST_UTIL.flush();
 472
 473       getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 474       getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
 475       getVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
 476
 477       scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 478       scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 2, 5);
 479       scanVersionRangeAndVerifyGreaterThan(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 4, 5);
 480     }
 481   }
 482
 483   /**
 484    * HBASE-52
 485    * Add a means of scanning over all versions
 486    */
 487   @Test public void testJiraTest52() throws Exception {
 488     final TableName tableName = name.getTableName();
 489     byte[][] VALUES = makeNAscii(VALUE, 7);
 490     long[] STAMPS = makeStamps(7);
 491
 492     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
 493
 494       // Insert lots versions
 495
 496       Put put = new Put(ROW);
 497       put.addColumn(FAMILY, QUALIFIER, STAMPS[0], VALUES[0]);
 498       put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
 499       put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 500       put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
 501       put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
 502       put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
 503       ht.put(put);
 504
 505       getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 506
 507       scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 508
 509       // Try same from storefile
 510       TEST_UTIL.flush();
 511
 512       getAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 513
 514       scanAllVersionsAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS, VALUES, 0, 5);
 515     }
 516   }
 517
 518   @Test
 519   @SuppressWarnings("checkstyle:MethodLength")
 520   public void testDuplicateVersions() throws Exception {
 521     final TableName tableName = name.getTableName();
 522
 523     long[] STAMPS = makeStamps(20);
 524     byte[][] VALUES = makeNAscii(VALUE, 20);
 525
 526     try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
 527
 528       // Insert 4 versions of same column
 529       Put put = new Put(ROW);
 530       put.addColumn(FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
 531       put.addColumn(FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 532       put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
 533       put.addColumn(FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
 534       ht.put(put);
 535
 536       // Verify we can get each one properly
 537       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
 538       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 539       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
 540       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
 541       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
 542       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 543       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
 544       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
 545
 546       // Verify we don't accidentally get others
 547       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
 548       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
 549       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
 550       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
 551       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
 552       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
 553
 554       // Ensure maxVersions in query is respected
 555       Get get = new Get(ROW);
 556       get.addColumn(FAMILY, QUALIFIER);
 557       get.readVersions(2);
 558       Result result = ht.get(get);
 559       assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
 560         new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
 561
 562       Scan scan = new Scan(ROW);
 563       scan.addColumn(FAMILY, QUALIFIER);
 564       scan.setMaxVersions(2);
 565       result = getSingleScanResult(ht, scan);
 566       assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
 567         new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
 568
 569       // Flush and redo
 570
 571       TEST_UTIL.flush();
 572
 573       // Verify we can get each one properly
 574       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
 575       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 576       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
 577       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
 578       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
 579       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 580       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[4]);
 581       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[5], VALUES[5]);
 582
 583       // Verify we don't accidentally get others
 584       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
 585       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
 586       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
 587       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
 588       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[3]);
 589       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[6]);
 590
 591       // Ensure maxVersions in query is respected
 592       get = new Get(ROW);
 593       get.addColumn(FAMILY, QUALIFIER);
 594       get.readVersions(2);
 595       result = ht.get(get);
 596       assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
 597         new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
 598
 599       scan = new Scan(ROW);
 600       scan.addColumn(FAMILY, QUALIFIER);
 601       scan.setMaxVersions(2);
 602       result = getSingleScanResult(ht, scan);
 603       assertNResult(result, ROW, FAMILY, QUALIFIER, new long[] { STAMPS[4], STAMPS[5] },
 604         new byte[][] { VALUES[4], VALUES[5] }, 0, 1);
 605
 606       // Add some memstore and retest
 607
 608       // Insert 4 more versions of same column and a dupe
 609       put = new Put(ROW);
 610       put.addColumn(FAMILY, QUALIFIER, STAMPS[3], VALUES[3]);
 611       put.addColumn(FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
 612       put.addColumn(FAMILY, QUALIFIER, STAMPS[6], VALUES[6]);
 613       put.addColumn(FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
 614       put.addColumn(FAMILY, QUALIFIER, STAMPS[8], VALUES[8]);
 615       ht.put(put);
 616
 617       // Ensure maxVersions in query is respected
 618       get = new Get(ROW);
 619       get.addColumn(FAMILY, QUALIFIER);
 620       get.readVersions(7);
 621       result = ht.get(get);
 622       assertNResult(result, ROW, FAMILY, QUALIFIER,
 623         new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
 624         new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
 625           VALUES[8] }, 0, 6);
 626
 627       scan = new Scan(ROW);
 628       scan.addColumn(FAMILY, QUALIFIER);
 629       scan.setMaxVersions(7);
 630       result = getSingleScanResult(ht, scan);
 631       assertNResult(result, ROW, FAMILY, QUALIFIER,
 632         new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
 633         new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
 634           VALUES[8] }, 0, 6);
 635
 636       get = new Get(ROW);
 637       get.readVersions(7);
 638       result = ht.get(get);
 639       assertNResult(result, ROW, FAMILY, QUALIFIER,
 640         new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
 641         new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
 642           VALUES[8] }, 0, 6);
 643
 644       scan = new Scan(ROW);
 645       scan.setMaxVersions(7);
 646       result = getSingleScanResult(ht, scan);
 647       assertNResult(result, ROW, FAMILY, QUALIFIER,
 648         new long[] { STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8] },
 649         new byte[][] { VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7],
 650           VALUES[8] }, 0, 6);
 651
 652       // Verify we can get each one properly
 653       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
 654       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 655       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
 656       getVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
 657       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[1], VALUES[1]);
 658       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[2], VALUES[2]);
 659       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[4], VALUES[14]);
 660       scanVersionAndVerify(ht, ROW, FAMILY, QUALIFIER, STAMPS[7], VALUES[7]);
 661
 662       // Verify we don't accidentally get others
 663       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
 664       getVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
 665       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[0]);
 666       scanVersionAndVerifyMissing(ht, ROW, FAMILY, QUALIFIER, STAMPS[9]);
 667
 668       // Ensure maxVersions of table is respected
 669
 670       TEST_UTIL.flush();
 671
 672       // Insert 4 more versions of same column and a dupe
 673       put = new Put(ROW);
 674       put.addColumn(FAMILY, QUALIFIER, STAMPS[9], VALUES[9]);
 675       put.addColumn(FAMILY, QUALIFIER, STAMPS[11], VALUES[11]);
 676       put.addColumn(FAMILY, QUALIFIER, STAMPS[13], VALUES[13]);
 677       put.addColumn(FAMILY, QUALIFIER, STAMPS[15], VALUES[15]);
 678       ht.put(put);
 679
 680       get = new Get(ROW);
 681       get.addColumn(FAMILY, QUALIFIER);
 682       get.readVersions(Integer.MAX_VALUE);
 683       result = ht.get(get);
 684       assertNResult(result, ROW, FAMILY, QUALIFIER,
 685         new long[] { STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9],
 686           STAMPS[11], STAMPS[13], STAMPS[15] },
 687         new byte[][] { VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9],
 688           VALUES[11], VALUES[13], VALUES[15] }, 0, 9);
 689
 690       scan = new Scan(ROW);
 691       scan.addColumn(FAMILY, QUALIFIER);
 692       scan.setMaxVersions(Integer.MAX_VALUE);
 693       result = getSingleScanResult(ht, scan);
 694       assertNResult(result, ROW, FAMILY, QUALIFIER,
 695         new long[] { STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[7], STAMPS[8], STAMPS[9],
 696           STAMPS[11], STAMPS[13], STAMPS[15] },
 697         new byte[][] { VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[7], VALUES[8], VALUES[9],
 698           VALUES[11], VALUES[13], VALUES[15] }, 0, 9);
 699
 700       // Delete a version in the memstore and a version in a storefile
 701       Delete delete = new Delete(ROW);
 702       delete.addColumn(FAMILY, QUALIFIER, STAMPS[11]);
 703       delete.addColumn(FAMILY, QUALIFIER, STAMPS[7]);
 704       ht.delete(delete);
 705
 706       // Test that it's gone
 707       get = new Get(ROW);
 708       get.addColumn(FAMILY, QUALIFIER);
 709       get.readVersions(Integer.MAX_VALUE);
 710       result = ht.get(get);
 711       assertNResult(result, ROW, FAMILY, QUALIFIER,
 712         new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8],
 713           STAMPS[9], STAMPS[13], STAMPS[15] },
 714         new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8],
 715           VALUES[9], VALUES[13], VALUES[15] }, 0, 9);
 716
 717       scan = new Scan(ROW);
 718       scan.addColumn(FAMILY, QUALIFIER);
 719       scan.setMaxVersions(Integer.MAX_VALUE);
 720       result = getSingleScanResult(ht, scan);
 721       assertNResult(result, ROW, FAMILY, QUALIFIER,
 722         new long[] { STAMPS[1], STAMPS[2], STAMPS[3], STAMPS[4], STAMPS[5], STAMPS[6], STAMPS[8],
 723           STAMPS[9], STAMPS[13], STAMPS[15] },
 724         new byte[][] { VALUES[1], VALUES[2], VALUES[3], VALUES[14], VALUES[5], VALUES[6], VALUES[8],
 725           VALUES[9], VALUES[13], VALUES[15] }, 0, 9);
 726     }
 727   }
 728
 729   @Test public void testUpdates() throws Exception {
 730     final TableName tableName = name.getTableName();
 731     try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10)) {
 732
 733       // Write a column with values at timestamp 1, 2 and 3
 734       byte[] row = Bytes.toBytes("row1");
 735       byte[] qualifier = Bytes.toBytes("myCol");
 736       Put put = new Put(row);
 737       put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
 738       hTable.put(put);
 739
 740       put = new Put(row);
 741       put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
 742       hTable.put(put);
 743
 744       put = new Put(row);
 745       put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
 746       hTable.put(put);
 747
 748       Get get = new Get(row);
 749       get.addColumn(FAMILY, qualifier);
 750       get.readAllVersions();
 751
 752       // Check that the column indeed has the right values at timestamps 1 and
 753       // 2
 754       Result result = hTable.get(get);
 755       NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
 756       assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
 757       assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
 758
 759       // Update the value at timestamp 1
 760       put = new Put(row);
 761       put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
 762       hTable.put(put);
 763
 764       // Update the value at timestamp 2
 765       put = new Put(row);
 766       put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
 767       hTable.put(put);
 768
 769       // Check that the values at timestamp 2 and 1 got updated
 770       result = hTable.get(get);
 771       navigableMap = result.getMap().get(FAMILY).get(qualifier);
 772       assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
 773       assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
 774     }
 775   }
 776
 777   @Test public void testUpdatesWithMajorCompaction() throws Exception {
 778     final TableName tableName = name.getTableName();
 779     try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10);
 780         Admin admin = TEST_UTIL.getAdmin()) {
 781
 782       // Write a column with values at timestamp 1, 2 and 3
 783       byte[] row = Bytes.toBytes("row2");
 784       byte[] qualifier = Bytes.toBytes("myCol");
 785       Put put = new Put(row);
 786       put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
 787       hTable.put(put);
 788
 789       put = new Put(row);
 790       put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
 791       hTable.put(put);
 792
 793       put = new Put(row);
 794       put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
 795       hTable.put(put);
 796
 797       Get get = new Get(row);
 798       get.addColumn(FAMILY, qualifier);
 799       get.readAllVersions();
 800
 801       // Check that the column indeed has the right values at timestamps 1 and
 802       // 2
 803       Result result = hTable.get(get);
 804       NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
 805       assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
 806       assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
 807
 808       // Trigger a major compaction
 809       admin.flush(tableName);
 810       admin.majorCompact(tableName);
 811       Thread.sleep(6000);
 812
 813       // Update the value at timestamp 1
 814       put = new Put(row);
 815       put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
 816       hTable.put(put);
 817
 818       // Update the value at timestamp 2
 819       put = new Put(row);
 820       put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
 821       hTable.put(put);
 822
 823       // Trigger a major compaction
 824       admin.flush(tableName);
 825       admin.majorCompact(tableName);
 826       Thread.sleep(6000);
 827
 828       // Check that the values at timestamp 2 and 1 got updated
 829       result = hTable.get(get);
 830       navigableMap = result.getMap().get(FAMILY).get(qualifier);
 831       assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
 832       assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
 833     }
 834   }
 835
 836   @Test public void testMajorCompactionBetweenTwoUpdates() throws Exception {
 837     final TableName tableName = name.getTableName();
 838     try (Table hTable = TEST_UTIL.createTable(tableName, FAMILY, 10);
 839         Admin admin = TEST_UTIL.getAdmin()) {
 840
 841       // Write a column with values at timestamp 1, 2 and 3
 842       byte[] row = Bytes.toBytes("row3");
 843       byte[] qualifier = Bytes.toBytes("myCol");
 844       Put put = new Put(row);
 845       put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("AAA"));
 846       hTable.put(put);
 847
 848       put = new Put(row);
 849       put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("BBB"));
 850       hTable.put(put);
 851
 852       put = new Put(row);
 853       put.addColumn(FAMILY, qualifier, 3L, Bytes.toBytes("EEE"));
 854       hTable.put(put);
 855
 856       Get get = new Get(row);
 857       get.addColumn(FAMILY, qualifier);
 858       get.readAllVersions();
 859
 860       // Check that the column indeed has the right values at timestamps 1 and
 861       // 2
 862       Result result = hTable.get(get);
 863       NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY).get(qualifier);
 864       assertEquals("AAA", Bytes.toString(navigableMap.get(1L)));
 865       assertEquals("BBB", Bytes.toString(navigableMap.get(2L)));
 866
 867       // Trigger a major compaction
 868       admin.flush(tableName);
 869       admin.majorCompact(tableName);
 870       Thread.sleep(6000);
 871
 872       // Update the value at timestamp 1
 873       put = new Put(row);
 874       put.addColumn(FAMILY, qualifier, 1L, Bytes.toBytes("CCC"));
 875       hTable.put(put);
 876
 877       // Trigger a major compaction
 878       admin.flush(tableName);
 879       admin.majorCompact(tableName);
 880       Thread.sleep(6000);
 881
 882       // Update the value at timestamp 2
 883       put = new Put(row);
 884       put.addColumn(FAMILY, qualifier, 2L, Bytes.toBytes("DDD"));
 885       hTable.put(put);
 886
 887       // Trigger a major compaction
 888       admin.flush(tableName);
 889       admin.majorCompact(tableName);
 890       Thread.sleep(6000);
 891
 892       // Check that the values at timestamp 2 and 1 got updated
 893       result = hTable.get(get);
 894       navigableMap = result.getMap().get(FAMILY).get(qualifier);
 895
 896       assertEquals("CCC", Bytes.toString(navigableMap.get(1L)));
 897       assertEquals("DDD", Bytes.toString(navigableMap.get(2L)));
 898     }
 899   }
 900
 901   @Test public void testGet_EmptyTable() throws IOException {
 902     try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
 903       Get get = new Get(ROW);
 904       get.addFamily(FAMILY);
 905       Result r = table.get(get);
 906       assertTrue(r.isEmpty());
 907     }
 908   }
 909
 910   @Test public void testGet_NullQualifier() throws IOException {
 911     try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
 912       Put put = new Put(ROW);
 913       put.addColumn(FAMILY, QUALIFIER, VALUE);
 914       table.put(put);
 915
 916       put = new Put(ROW);
 917       put.addColumn(FAMILY, null, VALUE);
 918       table.put(put);
 919       LOG.info("Row put");
 920
 921       Get get = new Get(ROW);
 922       get.addColumn(FAMILY, null);
 923       Result r = table.get(get);
 924       assertEquals(1, r.size());
 925
 926       get = new Get(ROW);
 927       get.addFamily(FAMILY);
 928       r = table.get(get);
 929       assertEquals(2, r.size());
 930     }
 931   }
 932
 933   @Test public void testGet_NonExistentRow() throws IOException {
 934     try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
 935       Put put = new Put(ROW);
 936       put.addColumn(FAMILY, QUALIFIER, VALUE);
 937       table.put(put);
 938       LOG.info("Row put");
 939
 940       Get get = new Get(ROW);
 941       get.addFamily(FAMILY);
 942       Result r = table.get(get);
 943       assertFalse(r.isEmpty());
 944       System.out.println("Row retrieved successfully");
 945
 946       byte[] missingrow = Bytes.toBytes("missingrow");
 947       get = new Get(missingrow);
 948       get.addFamily(FAMILY);
 949       r = table.get(get);
 950       assertTrue(r.isEmpty());
 951       LOG.info("Row missing as it should be");
 952     }
 953   }
 954
 955   @Test public void testPut() throws IOException {
 956     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
 957     final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
 958     final byte[] row1 = Bytes.toBytes("row1");
 959     final byte[] row2 = Bytes.toBytes("row2");
 960     final byte[] value = Bytes.toBytes("abcd");
 961     try (Table table = TEST_UTIL
 962       .createTable(name.getTableName(), new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) {
 963       Put put = new Put(row1);
 964       put.addColumn(CONTENTS_FAMILY, null, value);
 965       table.put(put);
 966
 967       put = new Put(row2);
 968       put.addColumn(CONTENTS_FAMILY, null, value);
 969
 970       assertEquals(1, put.size());
 971       assertEquals(1, put.getFamilyCellMap().get(CONTENTS_FAMILY).size());
 972
 973       // KeyValue v1 expectation.  Cast for now until we go all Cell all the time. TODO
 974       KeyValue kv = (KeyValue) put.getFamilyCellMap().get(CONTENTS_FAMILY).get(0);
 975
 976       assertTrue(Bytes.equals(CellUtil.cloneFamily(kv), CONTENTS_FAMILY));
 977       // will it return null or an empty byte array?
 978       assertTrue(Bytes.equals(CellUtil.cloneQualifier(kv), new byte[0]));
 979
 980       assertTrue(Bytes.equals(CellUtil.cloneValue(kv), value));
 981
 982       table.put(put);
 983
 984       Scan scan = new Scan();
 985       scan.addColumn(CONTENTS_FAMILY, null);
 986       try (ResultScanner scanner = table.getScanner(scan)) {
 987         for (Result r : scanner) {
 988           for (Cell key : r.rawCells()) {
 989             System.out.println(Bytes.toString(r.getRow()) + ": " + key.toString());
 990           }
 991         }
 992       }
 993     }
 994   }
 995
 996   @Test public void testPutNoCF() throws IOException {
 997     final byte[] BAD_FAM = Bytes.toBytes("BAD_CF");
 998     final byte[] VAL = Bytes.toBytes(100);
 999     try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
1000       boolean caughtNSCFE = false;
1001
1002       try {
1003         Put p = new Put(ROW);
1004         p.addColumn(BAD_FAM, QUALIFIER, VAL);
1005         table.put(p);
1006       } catch (Exception e) {
1007         caughtNSCFE = e instanceof NoSuchColumnFamilyException;
1008       }
1009       assertTrue("Should throw NoSuchColumnFamilyException", caughtNSCFE);
1010     }
1011   }
1012
1013   @Test public void testRowsPut() throws IOException {
1014     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
1015     final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
1016     final int NB_BATCH_ROWS = 10;
1017     final byte[] value = Bytes.toBytes("abcd");
1018     try (Table table = TEST_UTIL
1019       .createTable(name.getTableName(), new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) {
1020       ArrayList<Put> rowsUpdate = new ArrayList<>();
1021       for (int i = 0; i < NB_BATCH_ROWS; i++) {
1022         byte[] row = Bytes.toBytes("row" + i);
1023         Put put = new Put(row);
1024         put.setDurability(Durability.SKIP_WAL);
1025         put.addColumn(CONTENTS_FAMILY, null, value);
1026         rowsUpdate.add(put);
1027       }
1028       table.put(rowsUpdate);
1029       Scan scan = new Scan();
1030       scan.addFamily(CONTENTS_FAMILY);
1031       try (ResultScanner scanner = table.getScanner(scan)) {
1032         int nbRows = 0;
1033         for (@SuppressWarnings("unused") Result row : scanner) {
1034           nbRows++;
1035         }
1036         assertEquals(NB_BATCH_ROWS, nbRows);
1037       }
1038     }
1039   }
1040
1041   @Test public void testRowsPutBufferedManyManyFlushes() throws IOException {
1042     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
1043     final byte[] SMALL_FAMILY = Bytes.toBytes("smallfam");
1044     final byte[] value = Bytes.toBytes("abcd");
1045     final int NB_BATCH_ROWS = 10;
1046     try (Table table = TEST_UTIL
1047       .createTable(name.getTableName(), new byte[][] { CONTENTS_FAMILY, SMALL_FAMILY })) {
1048       ArrayList<Put> rowsUpdate = new ArrayList<>();
1049       for (int i = 0; i < NB_BATCH_ROWS * 10; i++) {
1050         byte[] row = Bytes.toBytes("row" + i);
1051         Put put = new Put(row);
1052         put.setDurability(Durability.SKIP_WAL);
1053         put.addColumn(CONTENTS_FAMILY, null, value);
1054         rowsUpdate.add(put);
1055       }
1056       table.put(rowsUpdate);
1057
1058       Scan scan = new Scan();
1059       scan.addFamily(CONTENTS_FAMILY);
1060       try (ResultScanner scanner = table.getScanner(scan)) {
1061         int nbRows = 0;
1062         for (@SuppressWarnings("unused") Result row : scanner) {
1063           nbRows++;
1064         }
1065         assertEquals(NB_BATCH_ROWS * 10, nbRows);
1066       }
1067     }
1068   }
1069
1070   @Test public void testAddKeyValue() {
1071     final byte[] CONTENTS_FAMILY = Bytes.toBytes("contents");
1072     final byte[] value = Bytes.toBytes("abcd");
1073     final byte[] row1 = Bytes.toBytes("row1");
1074     final byte[] row2 = Bytes.toBytes("row2");
1075     byte[] qualifier = Bytes.toBytes("qf1");
1076     Put put = new Put(row1);
1077
1078     // Adding KeyValue with the same row
1079     KeyValue kv = new KeyValue(row1, CONTENTS_FAMILY, qualifier, value);
1080     boolean ok = true;
1081     try {
1082       put.add(kv);
1083     } catch (IOException e) {
1084       ok = false;
1085     }
1086     assertTrue(ok);
1087
1088     // Adding KeyValue with the different row
1089     kv = new KeyValue(row2, CONTENTS_FAMILY, qualifier, value);
1090     ok = false;
1091     try {
1092       put.add(kv);
1093     } catch (IOException e) {
1094       ok = true;
1095     }
1096     assertTrue(ok);
1097   }
1098
1099   /**
1100    * test for HBASE-737
1101    */
1102   @Test public void testHBase737() throws IOException {
1103     final byte[] FAM1 = Bytes.toBytes("fam1");
1104     final byte[] FAM2 = Bytes.toBytes("fam2");
1105     // Open table
1106     try (Table table = TEST_UTIL.createTable(name.getTableName(), new byte[][] { FAM1, FAM2 })) {
1107       // Insert some values
1108       Put put = new Put(ROW);
1109       put.addColumn(FAM1, Bytes.toBytes("letters"), Bytes.toBytes("abcdefg"));
1110       table.put(put);
1111       try {
1112         Thread.sleep(1000);
1113       } catch (InterruptedException i) {
1114         //ignore
1115       }
1116
1117       put = new Put(ROW);
1118       put.addColumn(FAM1, Bytes.toBytes("numbers"), Bytes.toBytes("123456"));
1119       table.put(put);
1120
1121       try {
1122         Thread.sleep(1000);
1123       } catch (InterruptedException i) {
1124         //ignore
1125       }
1126
1127       put = new Put(ROW);
1128       put.addColumn(FAM2, Bytes.toBytes("letters"), Bytes.toBytes("hijklmnop"));
1129       table.put(put);
1130
1131       long[] times = new long[3];
1132
1133       // First scan the memstore
1134
1135       Scan scan = new Scan();
1136       scan.addFamily(FAM1);
1137       scan.addFamily(FAM2);
1138       try (ResultScanner s = table.getScanner(scan)) {
1139         int index = 0;
1140         Result r;
1141         while ((r = s.next()) != null) {
1142           for (Cell key : r.rawCells()) {
1143             times[index++] = key.getTimestamp();
1144           }
1145         }
1146       }
1147       for (int i = 0; i < times.length - 1; i++) {
1148         for (int j = i + 1; j < times.length; j++) {
1149           assertTrue(times[j] > times[i]);
1150         }
1151       }
1152
1153       // Flush data to disk and try again
1154       TEST_UTIL.flush();
1155
1156       // Reset times
1157       Arrays.fill(times, 0);
1158
1159       try {
1160         Thread.sleep(1000);
1161       } catch (InterruptedException i) {
1162         //ignore
1163       }
1164       scan = new Scan();
1165       scan.addFamily(FAM1);
1166       scan.addFamily(FAM2);
1167       try (ResultScanner s = table.getScanner(scan)) {
1168         int index = 0;
1169         Result r = null;
1170         while ((r = s.next()) != null) {
1171           for (Cell key : r.rawCells()) {
1172             times[index++] = key.getTimestamp();
1173           }
1174         }
1175         for (int i = 0; i < times.length - 1; i++) {
1176           for (int j = i + 1; j < times.length; j++) {
1177             assertTrue(times[j] > times[i]);
1178           }
1179         }
1180       }
1181     }
1182   }
1183
1184   @Test public void testListTables() throws IOException {
1185     final String testTableName = name.getTableName().toString();
1186     final TableName tableName1 = TableName.valueOf(testTableName + "1");
1187     final TableName tableName2 = TableName.valueOf(testTableName + "2");
1188     final TableName tableName3 = TableName.valueOf(testTableName + "3");
1189     TableName[] tables = new TableName[] { tableName1, tableName2, tableName3 };
1190     for (TableName table : tables) {
1191       TEST_UTIL.createTable(table, FAMILY);
1192     }
1193     try (Admin admin = TEST_UTIL.getAdmin()) {
1194       List<TableDescriptor> ts = admin.listTableDescriptors();
1195       HashSet<TableDescriptor> result = new HashSet<>(ts);
1196       int size = result.size();
1197       assertTrue(size >= tables.length);
1198       for (TableName table : tables) {
1199         boolean found = false;
1200         for (TableDescriptor t : ts) {
1201           if (t.getTableName().equals(table)) {
1202             found = true;
1203             break;
1204           }
1205         }
1206         assertTrue("Not found: " + table, found);
1207       }
1208     }
1209   }
1210
1211   /**
1212    * simple test that just executes parts of the client
1213    * API that accept a pre-created Connection instance
1214    */
1215   @Test public void testUnmanagedHConnection() throws IOException {
1216     final TableName tableName = name.getTableName();
1217     TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
1218     try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
1219       Table t = conn.getTable(tableName);
1220       Admin admin = conn.getAdmin()) {
1221       assertTrue(admin.tableExists(tableName));
1222       assertTrue(t.get(new Get(ROW)).isEmpty());
1223     }
1224   }
1225
1226   /**
1227    * test of that unmanaged HConnections are able to reconnect
1228    * properly (see HBASE-5058)
1229    */
1230   @Test public void testUnmanagedHConnectionReconnect() throws Exception {
1231     Configuration conf = TEST_UTIL.getConfiguration();
1232     Class registryImpl = conf
1233       .getClass(HConstants.CLIENT_CONNECTION_REGISTRY_IMPL_CONF_KEY, ZKConnectionRegistry.class);
1234     // This test does not make sense for MasterRegistry since it stops the only master in the
1235     // cluster and starts a new master without populating the underlying config for the connection.
1236     Assume.assumeFalse(registryImpl.equals(MasterRegistry.class));
1237     final TableName tableName = name.getTableName();
1238     TEST_UTIL.createTable(tableName, HConstants.CATALOG_FAMILY);
1239     try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) {
1240       try (Table t = conn.getTable(tableName); Admin admin = conn.getAdmin()) {
1241         assertTrue(admin.tableExists(tableName));
1242         assertTrue(t.get(new Get(ROW)).isEmpty());
1243       }
1244
1245       // stop the master
1246       MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
1247       cluster.stopMaster(0, false);
1248       cluster.waitOnMaster(0);
1249
1250       // start up a new master
1251       cluster.startMaster();
1252       assertTrue(cluster.waitForActiveAndReadyMaster());
1253
1254       // test that the same unmanaged connection works with a new
1255       // Admin and can connect to the new master;
1256       boolean tablesOnMaster = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration());
1257       try (Admin admin = conn.getAdmin()) {
1258         assertTrue(admin.tableExists(tableName));
1259         assertEquals(
1260           admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().size(),
1261           SLAVES + (tablesOnMaster ? 1 : 0));
1262       }
1263     }
1264   }
1265
1266   @Test public void testMiscHTableStuff() throws IOException {
1267     final String testTableName = name.getTableName().toString();
1268     final TableName tableAname = TableName.valueOf(testTableName + "A");
1269     final TableName tableBname = TableName.valueOf(testTableName + "B");
1270     final byte[] attrName = Bytes.toBytes("TESTATTR");
1271     final byte[] attrValue = Bytes.toBytes("somevalue");
1272     byte[] value = Bytes.toBytes("value");
1273
1274     try (Table a = TEST_UTIL.createTable(tableAname, HConstants.CATALOG_FAMILY);
1275       Table b = TEST_UTIL.createTable(tableBname, HConstants.CATALOG_FAMILY)) {
1276       Put put = new Put(ROW);
1277       put.addColumn(HConstants.CATALOG_FAMILY, null, value);
1278       a.put(put);
1279
1280       // open a new connection to A and a connection to b
1281       try (Table newA = TEST_UTIL.getConnection().getTable(tableAname)) {
1282
1283         // copy data from A to B
1284         Scan scan = new Scan();
1285         scan.addFamily(HConstants.CATALOG_FAMILY);
1286         try (ResultScanner s = newA.getScanner(scan)) {
1287           for (Result r : s) {
1288             put = new Put(r.getRow());
1289             put.setDurability(Durability.SKIP_WAL);
1290             for (Cell kv : r.rawCells()) {
1291               put.add(kv);
1292             }
1293             b.put(put);
1294           }
1295         }
1296       }
1297
1298       // Opening a new connection to A will cause the tables to be reloaded
1299       try (Table anotherA = TEST_UTIL.getConnection().getTable(tableAname)) {
1300         Get get = new Get(ROW);
1301         get.addFamily(HConstants.CATALOG_FAMILY);
1302         anotherA.get(get);
1303       }
1304
1305       // We can still access A through newA because it has the table information
1306       // cached. And if it needs to recalibrate, that will cause the information
1307       // to be reloaded.
1308
1309       // Test user metadata
1310       Admin admin = TEST_UTIL.getAdmin();
1311       // make a modifiable descriptor
1312       HTableDescriptor desc = new HTableDescriptor(a.getDescriptor());
1313       // offline the table
1314       admin.disableTable(tableAname);
1315       // add a user attribute to HTD
1316       desc.setValue(attrName, attrValue);
1317       // add a user attribute to HCD
1318       for (HColumnDescriptor c : desc.getFamilies()) {
1319         c.setValue(attrName, attrValue);
1320       }
1321       // update metadata for all regions of this table
1322       admin.modifyTable(desc);
1323       // enable the table
1324       admin.enableTable(tableAname);
1325
1326       // Test that attribute changes were applied
1327       desc = new HTableDescriptor(a.getDescriptor());
1328       assertEquals("wrong table descriptor returned", desc.getTableName(), tableAname);
1329       // check HTD attribute
1330       value = desc.getValue(attrName);
1331       assertNotNull("missing HTD attribute value", value);
1332       assertFalse("HTD attribute value is incorrect", Bytes.compareTo(value, attrValue) != 0);
1333       // check HCD attribute
1334       for (HColumnDescriptor c : desc.getFamilies()) {
1335         value = c.getValue(attrName);
1336         assertNotNull("missing HCD attribute value", value);
1337         assertFalse("HCD attribute value is incorrect", Bytes.compareTo(value, attrValue) != 0);
1338       }
1339     }
1340   }
1341 }
1342