hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestIncrementsFromClientSide.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.client;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertNotEquals;
  22 import static org.junit.Assert.assertNotNull;
  23 import static org.junit.Assert.assertTrue;
  24 import static org.junit.Assert.fail;
  25
  26 import java.io.IOException;
  27 import java.util.ArrayList;
  28 import java.util.HashMap;
  29 import java.util.List;
  30 import java.util.Map;
  31 import org.apache.hadoop.conf.Configuration;
  32 import org.apache.hadoop.hbase.Cell;
  33 import org.apache.hadoop.hbase.CellUtil;
  34 import org.apache.hadoop.hbase.DoNotRetryIOException;
  35 import org.apache.hadoop.hbase.HBaseClassTestRule;
  36 import org.apache.hadoop.hbase.HBaseTestingUtility;
  37 import org.apache.hadoop.hbase.HConstants;
  38 import org.apache.hadoop.hbase.HTableDescriptor;
  39 import org.apache.hadoop.hbase.KeyValue;
  40 import org.apache.hadoop.hbase.TableName;
  41 import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
  42 import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
  43 import org.apache.hadoop.hbase.testclassification.LargeTests;
  44 import org.apache.hadoop.hbase.util.Bytes;
  45 import org.junit.AfterClass;
  46 import org.junit.BeforeClass;
  47 import org.junit.ClassRule;
  48 import org.junit.Rule;
  49 import org.junit.Test;
  50 import org.junit.experimental.categories.Category;
  51 import org.junit.rules.TestName;
  52 import org.slf4j.Logger;
  53 import org.slf4j.LoggerFactory;
  54
  55 /**
  56  * Run Increment tests that use the HBase clients; {@link HTable}.
  57  *
  58  * Test is parameterized to run the slow and fast increment code paths. If fast, in the @before, we
  59  * do a rolling restart of the single regionserver so that it can pick up the go fast configuration.
  60  * Doing it this way should be faster than starting/stopping a cluster per test.
  61  *
  62  * Test takes a long time because spin up a cluster between each run -- ugh.
  63  */
  64 @Category(LargeTests.class)
  65 public class TestIncrementsFromClientSide {
  66
  67   @ClassRule
  68   public static final HBaseClassTestRule CLASS_RULE =
  69       HBaseClassTestRule.forClass(TestIncrementsFromClientSide.class);
  70
  71   final Logger LOG = LoggerFactory.getLogger(getClass());
  72   protected final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
  73   private static byte [] ROW = Bytes.toBytes("testRow");
  74   private static byte [] FAMILY = Bytes.toBytes("testFamily");
  75   private static byte [] QUALIFIER = Bytes.toBytes("testQualifier");
  76   // This test depends on there being only one slave running at at a time. See the @Before
  77   // method where we do rolling restart.
  78   protected static int SLAVES = 1;
  79   @Rule public TestName name = new TestName();
  80
  81   @BeforeClass
  82   public static void beforeClass() throws Exception {
  83     Configuration conf = TEST_UTIL.getConfiguration();
  84     conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
  85         MultiRowMutationEndpoint.class.getName());
  86     conf.setBoolean("hbase.table.sanity.checks", true); // enable for below tests
  87     // We need more than one region server in this test
  88     TEST_UTIL.startMiniCluster(SLAVES);
  89   }
  90
  91   /**
  92    * @throws java.lang.Exception
  93    */
  94   @AfterClass
  95   public static void afterClass() throws Exception {
  96     TEST_UTIL.shutdownMiniCluster();
  97   }
  98
  99   /**
 100    * Test increment result when there are duplicate rpc request.
 101    */
 102   @Test
 103   public void testDuplicateIncrement() throws Exception {
 104     HTableDescriptor hdt = TEST_UTIL.createTableDescriptor(TableName.valueOf(name.getMethodName()));
 105     Map<String, String> kvs = new HashMap<>();
 106     kvs.put(SleepAtFirstRpcCall.SLEEP_TIME_CONF_KEY, "2000");
 107     hdt.addCoprocessor(SleepAtFirstRpcCall.class.getName(), null, 1, kvs);
 108     TEST_UTIL.createTable(hdt, new byte[][] { ROW }).close();
 109
 110     Configuration c = new Configuration(TEST_UTIL.getConfiguration());
 111     c.setInt(HConstants.HBASE_CLIENT_PAUSE, 50);
 112     // Client will retry beacuse rpc timeout is small than the sleep time of first rpc call
 113     c.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 1500);
 114
 115     try (Connection connection = ConnectionFactory.createConnection(c);
 116         Table table = connection.getTableBuilder(TableName.valueOf(name.getMethodName()), null)
 117           .setOperationTimeout(3 * 1000).build()) {
 118       Increment inc = new Increment(ROW);
 119       inc.addColumn(HBaseTestingUtility.fam1, QUALIFIER, 1);
 120       Result result = table.increment(inc);
 121
 122       Cell[] cells = result.rawCells();
 123       assertEquals(1, cells.length);
 124       assertIncrementKey(cells[0], ROW, HBaseTestingUtility.fam1, QUALIFIER, 1);
 125
 126       // Verify expected result
 127       Result readResult = table.get(new Get(ROW));
 128       cells = readResult.rawCells();
 129       assertEquals(1, cells.length);
 130       assertIncrementKey(cells[0], ROW, HBaseTestingUtility.fam1, QUALIFIER, 1);
 131     }
 132   }
 133
 134   @Test
 135   public void testIncrementWithDeletes() throws Exception {
 136     LOG.info("Starting " + this.name.getMethodName());
 137     final TableName TABLENAME =
 138         TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
 139     Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
 140     final byte[] COLUMN = Bytes.toBytes("column");
 141
 142     ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
 143     TEST_UTIL.flush(TABLENAME);
 144
 145     Delete del = new Delete(ROW);
 146     ht.delete(del);
 147
 148     ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
 149
 150     Get get = new Get(ROW);
 151     Result r = ht.get(get);
 152     assertEquals(1, r.size());
 153     assertEquals(5, Bytes.toLong(r.getValue(FAMILY, COLUMN)));
 154   }
 155
 156   @Test
 157   public void testIncrementingInvalidValue() throws Exception {
 158     LOG.info("Starting " + this.name.getMethodName());
 159     final TableName TABLENAME =
 160         TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
 161     Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
 162     final byte[] COLUMN = Bytes.toBytes("column");
 163     Put p = new Put(ROW);
 164     // write an integer here (not a Long)
 165     p.addColumn(FAMILY, COLUMN, Bytes.toBytes(5));
 166     ht.put(p);
 167     try {
 168       ht.incrementColumnValue(ROW, FAMILY, COLUMN, 5);
 169       fail("Should have thrown DoNotRetryIOException");
 170     } catch (DoNotRetryIOException iox) {
 171       // success
 172     }
 173     Increment inc = new Increment(ROW);
 174     inc.addColumn(FAMILY, COLUMN, 5);
 175     try {
 176       ht.increment(inc);
 177       fail("Should have thrown DoNotRetryIOException");
 178     } catch (DoNotRetryIOException iox) {
 179       // success
 180     }
 181   }
 182
 183   @Test
 184   public void testBatchIncrementsWithReturnResultFalse() throws Exception {
 185     LOG.info("Starting testBatchIncrementsWithReturnResultFalse");
 186     final TableName tableName = TableName.valueOf(name.getMethodName());
 187     Table table = TEST_UTIL.createTable(tableName, FAMILY);
 188     Increment inc1 = new Increment(Bytes.toBytes("row2"));
 189     inc1.setReturnResults(false);
 190     inc1.addColumn(FAMILY, Bytes.toBytes("f1"), 1);
 191     Increment inc2 = new Increment(Bytes.toBytes("row2"));
 192     inc2.setReturnResults(false);
 193     inc2.addColumn(FAMILY, Bytes.toBytes("f1"), 1);
 194     List<Increment> incs = new ArrayList<>();
 195     incs.add(inc1);
 196     incs.add(inc2);
 197     Object[] results = new Object[2];
 198     table.batch(incs, results);
 199     assertTrue(results.length == 2);
 200     for(Object r : results) {
 201       Result result = (Result)r;
 202       assertTrue(result.isEmpty());
 203     }
 204     table.close();
 205   }
 206
 207   @Test
 208   public void testIncrementInvalidArguments() throws Exception {
 209     LOG.info("Starting " + this.name.getMethodName());
 210     final TableName TABLENAME =
 211       TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
 212     Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
 213     final byte[] COLUMN = Bytes.toBytes("column");
 214     try {
 215       // try null row
 216       ht.incrementColumnValue(null, FAMILY, COLUMN, 5);
 217       fail("Should have thrown NPE/IOE");
 218     } catch (NullPointerException | IOException error) {
 219       // success
 220     }
 221     try {
 222       // try null family
 223       ht.incrementColumnValue(ROW, null, COLUMN, 5);
 224       fail("Should have thrown NPE/IOE");
 225     } catch (NullPointerException | IOException error) {
 226       // success
 227     }
 228     // try null row
 229     try {
 230       Increment incNoRow = new Increment((byte[]) null);
 231       incNoRow.addColumn(FAMILY, COLUMN, 5);
 232       fail("Should have thrown IAE/NPE");
 233     } catch (IllegalArgumentException | NullPointerException error) {
 234       // success
 235     }
 236     // try null family
 237     try {
 238       Increment incNoFamily = new Increment(ROW);
 239       incNoFamily.addColumn(null, COLUMN, 5);
 240       fail("Should have thrown IAE");
 241     } catch (IllegalArgumentException iax) {
 242       // success
 243     }
 244   }
 245
 246   @Test
 247   public void testIncrementOutOfOrder() throws Exception {
 248     LOG.info("Starting " + this.name.getMethodName());
 249     final TableName TABLENAME =
 250         TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
 251     Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
 252
 253     byte [][] QUALIFIERS = new byte [][] {
 254       Bytes.toBytes("B"), Bytes.toBytes("A"), Bytes.toBytes("C")
 255     };
 256
 257     Increment inc = new Increment(ROW);
 258     for (int i=0; i<QUALIFIERS.length; i++) {
 259       inc.addColumn(FAMILY, QUALIFIERS[i], 1);
 260     }
 261     ht.increment(inc);
 262
 263     // Verify expected results
 264     Get get = new Get(ROW);
 265     Result r = ht.get(get);
 266     Cell [] kvs = r.rawCells();
 267     assertEquals(3, kvs.length);
 268     assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[1], 1);
 269     assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[0], 1);
 270     assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 1);
 271
 272     // Now try multiple columns again
 273     inc = new Increment(ROW);
 274     for (int i=0; i<QUALIFIERS.length; i++) {
 275       inc.addColumn(FAMILY, QUALIFIERS[i], 1);
 276     }
 277     ht.increment(inc);
 278
 279     // Verify
 280     r = ht.get(get);
 281     kvs = r.rawCells();
 282     assertEquals(3, kvs.length);
 283     assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[1], 2);
 284     assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[0], 2);
 285     assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 2);
 286   }
 287
 288   @Test
 289   public void testIncrementOnSameColumn() throws Exception {
 290     LOG.info("Starting " + this.name.getMethodName());
 291     final byte[] TABLENAME = Bytes.toBytes(filterStringSoTableNameSafe(this.name.getMethodName()));
 292     Table ht = TEST_UTIL.createTable(TableName.valueOf(TABLENAME), FAMILY);
 293
 294     byte[][] QUALIFIERS =
 295         new byte[][] { Bytes.toBytes("A"), Bytes.toBytes("B"), Bytes.toBytes("C") };
 296
 297     Increment inc = new Increment(ROW);
 298     for (int i = 0; i < QUALIFIERS.length; i++) {
 299       inc.addColumn(FAMILY, QUALIFIERS[i], 1);
 300       inc.addColumn(FAMILY, QUALIFIERS[i], 1);
 301     }
 302     ht.increment(inc);
 303
 304     // Verify expected results
 305     Get get = new Get(ROW);
 306     Result r = ht.get(get);
 307     Cell[] kvs = r.rawCells();
 308     assertEquals(3, kvs.length);
 309     assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 1);
 310     assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 1);
 311     assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 1);
 312
 313     // Now try multiple columns again
 314     inc = new Increment(ROW);
 315     for (int i = 0; i < QUALIFIERS.length; i++) {
 316       inc.addColumn(FAMILY, QUALIFIERS[i], 1);
 317       inc.addColumn(FAMILY, QUALIFIERS[i], 1);
 318     }
 319     ht.increment(inc);
 320
 321     // Verify
 322     r = ht.get(get);
 323     kvs = r.rawCells();
 324     assertEquals(3, kvs.length);
 325     assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 2);
 326     assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 2);
 327     assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 2);
 328
 329     ht.close();
 330   }
 331
 332   @Test
 333   public void testIncrementIncrZeroAtFirst() throws Exception {
 334     LOG.info("Starting " + this.name.getMethodName());
 335     final TableName TABLENAME =
 336             TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
 337     Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
 338
 339     byte[] col1 = Bytes.toBytes("col1");
 340     byte[] col2 = Bytes.toBytes("col2");
 341     byte[] col3 = Bytes.toBytes("col3");
 342
 343     // Now increment zero at first time incr
 344     Increment inc = new Increment(ROW);
 345     inc.addColumn(FAMILY, col1, 0);
 346     ht.increment(inc);
 347
 348     // Verify expected results
 349     Get get = new Get(ROW);
 350     Result r = ht.get(get);
 351     Cell [] kvs = r.rawCells();
 352     assertEquals(1, kvs.length);
 353     assertNotNull(kvs[0]);
 354     assertIncrementKey(kvs[0], ROW, FAMILY, col1, 0);
 355
 356     // Now try multiple columns by different amounts
 357     inc = new Increment(ROW);
 358     inc.addColumn(FAMILY, col1, 1);
 359     inc.addColumn(FAMILY, col2, 0);
 360     inc.addColumn(FAMILY, col3, 2);
 361     ht.increment(inc);
 362     // Verify
 363     get = new Get(ROW);
 364     r = ht.get(get);
 365     kvs = r.rawCells();
 366     assertEquals(3, kvs.length);
 367     assertNotNull(kvs[0]);
 368     assertNotNull(kvs[1]);
 369     assertNotNull(kvs[2]);
 370     assertIncrementKey(kvs[0], ROW, FAMILY, col1, 1);
 371     assertIncrementKey(kvs[1], ROW, FAMILY, col2, 0);
 372     assertIncrementKey(kvs[2], ROW, FAMILY, col3, 2);
 373   }
 374
 375   @Test
 376   public void testIncrement() throws Exception {
 377     LOG.info("Starting " + this.name.getMethodName());
 378     final TableName TABLENAME =
 379         TableName.valueOf(filterStringSoTableNameSafe(this.name.getMethodName()));
 380     Table ht = TEST_UTIL.createTable(TABLENAME, FAMILY);
 381
 382     byte [][] ROWS = new byte [][] {
 383         Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"),
 384         Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
 385         Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i")
 386     };
 387     byte [][] QUALIFIERS = new byte [][] {
 388         Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c"),
 389         Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
 390         Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i")
 391     };
 392
 393     // Do some simple single-column increments
 394
 395     // First with old API
 396     ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[0], 1);
 397     ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[1], 2);
 398     ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[2], 3);
 399     ht.incrementColumnValue(ROW, FAMILY, QUALIFIERS[3], 4);
 400
 401     // Now increment things incremented with old and do some new
 402     Increment inc = new Increment(ROW);
 403     inc.addColumn(FAMILY, QUALIFIERS[1], 1);
 404     inc.addColumn(FAMILY, QUALIFIERS[3], 1);
 405     inc.addColumn(FAMILY, QUALIFIERS[4], 1);
 406     ht.increment(inc);
 407
 408     // Verify expected results
 409     Get get = new Get(ROW);
 410     Result r = ht.get(get);
 411     Cell [] kvs = r.rawCells();
 412     assertEquals(5, kvs.length);
 413     assertIncrementKey(kvs[0], ROW, FAMILY, QUALIFIERS[0], 1);
 414     assertIncrementKey(kvs[1], ROW, FAMILY, QUALIFIERS[1], 3);
 415     assertIncrementKey(kvs[2], ROW, FAMILY, QUALIFIERS[2], 3);
 416     assertIncrementKey(kvs[3], ROW, FAMILY, QUALIFIERS[3], 5);
 417     assertIncrementKey(kvs[4], ROW, FAMILY, QUALIFIERS[4], 1);
 418
 419     // Now try multiple columns by different amounts
 420     inc = new Increment(ROWS[0]);
 421     for (int i=0;i<QUALIFIERS.length;i++) {
 422       inc.addColumn(FAMILY, QUALIFIERS[i], i+1);
 423     }
 424     ht.increment(inc);
 425     // Verify
 426     get = new Get(ROWS[0]);
 427     r = ht.get(get);
 428     kvs = r.rawCells();
 429     assertEquals(QUALIFIERS.length, kvs.length);
 430     for (int i=0;i<QUALIFIERS.length;i++) {
 431       assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], i+1);
 432     }
 433
 434     // Re-increment them
 435     inc = new Increment(ROWS[0]);
 436     for (int i=0;i<QUALIFIERS.length;i++) {
 437       inc.addColumn(FAMILY, QUALIFIERS[i], i+1);
 438     }
 439     ht.increment(inc);
 440     // Verify
 441     r = ht.get(get);
 442     kvs = r.rawCells();
 443     assertEquals(QUALIFIERS.length, kvs.length);
 444     for (int i=0;i<QUALIFIERS.length;i++) {
 445       assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], 2*(i+1));
 446     }
 447
 448     // Verify that an Increment of an amount of zero, returns current count; i.e. same as for above
 449     // test, that is: 2 * (i + 1).
 450     inc = new Increment(ROWS[0]);
 451     for (int i = 0; i < QUALIFIERS.length; i++) {
 452       inc.addColumn(FAMILY, QUALIFIERS[i], 0);
 453     }
 454     ht.increment(inc);
 455     r = ht.get(get);
 456     kvs = r.rawCells();
 457     assertEquals(QUALIFIERS.length, kvs.length);
 458     for (int i = 0; i < QUALIFIERS.length; i++) {
 459       assertIncrementKey(kvs[i], ROWS[0], FAMILY, QUALIFIERS[i], 2*(i+1));
 460     }
 461   }
 462
 463   @Test
 464   public void testIncrementWithCustomTimestamp() throws IOException {
 465     TableName TABLENAME = TableName.valueOf(name.getMethodName());
 466     Table table = TEST_UTIL.createTable(TABLENAME, FAMILY);
 467     long timestamp = 999;
 468     Increment increment = new Increment(ROW);
 469     increment.add(CellUtil.createCell(ROW, FAMILY, QUALIFIER, timestamp, KeyValue.Type.Put.getCode(), Bytes.toBytes(100L)));
 470     Result r = table.increment(increment);
 471     assertEquals(1, r.size());
 472     assertEquals(timestamp, r.rawCells()[0].getTimestamp());
 473     r = table.get(new Get(ROW));
 474     assertEquals(1, r.size());
 475     assertEquals(timestamp, r.rawCells()[0].getTimestamp());
 476     r = table.increment(increment);
 477     assertEquals(1, r.size());
 478     assertNotEquals(timestamp, r.rawCells()[0].getTimestamp());
 479     r = table.get(new Get(ROW));
 480     assertEquals(1, r.size());
 481     assertNotEquals(timestamp, r.rawCells()[0].getTimestamp());
 482   }
 483
 484   /**
 485    * Call over to the adjacent class's method of same name.
 486    */
 487   static void assertIncrementKey(Cell key, byte [] row, byte [] family,
 488       byte [] qualifier, long value) throws Exception {
 489     TestFromClientSide.assertIncrementKey(key, row, family, qualifier, value);
 490   }
 491
 492   public static String filterStringSoTableNameSafe(final String str) {
 493     return str.replaceAll("\\[fast\\=(.*)\\]", ".FAST.is.$1");
 494   }
 495 }