hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift/TestThriftServer.java

   1 /*
   2  *
   3  * Licensed to the Apache Software Foundation (ASF) under one
   4  * or more contributor license agreements.  See the NOTICE file
   5  * distributed with this work for additional information
   6  * regarding copyright ownership.  The ASF licenses this file
   7  * to you under the Apache License, Version 2.0 (the
   8  * "License"); you may not use this file except in compliance
   9  * with the License.  You may obtain a copy of the License at
  10  *
  11  *     http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  */
  19 package org.apache.hadoop.hbase.thrift;
  20
  21 import static org.junit.Assert.assertArrayEquals;
  22 import static org.junit.Assert.assertEquals;
  23 import static org.junit.Assert.assertFalse;
  24 import static org.junit.Assert.assertTrue;
  25 import static org.junit.Assert.fail;
  26
  27 import java.io.IOException;
  28 import java.nio.ByteBuffer;
  29 import java.util.ArrayList;
  30 import java.util.Collection;
  31 import java.util.HashMap;
  32 import java.util.List;
  33 import java.util.Map;
  34
  35 import org.apache.commons.logging.Log;
  36 import org.apache.commons.logging.LogFactory;
  37 import org.apache.hadoop.conf.Configuration;
  38 import org.apache.hadoop.hbase.CompatibilityFactory;
  39 import org.apache.hadoop.hbase.HBaseTestingUtility;
  40 import org.apache.hadoop.hbase.HColumnDescriptor;
  41 import org.apache.hadoop.hbase.HConstants;
  42 import org.apache.hadoop.hbase.HRegionInfo;
  43 import org.apache.hadoop.hbase.HTableDescriptor;
  44 import org.apache.hadoop.hbase.TableName;
  45 import org.apache.hadoop.hbase.client.Put;
  46 import org.apache.hadoop.hbase.client.Table;
  47 import org.apache.hadoop.hbase.filter.ParseFilter;
  48 import org.apache.hadoop.hbase.security.UserProvider;
  49 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
  50 import org.apache.hadoop.hbase.testclassification.ClientTests;
  51 import org.apache.hadoop.hbase.testclassification.LargeTests;
  52 import org.apache.hadoop.hbase.thrift.ThriftServerRunner.HBaseHandler;
  53 import org.apache.hadoop.hbase.thrift.generated.BatchMutation;
  54 import org.apache.hadoop.hbase.thrift.generated.ColumnDescriptor;
  55 import org.apache.hadoop.hbase.thrift.generated.Hbase;
  56 import org.apache.hadoop.hbase.thrift.generated.IOError;
  57 import org.apache.hadoop.hbase.thrift.generated.Mutation;
  58 import org.apache.hadoop.hbase.thrift.generated.TAppend;
  59 import org.apache.hadoop.hbase.thrift.generated.TCell;
  60 import org.apache.hadoop.hbase.thrift.generated.TIncrement;
  61 import org.apache.hadoop.hbase.thrift.generated.TRegionInfo;
  62 import org.apache.hadoop.hbase.thrift.generated.TRowResult;
  63 import org.apache.hadoop.hbase.thrift.generated.TScan;
  64 import org.apache.hadoop.hbase.util.Bytes;
  65 import org.apache.hadoop.hbase.util.Threads;
  66 import org.junit.AfterClass;
  67 import org.junit.BeforeClass;
  68 import org.junit.Rule;
  69 import org.junit.Test;
  70 import org.junit.experimental.categories.Category;
  71 import org.junit.rules.TestName;
  72
  73 /**
  74  * Unit testing for ThriftServerRunner.HBaseHandler, a part of the
  75  * org.apache.hadoop.hbase.thrift package.
  76  */
  77 @Category({ClientTests.class, LargeTests.class})
  78 public class TestThriftServer {
  79   private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
  80   private static final Log LOG = LogFactory.getLog(TestThriftServer.class);
  81   private static final MetricsAssertHelper metricsHelper = CompatibilityFactory
  82       .getInstance(MetricsAssertHelper.class);
  83   protected static final int MAXVERSIONS = 3;
  84
  85   private static ByteBuffer asByteBuffer(String i) {
  86     return ByteBuffer.wrap(Bytes.toBytes(i));
  87   }
  88   private static ByteBuffer asByteBuffer(long l) {
  89     return ByteBuffer.wrap(Bytes.toBytes(l));
  90   }
  91
  92   // Static names for tables, columns, rows, and values
  93   private static ByteBuffer tableAname = asByteBuffer("tableA");
  94   private static ByteBuffer tableBname = asByteBuffer("tableB");
  95   private static ByteBuffer columnAname = asByteBuffer("columnA:");
  96   private static ByteBuffer columnAAname = asByteBuffer("columnA:A");
  97   private static ByteBuffer columnBname = asByteBuffer("columnB:");
  98   private static ByteBuffer rowAname = asByteBuffer("rowA");
  99   private static ByteBuffer rowBname = asByteBuffer("rowB");
 100   private static ByteBuffer valueAname = asByteBuffer("valueA");
 101   private static ByteBuffer valueBname = asByteBuffer("valueB");
 102   private static ByteBuffer valueCname = asByteBuffer("valueC");
 103   private static ByteBuffer valueDname = asByteBuffer("valueD");
 104   private static ByteBuffer valueEname = asByteBuffer(100l);
 105
 106   @Rule
 107   public TestName name = new TestName();
 108
 109   @BeforeClass
 110   public static void beforeClass() throws Exception {
 111     UTIL.getConfiguration().setBoolean(ThriftServerRunner.COALESCE_INC_KEY, true);
 112     UTIL.getConfiguration().setBoolean("hbase.table.sanity.checks", false);
 113     UTIL.getConfiguration().setInt("hbase.client.retries.number", 3);
 114     UTIL.startMiniCluster();
 115   }
 116
 117   @AfterClass
 118   public static void afterClass() throws Exception {
 119     UTIL.shutdownMiniCluster();
 120   }
 121
 122   /**
 123    * Runs all of the tests under a single JUnit test method.  We
 124    * consolidate all testing to one method because HBaseClusterTestCase
 125    * is prone to OutOfMemoryExceptions when there are three or more
 126    * JUnit test methods.
 127    *
 128    * @throws Exception
 129    */
 130   @Test
 131   public void testAll() throws Exception {
 132     // Run all tests
 133     doTestTableCreateDrop();
 134     doTestThriftMetrics();
 135     doTestTableMutations();
 136     doTestTableTimestampsAndColumns();
 137     doTestTableScanners();
 138     doTestGetTableRegions();
 139     doTestFilterRegistration();
 140     doTestGetRegionInfo();
 141     doTestIncrements();
 142     doTestAppend();
 143     doTestCheckAndPut();
 144   }
 145
 146   /**
 147    * Tests for creating, enabling, disabling, and deleting tables.  Also
 148    * tests that creating a table with an invalid column name yields an
 149    * IllegalArgument exception.
 150    *
 151    * @throws Exception
 152    */
 153   public void doTestTableCreateDrop() throws Exception {
 154     ThriftServerRunner.HBaseHandler handler =
 155       new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration(),
 156         UserProvider.instantiate(UTIL.getConfiguration()));
 157     doTestTableCreateDrop(handler);
 158   }
 159
 160   public static void doTestTableCreateDrop(Hbase.Iface handler) throws Exception {
 161     createTestTables(handler);
 162     dropTestTables(handler);
 163   }
 164
 165   public static final class MySlowHBaseHandler extends ThriftServerRunner.HBaseHandler
 166       implements Hbase.Iface {
 167
 168     protected MySlowHBaseHandler(Configuration c)
 169         throws IOException {
 170       super(c, UserProvider.instantiate(c));
 171     }
 172
 173     @Override
 174     public List<ByteBuffer> getTableNames() throws IOError {
 175       Threads.sleepWithoutInterrupt(3000);
 176       return super.getTableNames();
 177     }
 178   }
 179
 180   /**
 181    * TODO: These counts are supposed to be zero but sometimes they are not, they are equal to the
 182    * passed in maybe.  Investigate why.  My guess is they are set by the test that runs just
 183    * previous to this one.  Sometimes they are cleared.  Sometimes not.
 184    * @param name
 185    * @param maybe
 186    * @param metrics
 187    * @return
 188    */
 189   private int getCurrentCount(final String name, final int maybe, final ThriftMetrics metrics) {
 190     int currentCount = 0;
 191     try {
 192       metricsHelper.assertCounter(name, maybe, metrics.getSource());
 193       LOG.info("Shouldn't this be null? name=" + name + ", equals=" + maybe);
 194       currentCount = maybe;
 195     } catch (AssertionError e) {
 196       // Ignore
 197     }
 198     return currentCount;
 199   }
 200
 201   /**
 202    * Tests if the metrics for thrift handler work correctly
 203    */
 204   public void doTestThriftMetrics() throws Exception {
 205     LOG.info("START doTestThriftMetrics");
 206     Configuration conf = UTIL.getConfiguration();
 207     ThriftMetrics metrics = getMetrics(conf);
 208     Hbase.Iface handler = getHandlerForMetricsTest(metrics, conf);
 209     int currentCountCreateTable = getCurrentCount("createTable_num_ops", 2, metrics);
 210     int currentCountDeleteTable = getCurrentCount("deleteTable_num_ops", 2, metrics);
 211     int currentCountDisableTable = getCurrentCount("disableTable_num_ops", 2, metrics);
 212     createTestTables(handler);
 213     dropTestTables(handler);;
 214     metricsHelper.assertCounter("createTable_num_ops", currentCountCreateTable + 2,
 215       metrics.getSource());
 216     metricsHelper.assertCounter("deleteTable_num_ops", currentCountDeleteTable + 2,
 217       metrics.getSource());
 218     metricsHelper.assertCounter("disableTable_num_ops", currentCountDisableTable + 2,
 219       metrics.getSource());
 220     handler.getTableNames(); // This will have an artificial delay.
 221
 222     // 3 to 6 seconds (to account for potential slowness), measured in nanoseconds
 223    try {
 224      metricsHelper.assertGaugeGt("getTableNames_avg_time", 3L * 1000 * 1000 * 1000, metrics.getSource());
 225      metricsHelper.assertGaugeLt("getTableNames_avg_time",6L * 1000 * 1000 * 1000, metrics.getSource());
 226    } catch (AssertionError e) {
 227      LOG.info("Fix me!  Why does this happen?  A concurrent cluster running?", e);
 228    }
 229   }
 230
 231   private static Hbase.Iface getHandlerForMetricsTest(ThriftMetrics metrics, Configuration conf)
 232       throws Exception {
 233     Hbase.Iface handler = new MySlowHBaseHandler(conf);
 234     return HbaseHandlerMetricsProxy.newInstance(handler, metrics, conf);
 235   }
 236
 237   private static ThriftMetrics getMetrics(Configuration conf) throws Exception {
 238     return new ThriftMetrics( conf, ThriftMetrics.ThriftServerType.ONE);
 239   }
 240
 241
 242   public static void createTestTables(Hbase.Iface handler) throws Exception {
 243     // Create/enable/disable/delete tables, ensure methods act correctly
 244     assertEquals(handler.getTableNames().size(), 0);
 245     handler.createTable(tableAname, getColumnDescriptors());
 246     assertEquals(handler.getTableNames().size(), 1);
 247     assertEquals(handler.getColumnDescriptors(tableAname).size(), 2);
 248     assertTrue(handler.isTableEnabled(tableAname));
 249     handler.createTable(tableBname, getColumnDescriptors());
 250     assertEquals(handler.getTableNames().size(), 2);
 251   }
 252
 253   public static void checkTableList(Hbase.Iface handler) throws Exception {
 254     assertTrue(handler.getTableNames().contains(tableAname));
 255   }
 256
 257   public static void dropTestTables(Hbase.Iface handler) throws Exception {
 258     handler.disableTable(tableBname);
 259     assertFalse(handler.isTableEnabled(tableBname));
 260     handler.deleteTable(tableBname);
 261     assertEquals(handler.getTableNames().size(), 1);
 262     handler.disableTable(tableAname);
 263     assertFalse(handler.isTableEnabled(tableAname));
 264     /* TODO Reenable.
 265     assertFalse(handler.isTableEnabled(tableAname));
 266     handler.enableTable(tableAname);
 267     assertTrue(handler.isTableEnabled(tableAname));
 268     handler.disableTable(tableAname);*/
 269     handler.deleteTable(tableAname);
 270     assertEquals(handler.getTableNames().size(), 0);
 271   }
 272
 273   public void doTestIncrements() throws Exception {
 274     ThriftServerRunner.HBaseHandler handler =
 275       new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration(),
 276         UserProvider.instantiate(UTIL.getConfiguration()));
 277     createTestTables(handler);
 278     doTestIncrements(handler);
 279     dropTestTables(handler);
 280   }
 281
 282   public static void doTestIncrements(HBaseHandler handler) throws Exception {
 283     List<Mutation> mutations = new ArrayList<>(1);
 284     mutations.add(new Mutation(false, columnAAname, valueEname, true));
 285     mutations.add(new Mutation(false, columnAname, valueEname, true));
 286     handler.mutateRow(tableAname, rowAname, mutations, null);
 287     handler.mutateRow(tableAname, rowBname, mutations, null);
 288
 289     List<TIncrement> increments = new ArrayList<>(3);
 290     increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7));
 291     increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7));
 292     increments.add(new TIncrement(tableAname, rowBname, columnAAname, 7));
 293
 294     int numIncrements = 60000;
 295     for (int i = 0; i < numIncrements; i++) {
 296       handler.increment(new TIncrement(tableAname, rowAname, columnAname, 2));
 297       handler.incrementRows(increments);
 298     }
 299
 300     Thread.sleep(1000);
 301     long lv = handler.get(tableAname, rowAname, columnAname, null).get(0).value.getLong();
 302     // Wait on all increments being flushed
 303     while (handler.coalescer.getQueueSize() != 0) Threads.sleep(10);
 304     assertEquals((100 + (2 * numIncrements)), lv );
 305
 306
 307     lv = handler.get(tableAname, rowBname, columnAAname, null).get(0).value.getLong();
 308     assertEquals((100 + (3 * 7 * numIncrements)), lv);
 309
 310     assertTrue(handler.coalescer.getSuccessfulCoalescings() > 0);
 311
 312   }
 313
 314   /**
 315    * Tests adding a series of Mutations and BatchMutations, including a
 316    * delete mutation.  Also tests data retrieval, and getting back multiple
 317    * versions.
 318    *
 319    * @throws Exception
 320    */
 321   public void doTestTableMutations() throws Exception {
 322     ThriftServerRunner.HBaseHandler handler =
 323       new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration(),
 324         UserProvider.instantiate(UTIL.getConfiguration()));
 325     doTestTableMutations(handler);
 326   }
 327
 328   public static void doTestTableMutations(Hbase.Iface handler) throws Exception {
 329     // Setup
 330     handler.createTable(tableAname, getColumnDescriptors());
 331
 332     // Apply a few Mutations to rowA
 333     //     mutations.add(new Mutation(false, columnAname, valueAname));
 334     //     mutations.add(new Mutation(false, columnBname, valueBname));
 335     handler.mutateRow(tableAname, rowAname, getMutations(), null);
 336
 337     // Assert that the changes were made
 338     assertEquals(valueAname,
 339       handler.get(tableAname, rowAname, columnAname, null).get(0).value);
 340     TRowResult rowResult1 = handler.getRow(tableAname, rowAname, null).get(0);
 341     assertEquals(rowAname, rowResult1.row);
 342     assertEquals(valueBname,
 343       rowResult1.columns.get(columnBname).value);
 344
 345     // Apply a few BatchMutations for rowA and rowB
 346     // rowAmutations.add(new Mutation(true, columnAname, null));
 347     // rowAmutations.add(new Mutation(false, columnBname, valueCname));
 348     // batchMutations.add(new BatchMutation(rowAname, rowAmutations));
 349     // Mutations to rowB
 350     // rowBmutations.add(new Mutation(false, columnAname, valueCname));
 351     // rowBmutations.add(new Mutation(false, columnBname, valueDname));
 352     // batchMutations.add(new BatchMutation(rowBname, rowBmutations));
 353     handler.mutateRows(tableAname, getBatchMutations(), null);
 354
 355     // Assert that changes were made to rowA
 356     List<TCell> cells = handler.get(tableAname, rowAname, columnAname, null);
 357     assertFalse(cells.size() > 0);
 358     assertEquals(valueCname, handler.get(tableAname, rowAname, columnBname, null).get(0).value);
 359     List<TCell> versions = handler.getVer(tableAname, rowAname, columnBname, MAXVERSIONS, null);
 360     assertEquals(valueCname, versions.get(0).value);
 361     assertEquals(valueBname, versions.get(1).value);
 362
 363     // Assert that changes were made to rowB
 364     TRowResult rowResult2 = handler.getRow(tableAname, rowBname, null).get(0);
 365     assertEquals(rowBname, rowResult2.row);
 366     assertEquals(valueCname, rowResult2.columns.get(columnAname).value);
 367     assertEquals(valueDname, rowResult2.columns.get(columnBname).value);
 368
 369     // Apply some deletes
 370     handler.deleteAll(tableAname, rowAname, columnBname, null);
 371     handler.deleteAllRow(tableAname, rowBname, null);
 372
 373     // Assert that the deletes were applied
 374     int size = handler.get(tableAname, rowAname, columnBname, null).size();
 375     assertEquals(0, size);
 376     size = handler.getRow(tableAname, rowBname, null).size();
 377     assertEquals(0, size);
 378
 379     // Try null mutation
 380     List<Mutation> mutations = new ArrayList<>(1);
 381     mutations.add(new Mutation(false, columnAname, null, true));
 382     handler.mutateRow(tableAname, rowAname, mutations, null);
 383     TRowResult rowResult3 = handler.getRow(tableAname, rowAname, null).get(0);
 384     assertEquals(rowAname, rowResult3.row);
 385     assertEquals(0, rowResult3.columns.get(columnAname).value.remaining());
 386
 387     // Teardown
 388     handler.disableTable(tableAname);
 389     handler.deleteTable(tableAname);
 390   }
 391
 392   /**
 393    * Similar to testTableMutations(), except Mutations are applied with
 394    * specific timestamps and data retrieval uses these timestamps to
 395    * extract specific versions of data.
 396    *
 397    * @throws Exception
 398    */
 399   public void doTestTableTimestampsAndColumns() throws Exception {
 400     // Setup
 401     ThriftServerRunner.HBaseHandler handler =
 402       new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration(),
 403         UserProvider.instantiate(UTIL.getConfiguration()));
 404     handler.createTable(tableAname, getColumnDescriptors());
 405
 406     // Apply timestamped Mutations to rowA
 407     long time1 = System.currentTimeMillis();
 408     handler.mutateRowTs(tableAname, rowAname, getMutations(), time1, null);
 409
 410     Thread.sleep(1000);
 411
 412     // Apply timestamped BatchMutations for rowA and rowB
 413     long time2 = System.currentTimeMillis();
 414     handler.mutateRowsTs(tableAname, getBatchMutations(), time2, null);
 415
 416     // Apply an overlapping timestamped mutation to rowB
 417     handler.mutateRowTs(tableAname, rowBname, getMutations(), time2, null);
 418
 419     // the getVerTs is [inf, ts) so you need to increment one.
 420     time1 += 1;
 421     time2 += 2;
 422
 423     // Assert that the timestamp-related methods retrieve the correct data
 424     assertEquals(2, handler.getVerTs(tableAname, rowAname, columnBname, time2,
 425       MAXVERSIONS, null).size());
 426     assertEquals(1, handler.getVerTs(tableAname, rowAname, columnBname, time1,
 427       MAXVERSIONS, null).size());
 428
 429     TRowResult rowResult1 = handler.getRowTs(tableAname, rowAname, time1, null).get(0);
 430     TRowResult rowResult2 = handler.getRowTs(tableAname, rowAname, time2, null).get(0);
 431     // columnA was completely deleted
 432     //assertTrue(Bytes.equals(rowResult1.columns.get(columnAname).value, valueAname));
 433     assertEquals(rowResult1.columns.get(columnBname).value, valueBname);
 434     assertEquals(rowResult2.columns.get(columnBname).value, valueCname);
 435
 436     // ColumnAname has been deleted, and will never be visible even with a getRowTs()
 437     assertFalse(rowResult2.columns.containsKey(columnAname));
 438
 439     List<ByteBuffer> columns = new ArrayList<>(1);
 440     columns.add(columnBname);
 441
 442     rowResult1 = handler.getRowWithColumns(tableAname, rowAname, columns, null).get(0);
 443     assertEquals(rowResult1.columns.get(columnBname).value, valueCname);
 444     assertFalse(rowResult1.columns.containsKey(columnAname));
 445
 446     rowResult1 = handler.getRowWithColumnsTs(tableAname, rowAname, columns, time1, null).get(0);
 447     assertEquals(rowResult1.columns.get(columnBname).value, valueBname);
 448     assertFalse(rowResult1.columns.containsKey(columnAname));
 449
 450     // Apply some timestamped deletes
 451     // this actually deletes _everything_.
 452     // nukes everything in columnB: forever.
 453     handler.deleteAllTs(tableAname, rowAname, columnBname, time1, null);
 454     handler.deleteAllRowTs(tableAname, rowBname, time2, null);
 455
 456     // Assert that the timestamp-related methods retrieve the correct data
 457     int size = handler.getVerTs(tableAname, rowAname, columnBname, time1, MAXVERSIONS, null).size();
 458     assertEquals(0, size);
 459
 460     size = handler.getVerTs(tableAname, rowAname, columnBname, time2, MAXVERSIONS, null).size();
 461     assertEquals(1, size);
 462
 463     // should be available....
 464     assertEquals(handler.get(tableAname, rowAname, columnBname, null).get(0).value, valueCname);
 465
 466     assertEquals(0, handler.getRow(tableAname, rowBname, null).size());
 467
 468     // Teardown
 469     handler.disableTable(tableAname);
 470     handler.deleteTable(tableAname);
 471   }
 472
 473   /**
 474    * Tests the four different scanner-opening methods (with and without
 475    * a stoprow, with and without a timestamp).
 476    *
 477    * @throws Exception
 478    */
 479   public void doTestTableScanners() throws Exception {
 480     // Setup
 481     ThriftServerRunner.HBaseHandler handler =
 482       new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration(),
 483         UserProvider.instantiate(UTIL.getConfiguration()));
 484     handler.createTable(tableAname, getColumnDescriptors());
 485
 486     // Apply timestamped Mutations to rowA
 487     long time1 = System.currentTimeMillis();
 488     handler.mutateRowTs(tableAname, rowAname, getMutations(), time1, null);
 489
 490     // Sleep to assure that 'time1' and 'time2' will be different even with a
 491     // coarse grained system timer.
 492     Thread.sleep(1000);
 493
 494     // Apply timestamped BatchMutations for rowA and rowB
 495     long time2 = System.currentTimeMillis();
 496     handler.mutateRowsTs(tableAname, getBatchMutations(), time2, null);
 497
 498     time1 += 1;
 499
 500     // Test a scanner on all rows and all columns, no timestamp
 501     int scanner1 = handler.scannerOpen(tableAname, rowAname, getColumnList(true, true), null);
 502     TRowResult rowResult1a = handler.scannerGet(scanner1).get(0);
 503     assertEquals(rowResult1a.row, rowAname);
 504     // This used to be '1'.  I don't know why when we are asking for two columns
 505     // and when the mutations above would seem to add two columns to the row.
 506     // -- St.Ack 05/12/2009
 507     assertEquals(rowResult1a.columns.size(), 1);
 508     assertEquals(rowResult1a.columns.get(columnBname).value, valueCname);
 509
 510     TRowResult rowResult1b = handler.scannerGet(scanner1).get(0);
 511     assertEquals(rowResult1b.row, rowBname);
 512     assertEquals(rowResult1b.columns.size(), 2);
 513     assertEquals(rowResult1b.columns.get(columnAname).value, valueCname);
 514     assertEquals(rowResult1b.columns.get(columnBname).value, valueDname);
 515     closeScanner(scanner1, handler);
 516
 517     // Test a scanner on all rows and all columns, with timestamp
 518     int scanner2 = handler.scannerOpenTs(tableAname, rowAname, getColumnList(true, true), time1, null);
 519     TRowResult rowResult2a = handler.scannerGet(scanner2).get(0);
 520     assertEquals(rowResult2a.columns.size(), 1);
 521     // column A deleted, does not exist.
 522     //assertTrue(Bytes.equals(rowResult2a.columns.get(columnAname).value, valueAname));
 523     assertEquals(rowResult2a.columns.get(columnBname).value, valueBname);
 524     closeScanner(scanner2, handler);
 525
 526     // Test a scanner on the first row and first column only, no timestamp
 527     int scanner3 = handler.scannerOpenWithStop(tableAname, rowAname, rowBname,
 528         getColumnList(true, false), null);
 529     closeScanner(scanner3, handler);
 530
 531     // Test a scanner on the first row and second column only, with timestamp
 532     int scanner4 = handler.scannerOpenWithStopTs(tableAname, rowAname, rowBname,
 533         getColumnList(false, true), time1, null);
 534     TRowResult rowResult4a = handler.scannerGet(scanner4).get(0);
 535     assertEquals(rowResult4a.columns.size(), 1);
 536     assertEquals(rowResult4a.columns.get(columnBname).value, valueBname);
 537
 538     // Test scanner using a TScan object once with sortColumns False and once with sortColumns true
 539     TScan scanNoSortColumns = new TScan();
 540     scanNoSortColumns.setStartRow(rowAname);
 541     scanNoSortColumns.setStopRow(rowBname);
 542
 543     int scanner5 = handler.scannerOpenWithScan(tableAname , scanNoSortColumns, null);
 544     TRowResult rowResult5 = handler.scannerGet(scanner5).get(0);
 545     assertEquals(rowResult5.columns.size(), 1);
 546     assertEquals(rowResult5.columns.get(columnBname).value, valueCname);
 547
 548     TScan scanSortColumns = new TScan();
 549     scanSortColumns.setStartRow(rowAname);
 550     scanSortColumns.setStopRow(rowBname);
 551     scanSortColumns = scanSortColumns.setSortColumns(true);
 552
 553     int scanner6 = handler.scannerOpenWithScan(tableAname ,scanSortColumns, null);
 554     TRowResult rowResult6 = handler.scannerGet(scanner6).get(0);
 555     assertEquals(rowResult6.sortedColumns.size(), 1);
 556     assertEquals(rowResult6.sortedColumns.get(0).getCell().value, valueCname);
 557
 558     List<Mutation> rowBmutations = new ArrayList<>(20);
 559     for (int i = 0; i < 20; i++) {
 560       rowBmutations.add(new Mutation(false, asByteBuffer("columnA:" + i), valueCname, true));
 561     }
 562     ByteBuffer rowC = asByteBuffer("rowC");
 563     handler.mutateRow(tableAname, rowC, rowBmutations, null);
 564
 565     TScan scanSortMultiColumns = new TScan();
 566     scanSortMultiColumns.setStartRow(rowC);
 567     scanSortMultiColumns = scanSortMultiColumns.setSortColumns(true);
 568     int scanner7 = handler.scannerOpenWithScan(tableAname, scanSortMultiColumns, null);
 569     TRowResult rowResult7 = handler.scannerGet(scanner7).get(0);
 570
 571     ByteBuffer smallerColumn = asByteBuffer("columnA:");
 572     for (int i = 0; i < 20; i++) {
 573       ByteBuffer currentColumn = rowResult7.sortedColumns.get(i).columnName;
 574       assertTrue(Bytes.compareTo(smallerColumn.array(), currentColumn.array()) < 0);
 575       smallerColumn = currentColumn;
 576     }
 577
 578     TScan reversedScan = new TScan();
 579     reversedScan.setReversed(true);
 580     reversedScan.setStartRow(rowBname);
 581     reversedScan.setStopRow(rowAname);
 582
 583     int scanner8 = handler.scannerOpenWithScan(tableAname , reversedScan, null);
 584     List<TRowResult> results = handler.scannerGet(scanner8);
 585     handler.scannerClose(scanner8);
 586     assertEquals(results.size(), 1);
 587     assertEquals(ByteBuffer.wrap(results.get(0).getRow()), rowBname);
 588
 589     // Teardown
 590     handler.disableTable(tableAname);
 591     handler.deleteTable(tableAname);
 592   }
 593
 594   /**
 595    * For HBASE-2556
 596    * Tests for GetTableRegions
 597    *
 598    * @throws Exception
 599    */
 600   public void doTestGetTableRegions() throws Exception {
 601     ThriftServerRunner.HBaseHandler handler =
 602       new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration(),
 603         UserProvider.instantiate(UTIL.getConfiguration()));
 604     doTestGetTableRegions(handler);
 605   }
 606
 607   public static void doTestGetTableRegions(Hbase.Iface handler)
 608       throws Exception {
 609     assertEquals(handler.getTableNames().size(), 0);
 610     handler.createTable(tableAname, getColumnDescriptors());
 611     assertEquals(handler.getTableNames().size(), 1);
 612     List<TRegionInfo> regions = handler.getTableRegions(tableAname);
 613     int regionCount = regions.size();
 614     assertEquals("empty table should have only 1 region, " +
 615             "but found " + regionCount, regionCount, 1);
 616     LOG.info("Region found:" + regions.get(0));
 617     handler.disableTable(tableAname);
 618     handler.deleteTable(tableAname);
 619     regionCount = handler.getTableRegions(tableAname).size();
 620     assertEquals("non-existing table should have 0 region, " +
 621             "but found " + regionCount, regionCount, 0);
 622   }
 623
 624   public void doTestFilterRegistration() throws Exception {
 625     Configuration conf = UTIL.getConfiguration();
 626
 627     conf.set("hbase.thrift.filters", "MyFilter:filterclass");
 628
 629     ThriftServerRunner.registerFilters(conf);
 630
 631     Map<String, String> registeredFilters = ParseFilter.getAllFilters();
 632
 633     assertEquals("filterclass", registeredFilters.get("MyFilter"));
 634   }
 635
 636   public void doTestGetRegionInfo() throws Exception {
 637     ThriftServerRunner.HBaseHandler handler =
 638       new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration(),
 639         UserProvider.instantiate(UTIL.getConfiguration()));
 640     doTestGetRegionInfo(handler);
 641   }
 642
 643   public static void doTestGetRegionInfo(Hbase.Iface handler) throws Exception {
 644     // Create tableA and add two columns to rowA
 645     handler.createTable(tableAname, getColumnDescriptors());
 646     try {
 647       handler.mutateRow(tableAname, rowAname, getMutations(), null);
 648       byte[] searchRow = HRegionInfo.createRegionName(
 649           TableName.valueOf(tableAname.array()), rowAname.array(),
 650           HConstants.NINES, false);
 651       TRegionInfo regionInfo = handler.getRegionInfo(ByteBuffer.wrap(searchRow));
 652       assertTrue(Bytes.toStringBinary(regionInfo.getName()).startsWith(
 653             Bytes.toStringBinary(tableAname)));
 654     } finally {
 655       handler.disableTable(tableAname);
 656       handler.deleteTable(tableAname);
 657     }
 658   }
 659
 660   /**
 661    * Appends the value to a cell and checks that the cell value is updated properly.
 662    *
 663    * @throws Exception
 664    */
 665   public static void doTestAppend() throws Exception {
 666     ThriftServerRunner.HBaseHandler handler =
 667       new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration(),
 668         UserProvider.instantiate(UTIL.getConfiguration()));
 669     handler.createTable(tableAname, getColumnDescriptors());
 670     try {
 671       List<Mutation> mutations = new ArrayList<>(1);
 672       mutations.add(new Mutation(false, columnAname, valueAname, true));
 673       handler.mutateRow(tableAname, rowAname, mutations, null);
 674
 675       List<ByteBuffer> columnList = new ArrayList<>(1);
 676       columnList.add(columnAname);
 677       List<ByteBuffer> valueList = new ArrayList<>(1);
 678       valueList.add(valueBname);
 679
 680       TAppend append = new TAppend(tableAname, rowAname, columnList, valueList);
 681       handler.append(append);
 682
 683       TRowResult rowResult = handler.getRow(tableAname, rowAname, null).get(0);
 684       assertEquals(rowAname, rowResult.row);
 685       assertArrayEquals(Bytes.add(valueAname.array(), valueBname.array()),
 686         rowResult.columns.get(columnAname).value.array());
 687     } finally {
 688       handler.disableTable(tableAname);
 689       handler.deleteTable(tableAname);
 690     }
 691   }
 692
 693   /**
 694    * Check that checkAndPut fails if the cell does not exist, then put in the cell, then check that
 695    * the checkAndPut succeeds.
 696    *
 697    * @throws Exception
 698    */
 699   public static void doTestCheckAndPut() throws Exception {
 700     ThriftServerRunner.HBaseHandler handler =
 701       new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration(),
 702         UserProvider.instantiate(UTIL.getConfiguration()));
 703     handler.createTable(tableAname, getColumnDescriptors());
 704     try {
 705       List<Mutation> mutations = new ArrayList<>(1);
 706       mutations.add(new Mutation(false, columnAname, valueAname, true));
 707       Mutation putB = (new Mutation(false, columnBname, valueBname, true));
 708
 709       assertFalse(handler.checkAndPut(tableAname, rowAname, columnAname, valueAname, putB, null));
 710
 711       handler.mutateRow(tableAname, rowAname, mutations, null);
 712
 713       assertTrue(handler.checkAndPut(tableAname, rowAname, columnAname, valueAname, putB, null));
 714
 715       TRowResult rowResult = handler.getRow(tableAname, rowAname, null).get(0);
 716       assertEquals(rowAname, rowResult.row);
 717       assertEquals(valueBname, rowResult.columns.get(columnBname).value);
 718     } finally {
 719       handler.disableTable(tableAname);
 720       handler.deleteTable(tableAname);
 721     }
 722   }
 723
 724   @Test
 725   public void testMetricsWithException() throws Exception {
 726     String rowkey = "row1";
 727     String family = "f";
 728     String col = "c";
 729     // create a table which will throw exceptions for requests
 730     final TableName tableName = TableName.valueOf(name.getMethodName());
 731     HTableDescriptor tableDesc = new HTableDescriptor(tableName);
 732     tableDesc.addCoprocessor(ErrorThrowingGetObserver.class.getName());
 733     tableDesc.addFamily(new HColumnDescriptor(family));
 734
 735     Table table = UTIL.createTable(tableDesc, null);
 736     long now = System.currentTimeMillis();
 737     table.put(new Put(Bytes.toBytes(rowkey))
 738         .addColumn(Bytes.toBytes(family), Bytes.toBytes(col), now, Bytes.toBytes("val1")));
 739
 740     Configuration conf = UTIL.getConfiguration();
 741     ThriftMetrics metrics = getMetrics(conf);
 742     ThriftServerRunner.HBaseHandler hbaseHandler =
 743         new ThriftServerRunner.HBaseHandler(UTIL.getConfiguration(),
 744             UserProvider.instantiate(UTIL.getConfiguration()));
 745     Hbase.Iface handler = HbaseHandlerMetricsProxy.newInstance(hbaseHandler, metrics, conf);
 746
 747     ByteBuffer tTableName = asByteBuffer(tableName.getNameAsString());
 748
 749     // check metrics increment with a successful get
 750     long preGetCounter = metricsHelper.checkCounterExists("getRow_num_ops", metrics.getSource()) ?
 751         metricsHelper.getCounter("getRow_num_ops", metrics.getSource()) :
 752         0;
 753     List<TRowResult> tRowResult = handler.getRow(tTableName, asByteBuffer(rowkey), null);
 754     assertEquals(1, tRowResult.size());
 755     TRowResult tResult = tRowResult.get(0);
 756
 757     TCell expectedColumnValue = new TCell(asByteBuffer("val1"), now);
 758
 759     assertArrayEquals(Bytes.toBytes(rowkey), tResult.getRow());
 760     Collection<TCell> returnedColumnValues = tResult.getColumns().values();
 761     assertEquals(1, returnedColumnValues.size());
 762     assertEquals(expectedColumnValue, returnedColumnValues.iterator().next());
 763
 764     metricsHelper.assertCounter("getRow_num_ops", preGetCounter + 1, metrics.getSource());
 765
 766     // check metrics increment when the get throws each exception type
 767     for (ErrorThrowingGetObserver.ErrorType type : ErrorThrowingGetObserver.ErrorType.values()) {
 768       testExceptionType(handler, metrics, tTableName, rowkey, type);
 769     }
 770   }
 771
 772   private void testExceptionType(Hbase.Iface handler, ThriftMetrics metrics,
 773                                  ByteBuffer tTableName, String rowkey,
 774                                  ErrorThrowingGetObserver.ErrorType errorType) throws Exception {
 775     long preGetCounter = metricsHelper.getCounter("getRow_num_ops", metrics.getSource());
 776     String exceptionKey = errorType.getMetricName();
 777     long preExceptionCounter = metricsHelper.checkCounterExists(exceptionKey, metrics.getSource()) ?
 778         metricsHelper.getCounter(exceptionKey, metrics.getSource()) :
 779         0;
 780     Map<ByteBuffer, ByteBuffer> attributes = new HashMap<>();
 781     attributes.put(asByteBuffer(ErrorThrowingGetObserver.SHOULD_ERROR_ATTRIBUTE),
 782         asByteBuffer(errorType.name()));
 783     try {
 784       List<TRowResult> tRowResult = handler.getRow(tTableName, asByteBuffer(rowkey), attributes);
 785       fail("Get with error attribute should have thrown an exception");
 786     } catch (IOError e) {
 787       LOG.info("Received exception: ", e);
 788       metricsHelper.assertCounter("getRow_num_ops", preGetCounter + 1, metrics.getSource());
 789       metricsHelper.assertCounter(exceptionKey, preExceptionCounter + 1, metrics.getSource());
 790     }
 791   }
 792
 793   /**
 794    *
 795    * @return a List of ColumnDescriptors for use in creating a table.  Has one
 796    * default ColumnDescriptor and one ColumnDescriptor with fewer versions
 797    */
 798   private static List<ColumnDescriptor> getColumnDescriptors() {
 799     ArrayList<ColumnDescriptor> cDescriptors = new ArrayList<>(2);
 800
 801     // A default ColumnDescriptor
 802     ColumnDescriptor cDescA = new ColumnDescriptor();
 803     cDescA.name = columnAname;
 804     cDescriptors.add(cDescA);
 805
 806     // A slightly customized ColumnDescriptor (only 2 versions)
 807     ColumnDescriptor cDescB = new ColumnDescriptor(columnBname, 2, "NONE",
 808         false, "NONE", 0, 0, false, -1);
 809     cDescriptors.add(cDescB);
 810
 811     return cDescriptors;
 812   }
 813
 814   /**
 815    *
 816    * @param includeA whether or not to include columnA
 817    * @param includeB whether or not to include columnB
 818    * @return a List of column names for use in retrieving a scanner
 819    */
 820   private List<ByteBuffer> getColumnList(boolean includeA, boolean includeB) {
 821     List<ByteBuffer> columnList = new ArrayList<>();
 822     if (includeA) columnList.add(columnAname);
 823     if (includeB) columnList.add(columnBname);
 824     return columnList;
 825   }
 826
 827   /**
 828    *
 829    * @return a List of Mutations for a row, with columnA having valueA
 830    * and columnB having valueB
 831    */
 832   private static List<Mutation> getMutations() {
 833     List<Mutation> mutations = new ArrayList<>(2);
 834     mutations.add(new Mutation(false, columnAname, valueAname, true));
 835     mutations.add(new Mutation(false, columnBname, valueBname, true));
 836     return mutations;
 837   }
 838
 839   /**
 840    *
 841    * @return a List of BatchMutations with the following effects:
 842    * (rowA, columnA): delete
 843    * (rowA, columnB): place valueC
 844    * (rowB, columnA): place valueC
 845    * (rowB, columnB): place valueD
 846    */
 847   private static List<BatchMutation> getBatchMutations() {
 848     List<BatchMutation> batchMutations = new ArrayList<>(3);
 849
 850     // Mutations to rowA.  You can't mix delete and put anymore.
 851     List<Mutation> rowAmutations = new ArrayList<>(1);
 852     rowAmutations.add(new Mutation(true, columnAname, null, true));
 853     batchMutations.add(new BatchMutation(rowAname, rowAmutations));
 854
 855     rowAmutations = new ArrayList<>(1);
 856     rowAmutations.add(new Mutation(false, columnBname, valueCname, true));
 857     batchMutations.add(new BatchMutation(rowAname, rowAmutations));
 858
 859     // Mutations to rowB
 860     List<Mutation> rowBmutations = new ArrayList<>(2);
 861     rowBmutations.add(new Mutation(false, columnAname, valueCname, true));
 862     rowBmutations.add(new Mutation(false, columnBname, valueDname, true));
 863     batchMutations.add(new BatchMutation(rowBname, rowBmutations));
 864
 865     return batchMutations;
 866   }
 867
 868   /**
 869    * Asserts that the passed scanner is exhausted, and then closes
 870    * the scanner.
 871    *
 872    * @param scannerId the scanner to close
 873    * @param handler the HBaseHandler interfacing to HBase
 874    * @throws Exception
 875    */
 876   private void closeScanner(
 877       int scannerId, ThriftServerRunner.HBaseHandler handler) throws Exception {
 878     handler.scannerGet(scannerId);
 879     handler.scannerClose(scannerId);
 880   }
 881 }