hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandler.java

   1 /*
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.thrift2;
  19
  20 import static java.nio.ByteBuffer.wrap;
  21 import static org.apache.hadoop.hbase.HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD;
  22 import static org.apache.hadoop.hbase.HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD;
  23 import static org.apache.hadoop.hbase.thrift.HBaseServiceHandler.CLEANUP_INTERVAL;
  24 import static org.apache.hadoop.hbase.thrift.HBaseServiceHandler.MAX_IDLETIME;
  25 import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.deleteFromThrift;
  26 import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.getFromThrift;
  27 import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.incrementFromThrift;
  28 import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.putFromThrift;
  29 import static org.apache.hadoop.hbase.thrift2.ThriftUtilities.scanFromThrift;
  30 import static org.junit.Assert.assertArrayEquals;
  31 import static org.junit.Assert.assertEquals;
  32 import static org.junit.Assert.assertFalse;
  33 import static org.junit.Assert.assertNull;
  34 import static org.junit.Assert.assertTrue;
  35 import static org.junit.Assert.fail;
  36
  37 import java.io.IOException;
  38 import java.io.InterruptedIOException;
  39 import java.net.InetAddress;
  40 import java.nio.ByteBuffer;
  41 import java.util.ArrayList;
  42 import java.util.Arrays;
  43 import java.util.Collection;
  44 import java.util.Collections;
  45 import java.util.Comparator;
  46 import java.util.HashMap;
  47 import java.util.HashSet;
  48 import java.util.List;
  49 import java.util.Map;
  50 import java.util.Optional;
  51 import java.util.Set;
  52 import java.util.concurrent.TimeUnit;
  53 import org.apache.hadoop.conf.Configuration;
  54 import org.apache.hadoop.hbase.Cell;
  55 import org.apache.hadoop.hbase.CompatibilityFactory;
  56 import org.apache.hadoop.hbase.CoprocessorEnvironment;
  57 import org.apache.hadoop.hbase.HBaseClassTestRule;
  58 import org.apache.hadoop.hbase.HBaseTestingUtil;
  59 import org.apache.hadoop.hbase.ServerName;
  60 import org.apache.hadoop.hbase.TableName;
  61 import org.apache.hadoop.hbase.client.Admin;
  62 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
  63 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
  64 import org.apache.hadoop.hbase.client.Consistency;
  65 import org.apache.hadoop.hbase.client.Delete;
  66 import org.apache.hadoop.hbase.client.Durability;
  67 import org.apache.hadoop.hbase.client.Get;
  68 import org.apache.hadoop.hbase.client.Increment;
  69 import org.apache.hadoop.hbase.client.LogQueryFilter;
  70 import org.apache.hadoop.hbase.client.Put;
  71 import org.apache.hadoop.hbase.client.Scan;
  72 import org.apache.hadoop.hbase.client.Table;
  73 import org.apache.hadoop.hbase.client.TableDescriptor;
  74 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
  75 import org.apache.hadoop.hbase.coprocessor.ObserverContext;
  76 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
  77 import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
  78 import org.apache.hadoop.hbase.coprocessor.RegionObserver;
  79 import org.apache.hadoop.hbase.filter.ParseFilter;
  80 import org.apache.hadoop.hbase.security.UserProvider;
  81 import org.apache.hadoop.hbase.security.access.AccessControlClient;
  82 import org.apache.hadoop.hbase.security.access.Permission;
  83 import org.apache.hadoop.hbase.security.access.UserPermission;
  84 import org.apache.hadoop.hbase.test.MetricsAssertHelper;
  85 import org.apache.hadoop.hbase.testclassification.ClientTests;
  86 import org.apache.hadoop.hbase.testclassification.MediumTests;
  87 import org.apache.hadoop.hbase.thrift.ErrorThrowingGetObserver;
  88 import org.apache.hadoop.hbase.thrift.HBaseThriftTestingUtility;
  89 import org.apache.hadoop.hbase.thrift.HbaseHandlerMetricsProxy;
  90 import org.apache.hadoop.hbase.thrift.ThriftMetrics;
  91 import org.apache.hadoop.hbase.thrift.ThriftMetrics.ThriftServerType;
  92 import org.apache.hadoop.hbase.thrift2.generated.TAccessControlEntity;
  93 import org.apache.hadoop.hbase.thrift2.generated.TAppend;
  94 import org.apache.hadoop.hbase.thrift2.generated.TColumn;
  95 import org.apache.hadoop.hbase.thrift2.generated.TColumnFamilyDescriptor;
  96 import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement;
  97 import org.apache.hadoop.hbase.thrift2.generated.TColumnValue;
  98 import org.apache.hadoop.hbase.thrift2.generated.TCompareOperator;
  99 import org.apache.hadoop.hbase.thrift2.generated.TConsistency;
 100 import org.apache.hadoop.hbase.thrift2.generated.TDataBlockEncoding;
 101 import org.apache.hadoop.hbase.thrift2.generated.TDelete;
 102 import org.apache.hadoop.hbase.thrift2.generated.TDeleteType;
 103 import org.apache.hadoop.hbase.thrift2.generated.TDurability;
 104 import org.apache.hadoop.hbase.thrift2.generated.TFilterByOperator;
 105 import org.apache.hadoop.hbase.thrift2.generated.TGet;
 106 import org.apache.hadoop.hbase.thrift2.generated.THBaseService;
 107 import org.apache.hadoop.hbase.thrift2.generated.TIOError;
 108 import org.apache.hadoop.hbase.thrift2.generated.TIllegalArgument;
 109 import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
 110 import org.apache.hadoop.hbase.thrift2.generated.TLogQueryFilter;
 111 import org.apache.hadoop.hbase.thrift2.generated.TMutation;
 112 import org.apache.hadoop.hbase.thrift2.generated.TNamespaceDescriptor;
 113 import org.apache.hadoop.hbase.thrift2.generated.TOnlineLogRecord;
 114 import org.apache.hadoop.hbase.thrift2.generated.TPermissionScope;
 115 import org.apache.hadoop.hbase.thrift2.generated.TPut;
 116 import org.apache.hadoop.hbase.thrift2.generated.TReadType;
 117 import org.apache.hadoop.hbase.thrift2.generated.TResult;
 118 import org.apache.hadoop.hbase.thrift2.generated.TRowMutations;
 119 import org.apache.hadoop.hbase.thrift2.generated.TScan;
 120 import org.apache.hadoop.hbase.thrift2.generated.TServerName;
 121 import org.apache.hadoop.hbase.thrift2.generated.TTableDescriptor;
 122 import org.apache.hadoop.hbase.thrift2.generated.TTableName;
 123 import org.apache.hadoop.hbase.thrift2.generated.TThriftServerType;
 124 import org.apache.hadoop.hbase.thrift2.generated.TTimeRange;
 125 import org.apache.hadoop.hbase.util.Bytes;
 126 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
 127 import org.apache.thrift.TException;
 128 import org.apache.thrift.protocol.TBinaryProtocol;
 129 import org.apache.thrift.protocol.TProtocol;
 130 import org.apache.thrift.transport.TSocket;
 131 import org.apache.thrift.transport.TTransport;
 132 import org.junit.AfterClass;
 133 import org.junit.Assert;
 134 import org.junit.Before;
 135 import org.junit.BeforeClass;
 136 import org.junit.ClassRule;
 137 import org.junit.Rule;
 138 import org.junit.Test;
 139 import org.junit.experimental.categories.Category;
 140 import org.junit.rules.TestName;
 141 import org.slf4j.Logger;
 142 import org.slf4j.LoggerFactory;
 143
 144 import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
 145 import org.apache.hbase.thirdparty.org.apache.commons.collections4.CollectionUtils;
 146
 147 /**
 148  * Unit testing for ThriftServer.HBaseServiceHandler, a part of the org.apache.hadoop.hbase.thrift2
 149  * package.
 150  */
 151 @Category({ClientTests.class, MediumTests.class})
 152 public class TestThriftHBaseServiceHandler {
 153
 154   @ClassRule
 155   public static final HBaseClassTestRule CLASS_RULE =
 156       HBaseClassTestRule.forClass(TestThriftHBaseServiceHandler.class);
 157
 158   private static final Logger LOG = LoggerFactory.getLogger(TestThriftHBaseServiceHandler.class);
 159   private static final HBaseTestingUtil UTIL = new HBaseTestingUtil();
 160
 161   // Static names for tables, columns, rows, and values
 162   private static byte[] tableAname = Bytes.toBytes("tableA");
 163   private static byte[] familyAname = Bytes.toBytes("familyA");
 164   private static byte[] familyBname = Bytes.toBytes("familyB");
 165   private static byte[] qualifierAname = Bytes.toBytes("qualifierA");
 166   private static byte[] qualifierBname = Bytes.toBytes("qualifierB");
 167   private static byte[] valueAname = Bytes.toBytes("valueA");
 168   private static byte[] valueBname = Bytes.toBytes("valueB");
 169   private static ColumnFamilyDescriptor[] families = new ColumnFamilyDescriptor[] {
 170     ColumnFamilyDescriptorBuilder.newBuilder(familyAname).setMaxVersions(3).build(),
 171     ColumnFamilyDescriptorBuilder.newBuilder(familyBname).setMaxVersions(2).build() };
 172
 173
 174   private static final MetricsAssertHelper metricsHelper =
 175       CompatibilityFactory.getInstance(MetricsAssertHelper.class);
 176
 177   @Rule
 178   public TestName name = new TestName();
 179
 180
 181   public void assertTColumnValuesEqual(List<TColumnValue> columnValuesA,
 182       List<TColumnValue> columnValuesB) {
 183     assertEquals(columnValuesA.size(), columnValuesB.size());
 184     Comparator<TColumnValue> comparator = new Comparator<TColumnValue>() {
 185       @Override
 186       public int compare(TColumnValue o1, TColumnValue o2) {
 187         return Bytes.compareTo(Bytes.add(o1.getFamily(), o1.getQualifier()),
 188             Bytes.add(o2.getFamily(), o2.getQualifier()));
 189       }
 190     };
 191     Collections.sort(columnValuesA, comparator);
 192     Collections.sort(columnValuesB, comparator);
 193
 194     for (int i = 0; i < columnValuesA.size(); i++) {
 195       TColumnValue a = columnValuesA.get(i);
 196       TColumnValue b = columnValuesB.get(i);
 197       assertTColumnValueEqual(a, b);
 198     }
 199   }
 200
 201   public void assertTColumnValueEqual(TColumnValue a, TColumnValue b) {
 202     assertArrayEquals(a.getFamily(), b.getFamily());
 203     assertArrayEquals(a.getQualifier(), b.getQualifier());
 204     assertArrayEquals(a.getValue(), b.getValue());
 205   }
 206
 207   @BeforeClass
 208   public static void beforeClass() throws Exception {
 209     UTIL.getConfiguration().set("hbase.client.retries.number", "3");
 210     UTIL.getConfiguration().setBoolean("hbase.regionserver.slowlog.buffer.enabled", true);
 211
 212     UTIL.getConfiguration().set("hbase.client.retries.number", "3");
 213     UTIL.getConfiguration().setBoolean("hbase.security.authorization", true);
 214     UTIL.getConfiguration().set("hbase.coprocessor.master.classes",
 215       "org.apache.hadoop.hbase.security.access.AccessController");
 216     UTIL.getConfiguration().set("hbase.coprocessor.region.classes",
 217       "org.apache.hadoop.hbase.security.access.AccessController");
 218     UTIL.getConfiguration().set("hbase.coprocessor.regionserver.classes",
 219       "org.apache.hadoop.hbase.security.access.AccessController");
 220
 221     // as we opened access control, we need to start as a superuser. Otherwise, we will not have
 222     // sufficient permission to do operations.
 223     UTIL.getConfiguration().set("hbase.superuser", System.getProperty("user.name"));
 224
 225     UTIL.startMiniCluster();
 226     TableDescriptor tableDescriptor = TableDescriptorBuilder
 227       .newBuilder(TableName.valueOf(tableAname)).setColumnFamilies(Arrays.asList(families)).build();
 228     try (Admin admin = UTIL.getAdmin()) {
 229       admin.createTable(tableDescriptor);
 230     }
 231   }
 232
 233   @AfterClass
 234   public static void afterClass() throws Exception {
 235     UTIL.shutdownMiniCluster();
 236   }
 237
 238   @Before
 239   public void setup() throws Exception {
 240
 241   }
 242
 243   private ThriftHBaseServiceHandler createHandler() throws TException {
 244     try {
 245       Configuration conf = UTIL.getConfiguration();
 246       return new ThriftHBaseServiceHandler(conf, UserProvider.instantiate(conf));
 247     } catch (IOException ie) {
 248       throw new TException(ie);
 249     }
 250   }
 251
 252   @Test
 253   public void testExists() throws TIOError, TException {
 254     ThriftHBaseServiceHandler handler = createHandler();
 255     byte[] rowName = Bytes.toBytes("testExists");
 256     ByteBuffer table = wrap(tableAname);
 257
 258     TGet get = new TGet(wrap(rowName));
 259     assertFalse(handler.exists(table, get));
 260
 261     List<TColumnValue> columnValues = new ArrayList<>(2);
 262     columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
 263     columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
 264     TPut put = new TPut(wrap(rowName), columnValues);
 265     put.setColumnValues(columnValues);
 266
 267     handler.put(table, put);
 268
 269     assertTrue(handler.exists(table, get));
 270   }
 271
 272   @Test
 273   public void testExistsAll() throws TIOError, TException {
 274     ThriftHBaseServiceHandler handler = createHandler();
 275     byte[] rowName1 = Bytes.toBytes("testExistsAll1");
 276     byte[] rowName2 = Bytes.toBytes("testExistsAll2");
 277     ByteBuffer table = wrap(tableAname);
 278
 279     List<TGet> gets = new ArrayList<>();
 280     gets.add(new TGet(wrap(rowName2)));
 281     gets.add(new TGet(wrap(rowName2)));
 282     List<Boolean> existsResult1 = handler.existsAll(table, gets);
 283     assertFalse(existsResult1.get(0));
 284     assertFalse(existsResult1.get(1));
 285
 286     List<TColumnValue> columnValues = new ArrayList<TColumnValue>();
 287     columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
 288     columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
 289     List<TPut> puts = new ArrayList<TPut>();
 290     puts.add(new TPut(wrap(rowName1), columnValues));
 291     puts.add(new TPut(wrap(rowName2), columnValues));
 292
 293     handler.putMultiple(table, puts);
 294     List<Boolean> existsResult2 = handler.existsAll(table, gets);
 295
 296     assertTrue(existsResult2.get(0));
 297     assertTrue(existsResult2.get(1));
 298   }
 299
 300   @Test
 301   public void testPutGet() throws Exception {
 302     ThriftHBaseServiceHandler handler = createHandler();
 303     byte[] rowName = Bytes.toBytes("testPutGet");
 304     ByteBuffer table = wrap(tableAname);
 305
 306     List<TColumnValue> columnValues = new ArrayList<>(2);
 307     columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
 308     columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
 309     TPut put = new TPut(wrap(rowName), columnValues);
 310
 311     put.setColumnValues(columnValues);
 312
 313     handler.put(table, put);
 314
 315     TGet get = new TGet(wrap(rowName));
 316
 317     TResult result = handler.get(table, get);
 318     assertArrayEquals(rowName, result.getRow());
 319     List<TColumnValue> returnedColumnValues = result.getColumnValues();
 320     assertTColumnValuesEqual(columnValues, returnedColumnValues);
 321   }
 322
 323   @Test
 324   public void testPutGetMultiple() throws Exception {
 325     ThriftHBaseServiceHandler handler = createHandler();
 326     ByteBuffer table = wrap(tableAname);
 327     byte[] rowName1 = Bytes.toBytes("testPutGetMultiple1");
 328     byte[] rowName2 = Bytes.toBytes("testPutGetMultiple2");
 329
 330     List<TColumnValue> columnValues = new ArrayList<>(2);
 331     columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
 332     columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
 333     List<TPut> puts = new ArrayList<>(2);
 334     puts.add(new TPut(wrap(rowName1), columnValues));
 335     puts.add(new TPut(wrap(rowName2), columnValues));
 336
 337     handler.putMultiple(table, puts);
 338
 339     List<TGet> gets = new ArrayList<>(2);
 340     gets.add(new TGet(wrap(rowName1)));
 341     gets.add(new TGet(wrap(rowName2)));
 342
 343     List<TResult> results = handler.getMultiple(table, gets);
 344     assertEquals(2, results.size());
 345
 346     assertArrayEquals(rowName1, results.get(0).getRow());
 347     assertTColumnValuesEqual(columnValues, results.get(0).getColumnValues());
 348
 349     assertArrayEquals(rowName2, results.get(1).getRow());
 350     assertTColumnValuesEqual(columnValues, results.get(1).getColumnValues());
 351   }
 352
 353   @Test
 354   public void testDeleteMultiple() throws Exception {
 355     ThriftHBaseServiceHandler handler = createHandler();
 356     ByteBuffer table = wrap(tableAname);
 357     byte[] rowName1 = Bytes.toBytes("testDeleteMultiple1");
 358     byte[] rowName2 = Bytes.toBytes("testDeleteMultiple2");
 359
 360     List<TColumnValue> columnValues = new ArrayList<>(2);
 361     columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
 362     columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname)));
 363     List<TPut> puts = new ArrayList<>(2);
 364     puts.add(new TPut(wrap(rowName1), columnValues));
 365     puts.add(new TPut(wrap(rowName2), columnValues));
 366
 367     handler.putMultiple(table, puts);
 368
 369     List<TDelete> deletes = new ArrayList<>(2);
 370     deletes.add(new TDelete(wrap(rowName1)));
 371     deletes.add(new TDelete(wrap(rowName2)));
 372
 373     List<TDelete> deleteResults = handler.deleteMultiple(table, deletes);
 374     // 0 means they were all successfully applies
 375     assertEquals(0, deleteResults.size());
 376
 377     assertFalse(handler.exists(table, new TGet(wrap(rowName1))));
 378     assertFalse(handler.exists(table, new TGet(wrap(rowName2))));
 379   }
 380
 381   @Test
 382   public void testDelete() throws Exception {
 383     ThriftHBaseServiceHandler handler = createHandler();
 384     byte[] rowName = Bytes.toBytes("testDelete");
 385     ByteBuffer table = wrap(tableAname);
 386
 387     List<TColumnValue> columnValues = new ArrayList<>(2);
 388     TColumnValue columnValueA = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
 389       wrap(valueAname));
 390     TColumnValue columnValueB = new TColumnValue(wrap(familyBname), wrap(qualifierBname),
 391       wrap(valueBname));
 392     columnValues.add(columnValueA);
 393     columnValues.add(columnValueB);
 394     TPut put = new TPut(wrap(rowName), columnValues);
 395
 396     put.setColumnValues(columnValues);
 397
 398     handler.put(table, put);
 399
 400     TDelete delete = new TDelete(wrap(rowName));
 401     List<TColumn> deleteColumns = new ArrayList<>(1);
 402     TColumn deleteColumn = new TColumn(wrap(familyAname));
 403     deleteColumn.setQualifier(qualifierAname);
 404     deleteColumns.add(deleteColumn);
 405     delete.setColumns(deleteColumns);
 406
 407     handler.deleteSingle(table, delete);
 408
 409     TGet get = new TGet(wrap(rowName));
 410     TResult result = handler.get(table, get);
 411     assertArrayEquals(rowName, result.getRow());
 412     List<TColumnValue> returnedColumnValues = result.getColumnValues();
 413     List<TColumnValue> expectedColumnValues = new ArrayList<>(1);
 414     expectedColumnValues.add(columnValueB);
 415     assertTColumnValuesEqual(expectedColumnValues, returnedColumnValues);
 416   }
 417
 418   @Test
 419   public void testDeleteAllTimestamps() throws Exception {
 420     ThriftHBaseServiceHandler handler = createHandler();
 421     byte[] rowName = Bytes.toBytes("testDeleteAllTimestamps");
 422     ByteBuffer table = wrap(tableAname);
 423
 424     List<TColumnValue> columnValues = new ArrayList<>(1);
 425     TColumnValue columnValueA = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
 426       wrap(valueAname));
 427     columnValueA.setTimestamp(EnvironmentEdgeManager.currentTime() - 10);
 428     columnValues.add(columnValueA);
 429     TPut put = new TPut(wrap(rowName), columnValues);
 430
 431     put.setColumnValues(columnValues);
 432
 433     handler.put(table, put);
 434     columnValueA.setTimestamp(EnvironmentEdgeManager.currentTime());
 435     handler.put(table, put);
 436
 437     TGet get = new TGet(wrap(rowName));
 438     get.setMaxVersions(2);
 439     TResult result = handler.get(table, get);
 440     assertEquals(2, result.getColumnValuesSize());
 441
 442     TDelete delete = new TDelete(wrap(rowName));
 443     List<TColumn> deleteColumns = new ArrayList<>(1);
 444     TColumn deleteColumn = new TColumn(wrap(familyAname));
 445     deleteColumn.setQualifier(qualifierAname);
 446     deleteColumns.add(deleteColumn);
 447     delete.setColumns(deleteColumns);
 448     delete.setDeleteType(TDeleteType.DELETE_COLUMNS); // This is the default anyway.
 449
 450     handler.deleteSingle(table, delete);
 451
 452     get = new TGet(wrap(rowName));
 453     result = handler.get(table, get);
 454     assertNull(result.getRow());
 455     assertEquals(0, result.getColumnValuesSize());
 456   }
 457
 458   @Test
 459   public void testDeleteSingleTimestamp() throws Exception {
 460     ThriftHBaseServiceHandler handler = createHandler();
 461     byte[] rowName = Bytes.toBytes("testDeleteSingleTimestamp");
 462     ByteBuffer table = wrap(tableAname);
 463
 464     long timestamp1 = EnvironmentEdgeManager.currentTime() - 10;
 465     long timestamp2 = EnvironmentEdgeManager.currentTime();
 466
 467     List<TColumnValue> columnValues = new ArrayList<>(1);
 468     TColumnValue columnValueA = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
 469       wrap(valueAname));
 470     columnValueA.setTimestamp(timestamp1);
 471     columnValues.add(columnValueA);
 472     TPut put = new TPut(wrap(rowName), columnValues);
 473
 474     put.setColumnValues(columnValues);
 475
 476     handler.put(table, put);
 477     columnValueA.setTimestamp(timestamp2);
 478     handler.put(table, put);
 479
 480     TGet get = new TGet(wrap(rowName));
 481     get.setMaxVersions(2);
 482     TResult result = handler.get(table, get);
 483     assertEquals(2, result.getColumnValuesSize());
 484
 485     TDelete delete = new TDelete(wrap(rowName));
 486     List<TColumn> deleteColumns = new ArrayList<>(1);
 487     TColumn deleteColumn = new TColumn(wrap(familyAname));
 488     deleteColumn.setQualifier(qualifierAname);
 489     deleteColumns.add(deleteColumn);
 490     delete.setColumns(deleteColumns);
 491     delete.setDeleteType(TDeleteType.DELETE_COLUMN);
 492
 493     handler.deleteSingle(table, delete);
 494
 495     get = new TGet(wrap(rowName));
 496     result = handler.get(table, get);
 497     assertArrayEquals(rowName, result.getRow());
 498     assertEquals(1, result.getColumnValuesSize());
 499     // the older timestamp should remain.
 500     assertEquals(timestamp1, result.getColumnValues().get(0).getTimestamp());
 501   }
 502
 503   @Test
 504   public void testDeleteFamily() throws Exception {
 505     ThriftHBaseServiceHandler handler = createHandler();
 506     byte[] rowName = Bytes.toBytes("testDeleteFamily");
 507     ByteBuffer table = wrap(tableAname);
 508
 509     long timestamp1 = EnvironmentEdgeManager.currentTime() - 10;
 510     long timestamp2 = EnvironmentEdgeManager.currentTime();
 511
 512     List<TColumnValue> columnValues = new ArrayList<>();
 513     TColumnValue columnValueA =
 514         new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
 515     columnValueA.setTimestamp(timestamp1);
 516     columnValues.add(columnValueA);
 517     TPut put = new TPut(wrap(rowName), columnValues);
 518
 519     put.setColumnValues(columnValues);
 520
 521     handler.put(table, put);
 522     columnValueA.setTimestamp(timestamp2);
 523     handler.put(table, put);
 524
 525     TGet get = new TGet(wrap(rowName));
 526     get.setMaxVersions(2);
 527     TResult result = handler.get(table, get);
 528     assertEquals(2, result.getColumnValuesSize());
 529
 530     TDelete delete = new TDelete(wrap(rowName));
 531     List<TColumn> deleteColumns = new ArrayList<>();
 532     TColumn deleteColumn = new TColumn(wrap(familyAname));
 533     deleteColumns.add(deleteColumn);
 534     delete.setColumns(deleteColumns);
 535     delete.setDeleteType(TDeleteType.DELETE_FAMILY);
 536
 537     handler.deleteSingle(table, delete);
 538
 539     get = new TGet(wrap(rowName));
 540     result = handler.get(table, get);
 541     assertArrayEquals(null, result.getRow());
 542     assertEquals(0, result.getColumnValuesSize());
 543   }
 544
 545   @Test
 546   public void testDeleteFamilyVersion() throws Exception {
 547     ThriftHBaseServiceHandler handler = createHandler();
 548     byte[] rowName = Bytes.toBytes("testDeleteFamilyVersion");
 549     ByteBuffer table = wrap(tableAname);
 550
 551     long timestamp1 = EnvironmentEdgeManager.currentTime() - 10;
 552     long timestamp2 = EnvironmentEdgeManager.currentTime();
 553
 554     List<TColumnValue> columnValues = new ArrayList<>();
 555     TColumnValue columnValueA =
 556         new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
 557     columnValueA.setTimestamp(timestamp1);
 558     columnValues.add(columnValueA);
 559     TPut put = new TPut(wrap(rowName), columnValues);
 560
 561     put.setColumnValues(columnValues);
 562
 563     handler.put(table, put);
 564     columnValueA.setTimestamp(timestamp2);
 565     handler.put(table, put);
 566
 567     TGet get = new TGet(wrap(rowName));
 568     get.setMaxVersions(2);
 569     TResult result = handler.get(table, get);
 570     assertEquals(2, result.getColumnValuesSize());
 571
 572     TDelete delete = new TDelete(wrap(rowName));
 573     List<TColumn> deleteColumns = new ArrayList<>();
 574     TColumn deleteColumn = new TColumn(wrap(familyAname));
 575     deleteColumn.setTimestamp(timestamp1);
 576     deleteColumns.add(deleteColumn);
 577     delete.setColumns(deleteColumns);
 578     delete.setDeleteType(TDeleteType.DELETE_FAMILY_VERSION);
 579
 580     handler.deleteSingle(table, delete);
 581
 582     get = new TGet(wrap(rowName));
 583     result = handler.get(table, get);
 584     assertArrayEquals(rowName, result.getRow());
 585     assertEquals(1, result.getColumnValuesSize());
 586     assertEquals(timestamp2, result.getColumnValues().get(0).getTimestamp());
 587   }
 588
 589   @Test
 590   public void testIncrement() throws Exception {
 591     ThriftHBaseServiceHandler handler = createHandler();
 592     byte[] rowName = Bytes.toBytes("testIncrement");
 593     ByteBuffer table = wrap(tableAname);
 594
 595     List<TColumnValue> columnValues = new ArrayList<>(1);
 596     columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
 597       wrap(Bytes.toBytes(1L))));
 598     TPut put = new TPut(wrap(rowName), columnValues);
 599     put.setColumnValues(columnValues);
 600     handler.put(table, put);
 601
 602     List<TColumnIncrement> incrementColumns = new ArrayList<>(1);
 603     incrementColumns.add(new TColumnIncrement(wrap(familyAname), wrap(qualifierAname)));
 604     TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
 605     handler.increment(table, increment);
 606
 607     TGet get = new TGet(wrap(rowName));
 608     TResult result = handler.get(table, get);
 609
 610     assertArrayEquals(rowName, result.getRow());
 611     assertEquals(1, result.getColumnValuesSize());
 612     TColumnValue columnValue = result.getColumnValues().get(0);
 613     assertArrayEquals(Bytes.toBytes(2L), columnValue.getValue());
 614   }
 615
 616   @Test
 617   public void testAppend() throws Exception {
 618     ThriftHBaseServiceHandler handler = createHandler();
 619     byte[] rowName = Bytes.toBytes("testAppend");
 620     ByteBuffer table = wrap(tableAname);
 621     byte[] v1 = Bytes.toBytes("42");
 622     byte[] v2 = Bytes.toBytes("23");
 623     List<TColumnValue> columnValues = new ArrayList<>(1);
 624     columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(v1)));
 625     TPut put = new TPut(wrap(rowName), columnValues);
 626     put.setColumnValues(columnValues);
 627     handler.put(table, put);
 628
 629     List<TColumnValue> appendColumns = new ArrayList<>(1);
 630     appendColumns.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(v2)));
 631     TAppend append = new TAppend(wrap(rowName), appendColumns);
 632     handler.append(table, append);
 633
 634     TGet get = new TGet(wrap(rowName));
 635     TResult result = handler.get(table, get);
 636
 637     assertArrayEquals(rowName, result.getRow());
 638     assertEquals(1, result.getColumnValuesSize());
 639     TColumnValue columnValue = result.getColumnValues().get(0);
 640     assertArrayEquals(Bytes.add(v1, v2), columnValue.getValue());
 641   }
 642
 643   /**
 644    * check that checkAndPut fails if the cell does not exist, then put in the cell, then check
 645    * that the checkAndPut succeeds.
 646    */
 647   @Test
 648   public void testCheckAndPut() throws Exception {
 649     ThriftHBaseServiceHandler handler = createHandler();
 650     byte[] rowName = Bytes.toBytes("testCheckAndPut");
 651     ByteBuffer table = wrap(tableAname);
 652
 653     List<TColumnValue> columnValuesA = new ArrayList<>(1);
 654     TColumnValue columnValueA = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
 655       wrap(valueAname));
 656     columnValuesA.add(columnValueA);
 657     TPut putA = new TPut(wrap(rowName), columnValuesA);
 658     putA.setColumnValues(columnValuesA);
 659
 660     List<TColumnValue> columnValuesB = new ArrayList<>(1);
 661     TColumnValue columnValueB = new TColumnValue(wrap(familyBname), wrap(qualifierBname),
 662       wrap(valueBname));
 663     columnValuesB.add(columnValueB);
 664     TPut putB = new TPut(wrap(rowName), columnValuesB);
 665     putB.setColumnValues(columnValuesB);
 666
 667     assertFalse(handler.checkAndPut(table, wrap(rowName), wrap(familyAname),
 668       wrap(qualifierAname), wrap(valueAname), putB));
 669
 670     TGet get = new TGet(wrap(rowName));
 671     TResult result = handler.get(table, get);
 672     assertEquals(0, result.getColumnValuesSize());
 673
 674     handler.put(table, putA);
 675
 676     assertTrue(handler.checkAndPut(table, wrap(rowName), wrap(familyAname),
 677       wrap(qualifierAname), wrap(valueAname), putB));
 678
 679     result = handler.get(table, get);
 680     assertArrayEquals(rowName, result.getRow());
 681     List<TColumnValue> returnedColumnValues = result.getColumnValues();
 682     List<TColumnValue> expectedColumnValues = new ArrayList<>(2);
 683     expectedColumnValues.add(columnValueA);
 684     expectedColumnValues.add(columnValueB);
 685     assertTColumnValuesEqual(expectedColumnValues, returnedColumnValues);
 686   }
 687
 688   /**
 689    * check that checkAndDelete fails if the cell does not exist, then put in the cell, then
 690    * check that the checkAndDelete succeeds.
 691    */
 692   @Test
 693   public void testCheckAndDelete() throws Exception {
 694     ThriftHBaseServiceHandler handler = createHandler();
 695     byte[] rowName = Bytes.toBytes("testCheckAndDelete");
 696     ByteBuffer table = wrap(tableAname);
 697
 698     List<TColumnValue> columnValuesA = new ArrayList<>(1);
 699     TColumnValue columnValueA = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
 700       wrap(valueAname));
 701     columnValuesA.add(columnValueA);
 702     TPut putA = new TPut(wrap(rowName), columnValuesA);
 703     putA.setColumnValues(columnValuesA);
 704
 705     List<TColumnValue> columnValuesB = new ArrayList<>(1);
 706     TColumnValue columnValueB = new TColumnValue(wrap(familyBname), wrap(qualifierBname),
 707       wrap(valueBname));
 708     columnValuesB.add(columnValueB);
 709     TPut putB = new TPut(wrap(rowName), columnValuesB);
 710     putB.setColumnValues(columnValuesB);
 711
 712     // put putB so that we know whether the row has been deleted or not
 713     handler.put(table, putB);
 714
 715     TDelete delete = new TDelete(wrap(rowName));
 716
 717     assertFalse(handler.checkAndDelete(table, wrap(rowName), wrap(familyAname),
 718         wrap(qualifierAname), wrap(valueAname), delete));
 719
 720     TGet get = new TGet(wrap(rowName));
 721     TResult result = handler.get(table, get);
 722     assertArrayEquals(rowName, result.getRow());
 723     assertTColumnValuesEqual(columnValuesB, result.getColumnValues());
 724
 725     handler.put(table, putA);
 726
 727     assertTrue(handler.checkAndDelete(table, wrap(rowName), wrap(familyAname),
 728       wrap(qualifierAname), wrap(valueAname), delete));
 729
 730     result = handler.get(table, get);
 731     assertFalse(result.isSetRow());
 732     assertEquals(0, result.getColumnValuesSize());
 733   }
 734
 735   @Test
 736   public void testScan() throws Exception {
 737     ThriftHBaseServiceHandler handler = createHandler();
 738     ByteBuffer table = wrap(tableAname);
 739
 740     // insert data
 741     TColumnValue columnValue = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
 742       wrap(valueAname));
 743     List<TColumnValue> columnValues = new ArrayList<>(1);
 744     columnValues.add(columnValue);
 745     for (int i = 0; i < 10; i++) {
 746       TPut put = new TPut(wrap(Bytes.toBytes("testScan" + i)), columnValues);
 747       handler.put(table, put);
 748     }
 749
 750     // create scan instance
 751     TScan scan = new TScan();
 752     List<TColumn> columns = new ArrayList<>(1);
 753     TColumn column = new TColumn();
 754     column.setFamily(familyAname);
 755     column.setQualifier(qualifierAname);
 756     columns.add(column);
 757     scan.setColumns(columns);
 758     scan.setStartRow(Bytes.toBytes("testScan"));
 759     scan.setStopRow(Bytes.toBytes("testScan\uffff"));
 760
 761     // get scanner and rows
 762     int scanId = handler.openScanner(table, scan);
 763     List<TResult> results = handler.getScannerRows(scanId, 10);
 764     assertEquals(10, results.size());
 765     for (int i = 0; i < 10; i++) {
 766       // check if the rows are returned and in order
 767       assertArrayEquals(Bytes.toBytes("testScan" + i), results.get(i).getRow());
 768     }
 769
 770     // check that we are at the end of the scan
 771     results = handler.getScannerRows(scanId, 10);
 772     assertEquals(0, results.size());
 773
 774     // close scanner and check that it was indeed closed
 775     handler.closeScanner(scanId);
 776     try {
 777       handler.getScannerRows(scanId, 10);
 778       fail("Scanner id should be invalid");
 779     } catch (TIllegalArgument e) {
 780     }
 781   }
 782
 783   /**
 784    * Tests keeping a HBase scanner alive for long periods of time. Each call to getScannerRow()
 785    * should reset the ConnectionCache timeout for the scanner's connection.
 786    */
 787   @org.junit.Ignore @Test // Flakey. Diasabled by HBASE-24079. Renable with Fails with HBASE-24083.
 788   //  Caused by: java.util.concurrent.RejectedExecutionException:
 789   //  Task org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture@e385431
 790   //  rejected from java.util.concurrent.ThreadPoolExecutor@   52b027d[Terminated, pool size = 0,
 791   //  active threads = 0, queued tasks = 0, completed tasks = 1]
 792   //  at org.apache.hadoop.hbase.thrift2.TestThriftHBaseServiceHandler.
 793   //  testLongLivedScan(TestThriftHBaseServiceHandler.java:804)
 794   public void testLongLivedScan() throws Exception {
 795     int numTrials = 6;
 796     int trialPause = 1000;
 797     int cleanUpInterval = 100;
 798     Configuration conf = new Configuration(UTIL.getConfiguration());
 799     // Set the ConnectionCache timeout to trigger halfway through the trials
 800     conf.setInt(MAX_IDLETIME, (numTrials / 2) * trialPause);
 801     conf.setInt(CLEANUP_INTERVAL, cleanUpInterval);
 802     ThriftHBaseServiceHandler handler = new ThriftHBaseServiceHandler(conf,
 803         UserProvider.instantiate(conf));
 804
 805     ByteBuffer table = wrap(tableAname);
 806     // insert data
 807     TColumnValue columnValue = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
 808         wrap(valueAname));
 809     List<TColumnValue> columnValues = new ArrayList<>(1);
 810     columnValues.add(columnValue);
 811     for (int i = 0; i < numTrials; i++) {
 812       TPut put = new TPut(wrap(Bytes.toBytes("testScan" + i)), columnValues);
 813       handler.put(table, put);
 814     }
 815
 816     // create scan instance
 817     TScan scan = new TScan();
 818     List<TColumn> columns = new ArrayList<>(1);
 819     TColumn column = new TColumn();
 820     column.setFamily(familyAname);
 821     column.setQualifier(qualifierAname);
 822     columns.add(column);
 823     scan.setColumns(columns);
 824     scan.setStartRow(Bytes.toBytes("testScan"));
 825     scan.setStopRow(Bytes.toBytes("testScan\uffff"));
 826     // Prevent the scanner from caching results
 827     scan.setCaching(1);
 828
 829     // get scanner and rows
 830     int scanId = handler.openScanner(table, scan);
 831     for (int i = 0; i < numTrials; i++) {
 832       // Make sure that the Scanner doesn't throw an exception after the ConnectionCache timeout
 833       List<TResult> results = handler.getScannerRows(scanId, 1);
 834       assertArrayEquals(Bytes.toBytes("testScan" + i), results.get(0).getRow());
 835       Thread.sleep(trialPause);
 836     }
 837   }
 838
 839   @Test
 840   public void testReverseScan() throws Exception {
 841     ThriftHBaseServiceHandler handler = createHandler();
 842     ByteBuffer table = wrap(tableAname);
 843
 844     // insert data
 845     TColumnValue columnValue = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
 846       wrap(valueAname));
 847     List<TColumnValue> columnValues = new ArrayList<>(1);
 848     columnValues.add(columnValue);
 849     for (int i = 0; i < 10; i++) {
 850       TPut put = new TPut(wrap(Bytes.toBytes("testReverseScan" + i)), columnValues);
 851       handler.put(table, put);
 852     }
 853
 854     // create reverse scan instance
 855     TScan scan = new TScan();
 856     scan.setReversed(true);
 857     List<TColumn> columns = new ArrayList<>(1);
 858     TColumn column = new TColumn();
 859     column.setFamily(familyAname);
 860     column.setQualifier(qualifierAname);
 861     columns.add(column);
 862     scan.setColumns(columns);
 863     scan.setStartRow(Bytes.toBytes("testReverseScan\uffff"));
 864     scan.setStopRow(Bytes.toBytes("testReverseScan"));
 865
 866     // get scanner and rows
 867     int scanId = handler.openScanner(table, scan);
 868     List<TResult> results = handler.getScannerRows(scanId, 10);
 869     assertEquals(10, results.size());
 870     for (int i = 0; i < 10; i++) {
 871       // check if the rows are returned and in order
 872       assertArrayEquals(Bytes.toBytes("testReverseScan" + (9 - i)), results.get(i).getRow());
 873     }
 874
 875     // check that we are at the end of the scan
 876     results = handler.getScannerRows(scanId, 10);
 877     assertEquals(0, results.size());
 878
 879     // close scanner and check that it was indeed closed
 880     handler.closeScanner(scanId);
 881     try {
 882       handler.getScannerRows(scanId, 10);
 883       fail("Scanner id should be invalid");
 884     } catch (TIllegalArgument e) {
 885     }
 886   }
 887
 888   @Test
 889   public void testScanWithFilter() throws Exception {
 890     ThriftHBaseServiceHandler handler = createHandler();
 891     ByteBuffer table = wrap(tableAname);
 892
 893     // insert data
 894     TColumnValue columnValue = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
 895       wrap(valueAname));
 896     List<TColumnValue> columnValues = new ArrayList<>(1);
 897     columnValues.add(columnValue);
 898     for (int i = 0; i < 10; i++) {
 899       TPut put = new TPut(wrap(Bytes.toBytes("testScanWithFilter" + i)), columnValues);
 900       handler.put(table, put);
 901     }
 902
 903     // create scan instance with filter
 904     TScan scan = new TScan();
 905     List<TColumn> columns = new ArrayList<>(1);
 906     TColumn column = new TColumn();
 907     column.setFamily(familyAname);
 908     column.setQualifier(qualifierAname);
 909     columns.add(column);
 910     scan.setColumns(columns);
 911     scan.setStartRow(Bytes.toBytes("testScanWithFilter"));
 912     scan.setStopRow(Bytes.toBytes("testScanWithFilter\uffff"));
 913     // only get the key part
 914     scan.setFilterString(wrap(Bytes.toBytes("KeyOnlyFilter()")));
 915
 916     // get scanner and rows
 917     int scanId = handler.openScanner(table, scan);
 918     List<TResult> results = handler.getScannerRows(scanId, 10);
 919     assertEquals(10, results.size());
 920     for (int i = 0; i < 10; i++) {
 921       // check if the rows are returned and in order
 922       assertArrayEquals(Bytes.toBytes("testScanWithFilter" + i), results.get(i).getRow());
 923       // check that the value is indeed stripped by the filter
 924       assertEquals(0, results.get(i).getColumnValues().get(0).getValue().length);
 925     }
 926
 927     // check that we are at the end of the scan
 928     results = handler.getScannerRows(scanId, 10);
 929     assertEquals(0, results.size());
 930
 931     // close scanner and check that it was indeed closed
 932     handler.closeScanner(scanId);
 933     try {
 934       handler.getScannerRows(scanId, 10);
 935       fail("Scanner id should be invalid");
 936     } catch (TIllegalArgument e) {
 937     }
 938   }
 939
 940   @Test
 941   public void testScanWithColumnFamilyTimeRange() throws Exception {
 942     ThriftHBaseServiceHandler handler = createHandler();
 943     ByteBuffer table = wrap(tableAname);
 944
 945     // insert data
 946     TColumnValue familyAColumnValue = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
 947         wrap(valueAname));
 948     TColumnValue familyBColumnValue = new TColumnValue(wrap(familyBname), wrap(qualifierBname),
 949         wrap(valueBname));
 950     long minTimestamp = EnvironmentEdgeManager.currentTime();
 951     for (int i = 0; i < 10; i++) {
 952       familyAColumnValue.setTimestamp(minTimestamp + i);
 953       familyBColumnValue.setTimestamp(minTimestamp + i);
 954       List<TColumnValue> columnValues = new ArrayList<>(2);
 955       columnValues.add(familyAColumnValue);
 956       columnValues.add(familyBColumnValue);
 957       TPut put = new TPut(wrap(Bytes.toBytes("testScanWithColumnFamilyTimeRange" + i)),
 958           columnValues);
 959       handler.put(table, put);
 960     }
 961
 962     // create scan instance with column family time range
 963     TScan scan = new TScan();
 964     Map<ByteBuffer,TTimeRange> colFamTimeRangeMap = new HashMap<>(2);
 965     colFamTimeRangeMap.put(wrap(familyAname), new TTimeRange(minTimestamp + 3, minTimestamp + 5));
 966     colFamTimeRangeMap.put(wrap(familyBname), new TTimeRange(minTimestamp + 6, minTimestamp + 9));
 967     scan.setColFamTimeRangeMap(colFamTimeRangeMap);
 968
 969     // get scanner and rows
 970     int scanId = handler.openScanner(table, scan);
 971     List<TResult> results = handler.getScannerRows(scanId, 5);
 972     assertEquals(5, results.size());
 973     int familyACount = 0;
 974     int familyBCount = 0;
 975     for (TResult result : results) {
 976       List<TColumnValue> columnValues = result.getColumnValues();
 977       if (CollectionUtils.isNotEmpty(columnValues)) {
 978         if (Bytes.equals(familyAname, columnValues.get(0).getFamily())) {
 979           familyACount++;
 980         } else if (Bytes.equals(familyBname, columnValues.get(0).getFamily())) {
 981           familyBCount++;
 982         }
 983       }
 984     }
 985     assertEquals(2, familyACount);
 986     assertEquals(3, familyBCount);
 987
 988     // check that we are at the end of the scan
 989     results = handler.getScannerRows(scanId, 1);
 990     assertEquals(0, results.size());
 991
 992     // close scanner and check that it was indeed closed
 993     handler.closeScanner(scanId);
 994     try {
 995       handler.getScannerRows(scanId, 1);
 996       fail("Scanner id should be invalid");
 997     } catch (TIllegalArgument e) {
 998     }
 999   }
1000
1001   @Test
1002   public void testSmallScan() throws Exception {
1003     ThriftHBaseServiceHandler handler = createHandler();
1004     ByteBuffer table = wrap(tableAname);
1005
1006     // insert data
1007     TColumnValue columnValue = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
1008             wrap(valueAname));
1009     List<TColumnValue> columnValues = new ArrayList<>();
1010     columnValues.add(columnValue);
1011     for (int i = 0; i < 10; i++) {
1012       TPut put = new TPut(wrap(Bytes.toBytes("testSmallScan" + i)), columnValues);
1013       handler.put(table, put);
1014     }
1015
1016     // small scan instance
1017     TScan scan = new TScan();
1018     scan.setStartRow(Bytes.toBytes("testSmallScan"));
1019     scan.setStopRow(Bytes.toBytes("testSmallScan\uffff"));
1020     scan.setReadType(TReadType.PREAD);
1021     scan.setCaching(2);
1022
1023     // get scanner and rows
1024     int scanId = handler.openScanner(table, scan);
1025     List<TResult> results = handler.getScannerRows(scanId, 10);
1026     assertEquals(10, results.size());
1027     for (int i = 0; i < 10; i++) {
1028       // check if the rows are returned and in order
1029       assertArrayEquals(Bytes.toBytes("testSmallScan" + i), results.get(i).getRow());
1030     }
1031
1032     // check that we are at the end of the scan
1033     results = handler.getScannerRows(scanId, 10);
1034     assertEquals(0, results.size());
1035
1036     // close scanner and check that it was indeed closed
1037     handler.closeScanner(scanId);
1038     try {
1039       handler.getScannerRows(scanId, 10);
1040       fail("Scanner id should be invalid");
1041     } catch (TIllegalArgument e) {
1042     }
1043   }
1044
1045   @Test
1046   public void testExpiredScanner() throws Exception {
1047     Configuration conf = UTIL.getConfiguration();
1048     conf.setLong(HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 1000);
1049     ThriftHBaseServiceHandler handler =
1050       new ThriftHBaseServiceHandler(conf, UserProvider.instantiate(conf));
1051
1052     TScan scan = new TScan();
1053     ByteBuffer table = wrap(tableAname);
1054
1055     int scannerId = handler.openScanner(table, scan);
1056     handler.getScannerRows(scannerId, 1);
1057     Thread.sleep(1000);
1058
1059     try {
1060       handler.getScannerRows(scannerId, 1);
1061       fail("The scanner should be expired and have an TIllegalArgument exception here.");
1062     } catch (TIllegalArgument e) {
1063       assertEquals("Invalid scanner Id", e.getMessage());
1064     } finally {
1065       conf.setLong(HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD,
1066         DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD);
1067     }
1068   }
1069
1070   @Test
1071   public void testPutTTL() throws Exception {
1072     ThriftHBaseServiceHandler handler = createHandler();
1073     byte[] rowName = Bytes.toBytes("testPutTTL");
1074     ByteBuffer table = wrap(tableAname);
1075     List<TColumnValue> columnValues = new ArrayList<>(1);
1076
1077     // Add some dummy data
1078     columnValues.add(
1079         new TColumnValue(
1080             wrap(familyAname),
1081             wrap(qualifierAname),
1082             wrap(Bytes.toBytes(1L))));
1083
1084
1085     TPut put = new TPut(wrap(rowName), columnValues);
1086     put.setColumnValues(columnValues);
1087
1088     Map<ByteBuffer, ByteBuffer> attributes = new HashMap<>();
1089
1090     // Time in ms for the kv's to live.
1091     long ttlTimeMs = 2000L;
1092
1093     // the _ttl attribute is a number of ms ttl for key values in this put.
1094     attributes.put(wrap(Bytes.toBytes("_ttl")), wrap(Bytes.toBytes(ttlTimeMs)));
1095     // Attach the attributes
1096     put.setAttributes(attributes);
1097     // Send it.
1098     handler.put(table, put);
1099
1100     // Now get the data back
1101     TGet getOne = new TGet(wrap(rowName));
1102     TResult resultOne = handler.get(table, getOne);
1103
1104     // It's there.
1105     assertArrayEquals(rowName, resultOne.getRow());
1106     assertEquals(1, resultOne.getColumnValuesSize());
1107
1108     // Sleep 30 seconds just to make 100% sure that the key value should be expired.
1109     Thread.sleep(ttlTimeMs * 15);
1110
1111     TGet getTwo = new TGet(wrap(rowName));
1112     TResult resultTwo = handler.get(table, getTwo);
1113
1114
1115     // Nothing should be there since it's ttl'd out.
1116     assertNull(resultTwo.getRow());
1117     assertEquals(0, resultTwo.getColumnValuesSize());
1118   }
1119
1120   /**
1121    * Padding numbers to make comparison of sort order easier in a for loop
1122    *
1123    * @param n  The number to pad.
1124    * @param pad  The length to pad up to.
1125    * @return The padded number as a string.
1126    */
1127   private String pad(int n, byte pad) {
1128     String res = Integer.toString(n);
1129     while (res.length() < pad) {
1130       res = "0" + res;
1131     }
1132     return res;
1133   }
1134
1135   @Test
1136   public void testScanWithBatchSize() throws Exception {
1137     ThriftHBaseServiceHandler handler = createHandler();
1138     ByteBuffer table = wrap(tableAname);
1139
1140     // insert data
1141     List<TColumnValue> columnValues = new ArrayList<>(100);
1142     for (int i = 0; i < 100; i++) {
1143       String colNum = pad(i, (byte) 3);
1144       TColumnValue columnValue = new TColumnValue(wrap(familyAname),
1145         wrap(Bytes.toBytes("col" + colNum)), wrap(Bytes.toBytes("val" + colNum)));
1146       columnValues.add(columnValue);
1147     }
1148     TPut put = new TPut(wrap(Bytes.toBytes("testScanWithBatchSize")), columnValues);
1149     handler.put(table, put);
1150
1151     // create scan instance
1152     TScan scan = new TScan();
1153     List<TColumn> columns = new ArrayList<>(1);
1154     TColumn column = new TColumn();
1155     column.setFamily(familyAname);
1156     columns.add(column);
1157     scan.setColumns(columns);
1158     scan.setStartRow(Bytes.toBytes("testScanWithBatchSize"));
1159     scan.setStopRow(Bytes.toBytes("testScanWithBatchSize\uffff"));
1160     // set batch size to 10 columns per call
1161     scan.setBatchSize(10);
1162
1163     // get scanner
1164     int scanId = handler.openScanner(table, scan);
1165     List<TResult> results = null;
1166     for (int i = 0; i < 10; i++) {
1167       // get batch for single row (10x10 is what we expect)
1168       results = handler.getScannerRows(scanId, 1);
1169       assertEquals(1, results.size());
1170       // check length of batch
1171       List<TColumnValue> cols = results.get(0).getColumnValues();
1172       assertEquals(10, cols.size());
1173       // check if the columns are returned and in order
1174       for (int y = 0; y < 10; y++) {
1175         int colNum = y + (10 * i);
1176         String colNumPad = pad(colNum, (byte) 3);
1177         assertArrayEquals(Bytes.toBytes("col" + colNumPad), cols.get(y).getQualifier());
1178       }
1179     }
1180
1181     // check that we are at the end of the scan
1182     results = handler.getScannerRows(scanId, 1);
1183     assertEquals(0, results.size());
1184
1185     // close scanner and check that it was indeed closed
1186     handler.closeScanner(scanId);
1187     try {
1188       handler.getScannerRows(scanId, 1);
1189       fail("Scanner id should be invalid");
1190     } catch (TIllegalArgument e) {
1191     }
1192   }
1193
1194   @Test
1195   public void testGetScannerResults() throws Exception {
1196     ThriftHBaseServiceHandler handler = createHandler();
1197     ByteBuffer table = wrap(tableAname);
1198
1199     // insert data
1200     TColumnValue columnValue =
1201         new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname));
1202     List<TColumnValue> columnValues = new ArrayList<>(1);
1203     columnValues.add(columnValue);
1204     for (int i = 0; i < 20; i++) {
1205       TPut put =
1206           new TPut(wrap(Bytes.toBytes("testGetScannerResults" + pad(i, (byte) 2))), columnValues);
1207       handler.put(table, put);
1208     }
1209
1210     // create scan instance
1211     TScan scan = new TScan();
1212     List<TColumn> columns = new ArrayList<>(1);
1213     TColumn column = new TColumn();
1214     column.setFamily(familyAname);
1215     column.setQualifier(qualifierAname);
1216     columns.add(column);
1217     scan.setColumns(columns);
1218     scan.setStartRow(Bytes.toBytes("testGetScannerResults"));
1219
1220     // get 5 rows and check the returned results
1221     scan.setStopRow(Bytes.toBytes("testGetScannerResults05"));
1222     List<TResult> results = handler.getScannerResults(table, scan, 5);
1223     assertEquals(5, results.size());
1224     for (int i = 0; i < 5; i++) {
1225       // check if the rows are returned and in order
1226       assertArrayEquals(Bytes.toBytes("testGetScannerResults" + pad(i, (byte) 2)), results.get(i)
1227           .getRow());
1228     }
1229
1230     // get 10 rows and check the returned results
1231     scan.setStopRow(Bytes.toBytes("testGetScannerResults10"));
1232     results = handler.getScannerResults(table, scan, 10);
1233     assertEquals(10, results.size());
1234     for (int i = 0; i < 10; i++) {
1235       // check if the rows are returned and in order
1236       assertArrayEquals(Bytes.toBytes("testGetScannerResults" + pad(i, (byte) 2)), results.get(i)
1237           .getRow());
1238     }
1239
1240     // get 20 rows and check the returned results
1241     scan.setStopRow(Bytes.toBytes("testGetScannerResults20"));
1242     results = handler.getScannerResults(table, scan, 20);
1243     assertEquals(20, results.size());
1244     for (int i = 0; i < 20; i++) {
1245       // check if the rows are returned and in order
1246       assertArrayEquals(Bytes.toBytes("testGetScannerResults" + pad(i, (byte) 2)), results.get(i)
1247           .getRow());
1248     }
1249
1250     // reverse scan
1251     scan = new TScan();
1252     scan.setColumns(columns);
1253     scan.setReversed(true);
1254     scan.setStartRow(Bytes.toBytes("testGetScannerResults20"));
1255     scan.setStopRow(Bytes.toBytes("testGetScannerResults"));
1256     results = handler.getScannerResults(table, scan, 20);
1257     assertEquals(20, results.size());
1258     for (int i = 0; i < 20; i++) {
1259       // check if the rows are returned and in order
1260       assertArrayEquals(Bytes.toBytes("testGetScannerResults" + pad(19 - i, (byte) 2)),
1261           results.get(i).getRow());
1262     }
1263   }
1264
1265   @Test
1266   public void testFilterRegistration() throws Exception {
1267     Configuration conf = UTIL.getConfiguration();
1268     conf.set("hbase.thrift.filters", "MyFilter:filterclass");
1269     ThriftServer.registerFilters(conf);
1270     Map<String, String> registeredFilters = ParseFilter.getAllFilters();
1271     assertEquals("filterclass", registeredFilters.get("MyFilter"));
1272   }
1273
1274   @Test
1275   public void testMetrics() throws Exception {
1276     Configuration conf = UTIL.getConfiguration();
1277     ThriftMetrics metrics = getMetrics(conf);
1278     ThriftHBaseServiceHandler hbaseHandler = createHandler();
1279     THBaseService.Iface handler =
1280         HbaseHandlerMetricsProxy.newInstance(hbaseHandler, metrics,  conf);
1281     byte[] rowName = Bytes.toBytes("testMetrics");
1282     ByteBuffer table = wrap(tableAname);
1283
1284     TGet get = new TGet(wrap(rowName));
1285     assertFalse(handler.exists(table, get));
1286
1287     List<TColumnValue> columnValues = new ArrayList<>(2);
1288     columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
1289     columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname),  wrap(valueBname)));
1290     TPut put = new TPut(wrap(rowName), columnValues);
1291     put.setColumnValues(columnValues);
1292
1293     handler.put(table, put);
1294
1295     assertTrue(handler.exists(table, get));
1296     metricsHelper.assertCounter("put_num_ops", 1, metrics.getSource());
1297     metricsHelper.assertCounter("exists_num_ops", 2, metrics.getSource());
1298   }
1299
1300   private static ThriftMetrics getMetrics(Configuration conf) throws Exception {
1301     ThriftMetrics m = new ThriftMetrics(conf, ThriftMetrics.ThriftServerType.TWO);
1302     m.getSource().init(); //Clear all the metrics
1303     return m;
1304   }
1305
1306   @Test
1307   public void testMetricsWithException() throws Exception {
1308     byte[] rowkey = Bytes.toBytes("row1");
1309     byte[] family = Bytes.toBytes("f");
1310     byte[] col = Bytes.toBytes("c");
1311     // create a table which will throw exceptions for requests
1312     TableName tableName = TableName.valueOf(name.getMethodName());
1313     TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName)
1314       .setCoprocessor(ErrorThrowingGetObserver.class.getName())
1315       .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build();
1316
1317     Table table = UTIL.createTable(tableDesc, null);
1318     table.put(new Put(rowkey).addColumn(family, col, Bytes.toBytes("val1")));
1319
1320     ThriftHBaseServiceHandler hbaseHandler = createHandler();
1321     ThriftMetrics metrics = getMetrics(UTIL.getConfiguration());
1322     THBaseService.Iface handler =
1323         HbaseHandlerMetricsProxy.newInstance(hbaseHandler, metrics, null);
1324     ByteBuffer tTableName = wrap(tableName.getName());
1325
1326     // check metrics increment with a successful get
1327     long preGetCounter = metricsHelper.checkCounterExists("get_num_ops", metrics.getSource()) ?
1328         metricsHelper.getCounter("get_num_ops", metrics.getSource()) :
1329         0;
1330     TGet tGet = new TGet(wrap(rowkey));
1331     TResult tResult = handler.get(tTableName, tGet);
1332
1333     List<TColumnValue> expectedColumnValues = Lists.newArrayList(
1334         new TColumnValue(wrap(family), wrap(col), wrap(Bytes.toBytes("val1")))
1335     );
1336     assertArrayEquals(rowkey, tResult.getRow());
1337     List<TColumnValue> returnedColumnValues = tResult.getColumnValues();
1338     assertTColumnValuesEqual(expectedColumnValues, returnedColumnValues);
1339
1340     metricsHelper.assertCounter("get_num_ops", preGetCounter + 1, metrics.getSource());
1341
1342     // check metrics increment when the get throws each exception type
1343     for (ErrorThrowingGetObserver.ErrorType type : ErrorThrowingGetObserver.ErrorType.values()) {
1344       testExceptionType(handler, metrics, tTableName, rowkey, type);
1345     }
1346   }
1347
1348   private void testExceptionType(THBaseService.Iface handler, ThriftMetrics metrics,
1349       ByteBuffer tTableName, byte[] rowkey, ErrorThrowingGetObserver.ErrorType errorType) {
1350     long preGetCounter = metricsHelper.getCounter("get_num_ops", metrics.getSource());
1351     String exceptionKey = errorType.getMetricName();
1352     long preExceptionCounter = metricsHelper.checkCounterExists(exceptionKey, metrics.getSource()) ?
1353         metricsHelper.getCounter(exceptionKey, metrics.getSource()) :
1354         0;
1355     TGet tGet = new TGet(wrap(rowkey));
1356     Map<ByteBuffer, ByteBuffer> attributes = new HashMap<>();
1357     attributes.put(wrap(Bytes.toBytes(ErrorThrowingGetObserver.SHOULD_ERROR_ATTRIBUTE)),
1358         wrap(Bytes.toBytes(errorType.name())));
1359     tGet.setAttributes(attributes);
1360     try {
1361       TResult tResult = handler.get(tTableName, tGet);
1362       fail("Get with error attribute should have thrown an exception");
1363     } catch (TException e) {
1364       LOG.info("Received exception: ", e);
1365       metricsHelper.assertCounter("get_num_ops", preGetCounter + 1, metrics.getSource());
1366       metricsHelper.assertCounter(exceptionKey, preExceptionCounter + 1, metrics.getSource());
1367     }
1368
1369   }
1370
1371   /**
1372    * See HBASE-17611
1373    *
1374    * Latency metrics were capped at ~ 2 seconds due to the use of an int variable to capture the
1375    * duration.
1376    */
1377   @Test
1378   public void testMetricsPrecision() throws Exception {
1379     byte[] rowkey = Bytes.toBytes("row1");
1380     byte[] family = Bytes.toBytes("f");
1381     byte[] col = Bytes.toBytes("c");
1382     // create a table which will throw exceptions for requests
1383     TableName tableName = TableName.valueOf("testMetricsPrecision");
1384     TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(tableName)
1385       .setCoprocessor(DelayingRegionObserver.class.getName())
1386       .setColumnFamily(ColumnFamilyDescriptorBuilder.of(family)).build();
1387
1388     Table table = null;
1389     try {
1390       table = UTIL.createTable(tableDescriptor, null);
1391
1392       table.put(new Put(rowkey).addColumn(family, col, Bytes.toBytes("val1")));
1393
1394       ThriftHBaseServiceHandler hbaseHandler = createHandler();
1395       ThriftMetrics metrics = getMetrics(UTIL.getConfiguration());
1396       THBaseService.Iface handler =
1397           HbaseHandlerMetricsProxy.newInstance(hbaseHandler, metrics, null);
1398       ByteBuffer tTableName = wrap(tableName.getName());
1399
1400       // check metrics latency with a successful get
1401       TGet tGet = new TGet(wrap(rowkey));
1402       TResult tResult = handler.get(tTableName, tGet);
1403
1404       List<TColumnValue> expectedColumnValues = Lists.newArrayList(
1405           new TColumnValue(wrap(family), wrap(col), wrap(Bytes.toBytes("val1")))
1406       );
1407       assertArrayEquals(rowkey, tResult.getRow());
1408       List<TColumnValue> returnedColumnValues = tResult.getColumnValues();
1409       assertTColumnValuesEqual(expectedColumnValues, returnedColumnValues);
1410
1411       metricsHelper.assertGaugeGt("get_max", 3000L, metrics.getSource());
1412     } finally {
1413       if (table != null) {
1414         try {
1415           table.close();
1416         } catch (IOException ignored) {
1417         }
1418         UTIL.deleteTable(tableName);
1419       }
1420     }
1421   }
1422
1423
1424   @Test
1425   public void testAttribute() throws Exception {
1426     byte[] rowName = Bytes.toBytes("testAttribute");
1427     byte[] attributeKey = Bytes.toBytes("attribute1");
1428     byte[] attributeValue = Bytes.toBytes("value1");
1429     Map<ByteBuffer, ByteBuffer> attributes = new HashMap<>();
1430     attributes.put(wrap(attributeKey), wrap(attributeValue));
1431
1432     TGet tGet = new TGet(wrap(rowName));
1433     tGet.setAttributes(attributes);
1434     Get get = getFromThrift(tGet);
1435     assertArrayEquals(get.getAttribute("attribute1"), attributeValue);
1436
1437     List<TColumnValue> columnValues = new ArrayList<>(1);
1438     columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
1439     TPut tPut = new TPut(wrap(rowName) , columnValues);
1440     tPut.setAttributes(attributes);
1441     Put put = putFromThrift(tPut);
1442     assertArrayEquals(put.getAttribute("attribute1"), attributeValue);
1443
1444     TScan tScan = new TScan();
1445     tScan.setAttributes(attributes);
1446     Scan scan = scanFromThrift(tScan);
1447     assertArrayEquals(scan.getAttribute("attribute1"), attributeValue);
1448
1449     List<TColumnIncrement> incrementColumns = new ArrayList<>(1);
1450     incrementColumns.add(new TColumnIncrement(wrap(familyAname), wrap(qualifierAname)));
1451     TIncrement tIncrement = new TIncrement(wrap(rowName), incrementColumns);
1452     tIncrement.setAttributes(attributes);
1453     Increment increment = incrementFromThrift(tIncrement);
1454     assertArrayEquals(increment.getAttribute("attribute1"), attributeValue);
1455
1456     TDelete tDelete = new TDelete(wrap(rowName));
1457     tDelete.setAttributes(attributes);
1458     Delete delete = deleteFromThrift(tDelete);
1459     assertArrayEquals(delete.getAttribute("attribute1"), attributeValue);
1460   }
1461
1462   /**
1463    * Put valueA to a row, make sure put has happened, then create a mutation object to put valueB
1464    * and delete ValueA, then check that the row value is only valueB.
1465    */
1466   @Test
1467   public void testMutateRow() throws Exception {
1468     ThriftHBaseServiceHandler handler = createHandler();
1469     byte[] rowName = Bytes.toBytes("testMutateRow");
1470     ByteBuffer table = wrap(tableAname);
1471
1472     List<TColumnValue> columnValuesA = new ArrayList<>(1);
1473     TColumnValue columnValueA = new TColumnValue(wrap(familyAname), wrap(qualifierAname),
1474         wrap(valueAname));
1475     columnValuesA.add(columnValueA);
1476     TPut putA = new TPut(wrap(rowName), columnValuesA);
1477     putA.setColumnValues(columnValuesA);
1478
1479     handler.put(table,putA);
1480
1481     TGet get = new TGet(wrap(rowName));
1482     TResult result = handler.get(table, get);
1483     assertArrayEquals(rowName, result.getRow());
1484     List<TColumnValue> returnedColumnValues = result.getColumnValues();
1485
1486     List<TColumnValue> expectedColumnValues = new ArrayList<>(1);
1487     expectedColumnValues.add(columnValueA);
1488     assertTColumnValuesEqual(expectedColumnValues, returnedColumnValues);
1489
1490     List<TColumnValue> columnValuesB = new ArrayList<>(1);
1491     TColumnValue columnValueB = new TColumnValue(wrap(familyAname), wrap(qualifierBname),
1492         wrap(valueBname));
1493     columnValuesB.add(columnValueB);
1494     TPut putB = new TPut(wrap(rowName), columnValuesB);
1495     putB.setColumnValues(columnValuesB);
1496
1497     TDelete delete = new TDelete(wrap(rowName));
1498     List<TColumn> deleteColumns = new ArrayList<>(1);
1499     TColumn deleteColumn = new TColumn(wrap(familyAname));
1500     deleteColumn.setQualifier(qualifierAname);
1501     deleteColumns.add(deleteColumn);
1502     delete.setColumns(deleteColumns);
1503
1504     List<TMutation> mutations = new ArrayList<>(2);
1505     TMutation mutationA = TMutation.put(putB);
1506     mutations.add(mutationA);
1507
1508     TMutation mutationB = TMutation.deleteSingle(delete);
1509     mutations.add(mutationB);
1510
1511     TRowMutations tRowMutations = new TRowMutations(wrap(rowName),mutations);
1512     handler.mutateRow(table,tRowMutations);
1513
1514     result = handler.get(table, get);
1515     assertArrayEquals(rowName, result.getRow());
1516     returnedColumnValues = result.getColumnValues();
1517
1518     expectedColumnValues = new ArrayList<>(1);
1519     expectedColumnValues.add(columnValueB);
1520     assertTColumnValuesEqual(expectedColumnValues, returnedColumnValues);
1521   }
1522
1523   /**
1524    * Create TPut, TDelete , TIncrement objects, set durability then call ThriftUtility
1525    * functions to get Put , Delete and Increment respectively. Use getDurability to make sure
1526    * the returned objects have the appropriate durability setting.
1527    */
1528   @Test
1529   public void testDurability() throws Exception {
1530     byte[] rowName = Bytes.toBytes("testDurability");
1531     List<TColumnValue> columnValues = new ArrayList<>(1);
1532     columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname)));
1533
1534     List<TColumnIncrement> incrementColumns = new ArrayList<>(1);
1535     incrementColumns.add(new TColumnIncrement(wrap(familyAname), wrap(qualifierAname)));
1536
1537     TDelete tDelete = new TDelete(wrap(rowName));
1538     tDelete.setDurability(TDurability.SKIP_WAL);
1539     Delete delete = deleteFromThrift(tDelete);
1540     assertEquals(Durability.SKIP_WAL, delete.getDurability());
1541
1542     tDelete.setDurability(TDurability.ASYNC_WAL);
1543     delete = deleteFromThrift(tDelete);
1544     assertEquals(Durability.ASYNC_WAL, delete.getDurability());
1545
1546     tDelete.setDurability(TDurability.SYNC_WAL);
1547     delete = deleteFromThrift(tDelete);
1548     assertEquals(Durability.SYNC_WAL, delete.getDurability());
1549
1550     tDelete.setDurability(TDurability.FSYNC_WAL);
1551     delete = deleteFromThrift(tDelete);
1552     assertEquals(Durability.FSYNC_WAL, delete.getDurability());
1553
1554     TPut tPut = new TPut(wrap(rowName), columnValues);
1555     tPut.setDurability(TDurability.SKIP_WAL);
1556     Put put = putFromThrift(tPut);
1557     assertEquals(Durability.SKIP_WAL, put.getDurability());
1558
1559     tPut.setDurability(TDurability.ASYNC_WAL);
1560     put = putFromThrift(tPut);
1561     assertEquals(Durability.ASYNC_WAL, put.getDurability());
1562
1563     tPut.setDurability(TDurability.SYNC_WAL);
1564     put = putFromThrift(tPut);
1565     assertEquals(Durability.SYNC_WAL, put.getDurability());
1566
1567     tPut.setDurability(TDurability.FSYNC_WAL);
1568     put = putFromThrift(tPut);
1569     assertEquals(Durability.FSYNC_WAL, put.getDurability());
1570
1571     TIncrement tIncrement = new TIncrement(wrap(rowName), incrementColumns);
1572
1573     tIncrement.setDurability(TDurability.SKIP_WAL);
1574     Increment increment = incrementFromThrift(tIncrement);
1575     assertEquals(Durability.SKIP_WAL, increment.getDurability());
1576
1577     tIncrement.setDurability(TDurability.ASYNC_WAL);
1578     increment = incrementFromThrift(tIncrement);
1579     assertEquals(Durability.ASYNC_WAL, increment.getDurability());
1580
1581     tIncrement.setDurability(TDurability.SYNC_WAL);
1582     increment = incrementFromThrift(tIncrement);
1583     assertEquals(Durability.SYNC_WAL, increment.getDurability());
1584
1585     tIncrement.setDurability(TDurability.FSYNC_WAL);
1586     increment = incrementFromThrift(tIncrement);
1587     assertEquals(Durability.FSYNC_WAL, increment.getDurability());
1588   }
1589
1590   @Test
1591   public void testCheckAndMutate() throws Exception {
1592     ThriftHBaseServiceHandler handler = createHandler();
1593     ByteBuffer table = wrap(tableAname);
1594     ByteBuffer row = wrap(Bytes.toBytes("row"));
1595     ByteBuffer family = wrap(familyAname);
1596     ByteBuffer qualifier = wrap(qualifierAname);
1597     ByteBuffer value = wrap(valueAname);
1598
1599     // Create a mutation to write to 'B', our "mutate" of "checkAndMutate"
1600     List<TColumnValue> columnValuesB = new ArrayList<>(1);
1601     TColumnValue columnValueB = new TColumnValue(family, wrap(qualifierBname), wrap(valueBname));
1602     columnValuesB.add(columnValueB);
1603     TPut putB = new TPut(row, columnValuesB);
1604     putB.setColumnValues(columnValuesB);
1605
1606     TRowMutations tRowMutations = new TRowMutations(row,
1607         Arrays.<TMutation> asList(TMutation.put(putB)));
1608
1609     // Empty table when we begin
1610     TResult result = handler.get(table, new TGet(row));
1611     assertEquals(0, result.getColumnValuesSize());
1612
1613     // checkAndMutate -- condition should fail because the value doesn't exist.
1614     assertFalse("Expected condition to not pass",
1615         handler.checkAndMutate(table, row, family, qualifier, TCompareOperator.EQUAL, value,
1616             tRowMutations));
1617
1618     List<TColumnValue> columnValuesA = new ArrayList<>(1);
1619     TColumnValue columnValueA = new TColumnValue(family, qualifier, value);
1620     columnValuesA.add(columnValueA);
1621
1622     // Put an update 'A'
1623     handler.put(table, new TPut(row, columnValuesA));
1624
1625     // Verify that the update is there
1626     result = handler.get(table, new TGet(row));
1627     assertEquals(1, result.getColumnValuesSize());
1628     assertTColumnValueEqual(columnValueA, result.getColumnValues().get(0));
1629
1630     // checkAndMutate -- condition should pass since we added the value
1631     assertTrue("Expected condition to pass",
1632         handler.checkAndMutate(table, row, family, qualifier, TCompareOperator.EQUAL, value,
1633             tRowMutations));
1634
1635     result = handler.get(table, new TGet(row));
1636     assertEquals(2, result.getColumnValuesSize());
1637     assertTColumnValueEqual(columnValueA, result.getColumnValues().get(0));
1638     assertTColumnValueEqual(columnValueB, result.getColumnValues().get(1));
1639   }
1640
1641   @Test
1642   public void testConsistency() throws Exception {
1643     byte[] rowName = Bytes.toBytes("testConsistency");
1644     TGet tGet = new TGet(wrap(rowName));
1645     tGet.setConsistency(TConsistency.STRONG);
1646     Get get = getFromThrift(tGet);
1647     assertEquals(Consistency.STRONG, get.getConsistency());
1648
1649     tGet.setConsistency(TConsistency.TIMELINE);
1650     tGet.setTargetReplicaId(1);
1651     get = getFromThrift(tGet);
1652     assertEquals(Consistency.TIMELINE, get.getConsistency());
1653     assertEquals(1, get.getReplicaId());
1654
1655     TScan tScan = new TScan();
1656     tScan.setConsistency(TConsistency.STRONG);
1657     Scan scan = scanFromThrift(tScan);
1658     assertEquals(Consistency.STRONG, scan.getConsistency());
1659
1660     tScan.setConsistency(TConsistency.TIMELINE);
1661     tScan.setTargetReplicaId(1);
1662     scan = scanFromThrift(tScan);
1663     assertEquals(Consistency.TIMELINE, scan.getConsistency());
1664     assertEquals(1, scan.getReplicaId());
1665
1666     TResult tResult = new TResult();
1667     assertFalse(tResult.isSetStale());
1668     tResult.setStale(true);
1669     assertTrue(tResult.isSetStale());
1670   }
1671
1672   @Test
1673   public void testDDLOpertions() throws Exception {
1674     String namespace = "testDDLOpertionsNamespace";
1675     String table = "testDDLOpertionsTable";
1676     TTableName tTableName = new TTableName();
1677     tTableName.setNs(Bytes.toBytes(namespace));
1678     tTableName.setQualifier(Bytes.toBytes(table));
1679     ThriftHBaseServiceHandler handler = createHandler();
1680     //create name space
1681     TNamespaceDescriptor namespaceDescriptor = new TNamespaceDescriptor();
1682     namespaceDescriptor.setName(namespace);
1683     namespaceDescriptor.putToConfiguration("key1", "value1");
1684     namespaceDescriptor.putToConfiguration("key2", "value2");
1685     handler.createNamespace(namespaceDescriptor);
1686     //list namespace
1687     List<TNamespaceDescriptor> namespaceDescriptors = handler.listNamespaceDescriptors();
1688     // should have 3 namespace, default hbase and testDDLOpertionsNamespace
1689     assertTrue(namespaceDescriptors.size() == 3);
1690     //modify namesapce
1691     namespaceDescriptor.putToConfiguration("kye3", "value3");
1692     handler.modifyNamespace(namespaceDescriptor);
1693     //get namespace
1694     TNamespaceDescriptor namespaceDescriptorReturned = handler.getNamespaceDescriptor(namespace);
1695     assertTrue(namespaceDescriptorReturned.getConfiguration().size() == 3);
1696     //create table
1697     TTableDescriptor tableDescriptor = new TTableDescriptor();
1698     tableDescriptor.setTableName(tTableName);
1699     TColumnFamilyDescriptor columnFamilyDescriptor1 = new TColumnFamilyDescriptor();
1700     columnFamilyDescriptor1.setName(familyAname);
1701     columnFamilyDescriptor1.setDataBlockEncoding(TDataBlockEncoding.DIFF);
1702     tableDescriptor.addToColumns(columnFamilyDescriptor1);
1703     List<ByteBuffer> splitKeys = new ArrayList<>();
1704     splitKeys.add(ByteBuffer.wrap(Bytes.toBytes(5)));
1705     handler.createTable(tableDescriptor, splitKeys);
1706     //modify table
1707     tableDescriptor.setDurability(TDurability.ASYNC_WAL);
1708     handler.modifyTable(tableDescriptor);
1709     //modify column family
1710     columnFamilyDescriptor1.setInMemory(true);
1711     handler.modifyColumnFamily(tTableName, columnFamilyDescriptor1);
1712     //add column family
1713     TColumnFamilyDescriptor columnFamilyDescriptor2 = new TColumnFamilyDescriptor();
1714     columnFamilyDescriptor2.setName(familyBname);
1715     columnFamilyDescriptor2.setDataBlockEncoding(TDataBlockEncoding.PREFIX);
1716     handler.addColumnFamily(tTableName, columnFamilyDescriptor2);
1717     //get table descriptor
1718     TTableDescriptor tableDescriptorReturned = handler.getTableDescriptor(tTableName);
1719     assertTrue(tableDescriptorReturned.getColumns().size() == 2);
1720     assertTrue(tableDescriptorReturned.getDurability() ==  TDurability.ASYNC_WAL);
1721     TColumnFamilyDescriptor columnFamilyDescriptor1Returned = tableDescriptorReturned.getColumns()
1722         .stream().filter(desc -> Bytes.equals(desc.getName(), familyAname)).findFirst().get();
1723     assertTrue(columnFamilyDescriptor1Returned.isInMemory() == true);
1724     //delete column family
1725     handler.deleteColumnFamily(tTableName, ByteBuffer.wrap(familyBname));
1726     tableDescriptorReturned = handler.getTableDescriptor(tTableName);
1727     assertTrue(tableDescriptorReturned.getColumns().size() == 1);
1728     //disable table
1729     handler.disableTable(tTableName);
1730     assertTrue(handler.isTableDisabled(tTableName));
1731     //enable table
1732     handler.enableTable(tTableName);
1733     assertTrue(handler.isTableEnabled(tTableName));
1734     assertTrue(handler.isTableAvailable(tTableName));
1735     //truncate table
1736     handler.disableTable(tTableName);
1737     handler.truncateTable(tTableName, true);
1738     assertTrue(handler.isTableAvailable(tTableName));
1739     //delete table
1740     handler.disableTable(tTableName);
1741     handler.deleteTable(tTableName);
1742     assertFalse(handler.tableExists(tTableName));
1743     //delete namespace
1744     handler.deleteNamespace(namespace);
1745     namespaceDescriptors = handler.listNamespaceDescriptors();
1746     // should have 2 namespace, default and hbase
1747     assertTrue(namespaceDescriptors.size() == 2);
1748   }
1749
1750   @Test
1751   public void testGetTableDescriptor() throws Exception {
1752     ThriftHBaseServiceHandler handler = createHandler();
1753     TTableDescriptor tableDescriptor = handler
1754         .getTableDescriptor(ThriftUtilities.tableNameFromHBase(TableName.valueOf(tableAname)));
1755     TableDescriptor table = ThriftUtilities.tableDescriptorFromThrift(tableDescriptor);
1756     assertTrue(table.getTableName().equals(TableName.valueOf(tableAname)));
1757     assertTrue(table.getColumnFamilies().length == 2);
1758     assertTrue(table.getColumnFamily(familyAname).getMaxVersions() == 3);
1759     assertTrue(table.getColumnFamily(familyBname).getMaxVersions() == 2);
1760   }
1761
1762   @Test
1763   public void testGetThriftServerType() throws Exception {
1764     ThriftHBaseServiceHandler handler = createHandler();
1765     assertEquals(TThriftServerType.TWO, handler.getThriftServerType());
1766   }
1767
1768   /**
1769    * Verify that thrift2 client calling thrift server can get the thrift server type correctly.
1770    */
1771   @Test
1772   public void testGetThriftServerOneType() throws Exception {
1773
1774     // start a thrift server
1775     HBaseThriftTestingUtility THRIFT_TEST_UTIL = new HBaseThriftTestingUtility();
1776
1777     LOG.info("Starting HBase Thrift server One");
1778     THRIFT_TEST_UTIL.startThriftServer(UTIL.getConfiguration(), ThriftServerType.ONE);
1779     try (TTransport transport = new TSocket(InetAddress.getLocalHost().getHostName(),
1780         THRIFT_TEST_UTIL.getServerPort())){
1781       TProtocol protocol = new TBinaryProtocol(transport);
1782       // This is our thrift2 client.
1783       THBaseService.Iface client = new THBaseService.Client(protocol);
1784       // open the transport
1785       transport.open();
1786       assertEquals(TThriftServerType.ONE.name(), client.getThriftServerType().name());
1787     } finally {
1788       THRIFT_TEST_UTIL.stopThriftServer();
1789     }
1790   }
1791
1792   @Test
1793   public void testSlowLogResponses() throws Exception {
1794
1795     // start a thrift server
1796     HBaseThriftTestingUtility THRIFT_TEST_UTIL = new HBaseThriftTestingUtility();
1797     Configuration configuration = UTIL.getConfiguration();
1798     configuration.setBoolean("hbase.regionserver.slowlog.buffer.enabled", true);
1799
1800     THRIFT_TEST_UTIL.startThriftServer(configuration, ThriftServerType.ONE);
1801     ThriftHBaseServiceHandler thriftHBaseServiceHandler =
1802       new ThriftHBaseServiceHandler(configuration,
1803         UserProvider.instantiate(configuration));
1804     Collection<ServerName> serverNames = UTIL.getAdmin().getRegionServers();
1805     Set<TServerName> tServerNames =
1806       ThriftUtilities.getServerNamesFromHBase(new HashSet<>(serverNames));
1807     List<Boolean> clearedResponses =
1808       thriftHBaseServiceHandler.clearSlowLogResponses(tServerNames);
1809     clearedResponses.forEach(Assert::assertTrue);
1810     TLogQueryFilter tLogQueryFilter = new TLogQueryFilter();
1811     tLogQueryFilter.setLimit(15);
1812     Assert.assertEquals(tLogQueryFilter.getFilterByOperator(), TFilterByOperator.OR);
1813     LogQueryFilter logQueryFilter = ThriftUtilities.getSlowLogQueryFromThrift(tLogQueryFilter);
1814     Assert.assertEquals(logQueryFilter.getFilterByOperator(), LogQueryFilter.FilterByOperator.OR);
1815     tLogQueryFilter.setFilterByOperator(TFilterByOperator.AND);
1816     logQueryFilter = ThriftUtilities.getSlowLogQueryFromThrift(tLogQueryFilter);
1817     Assert.assertEquals(logQueryFilter.getFilterByOperator(), LogQueryFilter.FilterByOperator.AND);
1818     List<TOnlineLogRecord> tLogRecords =
1819       thriftHBaseServiceHandler.getSlowLogResponses(tServerNames, tLogQueryFilter);
1820     assertEquals(tLogRecords.size(), 0);
1821   }
1822
1823   @Test
1824   public void testPerformTablePermissions() throws Throwable {
1825     // initialize fake objects.
1826     String fakeUser = "user";
1827     TAccessControlEntity tce = new TAccessControlEntity();
1828     tce.setActions("R");
1829     tce.setTableName(Bytes.toString(tableAname));
1830     tce.setScope(TPermissionScope.TABLE);
1831     tce.setUsername(fakeUser);
1832
1833     ThriftHBaseServiceHandler handler = createHandler();
1834     handler.grant(tce);
1835
1836     List<UserPermission> permissionList = AccessControlClient.getUserPermissions(UTIL.getConnection(),
1837         Bytes.toString(tableAname), fakeUser);
1838     // we only grant one R permission
1839     assertEquals(permissionList.size(), 1);
1840
1841     Permission.Action[] actions = permissionList.get(0).getPermission().getActions();
1842     assertEquals(actions.length, 1);
1843     assertEquals(actions[0], Permission.Action.READ);
1844
1845     // than revoke the permission
1846     handler.revoke(tce);
1847     permissionList = AccessControlClient.getUserPermissions(UTIL.getConnection(),
1848       Bytes.toString(tableAname), fakeUser);
1849
1850     // it should return an empty list
1851     assertEquals(0, permissionList.size());
1852   }
1853
1854
1855   @Test
1856   public void testPerformNamespacePermissions() throws Throwable {
1857     // initialize fake objects. We test the permission grant and revoke on default NS.
1858     String fakeUser = "user";
1859     String defaultNameSpace = "default";
1860     TAccessControlEntity tce = new TAccessControlEntity();
1861     tce.setActions("R");
1862     tce.setNsName(defaultNameSpace);
1863     tce.setScope(TPermissionScope.NAMESPACE);
1864     tce.setUsername(fakeUser);
1865
1866     ThriftHBaseServiceHandler handler = createHandler();
1867     handler.grant(tce);
1868
1869     List<UserPermission> permissionList = AccessControlClient.getUserPermissions(UTIL.getConnection(),
1870       "@" + defaultNameSpace, fakeUser);
1871
1872     // we only grant one R permission
1873     assertEquals(permissionList.size(), 1);
1874
1875     Permission.Action[] actions = permissionList.get(0).getPermission().getActions();
1876     assertEquals(actions.length, 1);
1877     assertEquals(actions[0], Permission.Action.READ);
1878
1879     // revoke the permission
1880     handler.revoke(tce);
1881     permissionList = AccessControlClient.getUserPermissions(UTIL.getConnection(),
1882       "@" + defaultNameSpace, fakeUser);
1883
1884     // it should return an empty list
1885     assertEquals(0, permissionList.size());
1886   }
1887
1888   public static class DelayingRegionObserver implements RegionCoprocessor, RegionObserver {
1889     private static final Logger LOG = LoggerFactory.getLogger(DelayingRegionObserver.class);
1890     // sleep time in msec
1891     private long delayMillis;
1892
1893     @Override
1894     public Optional<RegionObserver> getRegionObserver() {
1895       return Optional.of(this);
1896     }
1897
1898     @Override
1899     public void start(CoprocessorEnvironment e) throws IOException {
1900       this.delayMillis = e.getConfiguration()
1901           .getLong("delayingregionobserver.delay", 3000);
1902     }
1903
1904     @Override
1905     public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e, Get get,
1906                          List<Cell> results) throws IOException {
1907       try {
1908         long start = EnvironmentEdgeManager.currentTime();
1909         TimeUnit.MILLISECONDS.sleep(delayMillis);
1910         if (LOG.isTraceEnabled()) {
1911           LOG.trace("Slept for " + (EnvironmentEdgeManager.currentTime() - start) + " msec");
1912         }
1913       } catch (InterruptedException ie) {
1914         throw new InterruptedIOException("Interrupted while sleeping");
1915       }
1916     }
1917   }
1918 }
1919