hbase-thrift/src/test/java/org/apache/hadoop/hbase/thrift2/TestThriftHBaseServiceHandlerWithLabels.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.thrift2;
  19
  20 import static java.nio.ByteBuffer.wrap;
  21 import static org.junit.Assert.assertArrayEquals;
  22 import static org.junit.Assert.assertEquals;
  23 import static org.junit.Assert.assertNull;
  24 import static org.junit.Assert.fail;
  25
  26 import java.io.IOException;
  27 import java.nio.ByteBuffer;
  28 import java.security.PrivilegedExceptionAction;
  29 import java.util.ArrayList;
  30 import java.util.Collections;
  31 import java.util.Comparator;
  32 import java.util.List;
  33 import org.apache.hadoop.conf.Configuration;
  34 import org.apache.hadoop.hbase.HBaseClassTestRule;
  35 import org.apache.hadoop.hbase.HBaseTestingUtility;
  36 import org.apache.hadoop.hbase.TableName;
  37 import org.apache.hadoop.hbase.client.Admin;
  38 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
  39 import org.apache.hadoop.hbase.client.Connection;
  40 import org.apache.hadoop.hbase.client.ConnectionFactory;
  41 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
  42 import org.apache.hadoop.hbase.security.User;
  43 import org.apache.hadoop.hbase.security.UserProvider;
  44 import org.apache.hadoop.hbase.security.visibility.ScanLabelGenerator;
  45 import org.apache.hadoop.hbase.security.visibility.SimpleScanLabelGenerator;
  46 import org.apache.hadoop.hbase.security.visibility.VisibilityClient;
  47 import org.apache.hadoop.hbase.security.visibility.VisibilityConstants;
  48 import org.apache.hadoop.hbase.security.visibility.VisibilityTestUtil;
  49 import org.apache.hadoop.hbase.security.visibility.VisibilityUtils;
  50 import org.apache.hadoop.hbase.testclassification.ClientTests;
  51 import org.apache.hadoop.hbase.testclassification.MediumTests;
  52 import org.apache.hadoop.hbase.thrift2.generated.TAppend;
  53 import org.apache.hadoop.hbase.thrift2.generated.TAuthorization;
  54 import org.apache.hadoop.hbase.thrift2.generated.TCellVisibility;
  55 import org.apache.hadoop.hbase.thrift2.generated.TColumn;
  56 import org.apache.hadoop.hbase.thrift2.generated.TColumnIncrement;
  57 import org.apache.hadoop.hbase.thrift2.generated.TColumnValue;
  58 import org.apache.hadoop.hbase.thrift2.generated.TGet;
  59 import org.apache.hadoop.hbase.thrift2.generated.TIllegalArgument;
  60 import org.apache.hadoop.hbase.thrift2.generated.TIncrement;
  61 import org.apache.hadoop.hbase.thrift2.generated.TPut;
  62 import org.apache.hadoop.hbase.thrift2.generated.TResult;
  63 import org.apache.hadoop.hbase.thrift2.generated.TScan;
  64 import org.apache.hadoop.hbase.util.Bytes;
  65 import org.junit.AfterClass;
  66 import org.junit.Assert;
  67 import org.junit.Before;
  68 import org.junit.BeforeClass;
  69 import org.junit.ClassRule;
  70 import org.junit.Test;
  71 import org.junit.experimental.categories.Category;
  72 import org.slf4j.Logger;
  73 import org.slf4j.LoggerFactory;
  74
  75 import org.apache.hadoop.hbase.shaded.protobuf.generated.VisibilityLabelsProtos.VisibilityLabelsResponse;
  76
  77 @Category({ClientTests.class, MediumTests.class})
  78 public class TestThriftHBaseServiceHandlerWithLabels {
  79
  80   @ClassRule
  81   public static final HBaseClassTestRule CLASS_RULE =
  82       HBaseClassTestRule.forClass(TestThriftHBaseServiceHandlerWithLabels.class);
  83
  84   private static final Logger LOG = LoggerFactory
  85     .getLogger(TestThriftHBaseServiceHandlerWithLabels.class);
  86   private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
  87
  88   // Static names for tables, columns, rows, and values
  89   private static byte[] tableAname = Bytes.toBytes("tableA");
  90   private static byte[] familyAname = Bytes.toBytes("familyA");
  91   private static byte[] familyBname = Bytes.toBytes("familyB");
  92   private static byte[] qualifierAname = Bytes.toBytes("qualifierA");
  93   private static byte[] qualifierBname = Bytes.toBytes("qualifierB");
  94   private static byte[] valueAname = Bytes.toBytes("valueA");
  95   private static byte[] valueBname = Bytes.toBytes("valueB");
  96   private static ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor[] families =
  97     new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor[]{
  98       new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(familyAname)
  99         .setMaxVersions(3),
 100       new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(familyBname)
 101         .setMaxVersions(2)};
 102
 103   private final static String TOPSECRET = "topsecret";
 104   private final static String PUBLIC = "public";
 105   private final static String PRIVATE = "private";
 106   private final static String CONFIDENTIAL = "confidential";
 107   private final static String SECRET = "secret";
 108   private static User SUPERUSER;
 109
 110   private static Configuration conf;
 111
 112   public void assertTColumnValuesEqual(List<TColumnValue> columnValuesA,
 113       List<TColumnValue> columnValuesB) {
 114     assertEquals(columnValuesA.size(), columnValuesB.size());
 115     Comparator<TColumnValue> comparator = new Comparator<TColumnValue>() {
 116       @Override
 117       public int compare(TColumnValue o1, TColumnValue o2) {
 118         return Bytes.compareTo(Bytes.add(o1.getFamily(), o1.getQualifier()),
 119             Bytes.add(o2.getFamily(), o2.getQualifier()));
 120       }
 121     };
 122     Collections.sort(columnValuesA, comparator);
 123     Collections.sort(columnValuesB, comparator);
 124
 125     for (int i = 0; i < columnValuesA.size(); i++) {
 126       TColumnValue a = columnValuesA.get(i);
 127       TColumnValue b = columnValuesB.get(i);
 128       assertArrayEquals(a.getFamily(), b.getFamily());
 129       assertArrayEquals(a.getQualifier(), b.getQualifier());
 130       assertArrayEquals(a.getValue(), b.getValue());
 131     }
 132   }
 133
 134   @BeforeClass
 135   public static void beforeClass() throws Exception {
 136     SUPERUSER = User.createUserForTesting(conf, "admin",
 137         new String[] { "supergroup" });
 138     conf = UTIL.getConfiguration();
 139     conf.setClass(VisibilityUtils.VISIBILITY_LABEL_GENERATOR_CLASS,
 140         SimpleScanLabelGenerator.class, ScanLabelGenerator.class);
 141     conf.set("hbase.superuser", SUPERUSER.getShortName());
 142     VisibilityTestUtil.enableVisiblityLabels(conf);
 143     UTIL.startMiniCluster(1);
 144     // Wait for the labels table to become available
 145     UTIL.waitTableEnabled(VisibilityConstants.LABELS_TABLE_NAME.getName(), 50000);
 146     createLabels();
 147     Admin admin = UTIL.getAdmin();
 148     TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
 149       new TableDescriptorBuilder.ModifyableTableDescriptor(TableName.valueOf(tableAname));
 150     for (ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor family : families) {
 151       tableDescriptor.setColumnFamily(family);
 152     }
 153     admin.createTable(tableDescriptor);
 154     admin.close();
 155     setAuths();
 156   }
 157
 158   private static void createLabels() throws IOException, InterruptedException {
 159     PrivilegedExceptionAction<VisibilityLabelsResponse> action =
 160         new PrivilegedExceptionAction<VisibilityLabelsResponse>() {
 161       @Override
 162       public VisibilityLabelsResponse run() throws Exception {
 163         String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
 164         try (Connection conn = ConnectionFactory.createConnection(conf)) {
 165           VisibilityClient.addLabels(conn, labels);
 166         } catch (Throwable t) {
 167           throw new IOException(t);
 168         }
 169         return null;
 170       }
 171     };
 172     SUPERUSER.runAs(action);
 173   }
 174
 175   private static void setAuths() throws IOException {
 176     String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, PUBLIC, TOPSECRET };
 177     try {
 178       VisibilityClient.setAuths(UTIL.getConnection(), labels, User.getCurrent().getShortName());
 179     } catch (Throwable t) {
 180       throw new IOException(t);
 181     }
 182   }
 183
 184   @AfterClass
 185   public static void afterClass() throws Exception {
 186     UTIL.shutdownMiniCluster();
 187   }
 188
 189   @Before
 190   public void setup() throws Exception {
 191
 192   }
 193
 194   private ThriftHBaseServiceHandler createHandler() throws IOException {
 195     return new ThriftHBaseServiceHandler(conf, UserProvider.instantiate(conf));
 196   }
 197
 198   @Test
 199   public void testScanWithVisibilityLabels() throws Exception {
 200     ThriftHBaseServiceHandler handler = createHandler();
 201     ByteBuffer table = wrap(tableAname);
 202
 203     // insert data
 204     TColumnValue columnValue = new TColumnValue(wrap(familyAname),
 205         wrap(qualifierAname), wrap(valueAname));
 206     List<TColumnValue> columnValues = new ArrayList<>(1);
 207     columnValues.add(columnValue);
 208     for (int i = 0; i < 10; i++) {
 209       TPut put = new TPut(wrap(Bytes.toBytes("testScan" + i)), columnValues);
 210       if (i == 5) {
 211         put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
 212       } else {
 213         put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET
 214             + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
 215       }
 216       handler.put(table, put);
 217     }
 218
 219     // create scan instance
 220     TScan scan = new TScan();
 221     List<TColumn> columns = new ArrayList<>(1);
 222     TColumn column = new TColumn();
 223     column.setFamily(familyAname);
 224     column.setQualifier(qualifierAname);
 225     columns.add(column);
 226     scan.setColumns(columns);
 227     scan.setStartRow(Bytes.toBytes("testScan"));
 228     scan.setStopRow(Bytes.toBytes("testScan\uffff"));
 229
 230     TAuthorization tauth = new TAuthorization();
 231     List<String> labels = new ArrayList<>(2);
 232     labels.add(SECRET);
 233     labels.add(PRIVATE);
 234     tauth.setLabels(labels);
 235     scan.setAuthorizations(tauth);
 236     // get scanner and rows
 237     int scanId = handler.openScanner(table, scan);
 238     List<TResult> results = handler.getScannerRows(scanId, 10);
 239     assertEquals(9, results.size());
 240     Assert.assertFalse(Bytes.equals(results.get(5).getRow(), Bytes.toBytes("testScan" + 5)));
 241     for (int i = 0; i < 9; i++) {
 242       if (i < 5) {
 243         assertArrayEquals(Bytes.toBytes("testScan" + i), results.get(i).getRow());
 244       } else if (i == 5) {
 245         continue;
 246       } else {
 247         assertArrayEquals(Bytes.toBytes("testScan" + (i + 1)), results.get(i)
 248             .getRow());
 249       }
 250     }
 251
 252     // check that we are at the end of the scan
 253     results = handler.getScannerRows(scanId, 9);
 254     assertEquals(0, results.size());
 255
 256     // close scanner and check that it was indeed closed
 257     handler.closeScanner(scanId);
 258     try {
 259       handler.getScannerRows(scanId, 9);
 260       fail("Scanner id should be invalid");
 261     } catch (TIllegalArgument e) {
 262     }
 263   }
 264
 265   @Test
 266   public void testGetScannerResultsWithAuthorizations() throws Exception {
 267     ThriftHBaseServiceHandler handler = createHandler();
 268     ByteBuffer table = wrap(tableAname);
 269
 270     // insert data
 271     TColumnValue columnValue = new TColumnValue(wrap(familyAname),
 272         wrap(qualifierAname), wrap(valueAname));
 273     List<TColumnValue> columnValues = new ArrayList<>(1);
 274     columnValues.add(columnValue);
 275     for (int i = 0; i < 20; i++) {
 276       TPut put = new TPut(
 277           wrap(Bytes.toBytes("testGetScannerResults" + pad(i, (byte) 2))), columnValues);
 278       if (i == 3) {
 279         put.setCellVisibility(new TCellVisibility().setExpression(PUBLIC));
 280       } else {
 281         put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET
 282             + "|" + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
 283       }
 284       handler.put(table, put);
 285     }
 286
 287     // create scan instance
 288     TScan scan = new TScan();
 289     List<TColumn> columns = new ArrayList<>(1);
 290     TColumn column = new TColumn();
 291     column.setFamily(familyAname);
 292     column.setQualifier(qualifierAname);
 293     columns.add(column);
 294     scan.setColumns(columns);
 295     scan.setStartRow(Bytes.toBytes("testGetScannerResults"));
 296
 297     // get 5 rows and check the returned results
 298     scan.setStopRow(Bytes.toBytes("testGetScannerResults05"));
 299     TAuthorization tauth = new TAuthorization();
 300     List<String> labels = new ArrayList<>(2);
 301     labels.add(SECRET);
 302     labels.add(PRIVATE);
 303     tauth.setLabels(labels);
 304     scan.setAuthorizations(tauth);
 305     List<TResult> results = handler.getScannerResults(table, scan, 5);
 306     assertEquals(4, results.size());
 307     for (int i = 0; i < 4; i++) {
 308       if (i < 3) {
 309         assertArrayEquals(
 310             Bytes.toBytes("testGetScannerResults" + pad(i, (byte) 2)), results.get(i).getRow());
 311       } else if (i == 3) {
 312         continue;
 313       } else {
 314         assertArrayEquals(
 315             Bytes.toBytes("testGetScannerResults" + pad(i + 1, (byte) 2)), results.get(i).getRow());
 316       }
 317     }
 318   }
 319
 320   @Test
 321   public void testGetsWithLabels() throws Exception {
 322     ThriftHBaseServiceHandler handler = createHandler();
 323     byte[] rowName = Bytes.toBytes("testPutGet");
 324     ByteBuffer table = wrap(tableAname);
 325
 326     List<TColumnValue> columnValues = new ArrayList<>(2);
 327     columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
 328         wrap(valueAname)));
 329     columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname),
 330         wrap(valueBname)));
 331     TPut put = new TPut(wrap(rowName), columnValues);
 332
 333     put.setColumnValues(columnValues);
 334     put.setCellVisibility(new TCellVisibility().setExpression("(" + SECRET + "|"
 335         + CONFIDENTIAL + ")" + "&" + "!" + TOPSECRET));
 336     handler.put(table, put);
 337     TGet get = new TGet(wrap(rowName));
 338     TAuthorization tauth = new TAuthorization();
 339     List<String> labels = new ArrayList<>(2);
 340     labels.add(SECRET);
 341     labels.add(PRIVATE);
 342     tauth.setLabels(labels);
 343     get.setAuthorizations(tauth);
 344     TResult result = handler.get(table, get);
 345     assertArrayEquals(rowName, result.getRow());
 346     List<TColumnValue> returnedColumnValues = result.getColumnValues();
 347     assertTColumnValuesEqual(columnValues, returnedColumnValues);
 348   }
 349
 350   @Test
 351   public void testIncrementWithTags() throws Exception {
 352     ThriftHBaseServiceHandler handler = createHandler();
 353     byte[] rowName = Bytes.toBytes("testIncrementWithTags");
 354     ByteBuffer table = wrap(tableAname);
 355
 356     List<TColumnValue> columnValues = new ArrayList<>(1);
 357     columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
 358         wrap(Bytes.toBytes(1L))));
 359     TPut put = new TPut(wrap(rowName), columnValues);
 360     put.setColumnValues(columnValues);
 361     put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
 362     handler.put(table, put);
 363
 364     List<TColumnIncrement> incrementColumns = new ArrayList<>(1);
 365     incrementColumns.add(new TColumnIncrement(wrap(familyAname),
 366         wrap(qualifierAname)));
 367     TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
 368     increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
 369     handler.increment(table, increment);
 370
 371     TGet get = new TGet(wrap(rowName));
 372     TAuthorization tauth = new TAuthorization();
 373     List<String> labels = new ArrayList<>(1);
 374     labels.add(SECRET);
 375     tauth.setLabels(labels);
 376     get.setAuthorizations(tauth);
 377     TResult result = handler.get(table, get);
 378
 379     assertArrayEquals(rowName, result.getRow());
 380     assertEquals(1, result.getColumnValuesSize());
 381     TColumnValue columnValue = result.getColumnValues().get(0);
 382     assertArrayEquals(Bytes.toBytes(2L), columnValue.getValue());
 383   }
 384
 385   @Test
 386   public void testIncrementWithTagsWithNotMatchLabels() throws Exception {
 387     ThriftHBaseServiceHandler handler = createHandler();
 388     byte[] rowName = Bytes.toBytes("testIncrementWithTagsWithNotMatchLabels");
 389     ByteBuffer table = wrap(tableAname);
 390
 391     List<TColumnValue> columnValues = new ArrayList<>(1);
 392     columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
 393         wrap(Bytes.toBytes(1L))));
 394     TPut put = new TPut(wrap(rowName), columnValues);
 395     put.setColumnValues(columnValues);
 396     put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
 397     handler.put(table, put);
 398
 399     List<TColumnIncrement> incrementColumns = new ArrayList<>(1);
 400     incrementColumns.add(new TColumnIncrement(wrap(familyAname),
 401         wrap(qualifierAname)));
 402     TIncrement increment = new TIncrement(wrap(rowName), incrementColumns);
 403     increment.setCellVisibility(new TCellVisibility().setExpression(SECRET));
 404     handler.increment(table, increment);
 405
 406     TGet get = new TGet(wrap(rowName));
 407     TAuthorization tauth = new TAuthorization();
 408     List<String> labels = new ArrayList<>(1);
 409     labels.add(PUBLIC);
 410     tauth.setLabels(labels);
 411     get.setAuthorizations(tauth);
 412     TResult result = handler.get(table, get);
 413     assertNull(result.getRow());
 414   }
 415
 416   @Test
 417   public void testAppend() throws Exception {
 418     ThriftHBaseServiceHandler handler = createHandler();
 419     byte[] rowName = Bytes.toBytes("testAppend");
 420     ByteBuffer table = wrap(tableAname);
 421     byte[] v1 = Bytes.toBytes(1L);
 422     byte[] v2 = Bytes.toBytes(5L);
 423     List<TColumnValue> columnValues = new ArrayList<>(1);
 424     columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
 425         wrap(Bytes.toBytes(1L))));
 426     TPut put = new TPut(wrap(rowName), columnValues);
 427     put.setColumnValues(columnValues);
 428     put.setCellVisibility(new TCellVisibility().setExpression(PRIVATE));
 429     handler.put(table, put);
 430
 431     List<TColumnValue> appendColumns = new ArrayList<>(1);
 432     appendColumns.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname),
 433         wrap(v2)));
 434     TAppend append = new TAppend(wrap(rowName), appendColumns);
 435     append.setCellVisibility(new TCellVisibility().setExpression(SECRET));
 436     handler.append(table, append);
 437
 438     TGet get = new TGet(wrap(rowName));
 439     TAuthorization tauth = new TAuthorization();
 440     List<String> labels = new ArrayList<>(1);
 441     labels.add(SECRET);
 442     tauth.setLabels(labels);
 443     get.setAuthorizations(tauth);
 444     TResult result = handler.get(table, get);
 445
 446     assertArrayEquals(rowName, result.getRow());
 447     assertEquals(1, result.getColumnValuesSize());
 448     TColumnValue columnValue = result.getColumnValues().get(0);
 449     assertArrayEquals(Bytes.add(v1, v2), columnValue.getValue());
 450   }
 451
 452   /**
 453    * Padding numbers to make comparison of sort order easier in a for loop
 454    *
 455    * @param n
 456    *          The number to pad.
 457    * @param pad
 458    *          The length to pad up to.
 459    * @return The padded number as a string.
 460    */
 461   private String pad(int n, byte pad) {
 462     String res = Integer.toString(n);
 463
 464     while (res.length() < pad) {
 465       res = "0" + res;
 466     }
 467
 468     return res;
 469   }
 470 }