hbase-rest/src/test/java/org/apache/hadoop/hbase/rest/TestScannerResource.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.rest;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertNotNull;
  22 import static org.junit.Assert.assertNull;
  23 import static org.junit.Assert.assertTrue;
  24
  25 import java.io.ByteArrayInputStream;
  26 import java.io.IOException;
  27 import java.io.StringWriter;
  28 import java.util.ArrayList;
  29 import java.util.Iterator;
  30 import java.util.List;
  31 import java.util.Random;
  32 import javax.xml.bind.JAXBContext;
  33 import javax.xml.bind.JAXBException;
  34 import javax.xml.bind.Marshaller;
  35 import javax.xml.bind.Unmarshaller;
  36 import org.apache.hadoop.conf.Configuration;
  37 import org.apache.hadoop.hbase.CellUtil;
  38 import org.apache.hadoop.hbase.HBaseClassTestRule;
  39 import org.apache.hadoop.hbase.HBaseTestingUtil;
  40 import org.apache.hadoop.hbase.TableName;
  41 import org.apache.hadoop.hbase.client.Admin;
  42 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
  43 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
  44 import org.apache.hadoop.hbase.client.Connection;
  45 import org.apache.hadoop.hbase.client.ConnectionFactory;
  46 import org.apache.hadoop.hbase.client.Durability;
  47 import org.apache.hadoop.hbase.client.Put;
  48 import org.apache.hadoop.hbase.client.Table;
  49 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
  50 import org.apache.hadoop.hbase.rest.client.Client;
  51 import org.apache.hadoop.hbase.rest.client.Cluster;
  52 import org.apache.hadoop.hbase.rest.client.Response;
  53 import org.apache.hadoop.hbase.rest.model.CellModel;
  54 import org.apache.hadoop.hbase.rest.model.CellSetModel;
  55 import org.apache.hadoop.hbase.rest.model.RowModel;
  56 import org.apache.hadoop.hbase.rest.model.ScannerModel;
  57 import org.apache.hadoop.hbase.testclassification.MediumTests;
  58 import org.apache.hadoop.hbase.testclassification.RestTests;
  59 import org.apache.hadoop.hbase.util.Bytes;
  60 import org.apache.http.Header;
  61 import org.junit.AfterClass;
  62 import org.junit.BeforeClass;
  63 import org.junit.ClassRule;
  64 import org.junit.Test;
  65 import org.junit.experimental.categories.Category;
  66 import org.slf4j.Logger;
  67 import org.slf4j.LoggerFactory;
  68
  69 @Category({RestTests.class, MediumTests.class})
  70 public class TestScannerResource {
  71
  72   @ClassRule
  73   public static final HBaseClassTestRule CLASS_RULE =
  74       HBaseClassTestRule.forClass(TestScannerResource.class);
  75
  76   private static final Logger LOG = LoggerFactory.getLogger(TestScannerResource.class);
  77   private static final TableName TABLE = TableName.valueOf("TestScannerResource");
  78   private static final TableName TABLE_TO_BE_DISABLED = TableName.valueOf("ScannerResourceDisable");
  79   private static final String NONEXISTENT_TABLE = "ThisTableDoesNotExist";
  80   private static final String CFA = "a";
  81   private static final String CFB = "b";
  82   private static final String COLUMN_1 = CFA + ":1";
  83   private static final String COLUMN_2 = CFB + ":2";
  84
  85   private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
  86   private static final HBaseRESTTestingUtility REST_TEST_UTIL =
  87     new HBaseRESTTestingUtility();
  88   private static Client client;
  89   private static JAXBContext context;
  90   private static Marshaller marshaller;
  91   private static Unmarshaller unmarshaller;
  92   private static int expectedRows1;
  93   private static int expectedRows2;
  94   private static Configuration conf;
  95
  96   static int insertData(Configuration conf, TableName tableName, String column, double prob)
  97       throws IOException {
  98     Random rng = new Random();
  99     byte[] k = new byte[3];
 100     byte [][] famAndQf = CellUtil.parseColumn(Bytes.toBytes(column));
 101     List<Put> puts = new ArrayList<>();
 102     for (byte b1 = 'a'; b1 < 'z'; b1++) {
 103       for (byte b2 = 'a'; b2 < 'z'; b2++) {
 104         for (byte b3 = 'a'; b3 < 'z'; b3++) {
 105           if (rng.nextDouble() < prob) {
 106             k[0] = b1;
 107             k[1] = b2;
 108             k[2] = b3;
 109             Put put = new Put(k);
 110             put.setDurability(Durability.SKIP_WAL);
 111             put.addColumn(famAndQf[0], famAndQf[1], k);
 112             puts.add(put);
 113           }
 114         }
 115       }
 116     }
 117     try (Connection conn = ConnectionFactory.createConnection(conf);
 118         Table table = conn.getTable(tableName)) {
 119       table.put(puts);
 120     }
 121     return puts.size();
 122   }
 123
 124   static int countCellSet(CellSetModel model) {
 125     int count = 0;
 126     Iterator<RowModel> rows = model.getRows().iterator();
 127     while (rows.hasNext()) {
 128       RowModel row = rows.next();
 129       Iterator<CellModel> cells = row.getCells().iterator();
 130       while (cells.hasNext()) {
 131         cells.next();
 132         count++;
 133       }
 134     }
 135     return count;
 136   }
 137
 138   private static int fullTableScan(ScannerModel model) throws IOException {
 139     model.setBatch(100);
 140     Response response = client.put("/" + TABLE + "/scanner",
 141       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
 142     assertEquals(201, response.getCode());
 143     String scannerURI = response.getLocation();
 144     assertNotNull(scannerURI);
 145     int count = 0;
 146     while (true) {
 147       response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
 148       assertTrue(response.getCode() == 200 || response.getCode() == 204);
 149       if (response.getCode() == 200) {
 150         assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
 151         CellSetModel cellSet = new CellSetModel();
 152         cellSet.getObjectFromMessage(response.getBody());
 153         Iterator<RowModel> rows = cellSet.getRows().iterator();
 154         while (rows.hasNext()) {
 155           RowModel row = rows.next();
 156           Iterator<CellModel> cells = row.getCells().iterator();
 157           while (cells.hasNext()) {
 158             cells.next();
 159             count++;
 160           }
 161         }
 162       } else {
 163         break;
 164       }
 165     }
 166     // delete the scanner
 167     response = client.delete(scannerURI);
 168     assertEquals(200, response.getCode());
 169     return count;
 170   }
 171
 172   @BeforeClass
 173   public static void setUpBeforeClass() throws Exception {
 174     conf = TEST_UTIL.getConfiguration();
 175     TEST_UTIL.startMiniCluster();
 176     REST_TEST_UTIL.startServletContainer(conf);
 177     client = new Client(new Cluster().add("localhost",
 178       REST_TEST_UTIL.getServletPort()));
 179     context = JAXBContext.newInstance(
 180       CellModel.class,
 181       CellSetModel.class,
 182       RowModel.class,
 183       ScannerModel.class);
 184     marshaller = context.createMarshaller();
 185     unmarshaller = context.createUnmarshaller();
 186     Admin admin = TEST_UTIL.getAdmin();
 187     if (admin.tableExists(TABLE)) {
 188       return;
 189     }
 190     TableDescriptorBuilder tableDescriptorBuilder =
 191       TableDescriptorBuilder.newBuilder(TABLE);
 192     ColumnFamilyDescriptor columnFamilyDescriptor =
 193       ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(CFA)).build();
 194     tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
 195     columnFamilyDescriptor =
 196       ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(CFB)).build();
 197     tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
 198
 199     admin.createTable(tableDescriptorBuilder.build());
 200     expectedRows1 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_1, 1.0);
 201     expectedRows2 = insertData(TEST_UTIL.getConfiguration(), TABLE, COLUMN_2, 0.5);
 202
 203     tableDescriptorBuilder=TableDescriptorBuilder.newBuilder(TABLE_TO_BE_DISABLED);
 204     columnFamilyDescriptor =
 205       ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(CFA)).build();
 206     tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
 207     columnFamilyDescriptor =
 208       ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(CFB)).build();
 209     tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);
 210
 211     admin.createTable(tableDescriptorBuilder.build());
 212   }
 213
 214   @AfterClass
 215   public static void tearDownAfterClass() throws Exception {
 216     REST_TEST_UTIL.shutdownServletContainer();
 217     TEST_UTIL.shutdownMiniCluster();
 218   }
 219
 220   @Test
 221   public void testSimpleScannerXML() throws IOException, JAXBException {
 222     final int BATCH_SIZE = 5;
 223     // new scanner
 224     ScannerModel model = new ScannerModel();
 225     model.setBatch(BATCH_SIZE);
 226     model.addColumn(Bytes.toBytes(COLUMN_1));
 227     StringWriter writer = new StringWriter();
 228     marshaller.marshal(model, writer);
 229     byte[] body = Bytes.toBytes(writer.toString());
 230
 231     // test put operation is forbidden in read-only mode
 232     conf.set("hbase.rest.readonly", "true");
 233     Response response = client.put("/" + TABLE + "/scanner",
 234       Constants.MIMETYPE_XML, body);
 235     assertEquals(403, response.getCode());
 236     String scannerURI = response.getLocation();
 237     assertNull(scannerURI);
 238
 239     // recall previous put operation with read-only off
 240     conf.set("hbase.rest.readonly", "false");
 241     response = client.put("/" + TABLE + "/scanner", Constants.MIMETYPE_XML,
 242       body);
 243     assertEquals(201, response.getCode());
 244     scannerURI = response.getLocation();
 245     assertNotNull(scannerURI);
 246
 247     // get a cell set
 248     response = client.get(scannerURI, Constants.MIMETYPE_XML);
 249     assertEquals(200, response.getCode());
 250     assertEquals(Constants.MIMETYPE_XML, response.getHeader("content-type"));
 251     CellSetModel cellSet = (CellSetModel)
 252       unmarshaller.unmarshal(new ByteArrayInputStream(response.getBody()));
 253     // confirm batch size conformance
 254     assertEquals(BATCH_SIZE, countCellSet(cellSet));
 255
 256     // test delete scanner operation is forbidden in read-only mode
 257     conf.set("hbase.rest.readonly", "true");
 258     response = client.delete(scannerURI);
 259     assertEquals(403, response.getCode());
 260
 261     // recall previous delete scanner operation with read-only off
 262     conf.set("hbase.rest.readonly", "false");
 263     response = client.delete(scannerURI);
 264     assertEquals(200, response.getCode());
 265   }
 266
 267   @Test
 268   public void testSimpleScannerPB() throws IOException {
 269     final int BATCH_SIZE = 10;
 270     // new scanner
 271     ScannerModel model = new ScannerModel();
 272     model.setBatch(BATCH_SIZE);
 273     model.addColumn(Bytes.toBytes(COLUMN_1));
 274
 275     // test put operation is forbidden in read-only mode
 276     conf.set("hbase.rest.readonly", "true");
 277     Response response = client.put("/" + TABLE + "/scanner",
 278       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
 279     assertEquals(403, response.getCode());
 280     String scannerURI = response.getLocation();
 281     assertNull(scannerURI);
 282
 283     // recall previous put operation with read-only off
 284     conf.set("hbase.rest.readonly", "false");
 285     response = client.put("/" + TABLE + "/scanner",
 286       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
 287     assertEquals(201, response.getCode());
 288     scannerURI = response.getLocation();
 289     assertNotNull(scannerURI);
 290
 291     // get a cell set
 292     response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
 293     assertEquals(200, response.getCode());
 294     assertEquals(Constants.MIMETYPE_PROTOBUF, response.getHeader("content-type"));
 295     CellSetModel cellSet = new CellSetModel();
 296     cellSet.getObjectFromMessage(response.getBody());
 297     // confirm batch size conformance
 298     assertEquals(BATCH_SIZE, countCellSet(cellSet));
 299
 300     // test delete scanner operation is forbidden in read-only mode
 301     conf.set("hbase.rest.readonly", "true");
 302     response = client.delete(scannerURI);
 303     assertEquals(403, response.getCode());
 304
 305     // recall previous delete scanner operation with read-only off
 306     conf.set("hbase.rest.readonly", "false");
 307     response = client.delete(scannerURI);
 308     assertEquals(200, response.getCode());
 309   }
 310
 311   @Test
 312   public void testSimpleScannerBinary() throws IOException {
 313     // new scanner
 314     ScannerModel model = new ScannerModel();
 315     model.setBatch(1);
 316     model.addColumn(Bytes.toBytes(COLUMN_1));
 317
 318     // test put operation is forbidden in read-only mode
 319     conf.set("hbase.rest.readonly", "true");
 320     Response response = client.put("/" + TABLE + "/scanner",
 321       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
 322     assertEquals(403, response.getCode());
 323     String scannerURI = response.getLocation();
 324     assertNull(scannerURI);
 325
 326     // recall previous put operation with read-only off
 327     conf.set("hbase.rest.readonly", "false");
 328     response = client.put("/" + TABLE + "/scanner",
 329       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
 330     assertEquals(201, response.getCode());
 331     scannerURI = response.getLocation();
 332     assertNotNull(scannerURI);
 333
 334     // get a cell
 335     response = client.get(scannerURI, Constants.MIMETYPE_BINARY);
 336     assertEquals(200, response.getCode());
 337     assertEquals(Constants.MIMETYPE_BINARY, response.getHeader("content-type"));
 338     // verify that data was returned
 339     assertTrue(response.getBody().length > 0);
 340     // verify that the expected X-headers are present
 341     boolean foundRowHeader = false, foundColumnHeader = false,
 342       foundTimestampHeader = false;
 343     for (Header header: response.getHeaders()) {
 344       if (header.getName().equals("X-Row")) {
 345         foundRowHeader = true;
 346       } else if (header.getName().equals("X-Column")) {
 347         foundColumnHeader = true;
 348       } else if (header.getName().equals("X-Timestamp")) {
 349         foundTimestampHeader = true;
 350       }
 351     }
 352     assertTrue(foundRowHeader);
 353     assertTrue(foundColumnHeader);
 354     assertTrue(foundTimestampHeader);
 355
 356     // test delete scanner operation is forbidden in read-only mode
 357     conf.set("hbase.rest.readonly", "true");
 358     response = client.delete(scannerURI);
 359     assertEquals(403, response.getCode());
 360
 361     // recall previous delete scanner operation with read-only off
 362     conf.set("hbase.rest.readonly", "false");
 363     response = client.delete(scannerURI);
 364     assertEquals(200, response.getCode());
 365   }
 366
 367   @Test
 368   public void testFullTableScan() throws IOException {
 369     ScannerModel model = new ScannerModel();
 370     model.addColumn(Bytes.toBytes(COLUMN_1));
 371     assertEquals(expectedRows1, fullTableScan(model));
 372
 373     model = new ScannerModel();
 374     model.addColumn(Bytes.toBytes(COLUMN_2));
 375     assertEquals(expectedRows2, fullTableScan(model));
 376   }
 377
 378   @Test
 379   public void testTableDoesNotExist() throws IOException, JAXBException {
 380     ScannerModel model = new ScannerModel();
 381     StringWriter writer = new StringWriter();
 382     marshaller.marshal(model, writer);
 383     byte[] body = Bytes.toBytes(writer.toString());
 384     Response response = client.put("/" + NONEXISTENT_TABLE +
 385       "/scanner", Constants.MIMETYPE_XML, body);
 386     String scannerURI = response.getLocation();
 387     assertNotNull(scannerURI);
 388     response = client.get(scannerURI, Constants.MIMETYPE_XML);
 389     assertEquals(404, response.getCode());
 390   }
 391
 392   @Test
 393   public void testTableScanWithTableDisable() throws IOException {
 394     TEST_UTIL.getAdmin().disableTable(TABLE_TO_BE_DISABLED);
 395     ScannerModel model = new ScannerModel();
 396     model.addColumn(Bytes.toBytes(COLUMN_1));
 397     model.setCaching(1);
 398     Response response = client.put("/" + TABLE_TO_BE_DISABLED + "/scanner",
 399       Constants.MIMETYPE_PROTOBUF, model.createProtobufOutput());
 400     // we will see the exception when we actually want to get the result.
 401     assertEquals(201, response.getCode());
 402     String scannerURI = response.getLocation();
 403     assertNotNull(scannerURI);
 404     response = client.get(scannerURI, Constants.MIMETYPE_PROTOBUF);
 405     assertEquals(410, response.getCode());
 406   }
 407 }
 408