?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
HBASE-23723 Ensure MOB compaction works in optimized mode after snapshot clone (...
[hbase.git] / hbase-server / src / test / java / org / apache / hadoop / hbase / client / TestFromClientSide5.java
blob38ddf2ffb37ac022bb61d0c526a9e8aaec4155e2
1 /*
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18 package org.apache.hadoop.hbase.client;
19
20 import static org.junit.Assert.assertArrayEquals;
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertNotNull;
24 import static org.junit.Assert.assertNull;
25 import static org.junit.Assert.assertTrue;
26 import static org.junit.Assert.fail;
27
28 import java.io.IOException;
29 import java.util.ArrayList;
30 import java.util.Arrays;
31 import java.util.Collection;
32 import java.util.LinkedList;
33 import java.util.List;
34 import java.util.Map;
35 import java.util.NavigableMap;
36 import java.util.concurrent.Callable;
37 import java.util.concurrent.ExecutorService;
38 import java.util.concurrent.Executors;
39 import java.util.concurrent.atomic.AtomicReference;
40 import org.apache.commons.lang3.ArrayUtils;
41 import org.apache.hadoop.conf.Configuration;
42 import org.apache.hadoop.fs.Path;
43 import org.apache.hadoop.hbase.Cell;
44 import org.apache.hadoop.hbase.CellScanner;
45 import org.apache.hadoop.hbase.CellUtil;
46 import org.apache.hadoop.hbase.CompareOperator;
47 import org.apache.hadoop.hbase.DoNotRetryIOException;
48 import org.apache.hadoop.hbase.HBaseClassTestRule;
49 import org.apache.hadoop.hbase.HConstants;
50 import org.apache.hadoop.hbase.HRegionLocation;
51 import org.apache.hadoop.hbase.KeyValue;
52 import org.apache.hadoop.hbase.PrivateCellUtil;
53 import org.apache.hadoop.hbase.ServerName;
54 import org.apache.hadoop.hbase.TableName;
55 import org.apache.hadoop.hbase.TableNameTestRule;
56 import org.apache.hadoop.hbase.Waiter;
57 import org.apache.hadoop.hbase.client.metrics.ScanMetrics;
58 import org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint;
59 import org.apache.hadoop.hbase.filter.BinaryComparator;
60 import org.apache.hadoop.hbase.filter.Filter;
61 import org.apache.hadoop.hbase.filter.FilterList;
62 import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
63 import org.apache.hadoop.hbase.filter.InclusiveStopFilter;
64 import org.apache.hadoop.hbase.filter.KeyOnlyFilter;
65 import org.apache.hadoop.hbase.filter.QualifierFilter;
66 import org.apache.hadoop.hbase.filter.RegexStringComparator;
67 import org.apache.hadoop.hbase.filter.RowFilter;
68 import org.apache.hadoop.hbase.filter.SubstringComparator;
69 import org.apache.hadoop.hbase.filter.ValueFilter;
70 import org.apache.hadoop.hbase.io.TimeRange;
71 import org.apache.hadoop.hbase.io.hfile.BlockCache;
72 import org.apache.hadoop.hbase.io.hfile.CacheConfig;
73 import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
74 import org.apache.hadoop.hbase.regionserver.HRegion;
75 import org.apache.hadoop.hbase.regionserver.HRegionServer;
76 import org.apache.hadoop.hbase.regionserver.HStore;
77 import org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException;
78 import org.apache.hadoop.hbase.testclassification.ClientTests;
79 import org.apache.hadoop.hbase.testclassification.LargeTests;
80 import org.apache.hadoop.hbase.util.Bytes;
81 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
82 import org.apache.hadoop.hbase.util.FSUtils;
83 import org.junit.AfterClass;
84 import org.junit.ClassRule;
85 import org.junit.Ignore;
86 import org.junit.Rule;
87 import org.junit.Test;
88 import org.junit.experimental.categories.Category;
89 import org.junit.runner.RunWith;
90 import org.junit.runners.Parameterized;
91 import org.slf4j.Logger;
92 import org.slf4j.LoggerFactory;
93
94 import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
95 import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto;
96 import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutationProto.MutationType;
97 import org.apache.hadoop.hbase.shaded.protobuf.generated.MultiRowMutationProtos.MultiRowMutationService;
98 import org.apache.hadoop.hbase.shaded.protobuf.generated.MultiRowMutationProtos.MutateRowsRequest;
99
100 /**
101 * Run tests that use the HBase clients; {@link Table}.
102 * Sets up the HBase mini cluster once at start and runs through all client tests.
103 * Each creates a table named for the method and does its stuff against that.
104 *
105 * Parameterized to run with different registry implementations.
106 */
107 @Category({LargeTests.class, ClientTests.class})
108 @SuppressWarnings ("deprecation")
109 @RunWith(Parameterized.class)
110 public class TestFromClientSide5 extends FromClientSideBase {
111 private static final Logger LOG = LoggerFactory.getLogger(TestFromClientSide5.class);
112
113 @ClassRule
114 public static final HBaseClassTestRule CLASS_RULE =
115 HBaseClassTestRule.forClass(TestFromClientSide5.class);
116 @Rule
117 public TableNameTestRule name = new TableNameTestRule();
118
119 // To keep the child classes happy.
120 TestFromClientSide5() {}
121
122 public TestFromClientSide5(Class registry, int numHedgedReqs) throws Exception {
123 initialize(registry, numHedgedReqs, MultiRowMutationEndpoint.class);
124 }
125
126 @Parameterized.Parameters
127 public static Collection parameters() {
128 return Arrays.asList(new Object[][] {
129 { MasterRegistry.class, 1},
130 { MasterRegistry.class, 2},
131 { ZKConnectionRegistry.class, 1}
132 });
133 }
134
135 @AfterClass public static void tearDownAfterClass() throws Exception {
136 afterClass();
137 }
138
139 @Test
140 public void testGetClosestRowBefore() throws IOException, InterruptedException {
141 final TableName tableName = name.getTableName();
142 final byte[] firstRow = Bytes.toBytes("row111");
143 final byte[] secondRow = Bytes.toBytes("row222");
144 final byte[] thirdRow = Bytes.toBytes("row333");
145 final byte[] forthRow = Bytes.toBytes("row444");
146 final byte[] beforeFirstRow = Bytes.toBytes("row");
147 final byte[] beforeSecondRow = Bytes.toBytes("row22");
148 final byte[] beforeThirdRow = Bytes.toBytes("row33");
149 final byte[] beforeForthRow = Bytes.toBytes("row44");
150
151 try (Table table =
152 TEST_UTIL.createTable(tableName,
153 new byte[][] { HConstants.CATALOG_FAMILY, Bytes.toBytes("info2") }, 1, 1024);
154 RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
155
156 // set block size to 64 to making 2 kvs into one block, bypassing the walkForwardInSingleRow
157 // in Store.rowAtOrBeforeFromStoreFile
158 String regionName = locator.getAllRegionLocations().get(0).getRegion().getEncodedName();
159 HRegion region = TEST_UTIL.getRSForFirstRegionInTable(tableName).getRegion(regionName);
160 Put put1 = new Put(firstRow);
161 Put put2 = new Put(secondRow);
162 Put put3 = new Put(thirdRow);
163 Put put4 = new Put(forthRow);
164 byte[] one = new byte[] { 1 };
165 byte[] two = new byte[] { 2 };
166 byte[] three = new byte[] { 3 };
167 byte[] four = new byte[] { 4 };
168
169 put1.addColumn(HConstants.CATALOG_FAMILY, null, one);
170 put2.addColumn(HConstants.CATALOG_FAMILY, null, two);
171 put3.addColumn(HConstants.CATALOG_FAMILY, null, three);
172 put4.addColumn(HConstants.CATALOG_FAMILY, null, four);
173 table.put(put1);
174 table.put(put2);
175 table.put(put3);
176 table.put(put4);
177 region.flush(true);
178
179 Result result;
180
181 // Test before first that null is returned
182 result = getReverseScanResult(table, beforeFirstRow);
183 assertNull(result);
184
185 // Test at first that first is returned
186 result = getReverseScanResult(table, firstRow);
187 assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
188 assertTrue(Bytes.equals(result.getRow(), firstRow));
189 assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));
190
191 // Test in between first and second that first is returned
192 result = getReverseScanResult(table, beforeSecondRow);
193 assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
194 assertTrue(Bytes.equals(result.getRow(), firstRow));
195 assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), one));
196
197 // Test at second make sure second is returned
198 result = getReverseScanResult(table, secondRow);
199 assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
200 assertTrue(Bytes.equals(result.getRow(), secondRow));
201 assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), two));
202
203 // Test in second and third, make sure second is returned
204 result = getReverseScanResult(table, beforeThirdRow);
205 assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
206 assertTrue(Bytes.equals(result.getRow(), secondRow));
207 assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), two));
208
209 // Test at third make sure third is returned
210 result = getReverseScanResult(table, thirdRow);
211 assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
212 assertTrue(Bytes.equals(result.getRow(), thirdRow));
213 assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), three));
214
215 // Test in third and forth, make sure third is returned
216 result = getReverseScanResult(table, beforeForthRow);
217 assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
218 assertTrue(Bytes.equals(result.getRow(), thirdRow));
219 assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), three));
220
221 // Test at forth make sure forth is returned
222 result = getReverseScanResult(table, forthRow);
223 assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
224 assertTrue(Bytes.equals(result.getRow(), forthRow));
225 assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), four));
226
227 // Test after forth make sure forth is returned
228 result = getReverseScanResult(table, Bytes.add(forthRow, one));
229 assertTrue(result.containsColumn(HConstants.CATALOG_FAMILY, null));
230 assertTrue(Bytes.equals(result.getRow(), forthRow));
231 assertTrue(Bytes.equals(result.getValue(HConstants.CATALOG_FAMILY, null), four));
232 }
233 }
234
235 private Result getReverseScanResult(Table table, byte[] row) throws IOException {
236 Scan scan = new Scan().withStartRow(row);
237 scan.setSmall(true);
238 scan.setReversed(true);
239 scan.setCaching(1);
240 scan.addFamily(HConstants.CATALOG_FAMILY);
241 try (ResultScanner scanner = table.getScanner(scan)) {
242 return scanner.next();
243 }
244 }
245
246 /**
247 * For HBASE-2156
248 */
249 @Test
250 public void testScanVariableReuse() {
251 Scan scan = new Scan();
252 scan.addFamily(FAMILY);
253 scan.addColumn(FAMILY, ROW);
254
255 assertEquals(1, scan.getFamilyMap().get(FAMILY).size());
256
257 scan = new Scan();
258 scan.addFamily(FAMILY);
259
260 assertNull(scan.getFamilyMap().get(FAMILY));
261 assertTrue(scan.getFamilyMap().containsKey(FAMILY));
262 }
263
264 @Test
265 public void testMultiRowMutation() throws Exception {
266 LOG.info("Starting testMultiRowMutation");
267 final TableName tableName = name.getTableName();
268 final byte [] ROW1 = Bytes.toBytes("testRow1");
269
270 try (Table t = TEST_UTIL.createTable(tableName, FAMILY)) {
271 Put p = new Put(ROW);
272 p.addColumn(FAMILY, QUALIFIER, VALUE);
273 MutationProto m1 = ProtobufUtil.toMutation(MutationType.PUT, p);
274
275 p = new Put(ROW1);
276 p.addColumn(FAMILY, QUALIFIER, VALUE);
277 MutationProto m2 = ProtobufUtil.toMutation(MutationType.PUT, p);
278
279 MutateRowsRequest.Builder mrmBuilder = MutateRowsRequest.newBuilder();
280 mrmBuilder.addMutationRequest(m1);
281 mrmBuilder.addMutationRequest(m2);
282 MutateRowsRequest mrm = mrmBuilder.build();
283 CoprocessorRpcChannel channel = t.coprocessorService(ROW);
284 MultiRowMutationService.BlockingInterface service =
285 MultiRowMutationService.newBlockingStub(channel);
286 service.mutateRows(null, mrm);
287 Get g = new Get(ROW);
288 Result r = t.get(g);
289 assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)));
290 g = new Get(ROW1);
291 r = t.get(g);
292 assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIER)));
293 }
294 }
295
296 @Test
297 public void testRowMutation() throws Exception {
298 LOG.info("Starting testRowMutation");
299 final TableName tableName = name.getTableName();
300 try (Table t = TEST_UTIL.createTable(tableName, FAMILY)) {
301 byte[][] QUALIFIERS = new byte[][] { Bytes.toBytes("a"), Bytes.toBytes("b") };
302 RowMutations arm = new RowMutations(ROW);
303 Put p = new Put(ROW);
304 p.addColumn(FAMILY, QUALIFIERS[0], VALUE);
305 arm.add(p);
306 t.mutateRow(arm);
307
308 Get g = new Get(ROW);
309 Result r = t.get(g);
310 assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[0])));
311
312 arm = new RowMutations(ROW);
313 p = new Put(ROW);
314 p.addColumn(FAMILY, QUALIFIERS[1], VALUE);
315 arm.add(p);
316 Delete d = new Delete(ROW);
317 d.addColumns(FAMILY, QUALIFIERS[0]);
318 arm.add(d);
319 // TODO: Trying mutateRow again. The batch was failing with a one try only.
320 t.mutateRow(arm);
321 r = t.get(g);
322 assertEquals(0, Bytes.compareTo(VALUE, r.getValue(FAMILY, QUALIFIERS[1])));
323 assertNull(r.getValue(FAMILY, QUALIFIERS[0]));
324
325 // Test that we get a region level exception
326 try {
327 arm = new RowMutations(ROW);
328 p = new Put(ROW);
329 p.addColumn(new byte[] { 'b', 'o', 'g', 'u', 's' }, QUALIFIERS[0], VALUE);
330 arm.add(p);
331 t.mutateRow(arm);
332 fail("Expected NoSuchColumnFamilyException");
333 } catch (NoSuchColumnFamilyException e) {
334 return;
335 } catch (RetriesExhaustedWithDetailsException e) {
336 for (Throwable rootCause : e.getCauses()) {
337 if (rootCause instanceof NoSuchColumnFamilyException) {
338 return;
339 }
340 }
341 throw e;
342 }
343 }
344 }
345
346 @Test
347 public void testBatchAppendWithReturnResultFalse() throws Exception {
348 LOG.info("Starting testBatchAppendWithReturnResultFalse");
349 final TableName tableName = name.getTableName();
350 try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) {
351 Append append1 = new Append(Bytes.toBytes("row1"));
352 append1.setReturnResults(false);
353 append1.addColumn(FAMILY, Bytes.toBytes("f1"), Bytes.toBytes("value1"));
354 Append append2 = new Append(Bytes.toBytes("row1"));
355 append2.setReturnResults(false);
356 append2.addColumn(FAMILY, Bytes.toBytes("f1"), Bytes.toBytes("value2"));
357 List<Append> appends = new ArrayList<>();
358 appends.add(append1);
359 appends.add(append2);
360 Object[] results = new Object[2];
361 table.batch(appends, results);
362 assertEquals(2, results.length);
363 for (Object r : results) {
364 Result result = (Result) r;
365 assertTrue(result.isEmpty());
366 }
367 }
368 }
369
370 @Test
371 public void testAppend() throws Exception {
372 LOG.info("Starting testAppend");
373 final TableName tableName = name.getTableName();
374 try (Table t = TEST_UTIL.createTable(tableName, FAMILY)) {
375 byte[] v1 = Bytes.toBytes("42");
376 byte[] v2 = Bytes.toBytes("23");
377 byte[][] QUALIFIERS = new byte[][]{
378 Bytes.toBytes("b"), Bytes.toBytes("a"), Bytes.toBytes("c")
379 };
380 Append a = new Append(ROW);
381 a.addColumn(FAMILY, QUALIFIERS[0], v1);
382 a.addColumn(FAMILY, QUALIFIERS[1], v2);
383 a.setReturnResults(false);
384 assertEmptyResult(t.append(a));
385
386 a = new Append(ROW);
387 a.addColumn(FAMILY, QUALIFIERS[0], v2);
388 a.addColumn(FAMILY, QUALIFIERS[1], v1);
389 a.addColumn(FAMILY, QUALIFIERS[2], v2);
390 Result r = t.append(a);
391 assertEquals(0, Bytes.compareTo(Bytes.add(v1, v2), r.getValue(FAMILY, QUALIFIERS[0])));
392 assertEquals(0, Bytes.compareTo(Bytes.add(v2, v1), r.getValue(FAMILY, QUALIFIERS[1])));
393 // QUALIFIERS[2] previously not exist, verify both value and timestamp are correct
394 assertEquals(0, Bytes.compareTo(v2, r.getValue(FAMILY, QUALIFIERS[2])));
395 assertEquals(r.getColumnLatestCell(FAMILY, QUALIFIERS[0]).getTimestamp(),
396 r.getColumnLatestCell(FAMILY, QUALIFIERS[2]).getTimestamp());
397 }
398 }
399 private List<Result> doAppend(final boolean walUsed) throws IOException {
400 LOG.info("Starting testAppend, walUsed is " + walUsed);
401 final TableName TABLENAME =
402 TableName.valueOf(walUsed ? "testAppendWithWAL" : "testAppendWithoutWAL");
403 try (Table t = TEST_UTIL.createTable(TABLENAME, FAMILY)) {
404 final byte[] row1 = Bytes.toBytes("c");
405 final byte[] row2 = Bytes.toBytes("b");
406 final byte[] row3 = Bytes.toBytes("a");
407 final byte[] qual = Bytes.toBytes("qual");
408 Put put_0 = new Put(row2);
409 put_0.addColumn(FAMILY, qual, Bytes.toBytes("put"));
410 Put put_1 = new Put(row3);
411 put_1.addColumn(FAMILY, qual, Bytes.toBytes("put"));
412 Append append_0 = new Append(row1);
413 append_0.addColumn(FAMILY, qual, Bytes.toBytes("i"));
414 Append append_1 = new Append(row1);
415 append_1.addColumn(FAMILY, qual, Bytes.toBytes("k"));
416 Append append_2 = new Append(row1);
417 append_2.addColumn(FAMILY, qual, Bytes.toBytes("e"));
418 if (!walUsed) {
419 append_2.setDurability(Durability.SKIP_WAL);
420 }
421 Append append_3 = new Append(row1);
422 append_3.addColumn(FAMILY, qual, Bytes.toBytes("a"));
423 Scan s = new Scan();
424 s.setCaching(1);
425 t.append(append_0);
426 t.put(put_0);
427 t.put(put_1);
428 List<Result> results = new LinkedList<>();
429 try (ResultScanner scanner = t.getScanner(s)) {
430 t.append(append_1);
431 t.append(append_2);
432 t.append(append_3);
433 for (Result r : scanner) {
434 results.add(r);
435 }
436 }
437 TEST_UTIL.deleteTable(TABLENAME);
438 return results;
439 }
440 }
441
442 @Test
443 public void testAppendWithoutWAL() throws Exception {
444 List<Result> resultsWithWal = doAppend(true);
445 List<Result> resultsWithoutWal = doAppend(false);
446 assertEquals(resultsWithWal.size(), resultsWithoutWal.size());
447 for (int i = 0; i != resultsWithWal.size(); ++i) {
448 Result resultWithWal = resultsWithWal.get(i);
449 Result resultWithoutWal = resultsWithoutWal.get(i);
450 assertEquals(resultWithWal.rawCells().length, resultWithoutWal.rawCells().length);
451 for (int j = 0; j != resultWithWal.rawCells().length; ++j) {
452 Cell cellWithWal = resultWithWal.rawCells()[j];
453 Cell cellWithoutWal = resultWithoutWal.rawCells()[j];
454 assertArrayEquals(CellUtil.cloneRow(cellWithWal), CellUtil.cloneRow(cellWithoutWal));
455 assertArrayEquals(CellUtil.cloneFamily(cellWithWal), CellUtil.cloneFamily(cellWithoutWal));
456 assertArrayEquals(CellUtil.cloneQualifier(cellWithWal),
457 CellUtil.cloneQualifier(cellWithoutWal));
458 assertArrayEquals(CellUtil.cloneValue(cellWithWal), CellUtil.cloneValue(cellWithoutWal));
459 }
460 }
461 }
462
463 @Test
464 public void testClientPoolRoundRobin() throws IOException {
465 final TableName tableName = name.getTableName();
466
467 int poolSize = 3;
468 int numVersions = poolSize * 2;
469 Configuration conf = TEST_UTIL.getConfiguration();
470 conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "round-robin");
471 conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize);
472
473 try (Table table =
474 TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, Integer.MAX_VALUE)) {
475
476 final long ts = EnvironmentEdgeManager.currentTime();
477 Get get = new Get(ROW);
478 get.addColumn(FAMILY, QUALIFIER);
479 get.readAllVersions();
480
481 for (int versions = 1; versions <= numVersions; versions++) {
482 Put put = new Put(ROW);
483 put.addColumn(FAMILY, QUALIFIER, ts + versions, VALUE);
484 table.put(put);
485
486 Result result = table.get(get);
487 NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY)
488 .get(QUALIFIER);
489
490 assertEquals("The number of versions of '" + Bytes.toString(FAMILY) + ":"
491 + Bytes.toString(QUALIFIER) + " did not match", versions, navigableMap.size());
492 for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
493 assertTrue("The value at time " + entry.getKey()
494 + " did not match what was put",
495 Bytes.equals(VALUE, entry.getValue()));
496 }
497 }
498 }
499 }
500
501 @Ignore ("Flakey: HBASE-8989") @Test
502 public void testClientPoolThreadLocal() throws IOException {
503 final TableName tableName = name.getTableName();
504
505 int poolSize = Integer.MAX_VALUE;
506 int numVersions = 3;
507 Configuration conf = TEST_UTIL.getConfiguration();
508 conf.set(HConstants.HBASE_CLIENT_IPC_POOL_TYPE, "thread-local");
509 conf.setInt(HConstants.HBASE_CLIENT_IPC_POOL_SIZE, poolSize);
510
511 try (final Table table = TEST_UTIL.createTable(tableName, new byte[][] { FAMILY }, 3)) {
512
513 final long ts = EnvironmentEdgeManager.currentTime();
514 final Get get = new Get(ROW);
515 get.addColumn(FAMILY, QUALIFIER);
516 get.readAllVersions();
517
518 for (int versions = 1; versions <= numVersions; versions++) {
519 Put put = new Put(ROW);
520 put.addColumn(FAMILY, QUALIFIER, ts + versions, VALUE);
521 table.put(put);
522
523 Result result = table.get(get);
524 NavigableMap<Long, byte[]> navigableMap = result.getMap().get(FAMILY)
525 .get(QUALIFIER);
526
527 assertEquals("The number of versions of '" + Bytes.toString(FAMILY) + ":"
528 + Bytes.toString(QUALIFIER) + " did not match", versions, navigableMap.size());
529 for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
530 assertTrue("The value at time " + entry.getKey()
531 + " did not match what was put",
532 Bytes.equals(VALUE, entry.getValue()));
533 }
534 }
535
536 final Object waitLock = new Object();
537 ExecutorService executorService = Executors.newFixedThreadPool(numVersions);
538 final AtomicReference<AssertionError> error = new AtomicReference<>(null);
539 for (int versions = numVersions; versions < numVersions * 2; versions++) {
540 final int versionsCopy = versions;
541 executorService.submit((Callable<Void>) () -> {
542 try {
543 Put put = new Put(ROW);
544 put.addColumn(FAMILY, QUALIFIER, ts + versionsCopy, VALUE);
545 table.put(put);
546
547 Result result = table.get(get);
548 NavigableMap<Long, byte[]> navigableMap = result.getMap()
549 .get(FAMILY).get(QUALIFIER);
550
551 assertEquals("The number of versions of '" + Bytes.toString(FAMILY) + ":"
552 + Bytes.toString(QUALIFIER) + " did not match " + versionsCopy, versionsCopy,
553 navigableMap.size());
554 for (Map.Entry<Long, byte[]> entry : navigableMap.entrySet()) {
555 assertTrue("The value at time " + entry.getKey()
556 + " did not match what was put",
557 Bytes.equals(VALUE, entry.getValue()));
558 }
559 synchronized (waitLock) {
560 waitLock.wait();
561 }
562 } catch (Exception ignored) {
563 } catch (AssertionError e) {
564 // the error happens in a thread, it won't fail the test,
565 // need to pass it to the caller for proper handling.
566 error.set(e);
567 LOG.error(e.toString(), e);
568 }
569
570 return null;
571 });
572 }
573 synchronized (waitLock) {
574 waitLock.notifyAll();
575 }
576 executorService.shutdownNow();
577 assertNull(error.get());
578 }
579 }
580
581 @Test
582 public void testCheckAndPut() throws IOException {
583 final byte [] anotherrow = Bytes.toBytes("anotherrow");
584 final byte [] value2 = Bytes.toBytes("abcd");
585
586 try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
587 Put put1 = new Put(ROW);
588 put1.addColumn(FAMILY, QUALIFIER, VALUE);
589
590 // row doesn't exist, so using non-null value should be considered "not match".
591 boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
592 .ifEquals(VALUE).thenPut(put1);
593 assertFalse(ok);
594
595 // row doesn't exist, so using "ifNotExists" should be considered "match".
596 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifNotExists().thenPut(put1);
597 assertTrue(ok);
598
599 // row now exists, so using "ifNotExists" should be considered "not match".
600 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifNotExists().thenPut(put1);
601 assertFalse(ok);
602
603 Put put2 = new Put(ROW);
604 put2.addColumn(FAMILY, QUALIFIER, value2);
605
606 // row now exists, use the matching value to check
607 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifEquals(VALUE).thenPut(put2);
608 assertTrue(ok);
609
610 Put put3 = new Put(anotherrow);
611 put3.addColumn(FAMILY, QUALIFIER, VALUE);
612
613 // try to do CheckAndPut on different rows
614 try {
615 table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifEquals(value2).thenPut(put3);
616 fail("trying to check and modify different rows should have failed.");
617 } catch (Exception ignored) {
618 }
619 }
620 }
621
622 @Test
623 public void testCheckAndMutateWithTimeRange() throws IOException {
624 try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
625 final long ts = System.currentTimeMillis() / 2;
626 Put put = new Put(ROW);
627 put.addColumn(FAMILY, QUALIFIER, ts, VALUE);
628
629 boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
630 .ifNotExists()
631 .thenPut(put);
632 assertTrue(ok);
633
634 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
635 .timeRange(TimeRange.at(ts + 10000))
636 .ifEquals(VALUE)
637 .thenPut(put);
638 assertFalse(ok);
639
640 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
641 .timeRange(TimeRange.from(ts + 10000))
642 .ifEquals(VALUE)
643 .thenPut(put);
644 assertFalse(ok);
645
646 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
647 .timeRange(TimeRange.between(ts + 10000, ts + 20000))
648 .ifEquals(VALUE)
649 .thenPut(put);
650 assertFalse(ok);
651
652 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
653 .timeRange(TimeRange.until(ts))
654 .ifEquals(VALUE)
655 .thenPut(put);
656 assertFalse(ok);
657
658 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
659 .timeRange(TimeRange.at(ts))
660 .ifEquals(VALUE)
661 .thenPut(put);
662 assertTrue(ok);
663
664 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
665 .timeRange(TimeRange.from(ts))
666 .ifEquals(VALUE)
667 .thenPut(put);
668 assertTrue(ok);
669
670 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
671 .timeRange(TimeRange.between(ts, ts + 20000))
672 .ifEquals(VALUE)
673 .thenPut(put);
674 assertTrue(ok);
675
676 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
677 .timeRange(TimeRange.until(ts + 10000))
678 .ifEquals(VALUE)
679 .thenPut(put);
680 assertTrue(ok);
681
682 RowMutations rm = new RowMutations(ROW)
683 .add((Mutation) put);
684 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
685 .timeRange(TimeRange.at(ts + 10000))
686 .ifEquals(VALUE)
687 .thenMutate(rm);
688 assertFalse(ok);
689
690 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
691 .timeRange(TimeRange.at(ts))
692 .ifEquals(VALUE)
693 .thenMutate(rm);
694 assertTrue(ok);
695
696 Delete delete = new Delete(ROW)
697 .addColumn(FAMILY, QUALIFIER);
698
699 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
700 .timeRange(TimeRange.at(ts + 10000))
701 .ifEquals(VALUE)
702 .thenDelete(delete);
703 assertFalse(ok);
704
705 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
706 .timeRange(TimeRange.at(ts))
707 .ifEquals(VALUE)
708 .thenDelete(delete);
709 assertTrue(ok);
710 }
711 }
712
713 @Test
714 public void testCheckAndPutWithCompareOp() throws IOException {
715 final byte [] value1 = Bytes.toBytes("aaaa");
716 final byte [] value2 = Bytes.toBytes("bbbb");
717 final byte [] value3 = Bytes.toBytes("cccc");
718 final byte [] value4 = Bytes.toBytes("dddd");
719
720 try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
721
722 Put put2 = new Put(ROW);
723 put2.addColumn(FAMILY, QUALIFIER, value2);
724
725 Put put3 = new Put(ROW);
726 put3.addColumn(FAMILY, QUALIFIER, value3);
727
728 // row doesn't exist, so using "ifNotExists" should be considered "match".
729 boolean ok =
730 table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER).ifNotExists().thenPut(put2);
731 assertTrue(ok);
732
733 // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL
734 // turns out "match"
735 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
736 .ifMatches(CompareOperator.GREATER, value1).thenPut(put2);
737 assertFalse(ok);
738 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
739 .ifMatches(CompareOperator.EQUAL, value1).thenPut(put2);
740 assertFalse(ok);
741 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
742 .ifMatches(CompareOperator.GREATER_OR_EQUAL, value1).thenPut(put2);
743 assertFalse(ok);
744 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
745 .ifMatches(CompareOperator.LESS, value1).thenPut(put2);
746 assertTrue(ok);
747 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
748 .ifMatches(CompareOperator.LESS_OR_EQUAL, value1).thenPut(put2);
749 assertTrue(ok);
750 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
751 .ifMatches(CompareOperator.NOT_EQUAL, value1).thenPut(put3);
752 assertTrue(ok);
753
754 // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL
755 // turns out "match"
756 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
757 .ifMatches(CompareOperator.LESS, value4).thenPut(put3);
758 assertFalse(ok);
759 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
760 .ifMatches(CompareOperator.LESS_OR_EQUAL, value4).thenPut(put3);
761 assertFalse(ok);
762 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
763 .ifMatches(CompareOperator.EQUAL, value4).thenPut(put3);
764 assertFalse(ok);
765 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
766 .ifMatches(CompareOperator.GREATER, value4).thenPut(put3);
767 assertTrue(ok);
768 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
769 .ifMatches(CompareOperator.GREATER_OR_EQUAL, value4).thenPut(put3);
770 assertTrue(ok);
771 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
772 .ifMatches(CompareOperator.NOT_EQUAL, value4).thenPut(put2);
773 assertTrue(ok);
774
775 // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL
776 // turns out "match"
777 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
778 .ifMatches(CompareOperator.GREATER, value2).thenPut(put2);
779 assertFalse(ok);
780 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
781 .ifMatches(CompareOperator.NOT_EQUAL, value2).thenPut(put2);
782 assertFalse(ok);
783 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
784 .ifMatches(CompareOperator.LESS, value2).thenPut(put2);
785 assertFalse(ok);
786 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
787 .ifMatches(CompareOperator.GREATER_OR_EQUAL, value2).thenPut(put2);
788 assertTrue(ok);
789 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
790 .ifMatches(CompareOperator.LESS_OR_EQUAL, value2).thenPut(put2);
791 assertTrue(ok);
792 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
793 .ifMatches(CompareOperator.EQUAL, value2).thenPut(put3);
794 assertTrue(ok);
795 }
796 }
797
798 @Test
799 public void testCheckAndDelete() throws IOException {
800 final byte [] value1 = Bytes.toBytes("aaaa");
801
802 try (Table table = TEST_UTIL.createTable(name.getTableName(),
803 FAMILY)) {
804
805 Put put = new Put(ROW);
806 put.addColumn(FAMILY, QUALIFIER, value1);
807 table.put(put);
808
809 Delete delete = new Delete(ROW);
810 delete.addColumns(FAMILY, QUALIFIER);
811
812 boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
813 .ifEquals(value1).thenDelete(delete);
814 assertTrue(ok);
815 }
816 }
817
818 @Test
819 public void testCheckAndDeleteWithCompareOp() throws IOException {
820 final byte [] value1 = Bytes.toBytes("aaaa");
821 final byte [] value2 = Bytes.toBytes("bbbb");
822 final byte [] value3 = Bytes.toBytes("cccc");
823 final byte [] value4 = Bytes.toBytes("dddd");
824
825 try (Table table = TEST_UTIL.createTable(name.getTableName(),
826 FAMILY)) {
827
828 Put put2 = new Put(ROW);
829 put2.addColumn(FAMILY, QUALIFIER, value2);
830 table.put(put2);
831
832 Put put3 = new Put(ROW);
833 put3.addColumn(FAMILY, QUALIFIER, value3);
834
835 Delete delete = new Delete(ROW);
836 delete.addColumns(FAMILY, QUALIFIER);
837
838 // cell = "bbbb", using "aaaa" to compare only LESS/LESS_OR_EQUAL/NOT_EQUAL
839 // turns out "match"
840 boolean ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
841 .ifMatches(CompareOperator.GREATER, value1).thenDelete(delete);
842 assertFalse(ok);
843 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
844 .ifMatches(CompareOperator.EQUAL, value1).thenDelete(delete);
845 assertFalse(ok);
846 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
847 .ifMatches(CompareOperator.GREATER_OR_EQUAL, value1).thenDelete(delete);
848 assertFalse(ok);
849 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
850 .ifMatches(CompareOperator.LESS, value1).thenDelete(delete);
851 assertTrue(ok);
852 table.put(put2);
853 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
854 .ifMatches(CompareOperator.LESS_OR_EQUAL, value1).thenDelete(delete);
855 assertTrue(ok);
856 table.put(put2);
857 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
858 .ifMatches(CompareOperator.NOT_EQUAL, value1).thenDelete(delete);
859 assertTrue(ok);
860
861 // cell = "cccc", using "dddd" to compare only LARGER/LARGER_OR_EQUAL/NOT_EQUAL
862 // turns out "match"
863 table.put(put3);
864 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
865 .ifMatches(CompareOperator.LESS, value4).thenDelete(delete);
866 assertFalse(ok);
867 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
868 .ifMatches(CompareOperator.LESS_OR_EQUAL, value4).thenDelete(delete);
869 assertFalse(ok);
870 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
871 .ifMatches(CompareOperator.EQUAL, value4).thenDelete(delete);
872 assertFalse(ok);
873 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
874 .ifMatches(CompareOperator.GREATER, value4).thenDelete(delete);
875 assertTrue(ok);
876 table.put(put3);
877 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
878 .ifMatches(CompareOperator.GREATER_OR_EQUAL, value4).thenDelete(delete);
879 assertTrue(ok);
880 table.put(put3);
881 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
882 .ifMatches(CompareOperator.NOT_EQUAL, value4).thenDelete(delete);
883 assertTrue(ok);
884
885 // cell = "bbbb", using "bbbb" to compare only GREATER_OR_EQUAL/LESS_OR_EQUAL/EQUAL
886 // turns out "match"
887 table.put(put2);
888 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
889 .ifMatches(CompareOperator.GREATER, value2).thenDelete(delete);
890 assertFalse(ok);
891 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
892 .ifMatches(CompareOperator.NOT_EQUAL, value2).thenDelete(delete);
893 assertFalse(ok);
894 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
895 .ifMatches(CompareOperator.LESS, value2).thenDelete(delete);
896 assertFalse(ok);
897 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
898 .ifMatches(CompareOperator.GREATER_OR_EQUAL, value2).thenDelete(delete);
899 assertTrue(ok);
900 table.put(put2);
901 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
902 .ifMatches(CompareOperator.LESS_OR_EQUAL, value2).thenDelete(delete);
903 assertTrue(ok);
904 table.put(put2);
905 ok = table.checkAndMutate(ROW, FAMILY).qualifier(QUALIFIER)
906 .ifMatches(CompareOperator.EQUAL, value2).thenDelete(delete);
907 assertTrue(ok);
908 }
909 }
910
911 /**
912 * Test ScanMetrics
913 */
914 @Test
915 @SuppressWarnings({"unused", "checkstyle:EmptyBlock"})
916 public void testScanMetrics() throws Exception {
917 final TableName tableName = name.getTableName();
918
919 // Set up test table:
920 // Create table:
921 try (Table ht = TEST_UTIL.createMultiRegionTable(tableName, FAMILY)) {
922 int numOfRegions;
923 try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
924 numOfRegions = r.getStartKeys().length;
925 }
926 // Create 3 rows in the table, with rowkeys starting with "zzz*" so that
927 // scan are forced to hit all the regions.
928 Put put1 = new Put(Bytes.toBytes("zzz1"));
929 put1.addColumn(FAMILY, QUALIFIER, VALUE);
930 Put put2 = new Put(Bytes.toBytes("zzz2"));
931 put2.addColumn(FAMILY, QUALIFIER, VALUE);
932 Put put3 = new Put(Bytes.toBytes("zzz3"));
933 put3.addColumn(FAMILY, QUALIFIER, VALUE);
934 ht.put(Arrays.asList(put1, put2, put3));
935
936 Scan scan1 = new Scan();
937 int numRecords = 0;
938 try (ResultScanner scanner = ht.getScanner(scan1)) {
939 for (Result result : scanner) {
940 numRecords++;
941 }
942
943 LOG.info("test data has " + numRecords + " records.");
944
945 // by default, scan metrics collection is turned off
946 assertNull(scanner.getScanMetrics());
947 }
948
949 // turn on scan metrics
950 Scan scan2 = new Scan();
951 scan2.setScanMetricsEnabled(true);
952 scan2.setCaching(numRecords + 1);
953 try (ResultScanner scanner = ht.getScanner(scan2)) {
954 for (Result result : scanner.next(numRecords - 1)) {
955 }
956 scanner.close();
957 // closing the scanner will set the metrics.
958 assertNotNull(scanner.getScanMetrics());
959 }
960
961 // set caching to 1, because metrics are collected in each roundtrip only
962 scan2 = new Scan();
963 scan2.setScanMetricsEnabled(true);
964 scan2.setCaching(1);
965 try (ResultScanner scanner = ht.getScanner(scan2)) {
966 // per HBASE-5717, this should still collect even if you don't run all the way to
967 // the end of the scanner. So this is asking for 2 of the 3 rows we inserted.
968 for (Result result : scanner.next(numRecords - 1)) {
969 }
970 ScanMetrics scanMetrics = scanner.getScanMetrics();
971 assertEquals("Did not access all the regions in the table", numOfRegions,
972 scanMetrics.countOfRegions.get());
973 }
974
975 // check byte counters
976 scan2 = new Scan();
977 scan2.setScanMetricsEnabled(true);
978 scan2.setCaching(1);
979 try (ResultScanner scanner = ht.getScanner(scan2)) {
980 int numBytes = 0;
981 for (Result result : scanner.next(1)) {
982 for (Cell cell : result.listCells()) {
983 numBytes += PrivateCellUtil.estimatedSerializedSizeOf(cell);
984 }
985 }
986 scanner.close();
987 ScanMetrics scanMetrics = scanner.getScanMetrics();
988 assertEquals("Did not count the result bytes", numBytes,
989 scanMetrics.countOfBytesInResults.get());
990 }
991
992 // check byte counters on a small scan
993 scan2 = new Scan();
994 scan2.setScanMetricsEnabled(true);
995 scan2.setCaching(1);
996 scan2.setSmall(true);
997 try (ResultScanner scanner = ht.getScanner(scan2)) {
998 int numBytes = 0;
999 for (Result result : scanner.next(1)) {
1000 for (Cell cell : result.listCells()) {
1001 numBytes += PrivateCellUtil.estimatedSerializedSizeOf(cell);
1002 }
1003 }
1004 scanner.close();
1005 ScanMetrics scanMetrics = scanner.getScanMetrics();
1006 assertEquals("Did not count the result bytes", numBytes,
1007 scanMetrics.countOfBytesInResults.get());
1008 }
1009
1010 // now, test that the metrics are still collected even if you don't call close, but do
1011 // run past the end of all the records
1012 /** There seems to be a timing issue here. Comment out for now. Fix when time.
1013 Scan scanWithoutClose = new Scan();
1014 scanWithoutClose.setCaching(1);
1015 scanWithoutClose.setScanMetricsEnabled(true);
1016 ResultScanner scannerWithoutClose = ht.getScanner(scanWithoutClose);
1017 for (Result result : scannerWithoutClose.next(numRecords + 1)) {
1018 }
1019 ScanMetrics scanMetricsWithoutClose = getScanMetrics(scanWithoutClose);
1020 assertEquals("Did not access all the regions in the table", numOfRegions,
1021 scanMetricsWithoutClose.countOfRegions.get());
1022 */
1023
1024 // finally,
1025 // test that the metrics are collected correctly if you both run past all the records,
1026 // AND close the scanner
1027 Scan scanWithClose = new Scan();
1028 // make sure we can set caching up to the number of a scanned values
1029 scanWithClose.setCaching(numRecords);
1030 scanWithClose.setScanMetricsEnabled(true);
1031 try (ResultScanner scannerWithClose = ht.getScanner(scanWithClose)) {
1032 for (Result result : scannerWithClose.next(numRecords + 1)) {
1033 }
1034 scannerWithClose.close();
1035 ScanMetrics scanMetricsWithClose = scannerWithClose.getScanMetrics();
1036 assertEquals("Did not access all the regions in the table", numOfRegions,
1037 scanMetricsWithClose.countOfRegions.get());
1038 }
1039 }
1040 }
1041
1042 /**
1043 * Tests that cache on write works all the way up from the client-side.
1044 *
1045 * Performs inserts, flushes, and compactions, verifying changes in the block
1046 * cache along the way.
1047 */
1048 @Test
1049 public void testCacheOnWriteEvictOnClose() throws Exception {
1050 final TableName tableName = name.getTableName();
1051 byte [] data = Bytes.toBytes("data");
1052 try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) {
1053 try (RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
1054 // get the block cache and region
1055 String regionName = locator.getAllRegionLocations().get(0).getRegion().getEncodedName();
1056
1057 HRegion region = TEST_UTIL.getRSForFirstRegionInTable(tableName)
1058 .getRegion(regionName);
1059 HStore store = region.getStores().iterator().next();
1060 CacheConfig cacheConf = store.getCacheConfig();
1061 cacheConf.setCacheDataOnWrite(true);
1062 cacheConf.setEvictOnClose(true);
1063 BlockCache cache = cacheConf.getBlockCache().get();
1064
1065 // establish baseline stats
1066 long startBlockCount = cache.getBlockCount();
1067 long startBlockHits = cache.getStats().getHitCount();
1068 long startBlockMiss = cache.getStats().getMissCount();
1069
1070 // wait till baseline is stable, (minimal 500 ms)
1071 for (int i = 0; i < 5; i++) {
1072 Thread.sleep(100);
1073 if (startBlockCount != cache.getBlockCount()
1074 || startBlockHits != cache.getStats().getHitCount()
1075 || startBlockMiss != cache.getStats().getMissCount()) {
1076 startBlockCount = cache.getBlockCount();
1077 startBlockHits = cache.getStats().getHitCount();
1078 startBlockMiss = cache.getStats().getMissCount();
1079 i = -1;
1080 }
1081 }
1082
1083 // insert data
1084 Put put = new Put(ROW);
1085 put.addColumn(FAMILY, QUALIFIER, data);
1086 table.put(put);
1087 assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data));
1088
1089 // data was in memstore so don't expect any changes
1090 assertEquals(startBlockCount, cache.getBlockCount());
1091 assertEquals(startBlockHits, cache.getStats().getHitCount());
1092 assertEquals(startBlockMiss, cache.getStats().getMissCount());
1093
1094 // flush the data
1095 LOG.debug("Flushing cache");
1096 region.flush(true);
1097
1098 // expect two more blocks in cache - DATA and ROOT_INDEX
1099 // , no change in hits/misses
1100 long expectedBlockCount = startBlockCount + 2;
1101 long expectedBlockHits = startBlockHits;
1102 long expectedBlockMiss = startBlockMiss;
1103 assertEquals(expectedBlockCount, cache.getBlockCount());
1104 assertEquals(expectedBlockHits, cache.getStats().getHitCount());
1105 assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
1106 // read the data and expect same blocks, one new hit, no misses
1107 assertTrue(Bytes.equals(table.get(new Get(ROW)).value(), data));
1108 assertEquals(expectedBlockCount, cache.getBlockCount());
1109 assertEquals(++expectedBlockHits, cache.getStats().getHitCount());
1110 assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
1111 // insert a second column, read the row, no new blocks, one new hit
1112 byte[] QUALIFIER2 = Bytes.add(QUALIFIER, QUALIFIER);
1113 byte[] data2 = Bytes.add(data, data);
1114 put = new Put(ROW);
1115 put.addColumn(FAMILY, QUALIFIER2, data2);
1116 table.put(put);
1117 Result r = table.get(new Get(ROW));
1118 assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data));
1119 assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2));
1120 assertEquals(expectedBlockCount, cache.getBlockCount());
1121 assertEquals(++expectedBlockHits, cache.getStats().getHitCount());
1122 assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
1123 // flush, one new block
1124 System.out.println("Flushing cache");
1125 region.flush(true);
1126
1127 // + 1 for Index Block, +1 for data block
1128 expectedBlockCount += 2;
1129 assertEquals(expectedBlockCount, cache.getBlockCount());
1130 assertEquals(expectedBlockHits, cache.getStats().getHitCount());
1131 assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
1132 // compact, net minus two blocks, two hits, no misses
1133 System.out.println("Compacting");
1134 assertEquals(2, store.getStorefilesCount());
1135 store.triggerMajorCompaction();
1136 region.compact(true);
1137 store.closeAndArchiveCompactedFiles();
1138 waitForStoreFileCount(store, 1, 10000); // wait 10 seconds max
1139 assertEquals(1, store.getStorefilesCount());
1140 // evicted two data blocks and two index blocks and compaction does not cache new blocks
1141 expectedBlockCount = 0;
1142 assertEquals(expectedBlockCount, cache.getBlockCount());
1143 expectedBlockHits += 2;
1144 assertEquals(expectedBlockMiss, cache.getStats().getMissCount());
1145 assertEquals(expectedBlockHits, cache.getStats().getHitCount());
1146 // read the row, this should be a cache miss because we don't cache data
1147 // blocks on compaction
1148 r = table.get(new Get(ROW));
1149 assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER), data));
1150 assertTrue(Bytes.equals(r.getValue(FAMILY, QUALIFIER2), data2));
1151 expectedBlockCount += 1; // cached one data block
1152 assertEquals(expectedBlockCount, cache.getBlockCount());
1153 assertEquals(expectedBlockHits, cache.getStats().getHitCount());
1154 assertEquals(++expectedBlockMiss, cache.getStats().getMissCount());
1155 }
1156 }
1157 }
1158
1159 private void waitForStoreFileCount(HStore store, int count, int timeout)
1160 throws InterruptedException {
1161 long start = System.currentTimeMillis();
1162 while (start + timeout > System.currentTimeMillis() && store.getStorefilesCount() != count) {
1163 Thread.sleep(100);
1164 }
1165 System.out.println("start=" + start + ", now=" + System.currentTimeMillis() + ", cur=" +
1166 store.getStorefilesCount());
1167 assertEquals(count, store.getStorefilesCount());
1168 }
1169
1170 /**
1171 * Tests the non cached version of getRegionLocator by moving a region.
1172 */
1173 @Test
1174 public void testNonCachedGetRegionLocation() throws Exception {
1175 // Test Initialization.
1176 final TableName tableName = name.getTableName();
1177 byte [] family1 = Bytes.toBytes("f1");
1178 byte [] family2 = Bytes.toBytes("f2");
1179 try (Table ignored = TEST_UTIL.createTable(tableName, new byte[][] {family1, family2}, 10);
1180 Admin admin = TEST_UTIL.getAdmin();
1181 RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
1182 List<HRegionLocation> allRegionLocations = locator.getAllRegionLocations();
1183 assertEquals(1, allRegionLocations.size());
1184 RegionInfo regionInfo = allRegionLocations.get(0).getRegion();
1185 ServerName addrBefore = allRegionLocations.get(0).getServerName();
1186 // Verify region location before move.
1187 HRegionLocation addrCache = locator.getRegionLocation(regionInfo.getStartKey(), false);
1188 HRegionLocation addrNoCache = locator.getRegionLocation(regionInfo.getStartKey(), true);
1189
1190 assertEquals(addrBefore.getPort(), addrCache.getPort());
1191 assertEquals(addrBefore.getPort(), addrNoCache.getPort());
1192
1193
1194 // Make sure more than one server.
1195 if (TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size() <= 1) {
1196 TEST_UTIL.getMiniHBaseCluster().startRegionServer();
1197 Waiter.waitFor(TEST_UTIL.getConfiguration(), 30000, new Waiter.Predicate<Exception>() {
1198 @Override public boolean evaluate() throws Exception {
1199 return TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size() > 1;
1200 }
1201 });
1202 }
1203
1204 ServerName addrAfter = null;
1205 // Now move the region to a different server.
1206 for (int i = 0; i < TEST_UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size();
1207 i++) {
1208 HRegionServer regionServer = TEST_UTIL.getHBaseCluster().getRegionServer(i);
1209 ServerName addr = regionServer.getServerName();
1210 if (addr.getPort() != addrBefore.getPort()) {
1211 admin.move(regionInfo.getEncodedNameAsBytes(), addr);
1212 // Wait for the region to move.
1213 Thread.sleep(5000);
1214 addrAfter = addr;
1215 break;
1216 }
1217 }
1218
1219 // Verify the region was moved.
1220 addrCache = locator.getRegionLocation(regionInfo.getStartKey(), false);
1221 addrNoCache = locator.getRegionLocation(regionInfo.getStartKey(), true);
1222 assertNotNull(addrAfter);
1223 assertTrue(addrAfter.getPort() != addrCache.getPort());
1224 assertEquals(addrAfter.getPort(), addrNoCache.getPort());
1225 }
1226 }
1227
1228 /**
1229 * Tests getRegionsInRange by creating some regions over which a range of
1230 * keys spans; then changing the key range.
1231 */
1232 @Test
1233 public void testGetRegionsInRange() throws Exception {
1234 // Test Initialization.
1235 byte [] startKey = Bytes.toBytes("ddc");
1236 byte [] endKey = Bytes.toBytes("mmm");
1237 TableName tableName = name.getTableName();
1238 TEST_UTIL.createMultiRegionTable(tableName, new byte[][] { FAMILY }, 10);
1239
1240 int numOfRegions;
1241 try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
1242 numOfRegions = r.getStartKeys().length;
1243 }
1244 assertEquals(26, numOfRegions);
1245
1246 // Get the regions in this range
1247 List<HRegionLocation> regionsList = getRegionsInRange(tableName, startKey, endKey);
1248 assertEquals(10, regionsList.size());
1249
1250 // Change the start key
1251 startKey = Bytes.toBytes("fff");
1252 regionsList = getRegionsInRange(tableName, startKey, endKey);
1253 assertEquals(7, regionsList.size());
1254
1255 // Change the end key
1256 endKey = Bytes.toBytes("nnn");
1257 regionsList = getRegionsInRange(tableName, startKey, endKey);
1258 assertEquals(8, regionsList.size());
1259
1260 // Empty start key
1261 regionsList = getRegionsInRange(tableName, HConstants.EMPTY_START_ROW, endKey);
1262 assertEquals(13, regionsList.size());
1263
1264 // Empty end key
1265 regionsList = getRegionsInRange(tableName, startKey, HConstants.EMPTY_END_ROW);
1266 assertEquals(21, regionsList.size());
1267
1268 // Both start and end keys empty
1269 regionsList = getRegionsInRange(tableName, HConstants.EMPTY_START_ROW,
1270 HConstants.EMPTY_END_ROW);
1271 assertEquals(26, regionsList.size());
1272
1273 // Change the end key to somewhere in the last block
1274 endKey = Bytes.toBytes("zzz1");
1275 regionsList = getRegionsInRange(tableName, startKey, endKey);
1276 assertEquals(21, regionsList.size());
1277
1278 // Change the start key to somewhere in the first block
1279 startKey = Bytes.toBytes("aac");
1280 regionsList = getRegionsInRange(tableName, startKey, endKey);
1281 assertEquals(26, regionsList.size());
1282
1283 // Make start and end key the same
1284 startKey = Bytes.toBytes("ccc");
1285 endKey = Bytes.toBytes("ccc");
1286 regionsList = getRegionsInRange(tableName, startKey, endKey);
1287 assertEquals(1, regionsList.size());
1288 }
1289
1290 private List<HRegionLocation> getRegionsInRange(TableName tableName, byte[] startKey,
1291 byte[] endKey) throws IOException {
1292 List<HRegionLocation> regionsInRange = new ArrayList<>();
1293 byte[] currentKey = startKey;
1294 final boolean endKeyIsEndOfTable = Bytes.equals(endKey, HConstants.EMPTY_END_ROW);
1295 try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
1296 do {
1297 HRegionLocation regionLocation = r.getRegionLocation(currentKey);
1298 regionsInRange.add(regionLocation);
1299 currentKey = regionLocation.getRegion().getEndKey();
1300 } while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW)
1301 && (endKeyIsEndOfTable || Bytes.compareTo(currentKey, endKey) < 0));
1302 return regionsInRange;
1303 }
1304 }
1305
1306 @Test
1307 public void testJira6912() throws Exception {
1308 final TableName tableName = name.getTableName();
1309 try (Table foo = TEST_UTIL.createTable(tableName, new byte[][] {FAMILY}, 10)) {
1310
1311 List<Put> puts = new ArrayList<>();
1312 for (int i = 0; i != 100; i++) {
1313 Put put = new Put(Bytes.toBytes(i));
1314 put.addColumn(FAMILY, FAMILY, Bytes.toBytes(i));
1315 puts.add(put);
1316 }
1317 foo.put(puts);
1318 // If i comment this out it works
1319 TEST_UTIL.flush();
1320
1321 Scan scan = new Scan();
1322 scan.withStartRow(Bytes.toBytes(1));
1323 scan.setStopRow(Bytes.toBytes(3));
1324 scan.addColumn(FAMILY, FAMILY);
1325 scan.setFilter(new RowFilter(CompareOperator.NOT_EQUAL,
1326 new BinaryComparator(Bytes.toBytes(1))));
1327
1328 try (ResultScanner scanner = foo.getScanner(scan)) {
1329 Result[] bar = scanner.next(100);
1330 assertEquals(1, bar.length);
1331 }
1332 }
1333 }
1334
1335 @Test
1336 public void testScan_NullQualifier() throws IOException {
1337 try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
1338 Put put = new Put(ROW);
1339 put.addColumn(FAMILY, QUALIFIER, VALUE);
1340 table.put(put);
1341
1342 put = new Put(ROW);
1343 put.addColumn(FAMILY, null, VALUE);
1344 table.put(put);
1345 LOG.info("Row put");
1346
1347 Scan scan = new Scan();
1348 scan.addColumn(FAMILY, null);
1349
1350 ResultScanner scanner = table.getScanner(scan);
1351 Result[] bar = scanner.next(100);
1352 assertEquals(1, bar.length);
1353 assertEquals(1, bar[0].size());
1354
1355 scan = new Scan();
1356 scan.addFamily(FAMILY);
1357
1358 scanner = table.getScanner(scan);
1359 bar = scanner.next(100);
1360 assertEquals(1, bar.length);
1361 assertEquals(2, bar[0].size());
1362 }
1363 }
1364
1365 @Test
1366 public void testNegativeTimestamp() throws IOException {
1367 try (Table table = TEST_UTIL.createTable(name.getTableName(), FAMILY)) {
1368
1369 try {
1370 Put put = new Put(ROW, -1);
1371 put.addColumn(FAMILY, QUALIFIER, VALUE);
1372 table.put(put);
1373 fail("Negative timestamps should not have been allowed");
1374 } catch (IllegalArgumentException ex) {
1375 assertTrue(ex.getMessage().contains("negative"));
1376 }
1377
1378 try {
1379 Put put = new Put(ROW);
1380 long ts = -1;
1381 put.addColumn(FAMILY, QUALIFIER, ts, VALUE);
1382 table.put(put);
1383 fail("Negative timestamps should not have been allowed");
1384 } catch (IllegalArgumentException ex) {
1385 assertTrue(ex.getMessage().contains("negative"));
1386 }
1387
1388 try {
1389 Delete delete = new Delete(ROW, -1);
1390 table.delete(delete);
1391 fail("Negative timestamps should not have been allowed");
1392 } catch (IllegalArgumentException ex) {
1393 assertTrue(ex.getMessage().contains("negative"));
1394 }
1395
1396 try {
1397 Delete delete = new Delete(ROW);
1398 delete.addFamily(FAMILY, -1);
1399 table.delete(delete);
1400 fail("Negative timestamps should not have been allowed");
1401 } catch (IllegalArgumentException ex) {
1402 assertTrue(ex.getMessage().contains("negative"));
1403 }
1404
1405 try {
1406 Scan scan = new Scan();
1407 scan.setTimeRange(-1, 1);
1408 table.getScanner(scan);
1409 fail("Negative timestamps should not have been allowed");
1410 } catch (IllegalArgumentException ex) {
1411 assertTrue(ex.getMessage().contains("negative"));
1412 }
1413
1414 // KeyValue should allow negative timestamps for backwards compat. Otherwise, if the user
1415 // already has negative timestamps in cluster data, HBase won't be able to handle that
1416 try {
1417 new KeyValue(Bytes.toBytes(42), Bytes.toBytes(42), Bytes.toBytes(42), -1,
1418 Bytes.toBytes(42));
1419 } catch (IllegalArgumentException ex) {
1420 fail("KeyValue SHOULD allow negative timestamps");
1421 }
1422
1423 }
1424 }
1425
1426 @Test
1427 public void testRawScanRespectsVersions() throws Exception {
1428 final TableName tableName = name.getTableName();
1429 try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) {
1430 byte[] row = Bytes.toBytes("row");
1431
1432 // put the same row 4 times, with different values
1433 Put p = new Put(row);
1434 p.addColumn(FAMILY, QUALIFIER, 10, VALUE);
1435 table.put(p);
1436 p = new Put(row);
1437 p.addColumn(FAMILY, QUALIFIER, 11, ArrayUtils.add(VALUE, (byte) 2));
1438 table.put(p);
1439
1440 p = new Put(row);
1441 p.addColumn(FAMILY, QUALIFIER, 12, ArrayUtils.add(VALUE, (byte) 3));
1442 table.put(p);
1443
1444 p = new Put(row);
1445 p.addColumn(FAMILY, QUALIFIER, 13, ArrayUtils.add(VALUE, (byte) 4));
1446 table.put(p);
1447
1448 int versions = 4;
1449 Scan s = new Scan().withStartRow(row);
1450 // get all the possible versions
1451 s.readAllVersions();
1452 s.setRaw(true);
1453
1454 try (ResultScanner scanner = table.getScanner(s)) {
1455 int count = 0;
1456 for (Result r : scanner) {
1457 assertEquals("Found an unexpected number of results for the row!", versions,
1458 r.listCells().size());
1459 count++;
1460 }
1461 assertEquals("Found more than a single row when raw scanning the table with a single row!",
1462 1, count);
1463 }
1464
1465 // then if we decrease the number of versions, but keep the scan raw, we should see exactly
1466 // that number of versions
1467 versions = 2;
1468 s.readVersions(versions);
1469 try (ResultScanner scanner = table.getScanner(s)) {
1470 int count = 0;
1471 for (Result r : scanner) {
1472 assertEquals("Found an unexpected number of results for the row!", versions,
1473 r.listCells().size());
1474 count++;
1475 }
1476 assertEquals("Found more than a single row when raw scanning the table with a single row!",
1477 1, count);
1478 }
1479
1480 // finally, if we turn off raw scanning, but max out the number of versions, we should go back
1481 // to seeing just three
1482 versions = 3;
1483 s.readVersions(versions);
1484 try (ResultScanner scanner = table.getScanner(s)) {
1485 int count = 0;
1486 for (Result r : scanner) {
1487 assertEquals("Found an unexpected number of results for the row!", versions,
1488 r.listCells().size());
1489 count++;
1490 }
1491 assertEquals("Found more than a single row when raw scanning the table with a single row!",
1492 1, count);
1493 }
1494
1495 }
1496 TEST_UTIL.deleteTable(tableName);
1497 }
1498
1499 @Test
1500 public void testEmptyFilterList() throws Exception {
1501 // Test Initialization.
1502 final TableName tableName = name.getTableName();
1503 try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) {
1504
1505 // Insert one row each region
1506 Put put = new Put(Bytes.toBytes("row"));
1507 put.addColumn(FAMILY, QUALIFIER, VALUE);
1508 table.put(put);
1509
1510 List<Result> scanResults = new LinkedList<>();
1511 Scan scan = new Scan();
1512 scan.setFilter(new FilterList());
1513 try (ResultScanner scanner = table.getScanner(scan)) {
1514 for (Result r : scanner) {
1515 scanResults.add(r);
1516 }
1517 }
1518 assertEquals(1, scanResults.size());
1519 Get g = new Get(Bytes.toBytes("row"));
1520 g.setFilter(new FilterList());
1521 Result getResult = table.get(g);
1522 Result scanResult = scanResults.get(0);
1523 assertEquals(scanResult.rawCells().length, getResult.rawCells().length);
1524 for (int i = 0; i != scanResult.rawCells().length; ++i) {
1525 Cell scanCell = scanResult.rawCells()[i];
1526 Cell getCell = getResult.rawCells()[i];
1527 assertEquals(0, Bytes.compareTo(CellUtil.cloneRow(scanCell),
1528 CellUtil.cloneRow(getCell)));
1529 assertEquals(0, Bytes.compareTo(CellUtil.cloneFamily(scanCell),
1530 CellUtil.cloneFamily(getCell)));
1531 assertEquals(0, Bytes.compareTo(CellUtil.cloneQualifier(scanCell),
1532 CellUtil.cloneQualifier(getCell)));
1533 assertEquals(0, Bytes.compareTo(CellUtil.cloneValue(scanCell),
1534 CellUtil.cloneValue(getCell)));
1535 }
1536 }
1537 }
1538
1539 @Test
1540 public void testSmallScan() throws Exception {
1541 // Test Initialization.
1542 final TableName tableName = name.getTableName();
1543 try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) {
1544
1545 // Insert one row each region
1546 int insertNum = 10;
1547 for (int i = 0; i < 10; i++) {
1548 Put put = new Put(Bytes.toBytes("row" + String.format("%03d", i)));
1549 put.addColumn(FAMILY, QUALIFIER, VALUE);
1550 table.put(put);
1551 }
1552
1553 // normal scan
1554 try (ResultScanner scanner = table.getScanner(new Scan())) {
1555 int count = 0;
1556 for (Result r : scanner) {
1557 assertFalse(r.isEmpty());
1558 count++;
1559 }
1560 assertEquals(insertNum, count);
1561 }
1562
1563 // small scan
1564 Scan scan = new Scan().withStartRow(HConstants.EMPTY_START_ROW)
1565 .withStopRow(HConstants.EMPTY_END_ROW, true);
1566 scan.setSmall(true);
1567 scan.setCaching(2);
1568 try (ResultScanner scanner = table.getScanner(scan)) {
1569 int count = 0;
1570 for (Result r : scanner) {
1571 assertFalse(r.isEmpty());
1572 count++;
1573 }
1574 assertEquals(insertNum, count);
1575 }
1576 }
1577 }
1578
1579 @Test
1580 public void testSuperSimpleWithReverseScan() throws Exception {
1581 final TableName tableName = name.getTableName();
1582 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
1583 Put put = new Put(Bytes.toBytes("0-b11111-0000000000000000000"));
1584 put.addColumn(FAMILY, QUALIFIER, VALUE);
1585 ht.put(put);
1586 put = new Put(Bytes.toBytes("0-b11111-0000000000000000002"));
1587 put.addColumn(FAMILY, QUALIFIER, VALUE);
1588 ht.put(put);
1589 put = new Put(Bytes.toBytes("0-b11111-0000000000000000004"));
1590 put.addColumn(FAMILY, QUALIFIER, VALUE);
1591 ht.put(put);
1592 put = new Put(Bytes.toBytes("0-b11111-0000000000000000006"));
1593 put.addColumn(FAMILY, QUALIFIER, VALUE);
1594 ht.put(put);
1595 put = new Put(Bytes.toBytes("0-b11111-0000000000000000008"));
1596 put.addColumn(FAMILY, QUALIFIER, VALUE);
1597 ht.put(put);
1598 put = new Put(Bytes.toBytes("0-b22222-0000000000000000001"));
1599 put.addColumn(FAMILY, QUALIFIER, VALUE);
1600 ht.put(put);
1601 put = new Put(Bytes.toBytes("0-b22222-0000000000000000003"));
1602 put.addColumn(FAMILY, QUALIFIER, VALUE);
1603 ht.put(put);
1604 put = new Put(Bytes.toBytes("0-b22222-0000000000000000005"));
1605 put.addColumn(FAMILY, QUALIFIER, VALUE);
1606 ht.put(put);
1607 put = new Put(Bytes.toBytes("0-b22222-0000000000000000007"));
1608 put.addColumn(FAMILY, QUALIFIER, VALUE);
1609 ht.put(put);
1610 put = new Put(Bytes.toBytes("0-b22222-0000000000000000009"));
1611 put.addColumn(FAMILY, QUALIFIER, VALUE);
1612 ht.put(put);
1613 Scan scan = new Scan().withStartRow(Bytes.toBytes("0-b11111-9223372036854775807"))
1614 .withStopRow(Bytes.toBytes("0-b11111-0000000000000000000"), true);
1615 scan.setReversed(true);
1616 try (ResultScanner scanner = ht.getScanner(scan)) {
1617 Result result = scanner.next();
1618 assertTrue(Bytes.equals(result.getRow(),
1619 Bytes.toBytes("0-b11111-0000000000000000008")));
1620 }
1621 }
1622 }
1623
1624 @Test
1625 public void testFiltersWithReverseScan() throws Exception {
1626 final TableName tableName = name.getTableName();
1627 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
1628 byte[][] ROWS = makeN(ROW, 10);
1629 byte[][] QUALIFIERS = {Bytes.toBytes("col0-<d2v1>-<d3v2>"),
1630 Bytes.toBytes("col1-<d2v1>-<d3v2>"),
1631 Bytes.toBytes("col2-<d2v1>-<d3v2>"),
1632 Bytes.toBytes("col3-<d2v1>-<d3v2>"),
1633 Bytes.toBytes("col4-<d2v1>-<d3v2>"),
1634 Bytes.toBytes("col5-<d2v1>-<d3v2>"),
1635 Bytes.toBytes("col6-<d2v1>-<d3v2>"),
1636 Bytes.toBytes("col7-<d2v1>-<d3v2>"),
1637 Bytes.toBytes("col8-<d2v1>-<d3v2>"),
1638 Bytes.toBytes("col9-<d2v1>-<d3v2>")};
1639 for (int i = 0; i < 10; i++) {
1640 Put put = new Put(ROWS[i]);
1641 put.addColumn(FAMILY, QUALIFIERS[i], VALUE);
1642 ht.put(put);
1643 }
1644 Scan scan = new Scan();
1645 scan.setReversed(true);
1646 scan.addFamily(FAMILY);
1647 Filter filter = new QualifierFilter(CompareOperator.EQUAL,
1648 new RegexStringComparator("col[1-5]"));
1649 scan.setFilter(filter);
1650 try (ResultScanner scanner = ht.getScanner(scan)) {
1651 int expectedIndex = 5;
1652 for (Result result : scanner) {
1653 assertEquals(1, result.size());
1654 Cell c = result.rawCells()[0];
1655 assertTrue(Bytes.equals(c.getRowArray(), c.getRowOffset(), c.getRowLength(),
1656 ROWS[expectedIndex], 0, ROWS[expectedIndex].length));
1657 assertTrue(Bytes.equals(c.getQualifierArray(), c.getQualifierOffset(),
1658 c.getQualifierLength(), QUALIFIERS[expectedIndex], 0,
1659 QUALIFIERS[expectedIndex].length));
1660 expectedIndex--;
1661 }
1662 assertEquals(0, expectedIndex);
1663 }
1664 }
1665 }
1666
1667 @Test
1668 public void testKeyOnlyFilterWithReverseScan() throws Exception {
1669 final TableName tableName = name.getTableName();
1670 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
1671 byte[][] ROWS = makeN(ROW, 10);
1672 byte[][] QUALIFIERS = {Bytes.toBytes("col0-<d2v1>-<d3v2>"),
1673 Bytes.toBytes("col1-<d2v1>-<d3v2>"),
1674 Bytes.toBytes("col2-<d2v1>-<d3v2>"),
1675 Bytes.toBytes("col3-<d2v1>-<d3v2>"),
1676 Bytes.toBytes("col4-<d2v1>-<d3v2>"),
1677 Bytes.toBytes("col5-<d2v1>-<d3v2>"),
1678 Bytes.toBytes("col6-<d2v1>-<d3v2>"),
1679 Bytes.toBytes("col7-<d2v1>-<d3v2>"),
1680 Bytes.toBytes("col8-<d2v1>-<d3v2>"),
1681 Bytes.toBytes("col9-<d2v1>-<d3v2>")};
1682 for (int i = 0; i < 10; i++) {
1683 Put put = new Put(ROWS[i]);
1684 put.addColumn(FAMILY, QUALIFIERS[i], VALUE);
1685 ht.put(put);
1686 }
1687 Scan scan = new Scan();
1688 scan.setReversed(true);
1689 scan.addFamily(FAMILY);
1690 Filter filter = new KeyOnlyFilter(true);
1691 scan.setFilter(filter);
1692 try (ResultScanner ignored = ht.getScanner(scan)) {
1693 int count = 0;
1694 for (Result result : ht.getScanner(scan)) {
1695 assertEquals(1, result.size());
1696 assertEquals(Bytes.SIZEOF_INT, result.rawCells()[0].getValueLength());
1697 assertEquals(VALUE.length, Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0])));
1698 count++;
1699 }
1700 assertEquals(10, count);
1701 }
1702 }
1703 }
1704
1705 /**
1706 * Test simple table and non-existent row cases.
1707 */
1708 @Test
1709 public void testSimpleMissingWithReverseScan() throws Exception {
1710 final TableName tableName = name.getTableName();
1711 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
1712 byte[][] ROWS = makeN(ROW, 4);
1713
1714 // Try to get a row on an empty table
1715 Scan scan = new Scan();
1716 scan.setReversed(true);
1717 Result result = getSingleScanResult(ht, scan);
1718 assertNullResult(result);
1719
1720 scan = new Scan().withStartRow(ROWS[0]);
1721 scan.setReversed(true);
1722 result = getSingleScanResult(ht, scan);
1723 assertNullResult(result);
1724
1725 scan = new Scan().withStartRow(ROWS[0]).withStopRow(ROWS[1], true);
1726 scan.setReversed(true);
1727 result = getSingleScanResult(ht, scan);
1728 assertNullResult(result);
1729
1730 scan = new Scan();
1731 scan.setReversed(true);
1732 scan.addFamily(FAMILY);
1733 result = getSingleScanResult(ht, scan);
1734 assertNullResult(result);
1735
1736 scan = new Scan();
1737 scan.setReversed(true);
1738 scan.addColumn(FAMILY, QUALIFIER);
1739 result = getSingleScanResult(ht, scan);
1740 assertNullResult(result);
1741
1742 // Insert a row
1743
1744 Put put = new Put(ROWS[2]);
1745 put.addColumn(FAMILY, QUALIFIER, VALUE);
1746 ht.put(put);
1747
1748 // Make sure we can scan the row
1749 scan = new Scan();
1750 scan.setReversed(true);
1751 result = getSingleScanResult(ht, scan);
1752 assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
1753
1754 scan = new Scan().withStartRow(ROWS[3]).withStopRow(ROWS[0], true);
1755 scan.setReversed(true);
1756 result = getSingleScanResult(ht, scan);
1757 assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
1758
1759 scan = new Scan().withStartRow(ROWS[2]).withStopRow(ROWS[1], true);
1760 scan.setReversed(true);
1761 result = getSingleScanResult(ht, scan);
1762 assertSingleResult(result, ROWS[2], FAMILY, QUALIFIER, VALUE);
1763
1764 // Try to scan empty rows around it
1765 // Introduced MemStore#shouldSeekForReverseScan to fix the following
1766 scan = new Scan().withStartRow(ROWS[1]);
1767 scan.setReversed(true);
1768 result = getSingleScanResult(ht, scan);
1769 assertNullResult(result);
1770 }
1771 }
1772
1773 @Test
1774 public void testNullWithReverseScan() throws Exception {
1775 final TableName tableName = name.getTableName();
1776 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
1777 // Null qualifier (should work)
1778 Put put = new Put(ROW);
1779 put.addColumn(FAMILY, null, VALUE);
1780 ht.put(put);
1781 scanTestNull(ht, ROW, FAMILY, VALUE, true);
1782 Delete delete = new Delete(ROW);
1783 delete.addColumns(FAMILY, null);
1784 ht.delete(delete);
1785 }
1786
1787 // Use a new table
1788 try (Table ht =
1789 TEST_UTIL.createTable(TableName.valueOf(name.getTableName().toString() + "2"), FAMILY)) {
1790 // Empty qualifier, byte[0] instead of null (should work)
1791 Put put = new Put(ROW);
1792 put.addColumn(FAMILY, HConstants.EMPTY_BYTE_ARRAY, VALUE);
1793 ht.put(put);
1794 scanTestNull(ht, ROW, FAMILY, VALUE, true);
1795 TEST_UTIL.flush();
1796 scanTestNull(ht, ROW, FAMILY, VALUE, true);
1797 Delete delete = new Delete(ROW);
1798 delete.addColumns(FAMILY, HConstants.EMPTY_BYTE_ARRAY);
1799 ht.delete(delete);
1800 // Null value
1801 put = new Put(ROW);
1802 put.addColumn(FAMILY, QUALIFIER, null);
1803 ht.put(put);
1804 Scan scan = new Scan();
1805 scan.setReversed(true);
1806 scan.addColumn(FAMILY, QUALIFIER);
1807 Result result = getSingleScanResult(ht, scan);
1808 assertSingleResult(result, ROW, FAMILY, QUALIFIER, null);
1809 }
1810 }
1811
1812 @Test
1813 @SuppressWarnings("checkstyle:MethodLength")
1814 public void testDeletesWithReverseScan() throws Exception {
1815 final TableName tableName = name.getTableName();
1816 byte[][] ROWS = makeNAscii(ROW, 6);
1817 byte[][] FAMILIES = makeNAscii(FAMILY, 3);
1818 byte[][] VALUES = makeN(VALUE, 5);
1819 long[] ts = { 1000, 2000, 3000, 4000, 5000 };
1820 try (Table ht = TEST_UTIL.createTable(tableName, FAMILIES, 3)) {
1821
1822 Put put = new Put(ROW);
1823 put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
1824 put.addColumn(FAMILIES[0], QUALIFIER, ts[1], VALUES[1]);
1825 ht.put(put);
1826
1827 Delete delete = new Delete(ROW);
1828 delete.addFamily(FAMILIES[0], ts[0]);
1829 ht.delete(delete);
1830
1831 Scan scan = new Scan().withStartRow(ROW);
1832 scan.setReversed(true);
1833 scan.addFamily(FAMILIES[0]);
1834 scan.readVersions(Integer.MAX_VALUE);
1835 Result result = getSingleScanResult(ht, scan);
1836 assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[]{ts[1]},
1837 new byte[][]{VALUES[1]}, 0, 0);
1838
1839 // Test delete latest version
1840 put = new Put(ROW);
1841 put.addColumn(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
1842 put.addColumn(FAMILIES[0], QUALIFIER, ts[2], VALUES[2]);
1843 put.addColumn(FAMILIES[0], QUALIFIER, ts[3], VALUES[3]);
1844 put.addColumn(FAMILIES[0], null, ts[4], VALUES[4]);
1845 put.addColumn(FAMILIES[0], null, ts[2], VALUES[2]);
1846 put.addColumn(FAMILIES[0], null, ts[3], VALUES[3]);
1847 ht.put(put);
1848
1849 delete = new Delete(ROW);
1850 delete.addColumn(FAMILIES[0], QUALIFIER); // ts[4]
1851 ht.delete(delete);
1852
1853 scan = new Scan().withStartRow(ROW);
1854 scan.setReversed(true);
1855 scan.addColumn(FAMILIES[0], QUALIFIER);
1856 scan.readVersions(Integer.MAX_VALUE);
1857 result = getSingleScanResult(ht, scan);
1858 assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[]{ts[1],
1859 ts[2], ts[3]}, new byte[][]{VALUES[1], VALUES[2], VALUES[3]}, 0, 2);
1860
1861 // Test for HBASE-1847
1862 delete = new Delete(ROW);
1863 delete.addColumn(FAMILIES[0], null);
1864 ht.delete(delete);
1865
1866 // Cleanup null qualifier
1867 delete = new Delete(ROW);
1868 delete.addColumns(FAMILIES[0], null);
1869 ht.delete(delete);
1870
1871 // Expected client behavior might be that you can re-put deleted values
1872 // But alas, this is not to be. We can't put them back in either case.
1873
1874 put = new Put(ROW);
1875 put.addColumn(FAMILIES[0], QUALIFIER, ts[0], VALUES[0]);
1876 put.addColumn(FAMILIES[0], QUALIFIER, ts[4], VALUES[4]);
1877 ht.put(put);
1878
1879 // The Scanner returns the previous values, the expected-naive-unexpected
1880 // behavior
1881
1882 scan = new Scan().withStartRow(ROW);
1883 scan.setReversed(true);
1884 scan.addFamily(FAMILIES[0]);
1885 scan.readVersions(Integer.MAX_VALUE);
1886 result = getSingleScanResult(ht, scan);
1887 assertNResult(result, ROW, FAMILIES[0], QUALIFIER, new long[]{ts[1],
1888 ts[2], ts[3]}, new byte[][]{VALUES[1], VALUES[2], VALUES[3]}, 0, 2);
1889
1890 // Test deleting an entire family from one row but not the other various
1891 // ways
1892
1893 put = new Put(ROWS[0]);
1894 put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
1895 put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
1896 put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
1897 put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
1898 ht.put(put);
1899
1900 put = new Put(ROWS[1]);
1901 put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
1902 put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
1903 put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
1904 put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
1905 ht.put(put);
1906
1907 put = new Put(ROWS[2]);
1908 put.addColumn(FAMILIES[1], QUALIFIER, ts[0], VALUES[0]);
1909 put.addColumn(FAMILIES[1], QUALIFIER, ts[1], VALUES[1]);
1910 put.addColumn(FAMILIES[2], QUALIFIER, ts[2], VALUES[2]);
1911 put.addColumn(FAMILIES[2], QUALIFIER, ts[3], VALUES[3]);
1912 ht.put(put);
1913
1914 delete = new Delete(ROWS[0]);
1915 delete.addFamily(FAMILIES[2]);
1916 ht.delete(delete);
1917
1918 delete = new Delete(ROWS[1]);
1919 delete.addColumns(FAMILIES[1], QUALIFIER);
1920 ht.delete(delete);
1921
1922 delete = new Delete(ROWS[2]);
1923 delete.addColumn(FAMILIES[1], QUALIFIER);
1924 delete.addColumn(FAMILIES[1], QUALIFIER);
1925 delete.addColumn(FAMILIES[2], QUALIFIER);
1926 ht.delete(delete);
1927
1928 scan = new Scan().withStartRow(ROWS[0]);
1929 scan.setReversed(true);
1930 scan.addFamily(FAMILIES[1]);
1931 scan.addFamily(FAMILIES[2]);
1932 scan.readVersions(Integer.MAX_VALUE);
1933 result = getSingleScanResult(ht, scan);
1934 assertEquals("Expected 2 keys but received " + result.size(), 2, result.size());
1935 assertNResult(result, ROWS[0], FAMILIES[1], QUALIFIER, new long[]{ts[0],
1936 ts[1]}, new byte[][]{VALUES[0], VALUES[1]}, 0, 1);
1937
1938 scan = new Scan().withStartRow(ROWS[1]);
1939 scan.setReversed(true);
1940 scan.addFamily(FAMILIES[1]);
1941 scan.addFamily(FAMILIES[2]);
1942 scan.readVersions(Integer.MAX_VALUE);
1943 result = getSingleScanResult(ht, scan);
1944 assertEquals("Expected 2 keys but received " + result.size(), 2, result.size());
1945
1946 scan = new Scan().withStartRow(ROWS[2]);
1947 scan.setReversed(true);
1948 scan.addFamily(FAMILIES[1]);
1949 scan.addFamily(FAMILIES[2]);
1950 scan.readVersions(Integer.MAX_VALUE);
1951 result = getSingleScanResult(ht, scan);
1952 assertEquals(1, result.size());
1953 assertNResult(result, ROWS[2], FAMILIES[2], QUALIFIER,
1954 new long[]{ts[2]}, new byte[][]{VALUES[2]}, 0, 0);
1955
1956 // Test if we delete the family first in one row (HBASE-1541)
1957
1958 delete = new Delete(ROWS[3]);
1959 delete.addFamily(FAMILIES[1]);
1960 ht.delete(delete);
1961
1962 put = new Put(ROWS[3]);
1963 put.addColumn(FAMILIES[2], QUALIFIER, VALUES[0]);
1964 ht.put(put);
1965
1966 put = new Put(ROWS[4]);
1967 put.addColumn(FAMILIES[1], QUALIFIER, VALUES[1]);
1968 put.addColumn(FAMILIES[2], QUALIFIER, VALUES[2]);
1969 ht.put(put);
1970
1971 scan = new Scan().withStartRow(ROWS[4]);
1972 scan.setReversed(true);
1973 scan.addFamily(FAMILIES[1]);
1974 scan.addFamily(FAMILIES[2]);
1975 scan.readVersions(Integer.MAX_VALUE);
1976 ResultScanner scanner = ht.getScanner(scan);
1977 result = scanner.next();
1978 assertEquals("Expected 2 keys but received " + result.size(), 2, result.size());
1979 assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[4]));
1980 assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[1]), ROWS[4]));
1981 assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[1]));
1982 assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[1]), VALUES[2]));
1983 result = scanner.next();
1984 assertEquals("Expected 1 key but received " + result.size(), 1, result.size());
1985 assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[3]));
1986 assertTrue(Bytes.equals(CellUtil.cloneValue(result.rawCells()[0]), VALUES[0]));
1987 scanner.close();
1988 }
1989 }
1990
1991 /**
1992 * Tests reversed scan under multi regions
1993 */
1994 @Test
1995 public void testReversedScanUnderMultiRegions() throws Exception {
1996 // Test Initialization.
1997 final TableName tableName = name.getTableName();
1998 byte[] maxByteArray = ConnectionUtils.MAX_BYTE_ARRAY;
1999 byte[][] splitRows = new byte[][] { Bytes.toBytes("005"),
2000 Bytes.add(Bytes.toBytes("005"), Bytes.multiple(maxByteArray, 16)),
2001 Bytes.toBytes("006"),
2002 Bytes.add(Bytes.toBytes("006"), Bytes.multiple(maxByteArray, 8)),
2003 Bytes.toBytes("007"),
2004 Bytes.add(Bytes.toBytes("007"), Bytes.multiple(maxByteArray, 4)),
2005 Bytes.toBytes("008"), Bytes.multiple(maxByteArray, 2) };
2006 try (Table table = TEST_UTIL.createTable(tableName, FAMILY, splitRows)) {
2007 TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
2008
2009 try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
2010 assertEquals(splitRows.length + 1, l.getAllRegionLocations().size());
2011 }
2012 // Insert one row each region
2013 int insertNum = splitRows.length;
2014 for (byte[] splitRow : splitRows) {
2015 Put put = new Put(splitRow);
2016 put.addColumn(FAMILY, QUALIFIER, VALUE);
2017 table.put(put);
2018 }
2019
2020 // scan forward
2021 try (ResultScanner scanner = table.getScanner(new Scan())) {
2022 int count = 0;
2023 for (Result r : scanner) {
2024 assertFalse(r.isEmpty());
2025 count++;
2026 }
2027 assertEquals(insertNum, count);
2028 }
2029
2030 // scan backward
2031 Scan scan = new Scan();
2032 scan.setReversed(true);
2033 try (ResultScanner scanner = table.getScanner(scan)) {
2034 int count = 0;
2035 byte[] lastRow = null;
2036 for (Result r : scanner) {
2037 assertFalse(r.isEmpty());
2038 count++;
2039 byte[] thisRow = r.getRow();
2040 if (lastRow != null) {
2041 assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
2042 + ",this row=" + Bytes.toString(thisRow),
2043 Bytes.compareTo(thisRow, lastRow) < 0);
2044 }
2045 lastRow = thisRow;
2046 }
2047 assertEquals(insertNum, count);
2048 }
2049 }
2050 }
2051
2052 /**
2053 * Tests reversed scan under multi regions
2054 */
2055 @Test
2056 public void testSmallReversedScanUnderMultiRegions() throws Exception {
2057 // Test Initialization.
2058 final TableName tableName = name.getTableName();
2059 byte[][] splitRows = new byte[][]{
2060 Bytes.toBytes("000"), Bytes.toBytes("002"), Bytes.toBytes("004"),
2061 Bytes.toBytes("006"), Bytes.toBytes("008"), Bytes.toBytes("010")};
2062 try (Table table = TEST_UTIL.createTable(tableName, FAMILY, splitRows)) {
2063 TEST_UTIL.waitUntilAllRegionsAssigned(table.getName());
2064
2065 try (RegionLocator l = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
2066 assertEquals(splitRows.length + 1, l.getAllRegionLocations().size());
2067 }
2068 for (byte[] splitRow : splitRows) {
2069 Put put = new Put(splitRow);
2070 put.addColumn(FAMILY, QUALIFIER, VALUE);
2071 table.put(put);
2072
2073 byte[] nextRow = Bytes.copy(splitRow);
2074 nextRow[nextRow.length - 1]++;
2075
2076 put = new Put(nextRow);
2077 put.addColumn(FAMILY, QUALIFIER, VALUE);
2078 table.put(put);
2079 }
2080
2081 // scan forward
2082 try (ResultScanner scanner = table.getScanner(new Scan())) {
2083 int count = 0;
2084 for (Result r : scanner) {
2085 assertTrue(!r.isEmpty());
2086 count++;
2087 }
2088 assertEquals(12, count);
2089 }
2090
2091 reverseScanTest(table, false);
2092 reverseScanTest(table, true);
2093 }
2094 }
2095
2096 private void reverseScanTest(Table table, boolean small) throws IOException {
2097 // scan backward
2098 Scan scan = new Scan();
2099 scan.setReversed(true);
2100 try (ResultScanner scanner = table.getScanner(scan)) {
2101 int count = 0;
2102 byte[] lastRow = null;
2103 for (Result r : scanner) {
2104 assertTrue(!r.isEmpty());
2105 count++;
2106 byte[] thisRow = r.getRow();
2107 if (lastRow != null) {
2108 assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
2109 + ",this row=" + Bytes.toString(thisRow),
2110 Bytes.compareTo(thisRow, lastRow) < 0);
2111 }
2112 lastRow = thisRow;
2113 }
2114 assertEquals(12, count);
2115 }
2116
2117 scan = new Scan();
2118 scan.setSmall(small);
2119 scan.setReversed(true);
2120 scan.withStartRow(Bytes.toBytes("002"));
2121 try (ResultScanner scanner = table.getScanner(scan)) {
2122 int count = 0;
2123 byte[] lastRow = null;
2124 for (Result r : scanner) {
2125 assertTrue(!r.isEmpty());
2126 count++;
2127 byte[] thisRow = r.getRow();
2128 if (lastRow != null) {
2129 assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
2130 + ",this row=" + Bytes.toString(thisRow),
2131 Bytes.compareTo(thisRow, lastRow) < 0);
2132 }
2133 lastRow = thisRow;
2134 }
2135 assertEquals(3, count); // 000 001 002
2136 }
2137
2138 scan = new Scan();
2139 scan.setSmall(small);
2140 scan.setReversed(true);
2141 scan.withStartRow(Bytes.toBytes("002"));
2142 scan.setStopRow(Bytes.toBytes("000"));
2143 try (ResultScanner scanner = table.getScanner(scan)) {
2144 int count = 0;
2145 byte[] lastRow = null;
2146 for (Result r : scanner) {
2147 assertFalse(r.isEmpty());
2148 count++;
2149 byte[] thisRow = r.getRow();
2150 if (lastRow != null) {
2151 assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
2152 + ",this row=" + Bytes.toString(thisRow),
2153 Bytes.compareTo(thisRow, lastRow) < 0);
2154 }
2155 lastRow = thisRow;
2156 }
2157 assertEquals(2, count); // 001 002
2158 }
2159
2160 scan = new Scan();
2161 scan.setSmall(small);
2162 scan.setReversed(true);
2163 scan.withStartRow(Bytes.toBytes("001"));
2164 try (ResultScanner scanner = table.getScanner(scan)) {
2165 int count = 0;
2166 byte[] lastRow = null;
2167 for (Result r : scanner) {
2168 assertFalse(r.isEmpty());
2169 count++;
2170 byte[] thisRow = r.getRow();
2171 if (lastRow != null) {
2172 assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
2173 + ",this row=" + Bytes.toString(thisRow),
2174 Bytes.compareTo(thisRow, lastRow) < 0);
2175 }
2176 lastRow = thisRow;
2177 }
2178 assertEquals(2, count); // 000 001
2179 }
2180
2181 scan = new Scan();
2182 scan.setSmall(small);
2183 scan.setReversed(true);
2184 scan.withStartRow(Bytes.toBytes("000"));
2185 try (ResultScanner scanner = table.getScanner(scan)) {
2186 int count = 0;
2187 byte[] lastRow = null;
2188 for (Result r : scanner) {
2189 assertFalse(r.isEmpty());
2190 count++;
2191 byte[] thisRow = r.getRow();
2192 if (lastRow != null) {
2193 assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
2194 + ",this row=" + Bytes.toString(thisRow),
2195 Bytes.compareTo(thisRow, lastRow) < 0);
2196 }
2197 lastRow = thisRow;
2198 }
2199 assertEquals(1, count); // 000
2200 }
2201
2202 scan = new Scan();
2203 scan.setSmall(small);
2204 scan.setReversed(true);
2205 scan.withStartRow(Bytes.toBytes("006"));
2206 scan.setStopRow(Bytes.toBytes("002"));
2207 try (ResultScanner scanner = table.getScanner(scan)) {
2208 int count = 0;
2209 byte[] lastRow = null;
2210 for (Result r : scanner) {
2211 assertFalse(r.isEmpty());
2212 count++;
2213 byte[] thisRow = r.getRow();
2214 if (lastRow != null) {
2215 assertTrue("Error scan order, last row= " + Bytes.toString(lastRow)
2216 + ",this row=" + Bytes.toString(thisRow),
2217 Bytes.compareTo(thisRow, lastRow) < 0);
2218 }
2219 lastRow = thisRow;
2220 }
2221 assertEquals(4, count); // 003 004 005 006
2222 }
2223 }
2224
2225 @Test
2226 public void testFilterAllRecords() throws IOException {
2227 Scan scan = new Scan();
2228 scan.setBatch(1);
2229 scan.setCaching(1);
2230 // Filter out any records
2231 scan.setFilter(new FilterList(new FirstKeyOnlyFilter(), new InclusiveStopFilter(new byte[0])));
2232 try (Table table = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME)) {
2233 try (ResultScanner s = table.getScanner(scan)) {
2234 assertNull(s.next());
2235 }
2236 }
2237 }
2238
2239 @Test
2240 public void testCellSizeLimit() throws IOException {
2241 final TableName tableName = TableName.valueOf("testCellSizeLimit");
2242 TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
2243 new TableDescriptorBuilder.ModifyableTableDescriptor(tableName)
2244 .setValue(HRegion.HBASE_MAX_CELL_SIZE_KEY, Integer.toString(10 * 1024));
2245 ColumnFamilyDescriptor familyDescriptor =
2246 new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(FAMILY);
2247
2248 tableDescriptor.setColumnFamily(familyDescriptor);
2249 try (Admin admin = TEST_UTIL.getAdmin()) {
2250 admin.createTable(tableDescriptor);
2251 }
2252 // Will succeed
2253 try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
2254 t.put(new Put(ROW).addColumn(FAMILY, QUALIFIER, Bytes.toBytes(0L)));
2255 t.increment(new Increment(ROW).addColumn(FAMILY, QUALIFIER, 1L));
2256 }
2257 // Will succeed
2258 try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
2259 t.put(new Put(ROW).addColumn(FAMILY, QUALIFIER, new byte[9*1024]));
2260 }
2261 // Will fail
2262 try (Table t = TEST_UTIL.getConnection().getTable(tableName)) {
2263 try {
2264 t.put(new Put(ROW).addColumn(FAMILY, QUALIFIER, new byte[10 * 1024]));
2265 fail("Oversize cell failed to trigger exception");
2266 } catch (IOException e) {
2267 // expected
2268 }
2269 try {
2270 t.append(new Append(ROW).addColumn(FAMILY, QUALIFIER, new byte[2 * 1024]));
2271 fail("Oversize cell failed to trigger exception");
2272 } catch (IOException e) {
2273 // expected
2274 }
2275 }
2276 }
2277
2278 @Test
2279 public void testDeleteSpecifiedVersionOfSpecifiedColumn() throws Exception {
2280 final TableName tableName = name.getTableName();
2281
2282 byte[][] VALUES = makeN(VALUE, 5);
2283 long[] ts = {1000, 2000, 3000, 4000, 5000};
2284
2285 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 5)) {
2286
2287 Put put = new Put(ROW);
2288 // Put version 1000,2000,3000,4000 of column FAMILY:QUALIFIER
2289 for (int t = 0; t < 4; t++) {
2290 put.addColumn(FAMILY, QUALIFIER, ts[t], VALUES[t]);
2291 }
2292 ht.put(put);
2293
2294 Delete delete = new Delete(ROW);
2295 // Delete version 3000 of column FAMILY:QUALIFIER
2296 delete.addColumn(FAMILY, QUALIFIER, ts[2]);
2297 ht.delete(delete);
2298
2299 Get get = new Get(ROW);
2300 get.addColumn(FAMILY, QUALIFIER);
2301 get.readVersions(Integer.MAX_VALUE);
2302 Result result = ht.get(get);
2303 // verify version 1000,2000,4000 remains for column FAMILY:QUALIFIER
2304 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[0], ts[1], ts[3]}, new byte[][]{
2305 VALUES[0], VALUES[1], VALUES[3]}, 0, 2);
2306
2307 delete = new Delete(ROW);
2308 // Delete a version 5000 of column FAMILY:QUALIFIER which didn't exist
2309 delete.addColumn(FAMILY, QUALIFIER, ts[4]);
2310 ht.delete(delete);
2311
2312 get = new Get(ROW);
2313 get.addColumn(FAMILY, QUALIFIER);
2314 get.readVersions(Integer.MAX_VALUE);
2315 result = ht.get(get);
2316 // verify version 1000,2000,4000 remains for column FAMILY:QUALIFIER
2317 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[0], ts[1], ts[3]}, new byte[][]{
2318 VALUES[0], VALUES[1], VALUES[3]}, 0, 2);
2319 }
2320 }
2321
2322 @Test
2323 public void testDeleteLatestVersionOfSpecifiedColumn() throws Exception {
2324 final TableName tableName = name.getTableName();
2325 byte[][] VALUES = makeN(VALUE, 5);
2326 long[] ts = {1000, 2000, 3000, 4000, 5000};
2327 try (Table ht = TEST_UTIL.createTable(tableName, FAMILY, 5)) {
2328 Put put = new Put(ROW);
2329 // Put version 1000,2000,3000,4000 of column FAMILY:QUALIFIER
2330 for (int t = 0; t < 4; t++) {
2331 put.addColumn(FAMILY, QUALIFIER, ts[t], VALUES[t]);
2332 }
2333 ht.put(put);
2334
2335 Delete delete = new Delete(ROW);
2336 // Delete latest version of column FAMILY:QUALIFIER
2337 delete.addColumn(FAMILY, QUALIFIER);
2338 ht.delete(delete);
2339
2340 Get get = new Get(ROW);
2341 get.addColumn(FAMILY, QUALIFIER);
2342 get.readVersions(Integer.MAX_VALUE);
2343 Result result = ht.get(get);
2344 // verify version 1000,2000,3000 remains for column FAMILY:QUALIFIER
2345 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[0], ts[1], ts[2]}, new byte[][]{
2346 VALUES[0], VALUES[1], VALUES[2]}, 0, 2);
2347
2348 delete = new Delete(ROW);
2349 // Delete two latest version of column FAMILY:QUALIFIER
2350 delete.addColumn(FAMILY, QUALIFIER);
2351 delete.addColumn(FAMILY, QUALIFIER);
2352 ht.delete(delete);
2353
2354 get = new Get(ROW);
2355 get.addColumn(FAMILY, QUALIFIER);
2356 get.readVersions(Integer.MAX_VALUE);
2357 result = ht.get(get);
2358 // verify version 1000 remains for column FAMILY:QUALIFIER
2359 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[0]}, new byte[][]{VALUES[0]},
2360 0, 0);
2361
2362 put = new Put(ROW);
2363 // Put a version 5000 of column FAMILY:QUALIFIER
2364 put.addColumn(FAMILY, QUALIFIER, ts[4], VALUES[4]);
2365 ht.put(put);
2366
2367 get = new Get(ROW);
2368 get.addColumn(FAMILY, QUALIFIER);
2369 get.readVersions(Integer.MAX_VALUE);
2370 result = ht.get(get);
2371 // verify version 1000,5000 remains for column FAMILY:QUALIFIER
2372 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[0], ts[4]}, new byte[][]{
2373 VALUES[0], VALUES[4]}, 0, 1);
2374 }
2375 }
2376
2377 /**
2378 * Test for HBASE-17125
2379 */
2380 @Test
2381 public void testReadWithFilter() throws Exception {
2382 final TableName tableName = name.getTableName();
2383 try (Table table = TEST_UTIL.createTable(tableName, FAMILY, 3)) {
2384
2385 byte[] VALUEA = Bytes.toBytes("value-a");
2386 byte[] VALUEB = Bytes.toBytes("value-b");
2387 long[] ts = {1000, 2000, 3000, 4000};
2388
2389 Put put = new Put(ROW);
2390 // Put version 1000,2000,3000,4000 of column FAMILY:QUALIFIER
2391 for (int t = 0; t <= 3; t++) {
2392 if (t <= 1) {
2393 put.addColumn(FAMILY, QUALIFIER, ts[t], VALUEA);
2394 } else {
2395 put.addColumn(FAMILY, QUALIFIER, ts[t], VALUEB);
2396 }
2397 }
2398 table.put(put);
2399
2400 Scan scan =
2401 new Scan().setFilter(new ValueFilter(CompareOperator.EQUAL,
2402 new SubstringComparator("value-a")))
2403 .readVersions(3);
2404 ResultScanner scanner = table.getScanner(scan);
2405 Result result = scanner.next();
2406 // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
2407 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0,
2408 0);
2409
2410 Get get =
2411 new Get(ROW)
2412 .setFilter(new ValueFilter(CompareOperator.EQUAL,
2413 new SubstringComparator("value-a")))
2414 .readVersions(3);
2415 result = table.get(get);
2416 // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
2417 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0,
2418 0);
2419
2420 // Test with max versions 1, it should still read ts[1]
2421 scan =
2422 new Scan().setFilter(new ValueFilter(CompareOperator.EQUAL,
2423 new SubstringComparator("value-a")))
2424 .readVersions(1);
2425 scanner = table.getScanner(scan);
2426 result = scanner.next();
2427 // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
2428 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0,
2429 0);
2430
2431 // Test with max versions 1, it should still read ts[1]
2432 get =
2433 new Get(ROW)
2434 .setFilter(new ValueFilter(CompareOperator.EQUAL,
2435 new SubstringComparator("value-a")))
2436 .readVersions(1);
2437 result = table.get(get);
2438 // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
2439 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0,
2440 0);
2441
2442 // Test with max versions 5, it should still read ts[1]
2443 scan =
2444 new Scan().setFilter(new ValueFilter(CompareOperator.EQUAL,
2445 new SubstringComparator("value-a")))
2446 .readVersions(5);
2447 scanner = table.getScanner(scan);
2448 result = scanner.next();
2449 // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
2450 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0,
2451 0);
2452
2453 // Test with max versions 5, it should still read ts[1]
2454 get =
2455 new Get(ROW)
2456 .setFilter(new ValueFilter(CompareOperator.EQUAL,
2457 new SubstringComparator("value-a")))
2458 .readVersions(5);
2459 result = table.get(get);
2460 // ts[0] has gone from user view. Only read ts[2] which value is less or equal to 3
2461 assertNResult(result, ROW, FAMILY, QUALIFIER, new long[]{ts[1]}, new byte[][]{VALUEA}, 0,
2462 0);
2463 }
2464 }
2465
2466 @Test
2467 public void testCellUtilTypeMethods() throws IOException {
2468 final TableName tableName = name.getTableName();
2469 try (Table table = TEST_UTIL.createTable(tableName, FAMILY)) {
2470
2471 final byte[] row = Bytes.toBytes("p");
2472 Put p = new Put(row);
2473 p.addColumn(FAMILY, QUALIFIER, VALUE);
2474 table.put(p);
2475
2476 try (ResultScanner scanner = table.getScanner(new Scan())) {
2477 Result result = scanner.next();
2478 assertNotNull(result);
2479 CellScanner cs = result.cellScanner();
2480 assertTrue(cs.advance());
2481 Cell c = cs.current();
2482 assertTrue(CellUtil.isPut(c));
2483 assertFalse(CellUtil.isDelete(c));
2484 assertFalse(cs.advance());
2485 assertNull(scanner.next());
2486 }
2487
2488 Delete d = new Delete(row);
2489 d.addColumn(FAMILY, QUALIFIER);
2490 table.delete(d);
2491
2492 Scan scan = new Scan();
2493 scan.setRaw(true);
2494 try (ResultScanner scanner = table.getScanner(scan)) {
2495 Result result = scanner.next();
2496 assertNotNull(result);
2497 CellScanner cs = result.cellScanner();
2498 assertTrue(cs.advance());
2499
2500 // First cell should be the delete (masking the Put)
2501 Cell c = cs.current();
2502 assertTrue("Cell should be a Delete: " + c, CellUtil.isDelete(c));
2503 assertFalse("Cell should not be a Put: " + c, CellUtil.isPut(c));
2504
2505 // Second cell should be the original Put
2506 assertTrue(cs.advance());
2507 c = cs.current();
2508 assertFalse("Cell should not be a Delete: " + c, CellUtil.isDelete(c));
2509 assertTrue("Cell should be a Put: " + c, CellUtil.isPut(c));
2510
2511 // No more cells in this row
2512 assertFalse(cs.advance());
2513
2514 // No more results in this scan
2515 assertNull(scanner.next());
2516 }
2517 }
2518 }
2519
2520 @Test(expected = DoNotRetryIOException.class)
2521 public void testCreateTableWithZeroRegionReplicas() throws Exception {
2522 TableName tableName = name.getTableName();
2523 TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
2524 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("cf")))
2525 .setRegionReplication(0)
2526 .build();
2527
2528 TEST_UTIL.getAdmin().createTable(desc);
2529 }
2530
2531 @Test(expected = DoNotRetryIOException.class)
2532 public void testModifyTableWithZeroRegionReplicas() throws Exception {
2533 TableName tableName = name.getTableName();
2534 TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
2535 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(Bytes.toBytes("cf")))
2536 .build();
2537
2538 TEST_UTIL.getAdmin().createTable(desc);
2539 TableDescriptor newDesc = TableDescriptorBuilder.newBuilder(desc)
2540 .setRegionReplication(0)
2541 .build();
2542
2543 TEST_UTIL.getAdmin().modifyTable(newDesc);
2544 }
2545
2546 @Test(timeout = 60000)
2547 public void testModifyTableWithMemstoreData() throws Exception {
2548 TableName tableName = name.getTableName();
2549 createTableAndValidateTableSchemaModification(tableName, true);
2550 }
2551
2552 @Test(timeout = 60000)
2553 public void testDeleteCFWithMemstoreData() throws Exception {
2554 TableName tableName = name.getTableName();
2555 createTableAndValidateTableSchemaModification(tableName, false);
2556 }
2557
2558 /**
2559 * Create table and validate online schema modification
2560 * @param tableName Table name
2561 * @param modifyTable Modify table if true otherwise delete column family
2562 * @throws IOException in case of failures
2563 */
2564 private void createTableAndValidateTableSchemaModification(TableName tableName,
2565 boolean modifyTable) throws Exception {
2566 Admin admin = TEST_UTIL.getAdmin();
2567 // Create table with two Cfs
2568 byte[] cf1 = Bytes.toBytes("cf1");
2569 byte[] cf2 = Bytes.toBytes("cf2");
2570 TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName)
2571 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf1))
2572 .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf2)).build();
2573 admin.createTable(tableDesc);
2574
2575 Table t = TEST_UTIL.getConnection().getTable(tableName);
2576 // Insert few records and flush the table
2577 t.put(new Put(ROW).addColumn(cf1, QUALIFIER, Bytes.toBytes("val1")));
2578 t.put(new Put(ROW).addColumn(cf2, QUALIFIER, Bytes.toBytes("val2")));
2579 admin.flush(tableName);
2580 Path tableDir = FSUtils.getTableDir(TEST_UTIL.getDefaultRootDirPath(), tableName);
2581 List<Path> regionDirs = FSUtils.getRegionDirs(TEST_UTIL.getTestFileSystem(), tableDir);
2582 assertEquals(1, regionDirs.size());
2583 List<Path> familyDirs = FSUtils.getFamilyDirs(TEST_UTIL.getTestFileSystem(), regionDirs.get(0));
2584 assertEquals(2, familyDirs.size());
2585
2586 // Insert record but dont flush the table
2587 t.put(new Put(ROW).addColumn(cf1, QUALIFIER, Bytes.toBytes("val2")));
2588 t.put(new Put(ROW).addColumn(cf2, QUALIFIER, Bytes.toBytes("val2")));
2589
2590 if (modifyTable) {
2591 tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).removeColumnFamily(cf2).build();
2592 admin.modifyTable(tableDesc);
2593 } else {
2594 admin.deleteColumnFamily(tableName, cf2);
2595 }
2596 // After table modification or delete family there should be only one CF in FS
2597 familyDirs = FSUtils.getFamilyDirs(TEST_UTIL.getTestFileSystem(), regionDirs.get(0));
2598 assertEquals("CF dir count should be 1, but was " + familyDirs.size(), 1, familyDirs.size());
2599 }
2600 }
HBase