hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.regionserver;
  19
  20 import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
  21 import static org.junit.Assert.assertEquals;
  22 import static org.junit.Assert.assertTrue;
  23 import java.util.ArrayList;
  24 import java.util.List;
  25 import org.apache.hadoop.hbase.Cell;
  26 import org.apache.hadoop.hbase.CellUtil;
  27 import org.apache.hadoop.hbase.HBaseClassTestRule;
  28 import org.apache.hadoop.hbase.HBaseTestingUtility;
  29 import org.apache.hadoop.hbase.KeepDeletedCells;
  30 import org.apache.hadoop.hbase.TableName;
  31 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
  32 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
  33 import org.apache.hadoop.hbase.client.Delete;
  34 import org.apache.hadoop.hbase.client.Get;
  35 import org.apache.hadoop.hbase.client.Put;
  36 import org.apache.hadoop.hbase.client.Result;
  37 import org.apache.hadoop.hbase.client.TableDescriptor;
  38 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
  39 import org.apache.hadoop.hbase.filter.TimestampsFilter;
  40 import org.apache.hadoop.hbase.testclassification.MediumTests;
  41 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
  42 import org.apache.hadoop.hbase.util.Bytes;
  43 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
  44 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
  45 import org.junit.Assert;
  46 import org.junit.ClassRule;
  47 import org.junit.Rule;
  48 import org.junit.Test;
  49 import org.junit.experimental.categories.Category;
  50 import org.junit.rules.TestName;
  51
  52 /**
  53  * Test Minimum Versions feature (HBASE-4071).
  54  */
  55 @Category({RegionServerTests.class, MediumTests.class})
  56 public class TestMinVersions {
  57
  58   @ClassRule
  59   public static final HBaseClassTestRule CLASS_RULE =
  60       HBaseClassTestRule.forClass(TestMinVersions.class);
  61
  62   HBaseTestingUtility hbu = new HBaseTestingUtility();
  63   private final byte[] T0 = Bytes.toBytes("0");
  64   private final byte[] T1 = Bytes.toBytes("1");
  65   private final byte[] T2 = Bytes.toBytes("2");
  66   private final byte[] T3 = Bytes.toBytes("3");
  67   private final byte[] T4 = Bytes.toBytes("4");
  68   private final byte[] T5 = Bytes.toBytes("5");
  69
  70   private final byte[] c0 = COLUMNS[0];
  71
  72   @Rule public TestName name = new TestName();
  73
  74   /**
  75    * Verify behavior of getClosestBefore(...)
  76    */
  77   @Test
  78   public void testGetClosestBefore() throws Exception {
  79
  80     ColumnFamilyDescriptor cfd =
  81       ColumnFamilyDescriptorBuilder.newBuilder(c0)
  82       .setMinVersions(1).setMaxVersions(1000).setTimeToLive(1).
  83         setKeepDeletedCells(KeepDeletedCells.FALSE).build();
  84
  85     TableDescriptor htd = TableDescriptorBuilder.
  86       newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
  87     HRegion region = hbu.createLocalHRegion(htd, null, null);
  88     try {
  89
  90       // 2s in the past
  91       long ts = EnvironmentEdgeManager.currentTime() - 2000;
  92
  93       Put p = new Put(T1, ts);
  94       p.addColumn(c0, c0, T1);
  95       region.put(p);
  96
  97       p = new Put(T1, ts+1);
  98       p.addColumn(c0, c0, T4);
  99       region.put(p);
 100
 101       p = new Put(T3, ts);
 102       p.addColumn(c0, c0, T3);
 103       region.put(p);
 104
 105       // now make sure that getClosestBefore(...) get can
 106       // rows that would be expired without minVersion.
 107       // also make sure it gets the latest version
 108       Result r = hbu.getClosestRowBefore(region, T1, c0);
 109       checkResult(r, c0, T4);
 110
 111       r = hbu.getClosestRowBefore(region, T2, c0);
 112       checkResult(r, c0, T4);
 113
 114       // now flush/compact
 115       region.flush(true);
 116       region.compact(true);
 117
 118       r = hbu.getClosestRowBefore(region, T1, c0);
 119       checkResult(r, c0, T4);
 120
 121       r = hbu.getClosestRowBefore(region, T2, c0);
 122       checkResult(r, c0, T4);
 123     } finally {
 124       HBaseTestingUtility.closeRegionAndWAL(region);
 125     }
 126   }
 127
 128   /**
 129    * Test mixed memstore and storefile scanning
 130    * with minimum versions.
 131    */
 132   @Test
 133   public void testStoreMemStore() throws Exception {
 134     // keep 3 versions minimum
 135
 136     ColumnFamilyDescriptor cfd =
 137       ColumnFamilyDescriptorBuilder.newBuilder(c0)
 138         .setMinVersions(3).setMaxVersions(1000).setTimeToLive(1).
 139         setKeepDeletedCells(KeepDeletedCells.FALSE).build();
 140
 141     TableDescriptor htd = TableDescriptorBuilder.
 142       newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
 143
 144     HRegion region = hbu.createLocalHRegion(htd, null, null);
 145     // 2s in the past
 146     long ts = EnvironmentEdgeManager.currentTime() - 2000;
 147
 148     try {
 149       Put p = new Put(T1, ts-1);
 150       p.addColumn(c0, c0, T2);
 151       region.put(p);
 152
 153       p = new Put(T1, ts-3);
 154       p.addColumn(c0, c0, T0);
 155       region.put(p);
 156
 157       // now flush/compact
 158       region.flush(true);
 159       region.compact(true);
 160
 161       p = new Put(T1, ts);
 162       p.addColumn(c0, c0, T3);
 163       region.put(p);
 164
 165       p = new Put(T1, ts-2);
 166       p.addColumn(c0, c0, T1);
 167       region.put(p);
 168
 169       p = new Put(T1, ts-3);
 170       p.addColumn(c0, c0, T0);
 171       region.put(p);
 172
 173       // newest version in the memstore
 174       // the 2nd oldest in the store file
 175       // and the 3rd, 4th oldest also in the memstore
 176
 177       Get g = new Get(T1);
 178       g.readAllVersions();
 179       Result r = region.get(g); // this'll use ScanWildcardColumnTracker
 180       checkResult(r, c0, T3,T2,T1);
 181
 182       g = new Get(T1);
 183       g.readAllVersions();
 184       g.addColumn(c0, c0);
 185       r = region.get(g);  // this'll use ExplicitColumnTracker
 186       checkResult(r, c0, T3,T2,T1);
 187     } finally {
 188       HBaseTestingUtility.closeRegionAndWAL(region);
 189     }
 190   }
 191
 192   /**
 193    * Make sure the Deletes behave as expected with minimum versions
 194    */
 195   @Test
 196   public void testDelete() throws Exception {
 197     ColumnFamilyDescriptor cfd =
 198       ColumnFamilyDescriptorBuilder.newBuilder(c0)
 199         .setMinVersions(3).setMaxVersions(1000).setTimeToLive(1).
 200         setKeepDeletedCells(KeepDeletedCells.FALSE).build();
 201
 202     TableDescriptor htd = TableDescriptorBuilder.
 203       newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
 204
 205     HRegion region = hbu.createLocalHRegion(htd, null, null);
 206
 207     // 2s in the past
 208     long ts = EnvironmentEdgeManager.currentTime() - 2000;
 209
 210     try {
 211       Put p = new Put(T1, ts-2);
 212       p.addColumn(c0, c0, T1);
 213       region.put(p);
 214
 215       p = new Put(T1, ts-1);
 216       p.addColumn(c0, c0, T2);
 217       region.put(p);
 218
 219       p = new Put(T1, ts);
 220       p.addColumn(c0, c0, T3);
 221       region.put(p);
 222
 223       Delete d = new Delete(T1, ts-1);
 224       region.delete(d);
 225
 226       Get g = new Get(T1);
 227       g.readAllVersions();
 228       Result r = region.get(g);  // this'll use ScanWildcardColumnTracker
 229       checkResult(r, c0, T3);
 230
 231       g = new Get(T1);
 232       g.readAllVersions();
 233       g.addColumn(c0, c0);
 234       r = region.get(g);  // this'll use ExplicitColumnTracker
 235       checkResult(r, c0, T3);
 236
 237       // now flush/compact
 238       region.flush(true);
 239       region.compact(true);
 240
 241       // try again
 242       g = new Get(T1);
 243       g.readAllVersions();
 244       r = region.get(g);  // this'll use ScanWildcardColumnTracker
 245       checkResult(r, c0, T3);
 246
 247       g = new Get(T1);
 248       g.readAllVersions();
 249       g.addColumn(c0, c0);
 250       r = region.get(g);  // this'll use ExplicitColumnTracker
 251       checkResult(r, c0, T3);
 252     } finally {
 253       HBaseTestingUtility.closeRegionAndWAL(region);
 254     }
 255   }
 256
 257   /**
 258    * Make sure the memstor behaves correctly with minimum versions
 259    */
 260   @Test
 261   public void testMemStore() throws Exception {
 262     ColumnFamilyDescriptor cfd =
 263       ColumnFamilyDescriptorBuilder.newBuilder(c0)
 264         .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
 265         setKeepDeletedCells(KeepDeletedCells.FALSE).build();
 266
 267     TableDescriptor htd = TableDescriptorBuilder.
 268       newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
 269     HRegion region = hbu.createLocalHRegion(htd, null, null);
 270
 271     // 2s in the past
 272     long ts = EnvironmentEdgeManager.currentTime() - 2000;
 273
 274     try {
 275       // 2nd version
 276       Put p = new Put(T1, ts-2);
 277       p.addColumn(c0, c0, T2);
 278       region.put(p);
 279
 280       // 3rd version
 281       p = new Put(T1, ts-1);
 282       p.addColumn(c0, c0, T3);
 283       region.put(p);
 284
 285       // 4th version
 286       p = new Put(T1, ts);
 287       p.addColumn(c0, c0, T4);
 288       region.put(p);
 289
 290       // now flush/compact
 291       region.flush(true);
 292       region.compact(true);
 293
 294       // now put the first version (backdated)
 295       p = new Put(T1, ts-3);
 296       p.addColumn(c0, c0, T1);
 297       region.put(p);
 298
 299       // now the latest change is in the memstore,
 300       // but it is not the latest version
 301
 302       Result r = region.get(new Get(T1));
 303       checkResult(r, c0, T4);
 304
 305       Get g = new Get(T1);
 306       g.readAllVersions();
 307       r = region.get(g); // this'll use ScanWildcardColumnTracker
 308       checkResult(r, c0, T4,T3);
 309
 310       g = new Get(T1);
 311       g.readAllVersions();
 312       g.addColumn(c0, c0);
 313       r = region.get(g);  // this'll use ExplicitColumnTracker
 314       checkResult(r, c0, T4,T3);
 315
 316       p = new Put(T1, ts+1);
 317       p.addColumn(c0, c0, T5);
 318       region.put(p);
 319
 320       // now the latest version is in the memstore
 321
 322       g = new Get(T1);
 323       g.readAllVersions();
 324       r = region.get(g);  // this'll use ScanWildcardColumnTracker
 325       checkResult(r, c0, T5,T4);
 326
 327       g = new Get(T1);
 328       g.readAllVersions();
 329       g.addColumn(c0, c0);
 330       r = region.get(g);  // this'll use ExplicitColumnTracker
 331       checkResult(r, c0, T5,T4);
 332     } finally {
 333       HBaseTestingUtility.closeRegionAndWAL(region);
 334     }
 335   }
 336
 337   /**
 338    * Verify basic minimum versions functionality
 339    */
 340   @Test
 341   public void testBaseCase() throws Exception {
 342     // 2 version minimum, 1000 versions maximum, ttl = 1s
 343     ColumnFamilyDescriptor cfd =
 344       ColumnFamilyDescriptorBuilder.newBuilder(c0)
 345         .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
 346         setKeepDeletedCells(KeepDeletedCells.FALSE).build();
 347
 348     TableDescriptor htd = TableDescriptorBuilder.
 349       newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
 350     HRegion region = hbu.createLocalHRegion(htd, null, null);
 351     try {
 352
 353       // 2s in the past
 354       long ts = EnvironmentEdgeManager.currentTime() - 2000;
 355
 356        // 1st version
 357       Put p = new Put(T1, ts-3);
 358       p.addColumn(c0, c0, T1);
 359       region.put(p);
 360
 361       // 2nd version
 362       p = new Put(T1, ts-2);
 363       p.addColumn(c0, c0, T2);
 364       region.put(p);
 365
 366       // 3rd version
 367       p = new Put(T1, ts-1);
 368       p.addColumn(c0, c0, T3);
 369       region.put(p);
 370
 371       // 4th version
 372       p = new Put(T1, ts);
 373       p.addColumn(c0, c0, T4);
 374       region.put(p);
 375
 376       Result r = region.get(new Get(T1));
 377       checkResult(r, c0, T4);
 378
 379       Get g = new Get(T1);
 380       g.setTimeRange(0L, ts+1);
 381       r = region.get(g);
 382       checkResult(r, c0, T4);
 383
 384   // oldest version still exists
 385       g.setTimeRange(0L, ts-2);
 386       r = region.get(g);
 387       checkResult(r, c0, T1);
 388
 389       // gets see only available versions
 390       // even before compactions
 391       g = new Get(T1);
 392       g.readAllVersions();
 393       r = region.get(g); // this'll use ScanWildcardColumnTracker
 394       checkResult(r, c0, T4,T3);
 395
 396       g = new Get(T1);
 397       g.readAllVersions();
 398       g.addColumn(c0, c0);
 399       r = region.get(g);  // this'll use ExplicitColumnTracker
 400       checkResult(r, c0, T4,T3);
 401
 402       // now flush
 403       region.flush(true);
 404
 405       // with HBASE-4241 a flush will eliminate the expired rows
 406       g = new Get(T1);
 407       g.setTimeRange(0L, ts-2);
 408       r = region.get(g);
 409       assertTrue(r.isEmpty());
 410
 411       // major compaction
 412       region.compact(true);
 413
 414       // after compaction the 4th version is still available
 415       g = new Get(T1);
 416       g.setTimeRange(0L, ts+1);
 417       r = region.get(g);
 418       checkResult(r, c0, T4);
 419
 420       // so is the 3rd
 421       g.setTimeRange(0L, ts);
 422       r = region.get(g);
 423       checkResult(r, c0, T3);
 424
 425       // but the 2nd and earlier versions are gone
 426       g.setTimeRange(0L, ts-1);
 427       r = region.get(g);
 428       assertTrue(r.isEmpty());
 429     } finally {
 430       HBaseTestingUtility.closeRegionAndWAL(region);
 431     }
 432   }
 433
 434   /**
 435    * Verify that basic filters still behave correctly with
 436    * minimum versions enabled.
 437    */
 438   @Test
 439   public void testFilters() throws Exception {
 440     final byte [] c1 = COLUMNS[1];
 441     ColumnFamilyDescriptor cfd =
 442       ColumnFamilyDescriptorBuilder.newBuilder(c0)
 443         .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
 444         setKeepDeletedCells(KeepDeletedCells.FALSE).build();
 445
 446     ColumnFamilyDescriptor cfd2 =
 447       ColumnFamilyDescriptorBuilder.newBuilder(c1)
 448         .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
 449         setKeepDeletedCells(KeepDeletedCells.FALSE).build();
 450     List<ColumnFamilyDescriptor> cfdList = new ArrayList();
 451     cfdList.add(cfd);
 452     cfdList.add(cfd2);
 453
 454     TableDescriptor htd = TableDescriptorBuilder.
 455       newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamilies(cfdList).build();
 456     HRegion region = hbu.createLocalHRegion(htd, null, null);
 457
 458     // 2s in the past
 459     long ts = EnvironmentEdgeManager.currentTime() - 2000;
 460     try {
 461
 462       Put p = new Put(T1, ts-3);
 463       p.addColumn(c0, c0, T0);
 464       p.addColumn(c1, c1, T0);
 465       region.put(p);
 466
 467       p = new Put(T1, ts-2);
 468       p.addColumn(c0, c0, T1);
 469       p.addColumn(c1, c1, T1);
 470       region.put(p);
 471
 472       p = new Put(T1, ts-1);
 473       p.addColumn(c0, c0, T2);
 474       p.addColumn(c1, c1, T2);
 475       region.put(p);
 476
 477       p = new Put(T1, ts);
 478       p.addColumn(c0, c0, T3);
 479       p.addColumn(c1, c1, T3);
 480       region.put(p);
 481
 482       List<Long> tss = new ArrayList<>();
 483       tss.add(ts-1);
 484       tss.add(ts-2);
 485
 486       // Sholud only get T2, versions is 2, so T1 is gone from user view.
 487       Get g = new Get(T1);
 488       g.addColumn(c1,c1);
 489       g.setFilter(new TimestampsFilter(tss));
 490       g.readAllVersions();
 491       Result r = region.get(g);
 492       checkResult(r, c1, T2);
 493
 494       // Sholud only get T2, versions is 2, so T1 is gone from user view.
 495       g = new Get(T1);
 496       g.addColumn(c0,c0);
 497       g.setFilter(new TimestampsFilter(tss));
 498       g.readAllVersions();
 499       r = region.get(g);
 500       checkResult(r, c0, T2);
 501
 502       // now flush/compact
 503       region.flush(true);
 504       region.compact(true);
 505
 506       // After flush/compact, the result should be consistent with previous result
 507       g = new Get(T1);
 508       g.addColumn(c1,c1);
 509       g.setFilter(new TimestampsFilter(tss));
 510       g.readAllVersions();
 511       r = region.get(g);
 512       checkResult(r, c1, T2);
 513
 514       // After flush/compact, the result should be consistent with previous result
 515       g = new Get(T1);
 516       g.addColumn(c0,c0);
 517       g.setFilter(new TimestampsFilter(tss));
 518       g.readAllVersions();
 519       r = region.get(g);
 520       checkResult(r, c0, T2);
 521     } finally {
 522       HBaseTestingUtility.closeRegionAndWAL(region);
 523     }
 524   }
 525
 526   @Test
 527   public void testMinVersionsWithKeepDeletedCellsTTL() throws Exception {
 528     int ttl = 4;
 529     ColumnFamilyDescriptor cfd =
 530       ColumnFamilyDescriptorBuilder.newBuilder(c0)
 531         .setMinVersions(2).setMaxVersions(Integer.MAX_VALUE).setTimeToLive(ttl).
 532         setKeepDeletedCells(KeepDeletedCells.TTL).build();
 533
 534     TableDescriptor htd = TableDescriptorBuilder.
 535       newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
 536
 537     HRegion region = hbu.createLocalHRegion(htd, null, null);
 538
 539     long startTS = EnvironmentEdgeManager.currentTime();
 540     ManualEnvironmentEdge injectEdge = new ManualEnvironmentEdge();
 541     injectEdge.setValue(startTS);
 542     EnvironmentEdgeManager.injectEdge(injectEdge);
 543
 544     long ts = startTS - 2000;
 545     // 1st version
 546     Put p = new Put(T1, ts-3);
 547     p.addColumn(c0, c0, T1);
 548     region.put(p);
 549
 550     // 2nd version
 551     p = new Put(T1, ts-2);
 552     p.addColumn(c0, c0, T2);
 553     region.put(p);
 554
 555     // 3rd version
 556     p = new Put(T1, ts-1);
 557     p.addColumn(c0, c0, T3);
 558     region.put(p);
 559
 560     Get g;
 561     Result r;
 562
 563     //check we can still see all versions before compaction
 564     g = new Get(T1);
 565     g.readAllVersions();
 566     g.setTimeRange(0, ts);
 567     r = region.get(g);
 568     checkResult(r, c0, T3, T2, T1);
 569
 570     region.flush(true);
 571     region.compact(true);
 572     Assert.assertEquals(startTS, EnvironmentEdgeManager.currentTime());
 573     long expiredTime = EnvironmentEdgeManager.currentTime() - ts - 3;
 574     Assert.assertTrue("TTL for T1 has expired", expiredTime < (ttl * 1000));
 575     //check that nothing was purged yet
 576     g = new Get(T1);
 577     g.readAllVersions();
 578     g.setTimeRange(0, ts);
 579     r = region.get(g);
 580     checkResult(r, c0, T3, T2, T1);
 581
 582     g = new Get(T1);
 583     g.readAllVersions();
 584     g.setTimeRange(0, ts -1);
 585     r = region.get(g);
 586     checkResult(r, c0, T2, T1);
 587
 588     injectEdge.incValue(ttl * 1000);
 589
 590     region.flush(true);
 591     region.compact(true);
 592
 593     //check that after compaction (which is after TTL) that only T1 was purged
 594     g = new Get(T1);
 595     g.readAllVersions();
 596     g.setTimeRange(0, ts);
 597     r = region.get(g);
 598     checkResult(r, c0, T3, T2);
 599
 600     g = new Get(T1);
 601     g.readAllVersions();
 602     g.setTimestamp(ts -2);
 603     r = region.get(g);
 604     checkResult(r, c0, T2);
 605   }
 606
 607   private void checkResult(Result r, byte[] col, byte[] ... vals) {
 608     assertEquals(vals.length, r.size());
 609     List<Cell> kvs = r.getColumnCells(col, col);
 610     assertEquals(kvs.size(), vals.length);
 611     for (int i=0;i<vals.length;i++) {
 612       String expected = Bytes.toString(vals[i]);
 613       String actual = Bytes.toString(CellUtil.cloneValue(kvs.get(i)));
 614       assertTrue(expected + " was expected but doesn't match " + actual,
 615           CellUtil.matchingValue(kvs.get(i), vals[i]));
 616     }
 617   }
 618
 619 }
 620