hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java

   1 /**
   2  *
   3  * Licensed to the Apache Software Foundation (ASF) under one
   4  * or more contributor license agreements.  See the NOTICE file
   5  * distributed with this work for additional information
   6  * regarding copyright ownership.  The ASF licenses this file
   7  * to you under the Apache License, Version 2.0 (the
   8  * "License"); you may not use this file except in compliance
   9  * with the License.  You may obtain a copy of the License at
  10  *
  11  *     http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  */
  19 package org.apache.hadoop.hbase.regionserver;
  20
  21 import static org.apache.hadoop.hbase.HBaseTestingUtility.COLUMNS;
  22 import static org.junit.Assert.assertEquals;
  23 import static org.junit.Assert.assertTrue;
  24
  25 import java.util.ArrayList;
  26 import java.util.List;
  27
  28 import org.apache.hadoop.hbase.Cell;
  29 import org.apache.hadoop.hbase.CellUtil;
  30 import org.apache.hadoop.hbase.HBaseTestingUtility;
  31 import org.apache.hadoop.hbase.HTableDescriptor;
  32 import org.apache.hadoop.hbase.KeepDeletedCells;
  33 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
  34 import org.apache.hadoop.hbase.testclassification.SmallTests;
  35 import org.apache.hadoop.hbase.client.Delete;
  36 import org.apache.hadoop.hbase.client.Get;
  37 import org.apache.hadoop.hbase.client.Put;
  38 import org.apache.hadoop.hbase.client.Result;
  39 import org.apache.hadoop.hbase.filter.TimestampsFilter;
  40 import org.apache.hadoop.hbase.util.Bytes;
  41 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
  42 import org.junit.Rule;
  43 import org.junit.Test;
  44 import org.junit.experimental.categories.Category;
  45 import org.junit.rules.TestName;
  46
  47 /**
  48  * Test Minimum Versions feature (HBASE-4071).
  49  */
  50 @Category({RegionServerTests.class, SmallTests.class})
  51 public class TestMinVersions {
  52   HBaseTestingUtility hbu = HBaseTestingUtility.createLocalHTU();
  53   private final byte[] T0 = Bytes.toBytes("0");
  54   private final byte[] T1 = Bytes.toBytes("1");
  55   private final byte[] T2 = Bytes.toBytes("2");
  56   private final byte[] T3 = Bytes.toBytes("3");
  57   private final byte[] T4 = Bytes.toBytes("4");
  58   private final byte[] T5 = Bytes.toBytes("5");
  59
  60   private final byte[] c0 = COLUMNS[0];
  61
  62   @Rule public TestName name = new TestName();
  63
  64   /**
  65    * Verify behavior of getClosestBefore(...)
  66    */
  67   @Test
  68   public void testGetClosestBefore() throws Exception {
  69     HTableDescriptor htd =
  70         hbu.createTableDescriptor(name.getMethodName(), 1, 1000, 1, KeepDeletedCells.FALSE);
  71     Region region = hbu.createLocalHRegion(htd, null, null);
  72     try {
  73
  74       // 2s in the past
  75       long ts = EnvironmentEdgeManager.currentTime() - 2000;
  76
  77       Put p = new Put(T1, ts);
  78       p.addColumn(c0, c0, T1);
  79       region.put(p);
  80
  81       p = new Put(T1, ts+1);
  82       p.addColumn(c0, c0, T4);
  83       region.put(p);
  84
  85       p = new Put(T3, ts);
  86       p.addColumn(c0, c0, T3);
  87       region.put(p);
  88
  89       // now make sure that getClosestBefore(...) get can
  90       // rows that would be expired without minVersion.
  91       // also make sure it gets the latest version
  92       Result r = hbu.getClosestRowBefore(region, T1, c0);
  93       checkResult(r, c0, T4);
  94
  95       r = hbu.getClosestRowBefore(region, T2, c0);
  96       checkResult(r, c0, T4);
  97
  98       // now flush/compact
  99       region.flush(true);
 100       region.compact(true);
 101
 102       r = hbu.getClosestRowBefore(region, T1, c0);
 103       checkResult(r, c0, T4);
 104
 105       r = hbu.getClosestRowBefore(region, T2, c0);
 106       checkResult(r, c0, T4);
 107     } finally {
 108       HBaseTestingUtility.closeRegionAndWAL(region);
 109     }
 110   }
 111
 112   /**
 113    * Test mixed memstore and storefile scanning
 114    * with minimum versions.
 115    */
 116   @Test
 117   public void testStoreMemStore() throws Exception {
 118     // keep 3 versions minimum
 119     HTableDescriptor htd =
 120         hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE);
 121     Region region = hbu.createLocalHRegion(htd, null, null);
 122     // 2s in the past
 123     long ts = EnvironmentEdgeManager.currentTime() - 2000;
 124
 125     try {
 126       Put p = new Put(T1, ts-1);
 127       p.addColumn(c0, c0, T2);
 128       region.put(p);
 129
 130       p = new Put(T1, ts-3);
 131       p.addColumn(c0, c0, T0);
 132       region.put(p);
 133
 134       // now flush/compact
 135       region.flush(true);
 136       region.compact(true);
 137
 138       p = new Put(T1, ts);
 139       p.addColumn(c0, c0, T3);
 140       region.put(p);
 141
 142       p = new Put(T1, ts-2);
 143       p.addColumn(c0, c0, T1);
 144       region.put(p);
 145
 146       p = new Put(T1, ts-3);
 147       p.addColumn(c0, c0, T0);
 148       region.put(p);
 149
 150       // newest version in the memstore
 151       // the 2nd oldest in the store file
 152       // and the 3rd, 4th oldest also in the memstore
 153
 154       Get g = new Get(T1);
 155       g.setMaxVersions();
 156       Result r = region.get(g); // this'll use ScanWildcardColumnTracker
 157       checkResult(r, c0, T3,T2,T1);
 158
 159       g = new Get(T1);
 160       g.setMaxVersions();
 161       g.addColumn(c0, c0);
 162       r = region.get(g);  // this'll use ExplicitColumnTracker
 163       checkResult(r, c0, T3,T2,T1);
 164     } finally {
 165       HBaseTestingUtility.closeRegionAndWAL(region);
 166     }
 167   }
 168
 169   /**
 170    * Make sure the Deletes behave as expected with minimum versions
 171    */
 172   @Test
 173   public void testDelete() throws Exception {
 174     HTableDescriptor htd =
 175         hbu.createTableDescriptor(name.getMethodName(), 3, 1000, 1, KeepDeletedCells.FALSE);
 176     Region region = hbu.createLocalHRegion(htd, null, null);
 177
 178     // 2s in the past
 179     long ts = EnvironmentEdgeManager.currentTime() - 2000;
 180
 181     try {
 182       Put p = new Put(T1, ts-2);
 183       p.addColumn(c0, c0, T1);
 184       region.put(p);
 185
 186       p = new Put(T1, ts-1);
 187       p.addColumn(c0, c0, T2);
 188       region.put(p);
 189
 190       p = new Put(T1, ts);
 191       p.addColumn(c0, c0, T3);
 192       region.put(p);
 193
 194       Delete d = new Delete(T1, ts-1);
 195       region.delete(d);
 196
 197       Get g = new Get(T1);
 198       g.setMaxVersions();
 199       Result r = region.get(g);  // this'll use ScanWildcardColumnTracker
 200       checkResult(r, c0, T3);
 201
 202       g = new Get(T1);
 203       g.setMaxVersions();
 204       g.addColumn(c0, c0);
 205       r = region.get(g);  // this'll use ExplicitColumnTracker
 206       checkResult(r, c0, T3);
 207
 208       // now flush/compact
 209       region.flush(true);
 210       region.compact(true);
 211
 212       // try again
 213       g = new Get(T1);
 214       g.setMaxVersions();
 215       r = region.get(g);  // this'll use ScanWildcardColumnTracker
 216       checkResult(r, c0, T3);
 217
 218       g = new Get(T1);
 219       g.setMaxVersions();
 220       g.addColumn(c0, c0);
 221       r = region.get(g);  // this'll use ExplicitColumnTracker
 222       checkResult(r, c0, T3);
 223     } finally {
 224       HBaseTestingUtility.closeRegionAndWAL(region);
 225     }
 226   }
 227
 228   /**
 229    * Make sure the memstor behaves correctly with minimum versions
 230    */
 231   @Test
 232   public void testMemStore() throws Exception {
 233     HTableDescriptor htd =
 234         hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
 235     Region region = hbu.createLocalHRegion(htd, null, null);
 236
 237     // 2s in the past
 238     long ts = EnvironmentEdgeManager.currentTime() - 2000;
 239
 240     try {
 241       // 2nd version
 242       Put p = new Put(T1, ts-2);
 243       p.addColumn(c0, c0, T2);
 244       region.put(p);
 245
 246       // 3rd version
 247       p = new Put(T1, ts-1);
 248       p.addColumn(c0, c0, T3);
 249       region.put(p);
 250
 251       // 4th version
 252       p = new Put(T1, ts);
 253       p.addColumn(c0, c0, T4);
 254       region.put(p);
 255
 256       // now flush/compact
 257       region.flush(true);
 258       region.compact(true);
 259
 260       // now put the first version (backdated)
 261       p = new Put(T1, ts-3);
 262       p.addColumn(c0, c0, T1);
 263       region.put(p);
 264
 265       // now the latest change is in the memstore,
 266       // but it is not the latest version
 267
 268       Result r = region.get(new Get(T1));
 269       checkResult(r, c0, T4);
 270
 271       Get g = new Get(T1);
 272       g.setMaxVersions();
 273       r = region.get(g); // this'll use ScanWildcardColumnTracker
 274       checkResult(r, c0, T4,T3);
 275
 276       g = new Get(T1);
 277       g.setMaxVersions();
 278       g.addColumn(c0, c0);
 279       r = region.get(g);  // this'll use ExplicitColumnTracker
 280       checkResult(r, c0, T4,T3);
 281
 282       p = new Put(T1, ts+1);
 283       p.addColumn(c0, c0, T5);
 284       region.put(p);
 285
 286       // now the latest version is in the memstore
 287
 288       g = new Get(T1);
 289       g.setMaxVersions();
 290       r = region.get(g);  // this'll use ScanWildcardColumnTracker
 291       checkResult(r, c0, T5,T4);
 292
 293       g = new Get(T1);
 294       g.setMaxVersions();
 295       g.addColumn(c0, c0);
 296       r = region.get(g);  // this'll use ExplicitColumnTracker
 297       checkResult(r, c0, T5,T4);
 298     } finally {
 299       HBaseTestingUtility.closeRegionAndWAL(region);
 300     }
 301   }
 302
 303   /**
 304    * Verify basic minimum versions functionality
 305    */
 306   @Test
 307   public void testBaseCase() throws Exception {
 308     // 1 version minimum, 1000 versions maximum, ttl = 1s
 309     HTableDescriptor htd =
 310         hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
 311     Region region = hbu.createLocalHRegion(htd, null, null);
 312     try {
 313
 314       // 2s in the past
 315       long ts = EnvironmentEdgeManager.currentTime() - 2000;
 316
 317        // 1st version
 318       Put p = new Put(T1, ts-3);
 319       p.addColumn(c0, c0, T1);
 320       region.put(p);
 321
 322       // 2nd version
 323       p = new Put(T1, ts-2);
 324       p.addColumn(c0, c0, T2);
 325       region.put(p);
 326
 327       // 3rd version
 328       p = new Put(T1, ts-1);
 329       p.addColumn(c0, c0, T3);
 330       region.put(p);
 331
 332       // 4th version
 333       p = new Put(T1, ts);
 334       p.addColumn(c0, c0, T4);
 335       region.put(p);
 336
 337       Result r = region.get(new Get(T1));
 338       checkResult(r, c0, T4);
 339
 340       Get g = new Get(T1);
 341       g.setTimeRange(0L, ts+1);
 342       r = region.get(g);
 343       checkResult(r, c0, T4);
 344
 345   // oldest version still exists
 346       g.setTimeRange(0L, ts-2);
 347       r = region.get(g);
 348       checkResult(r, c0, T1);
 349
 350       // gets see only available versions
 351       // even before compactions
 352       g = new Get(T1);
 353       g.setMaxVersions();
 354       r = region.get(g); // this'll use ScanWildcardColumnTracker
 355       checkResult(r, c0, T4,T3);
 356
 357       g = new Get(T1);
 358       g.setMaxVersions();
 359       g.addColumn(c0, c0);
 360       r = region.get(g);  // this'll use ExplicitColumnTracker
 361       checkResult(r, c0, T4,T3);
 362
 363       // now flush
 364       region.flush(true);
 365
 366       // with HBASE-4241 a flush will eliminate the expired rows
 367       g = new Get(T1);
 368       g.setTimeRange(0L, ts-2);
 369       r = region.get(g);
 370       assertTrue(r.isEmpty());
 371
 372       // major compaction
 373       region.compact(true);
 374
 375       // after compaction the 4th version is still available
 376       g = new Get(T1);
 377       g.setTimeRange(0L, ts+1);
 378       r = region.get(g);
 379       checkResult(r, c0, T4);
 380
 381       // so is the 3rd
 382       g.setTimeRange(0L, ts);
 383       r = region.get(g);
 384       checkResult(r, c0, T3);
 385
 386       // but the 2nd and earlier versions are gone
 387       g.setTimeRange(0L, ts-1);
 388       r = region.get(g);
 389       assertTrue(r.isEmpty());
 390     } finally {
 391       HBaseTestingUtility.closeRegionAndWAL(region);
 392     }
 393   }
 394
 395   /**
 396    * Verify that basic filters still behave correctly with
 397    * minimum versions enabled.
 398    */
 399   @Test
 400   public void testFilters() throws Exception {
 401     HTableDescriptor htd =
 402         hbu.createTableDescriptor(name.getMethodName(), 2, 1000, 1, KeepDeletedCells.FALSE);
 403     Region region = hbu.createLocalHRegion(htd, null, null);
 404     final byte [] c1 = COLUMNS[1];
 405
 406     // 2s in the past
 407     long ts = EnvironmentEdgeManager.currentTime() - 2000;
 408     try {
 409
 410       Put p = new Put(T1, ts-3);
 411       p.addColumn(c0, c0, T0);
 412       p.addColumn(c1, c1, T0);
 413       region.put(p);
 414
 415       p = new Put(T1, ts-2);
 416       p.addColumn(c0, c0, T1);
 417       p.addColumn(c1, c1, T1);
 418       region.put(p);
 419
 420       p = new Put(T1, ts-1);
 421       p.addColumn(c0, c0, T2);
 422       p.addColumn(c1, c1, T2);
 423       region.put(p);
 424
 425       p = new Put(T1, ts);
 426       p.addColumn(c0, c0, T3);
 427       p.addColumn(c1, c1, T3);
 428       region.put(p);
 429
 430       List<Long> tss = new ArrayList<>();
 431       tss.add(ts-1);
 432       tss.add(ts-2);
 433
 434       Get g = new Get(T1);
 435       g.addColumn(c1,c1);
 436       g.setFilter(new TimestampsFilter(tss));
 437       g.setMaxVersions();
 438       Result r = region.get(g);
 439       checkResult(r, c1, T2,T1);
 440
 441       g = new Get(T1);
 442       g.addColumn(c0,c0);
 443       g.setFilter(new TimestampsFilter(tss));
 444       g.setMaxVersions();
 445       r = region.get(g);
 446       checkResult(r, c0, T2,T1);
 447
 448       // now flush/compact
 449       region.flush(true);
 450       region.compact(true);
 451
 452       g = new Get(T1);
 453       g.addColumn(c1,c1);
 454       g.setFilter(new TimestampsFilter(tss));
 455       g.setMaxVersions();
 456       r = region.get(g);
 457       checkResult(r, c1, T2);
 458
 459       g = new Get(T1);
 460       g.addColumn(c0,c0);
 461       g.setFilter(new TimestampsFilter(tss));
 462       g.setMaxVersions();
 463       r = region.get(g);
 464       checkResult(r, c0, T2);
 465     } finally {
 466       HBaseTestingUtility.closeRegionAndWAL(region);
 467     }
 468   }
 469
 470   private void checkResult(Result r, byte[] col, byte[] ... vals) {
 471     assertEquals(r.size(), vals.length);
 472     List<Cell> kvs = r.getColumnCells(col, col);
 473     assertEquals(kvs.size(), vals.length);
 474     for (int i=0;i<vals.length;i++) {
 475       assertTrue(CellUtil.matchingValue(kvs.get(i), vals[i]));
 476     }
 477   }
 478
 479 }
 480