hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMinVersions.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.regionserver;
  19
  20 import static org.apache.hadoop.hbase.HBaseTestingUtil.COLUMNS;
  21 import static org.junit.Assert.assertEquals;
  22 import static org.junit.Assert.assertTrue;
  23
  24 import java.io.IOException;
  25 import java.util.ArrayList;
  26 import java.util.List;
  27 import org.apache.hadoop.hbase.Cell;
  28 import org.apache.hadoop.hbase.CellUtil;
  29 import org.apache.hadoop.hbase.HBaseClassTestRule;
  30 import org.apache.hadoop.hbase.HBaseTestingUtil;
  31 import org.apache.hadoop.hbase.KeepDeletedCells;
  32 import org.apache.hadoop.hbase.TableName;
  33 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
  34 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
  35 import org.apache.hadoop.hbase.client.Delete;
  36 import org.apache.hadoop.hbase.client.Get;
  37 import org.apache.hadoop.hbase.client.Put;
  38 import org.apache.hadoop.hbase.client.Result;
  39 import org.apache.hadoop.hbase.client.TableDescriptor;
  40 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
  41 import org.apache.hadoop.hbase.filter.TimestampsFilter;
  42 import org.apache.hadoop.hbase.testclassification.MediumTests;
  43 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
  44 import org.apache.hadoop.hbase.util.Bytes;
  45 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
  46 import org.apache.hadoop.hbase.util.ManualEnvironmentEdge;
  47 import org.junit.Assert;
  48 import org.junit.ClassRule;
  49 import org.junit.Rule;
  50 import org.junit.Test;
  51 import org.junit.experimental.categories.Category;
  52 import org.junit.rules.TestName;
  53
  54 /**
  55  * Test Minimum Versions feature (HBASE-4071).
  56  */
  57 @Category({RegionServerTests.class, MediumTests.class})
  58 public class TestMinVersions {
  59
  60   @ClassRule
  61   public static final HBaseClassTestRule CLASS_RULE =
  62       HBaseClassTestRule.forClass(TestMinVersions.class);
  63
  64   HBaseTestingUtil hbu = new HBaseTestingUtil();
  65   private final byte[] T0 = Bytes.toBytes("0");
  66   private final byte[] T1 = Bytes.toBytes("1");
  67   private final byte[] T2 = Bytes.toBytes("2");
  68   private final byte[] T3 = Bytes.toBytes("3");
  69   private final byte[] T4 = Bytes.toBytes("4");
  70   private final byte[] T5 = Bytes.toBytes("5");
  71
  72   private final byte[] c0 = COLUMNS[0];
  73
  74   @Rule public TestName name = new TestName();
  75
  76   /**
  77    * Verify behavior of getClosestBefore(...)
  78    */
  79   @Test
  80   public void testGetClosestBefore() throws Exception {
  81
  82     ColumnFamilyDescriptor cfd =
  83       ColumnFamilyDescriptorBuilder.newBuilder(c0)
  84       .setMinVersions(1).setMaxVersions(1000).setTimeToLive(1).
  85         setKeepDeletedCells(KeepDeletedCells.FALSE).build();
  86
  87     TableDescriptor htd = TableDescriptorBuilder.
  88       newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
  89     HRegion region = hbu.createLocalHRegion(htd, null, null);
  90     try {
  91
  92       // 2s in the past
  93       long ts = EnvironmentEdgeManager.currentTime() - 2000;
  94
  95       Put p = new Put(T1, ts);
  96       p.addColumn(c0, c0, T1);
  97       region.put(p);
  98
  99       p = new Put(T1, ts+1);
 100       p.addColumn(c0, c0, T4);
 101       region.put(p);
 102
 103       p = new Put(T3, ts);
 104       p.addColumn(c0, c0, T3);
 105       region.put(p);
 106
 107       // now make sure that getClosestBefore(...) get can
 108       // rows that would be expired without minVersion.
 109       // also make sure it gets the latest version
 110       Result r = hbu.getClosestRowBefore(region, T1, c0);
 111       checkResult(r, c0, T4);
 112
 113       r = hbu.getClosestRowBefore(region, T2, c0);
 114       checkResult(r, c0, T4);
 115
 116       // now flush/compact
 117       region.flush(true);
 118       region.compact(true);
 119
 120       r = hbu.getClosestRowBefore(region, T1, c0);
 121       checkResult(r, c0, T4);
 122
 123       r = hbu.getClosestRowBefore(region, T2, c0);
 124       checkResult(r, c0, T4);
 125     } finally {
 126       HBaseTestingUtil.closeRegionAndWAL(region);
 127     }
 128   }
 129
 130   /**
 131    * Test mixed memstore and storefile scanning
 132    * with minimum versions.
 133    */
 134   @Test
 135   public void testStoreMemStore() throws Exception {
 136     // keep 3 versions minimum
 137
 138     ColumnFamilyDescriptor cfd =
 139       ColumnFamilyDescriptorBuilder.newBuilder(c0)
 140         .setMinVersions(3).setMaxVersions(1000).setTimeToLive(1).
 141         setKeepDeletedCells(KeepDeletedCells.FALSE).build();
 142
 143     TableDescriptor htd = TableDescriptorBuilder.
 144       newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
 145
 146     HRegion region = hbu.createLocalHRegion(htd, null, null);
 147     // 2s in the past
 148     long ts = EnvironmentEdgeManager.currentTime() - 2000;
 149
 150     try {
 151       Put p = new Put(T1, ts-1);
 152       p.addColumn(c0, c0, T2);
 153       region.put(p);
 154
 155       p = new Put(T1, ts-3);
 156       p.addColumn(c0, c0, T0);
 157       region.put(p);
 158
 159       // now flush/compact
 160       region.flush(true);
 161       region.compact(true);
 162
 163       p = new Put(T1, ts);
 164       p.addColumn(c0, c0, T3);
 165       region.put(p);
 166
 167       p = new Put(T1, ts-2);
 168       p.addColumn(c0, c0, T1);
 169       region.put(p);
 170
 171       p = new Put(T1, ts-3);
 172       p.addColumn(c0, c0, T0);
 173       region.put(p);
 174
 175       // newest version in the memstore
 176       // the 2nd oldest in the store file
 177       // and the 3rd, 4th oldest also in the memstore
 178
 179       Get g = new Get(T1);
 180       g.readAllVersions();
 181       Result r = region.get(g); // this'll use ScanWildcardColumnTracker
 182       checkResult(r, c0, T3,T2,T1);
 183
 184       g = new Get(T1);
 185       g.readAllVersions();
 186       g.addColumn(c0, c0);
 187       r = region.get(g);  // this'll use ExplicitColumnTracker
 188       checkResult(r, c0, T3,T2,T1);
 189     } finally {
 190       HBaseTestingUtil.closeRegionAndWAL(region);
 191     }
 192   }
 193
 194   /**
 195    * Make sure the Deletes behave as expected with minimum versions
 196    */
 197   @Test
 198   public void testDelete() throws Exception {
 199     ColumnFamilyDescriptor cfd =
 200       ColumnFamilyDescriptorBuilder.newBuilder(c0)
 201         .setMinVersions(3).setMaxVersions(1000).setTimeToLive(1).
 202         setKeepDeletedCells(KeepDeletedCells.FALSE).build();
 203
 204     TableDescriptor htd = TableDescriptorBuilder.
 205       newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
 206
 207     HRegion region = hbu.createLocalHRegion(htd, null, null);
 208
 209     // 2s in the past
 210     long ts = EnvironmentEdgeManager.currentTime() - 2000;
 211
 212     try {
 213       Put p = new Put(T1, ts-2);
 214       p.addColumn(c0, c0, T1);
 215       region.put(p);
 216
 217       p = new Put(T1, ts-1);
 218       p.addColumn(c0, c0, T2);
 219       region.put(p);
 220
 221       p = new Put(T1, ts);
 222       p.addColumn(c0, c0, T3);
 223       region.put(p);
 224
 225       Delete d = new Delete(T1, ts-1);
 226       region.delete(d);
 227
 228       Get g = new Get(T1);
 229       g.readAllVersions();
 230       Result r = region.get(g);  // this'll use ScanWildcardColumnTracker
 231       checkResult(r, c0, T3);
 232
 233       g = new Get(T1);
 234       g.readAllVersions();
 235       g.addColumn(c0, c0);
 236       r = region.get(g);  // this'll use ExplicitColumnTracker
 237       checkResult(r, c0, T3);
 238
 239       // now flush/compact
 240       region.flush(true);
 241       region.compact(true);
 242
 243       // try again
 244       g = new Get(T1);
 245       g.readAllVersions();
 246       r = region.get(g);  // this'll use ScanWildcardColumnTracker
 247       checkResult(r, c0, T3);
 248
 249       g = new Get(T1);
 250       g.readAllVersions();
 251       g.addColumn(c0, c0);
 252       r = region.get(g);  // this'll use ExplicitColumnTracker
 253       checkResult(r, c0, T3);
 254     } finally {
 255       HBaseTestingUtil.closeRegionAndWAL(region);
 256     }
 257   }
 258
 259   /**
 260    * Make sure the memstor behaves correctly with minimum versions
 261    */
 262   @Test
 263   public void testMemStore() throws Exception {
 264     ColumnFamilyDescriptor cfd =
 265       ColumnFamilyDescriptorBuilder.newBuilder(c0)
 266         .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
 267         setKeepDeletedCells(KeepDeletedCells.FALSE).build();
 268
 269     TableDescriptor htd = TableDescriptorBuilder.
 270       newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
 271     HRegion region = hbu.createLocalHRegion(htd, null, null);
 272
 273     // 2s in the past
 274     long ts = EnvironmentEdgeManager.currentTime() - 2000;
 275
 276     try {
 277       // 2nd version
 278       Put p = new Put(T1, ts-2);
 279       p.addColumn(c0, c0, T2);
 280       region.put(p);
 281
 282       // 3rd version
 283       p = new Put(T1, ts-1);
 284       p.addColumn(c0, c0, T3);
 285       region.put(p);
 286
 287       // 4th version
 288       p = new Put(T1, ts);
 289       p.addColumn(c0, c0, T4);
 290       region.put(p);
 291
 292       // now flush/compact
 293       region.flush(true);
 294       region.compact(true);
 295
 296       // now put the first version (backdated)
 297       p = new Put(T1, ts-3);
 298       p.addColumn(c0, c0, T1);
 299       region.put(p);
 300
 301       // now the latest change is in the memstore,
 302       // but it is not the latest version
 303
 304       Result r = region.get(new Get(T1));
 305       checkResult(r, c0, T4);
 306
 307       Get g = new Get(T1);
 308       g.readAllVersions();
 309       r = region.get(g); // this'll use ScanWildcardColumnTracker
 310       checkResult(r, c0, T4,T3);
 311
 312       g = new Get(T1);
 313       g.readAllVersions();
 314       g.addColumn(c0, c0);
 315       r = region.get(g);  // this'll use ExplicitColumnTracker
 316       checkResult(r, c0, T4,T3);
 317
 318       p = new Put(T1, ts+1);
 319       p.addColumn(c0, c0, T5);
 320       region.put(p);
 321
 322       // now the latest version is in the memstore
 323
 324       g = new Get(T1);
 325       g.readAllVersions();
 326       r = region.get(g);  // this'll use ScanWildcardColumnTracker
 327       checkResult(r, c0, T5,T4);
 328
 329       g = new Get(T1);
 330       g.readAllVersions();
 331       g.addColumn(c0, c0);
 332       r = region.get(g);  // this'll use ExplicitColumnTracker
 333       checkResult(r, c0, T5,T4);
 334     } finally {
 335       HBaseTestingUtil.closeRegionAndWAL(region);
 336     }
 337   }
 338
 339   /**
 340    * Verify basic minimum versions functionality
 341    */
 342   @Test
 343   public void testBaseCase() throws Exception {
 344     // 2 version minimum, 1000 versions maximum, ttl = 1s
 345     ColumnFamilyDescriptor cfd =
 346       ColumnFamilyDescriptorBuilder.newBuilder(c0)
 347         .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
 348         setKeepDeletedCells(KeepDeletedCells.FALSE).build();
 349
 350     TableDescriptor htd = TableDescriptorBuilder.
 351       newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
 352     HRegion region = hbu.createLocalHRegion(htd, null, null);
 353     try {
 354
 355       // 2s in the past
 356       long ts = EnvironmentEdgeManager.currentTime() - 2000;
 357
 358        // 1st version
 359       Put p = new Put(T1, ts-3);
 360       p.addColumn(c0, c0, T1);
 361       region.put(p);
 362
 363       // 2nd version
 364       p = new Put(T1, ts-2);
 365       p.addColumn(c0, c0, T2);
 366       region.put(p);
 367
 368       // 3rd version
 369       p = new Put(T1, ts-1);
 370       p.addColumn(c0, c0, T3);
 371       region.put(p);
 372
 373       // 4th version
 374       p = new Put(T1, ts);
 375       p.addColumn(c0, c0, T4);
 376       region.put(p);
 377
 378       Result r = region.get(new Get(T1));
 379       checkResult(r, c0, T4);
 380
 381       Get g = new Get(T1);
 382       g.setTimeRange(0L, ts+1);
 383       r = region.get(g);
 384       checkResult(r, c0, T4);
 385
 386   // oldest version still exists
 387       g.setTimeRange(0L, ts-2);
 388       r = region.get(g);
 389       checkResult(r, c0, T1);
 390
 391       // gets see only available versions
 392       // even before compactions
 393       g = new Get(T1);
 394       g.readAllVersions();
 395       r = region.get(g); // this'll use ScanWildcardColumnTracker
 396       checkResult(r, c0, T4,T3);
 397
 398       g = new Get(T1);
 399       g.readAllVersions();
 400       g.addColumn(c0, c0);
 401       r = region.get(g);  // this'll use ExplicitColumnTracker
 402       checkResult(r, c0, T4,T3);
 403
 404       // now flush
 405       region.flush(true);
 406
 407       // with HBASE-4241 a flush will eliminate the expired rows
 408       g = new Get(T1);
 409       g.setTimeRange(0L, ts-2);
 410       r = region.get(g);
 411       assertTrue(r.isEmpty());
 412
 413       // major compaction
 414       region.compact(true);
 415
 416       // after compaction the 4th version is still available
 417       g = new Get(T1);
 418       g.setTimeRange(0L, ts+1);
 419       r = region.get(g);
 420       checkResult(r, c0, T4);
 421
 422       // so is the 3rd
 423       g.setTimeRange(0L, ts);
 424       r = region.get(g);
 425       checkResult(r, c0, T3);
 426
 427       // but the 2nd and earlier versions are gone
 428       g.setTimeRange(0L, ts-1);
 429       r = region.get(g);
 430       assertTrue(r.isEmpty());
 431     } finally {
 432       HBaseTestingUtil.closeRegionAndWAL(region);
 433     }
 434   }
 435
 436   /**
 437    * Verify that basic filters still behave correctly with
 438    * minimum versions enabled.
 439    */
 440   @Test
 441   public void testFilters() throws Exception {
 442     final byte [] c1 = COLUMNS[1];
 443     ColumnFamilyDescriptor cfd =
 444       ColumnFamilyDescriptorBuilder.newBuilder(c0)
 445         .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
 446         setKeepDeletedCells(KeepDeletedCells.FALSE).build();
 447
 448     ColumnFamilyDescriptor cfd2 =
 449       ColumnFamilyDescriptorBuilder.newBuilder(c1)
 450         .setMinVersions(2).setMaxVersions(1000).setTimeToLive(1).
 451         setKeepDeletedCells(KeepDeletedCells.FALSE).build();
 452     List<ColumnFamilyDescriptor> cfdList = new ArrayList();
 453     cfdList.add(cfd);
 454     cfdList.add(cfd2);
 455
 456     TableDescriptor htd = TableDescriptorBuilder.
 457       newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamilies(cfdList).build();
 458     HRegion region = hbu.createLocalHRegion(htd, null, null);
 459
 460     // 2s in the past
 461     long ts = EnvironmentEdgeManager.currentTime() - 2000;
 462     try {
 463
 464       Put p = new Put(T1, ts-3);
 465       p.addColumn(c0, c0, T0);
 466       p.addColumn(c1, c1, T0);
 467       region.put(p);
 468
 469       p = new Put(T1, ts-2);
 470       p.addColumn(c0, c0, T1);
 471       p.addColumn(c1, c1, T1);
 472       region.put(p);
 473
 474       p = new Put(T1, ts-1);
 475       p.addColumn(c0, c0, T2);
 476       p.addColumn(c1, c1, T2);
 477       region.put(p);
 478
 479       p = new Put(T1, ts);
 480       p.addColumn(c0, c0, T3);
 481       p.addColumn(c1, c1, T3);
 482       region.put(p);
 483
 484       List<Long> tss = new ArrayList<>();
 485       tss.add(ts-1);
 486       tss.add(ts-2);
 487
 488       // Sholud only get T2, versions is 2, so T1 is gone from user view.
 489       Get g = new Get(T1);
 490       g.addColumn(c1,c1);
 491       g.setFilter(new TimestampsFilter(tss));
 492       g.readAllVersions();
 493       Result r = region.get(g);
 494       checkResult(r, c1, T2);
 495
 496       // Sholud only get T2, versions is 2, so T1 is gone from user view.
 497       g = new Get(T1);
 498       g.addColumn(c0,c0);
 499       g.setFilter(new TimestampsFilter(tss));
 500       g.readAllVersions();
 501       r = region.get(g);
 502       checkResult(r, c0, T2);
 503
 504       // now flush/compact
 505       region.flush(true);
 506       region.compact(true);
 507
 508       // After flush/compact, the result should be consistent with previous result
 509       g = new Get(T1);
 510       g.addColumn(c1,c1);
 511       g.setFilter(new TimestampsFilter(tss));
 512       g.readAllVersions();
 513       r = region.get(g);
 514       checkResult(r, c1, T2);
 515
 516       // After flush/compact, the result should be consistent with previous result
 517       g = new Get(T1);
 518       g.addColumn(c0,c0);
 519       g.setFilter(new TimestampsFilter(tss));
 520       g.readAllVersions();
 521       r = region.get(g);
 522       checkResult(r, c0, T2);
 523     } finally {
 524       HBaseTestingUtil.closeRegionAndWAL(region);
 525     }
 526   }
 527
 528   @Test
 529   public void testMinVersionsWithKeepDeletedCellsTTL() throws Exception {
 530     int ttl = 4;
 531     ColumnFamilyDescriptor cfd =
 532       ColumnFamilyDescriptorBuilder.newBuilder(c0)
 533         .setVersionsWithTimeToLive(ttl, 2).build();
 534     verifyVersionedCellKeyValues(ttl, cfd);
 535
 536     cfd = ColumnFamilyDescriptorBuilder.newBuilder(c0)
 537       .setMinVersions(2)
 538       .setMaxVersions(Integer.MAX_VALUE)
 539       .setTimeToLive(ttl)
 540       .setKeepDeletedCells(KeepDeletedCells.TTL)
 541       .build();
 542     verifyVersionedCellKeyValues(ttl, cfd);
 543   }
 544
 545   private void verifyVersionedCellKeyValues(int ttl, ColumnFamilyDescriptor cfd)
 546       throws IOException {
 547     TableDescriptor htd = TableDescriptorBuilder.
 548       newBuilder(TableName.valueOf(name.getMethodName())).setColumnFamily(cfd).build();
 549
 550     HRegion region = hbu.createLocalHRegion(htd, null, null);
 551
 552     try {
 553       long startTS = EnvironmentEdgeManager.currentTime();
 554       ManualEnvironmentEdge injectEdge = new ManualEnvironmentEdge();
 555       injectEdge.setValue(startTS);
 556       EnvironmentEdgeManager.injectEdge(injectEdge);
 557
 558       long ts = startTS - 2000;
 559       putFourVersions(region, ts);
 560
 561       Get get;
 562       Result result;
 563
 564       //check we can still see all versions before compaction
 565       get = new Get(T1);
 566       get.readAllVersions();
 567       get.setTimeRange(0, ts);
 568       result = region.get(get);
 569       checkResult(result, c0, T4, T3, T2, T1);
 570
 571       region.flush(true);
 572       region.compact(true);
 573       Assert.assertEquals(startTS, EnvironmentEdgeManager.currentTime());
 574       long expiredTime = EnvironmentEdgeManager.currentTime() - ts - 4;
 575       Assert.assertTrue("TTL for T1 has expired", expiredTime < (ttl * 1000));
 576       //check that nothing was purged yet
 577       verifyBeforeCompaction(region, ts);
 578
 579       injectEdge.incValue(ttl * 1000);
 580
 581       region.flush(true);
 582       region.compact(true);
 583       verifyAfterTtl(region, ts);
 584     } finally {
 585       HBaseTestingUtil.closeRegionAndWAL(region);
 586     }
 587   }
 588
 589   private void verifyAfterTtl(HRegion region, long ts) throws IOException {
 590     Get get;
 591     Result result;
 592     //check that after compaction (which is after TTL) that only T1 && T2 were purged
 593     get = new Get(T1);
 594     get.readAllVersions();
 595     get.setTimeRange(0, ts);
 596     result = region.get(get);
 597     checkResult(result, c0, T4, T3);
 598
 599     get = new Get(T1);
 600     get.readAllVersions();
 601     get.setTimeRange(0, ts - 1);
 602     result = region.get(get);
 603     checkResult(result, c0, T3);
 604
 605     get = new Get(T1);
 606     get.readAllVersions();
 607     get.setTimestamp(ts - 2);
 608     result = region.get(get);
 609     checkResult(result, c0, T3);
 610
 611     get = new Get(T1);
 612     get.readAllVersions();
 613     get.setTimestamp(ts - 3);
 614     result = region.get(get);
 615     Assert.assertEquals(result.getColumnCells(c0, c0).size(), 0);
 616
 617     get = new Get(T1);
 618     get.readAllVersions();
 619     get.setTimeRange(0, ts - 2);
 620     result = region.get(get);
 621     Assert.assertEquals(result.getColumnCells(c0, c0).size(), 0);
 622   }
 623
 624   private void verifyBeforeCompaction(HRegion region, long ts) throws IOException {
 625     Get get;
 626     Result result;
 627     get = new Get(T1);
 628     get.readAllVersions();
 629     get.setTimeRange(0, ts);
 630     result = region.get(get);
 631     checkResult(result, c0, T4, T3, T2, T1);
 632
 633     get = new Get(T1);
 634     get.readAllVersions();
 635     get.setTimeRange(0, ts - 1);
 636     result = region.get(get);
 637     checkResult(result, c0, T3, T2, T1);
 638
 639     get = new Get(T1);
 640     get.readAllVersions();
 641     get.setTimeRange(0, ts - 2);
 642     result = region.get(get);
 643     checkResult(result, c0, T2, T1);
 644
 645     get = new Get(T1);
 646     get.readAllVersions();
 647     get.setTimeRange(0, ts - 3);
 648     result = region.get(get);
 649     checkResult(result, c0, T1);
 650   }
 651
 652   private void putFourVersions(HRegion region, long ts) throws IOException {
 653     // 1st version
 654     Put put = new Put(T1, ts - 4);
 655     put.addColumn(c0, c0, T1);
 656     region.put(put);
 657
 658     // 2nd version
 659     put = new Put(T1, ts - 3);
 660     put.addColumn(c0, c0, T2);
 661     region.put(put);
 662
 663     // 3rd version
 664     put = new Put(T1, ts - 2);
 665     put.addColumn(c0, c0, T3);
 666     region.put(put);
 667
 668     // 4th version
 669     put = new Put(T1, ts - 1);
 670     put.addColumn(c0, c0, T4);
 671     region.put(put);
 672   }
 673
 674   private void checkResult(Result r, byte[] col, byte[] ... vals) {
 675     assertEquals(vals.length, r.size());
 676     List<Cell> kvs = r.getColumnCells(col, col);
 677     assertEquals(kvs.size(), vals.length);
 678     for (int i=0;i<vals.length;i++) {
 679       String expected = Bytes.toString(vals[i]);
 680       String actual = Bytes.toString(CellUtil.cloneValue(kvs.get(i)));
 681       assertTrue(expected + " was expected but doesn't match " + actual,
 682           CellUtil.matchingValue(kvs.get(i), vals[i]));
 683     }
 684   }
 685
 686 }
 687