hbase-server/src/test/java/org/apache/hadoop/hbase/TimestampTestBase.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18
  19 package org.apache.hadoop.hbase;
  20
  21 import java.io.IOException;
  22
  23 import org.apache.hadoop.hbase.client.Delete;
  24 import org.apache.hadoop.hbase.client.Get;
  25 import org.apache.hadoop.hbase.client.Put;
  26 import org.apache.hadoop.hbase.client.Result;
  27 import org.apache.hadoop.hbase.client.ResultScanner;
  28 import org.apache.hadoop.hbase.client.Scan;
  29 import org.apache.hadoop.hbase.client.Table;
  30 import org.apache.hadoop.hbase.client.Durability;
  31 import org.apache.hadoop.hbase.util.Bytes;
  32 import org.junit.Assert;
  33
  34 /**
  35  * Tests user specifiable time stamps putting, getting and scanning.  Also
  36  * tests same in presence of deletes.  Test cores are written so can be
  37  * run against an HRegion and against an HTable: i.e. both local and remote.
  38  */
  39 public class TimestampTestBase {
  40   private static final long T0 = 10L;
  41   private static final long T1 = 100L;
  42   private static final long T2 = 200L;
  43
  44   public static final byte [] FAMILY_NAME = Bytes.toBytes("colfamily11");
  45   private static final byte [] QUALIFIER_NAME = Bytes.toBytes("contents");
  46
  47   private static final byte [] ROW = Bytes.toBytes("row");
  48
  49   interface FlushCache {
  50     void flushcache() throws IOException;
  51   }
  52
  53   /*
  54    * Run test that delete works according to description in <a
  55    * href="https://issues.apache.org/jira/browse/HADOOP-1784">hadoop-1784</a>.
  56    * @param incommon
  57    * @param flusher
  58    * @throws IOException
  59    */
  60   public static void doTestDelete(final Table table, FlushCache flusher)
  61   throws IOException {
  62     // Add values at various timestamps (Values are timestampes as bytes).
  63     put(table, T0);
  64     put(table, T1);
  65     put(table, T2);
  66     put(table);
  67     // Verify that returned versions match passed timestamps.
  68     assertVersions(table, new long [] {HConstants.LATEST_TIMESTAMP, T2, T1});
  69
  70     // If I delete w/o specifying a timestamp, this means I'm deleting the latest.
  71     delete(table);
  72     // Verify that I get back T2 through T1 -- that the latest version has been deleted.
  73     assertVersions(table, new long [] {T2, T1, T0});
  74
  75     // Flush everything out to disk and then retry
  76     flusher.flushcache();
  77     assertVersions(table, new long [] {T2, T1, T0});
  78
  79     // Now add, back a latest so I can test remove other than the latest.
  80     put(table);
  81     assertVersions(table, new long [] {HConstants.LATEST_TIMESTAMP, T2, T1});
  82     delete(table, T2);
  83     assertVersions(table, new long [] {HConstants.LATEST_TIMESTAMP, T1, T0});
  84     // Flush everything out to disk and then retry
  85     flusher.flushcache();
  86     assertVersions(table, new long [] {HConstants.LATEST_TIMESTAMP, T1, T0});
  87
  88     // Now try deleting all from T2 back inclusive (We first need to add T2
  89     // back into the mix and to make things a little interesting, delete and then readd T1.
  90     put(table, T2);
  91     delete(table, T1);
  92     put(table, T1);
  93
  94     Delete delete = new Delete(ROW);
  95     delete.addColumns(FAMILY_NAME, QUALIFIER_NAME, T2);
  96     table.delete(delete);
  97
  98     // Should only be current value in set.  Assert this is so
  99     assertOnlyLatest(table, HConstants.LATEST_TIMESTAMP);
 100
 101     // Flush everything out to disk and then redo above tests
 102     flusher.flushcache();
 103     assertOnlyLatest(table, HConstants.LATEST_TIMESTAMP);
 104   }
 105
 106   private static void assertOnlyLatest(final Table incommon, final long currentTime)
 107   throws IOException {
 108     Get get = null;
 109     get = new Get(ROW);
 110     get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
 111     get.readVersions(3);
 112     Result result = incommon.get(get);
 113     Assert.assertEquals(1, result.size());
 114     long time = Bytes.toLong(CellUtil.cloneValue(result.rawCells()[0]));
 115     Assert.assertEquals(time, currentTime);
 116   }
 117
 118   /*
 119    * Assert that returned versions match passed in timestamps and that results
 120    * are returned in the right order.  Assert that values when converted to
 121    * longs match the corresponding passed timestamp.
 122    * @param r
 123    * @param tss
 124    * @throws IOException
 125    */
 126   public static void assertVersions(final Table incommon, final long [] tss)
 127   throws IOException {
 128     // Assert that 'latest' is what we expect.
 129     Get get = null;
 130     get = new Get(ROW);
 131     get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
 132     Result r = incommon.get(get);
 133     byte [] bytes = r.getValue(FAMILY_NAME, QUALIFIER_NAME);
 134     long t = Bytes.toLong(bytes);
 135     Assert.assertEquals(tss[0], t);
 136
 137     // Now assert that if we ask for multiple versions, that they come out in
 138     // order.
 139     get = new Get(ROW);
 140     get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
 141     get.readVersions(tss.length);
 142     Result result = incommon.get(get);
 143     Cell [] kvs = result.rawCells();
 144     Assert.assertEquals(kvs.length, tss.length);
 145     for(int i=0;i<kvs.length;i++) {
 146       t = Bytes.toLong(CellUtil.cloneValue(kvs[i]));
 147       Assert.assertEquals(tss[i], t);
 148     }
 149
 150     // Determine highest stamp to set as next max stamp
 151     long maxStamp = kvs[0].getTimestamp();
 152
 153     // Specify a timestamp get multiple versions.
 154     get = new Get(ROW);
 155     get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
 156     get.setTimeRange(0, maxStamp);
 157     get.readVersions(kvs.length - 1);
 158     result = incommon.get(get);
 159     kvs = result.rawCells();
 160     Assert.assertEquals(kvs.length, tss.length - 1);
 161     for(int i=1;i<kvs.length;i++) {
 162       t = Bytes.toLong(CellUtil.cloneValue(kvs[i-1]));
 163       Assert.assertEquals(tss[i], t);
 164     }
 165
 166     // Test scanner returns expected version
 167     assertScanContentTimestamp(incommon, tss[0]);
 168   }
 169
 170   /*
 171    * Run test scanning different timestamps.
 172    * @param incommon
 173    * @param flusher
 174    * @throws IOException
 175    */
 176   public static void doTestTimestampScanning(final Table incommon,
 177     final FlushCache flusher)
 178   throws IOException {
 179     // Add a couple of values for three different timestamps.
 180     put(incommon, T0);
 181     put(incommon, T1);
 182     put(incommon, HConstants.LATEST_TIMESTAMP);
 183     // Get count of latest items.
 184     int count = assertScanContentTimestamp(incommon,
 185       HConstants.LATEST_TIMESTAMP);
 186     // Assert I get same count when I scan at each timestamp.
 187     Assert.assertEquals(count, assertScanContentTimestamp(incommon, T0));
 188     Assert.assertEquals(count, assertScanContentTimestamp(incommon, T1));
 189     // Flush everything out to disk and then retry
 190     flusher.flushcache();
 191     Assert.assertEquals(count, assertScanContentTimestamp(incommon, T0));
 192     Assert.assertEquals(count, assertScanContentTimestamp(incommon, T1));
 193   }
 194
 195   /*
 196    * Assert that the scan returns only values < timestamp.
 197    * @param r
 198    * @param ts
 199    * @return Count of items scanned.
 200    * @throws IOException
 201    */
 202   public static int assertScanContentTimestamp(final Table in, final long ts)
 203   throws IOException {
 204     Scan scan = new Scan().withStartRow(HConstants.EMPTY_START_ROW);
 205     scan.addFamily(FAMILY_NAME);
 206     scan.setTimeRange(0, ts);
 207     ResultScanner scanner = in.getScanner(scan);
 208     int count = 0;
 209     try {
 210       // TODO FIX
 211 //      HStoreKey key = new HStoreKey();
 212 //      TreeMap<byte [], Cell>value =
 213 //        new TreeMap<byte [], Cell>(Bytes.BYTES_COMPARATOR);
 214 //      while (scanner.next(key, value)) {
 215 //        assertTrue(key.getTimestamp() <= ts);
 216 //        // Content matches the key or HConstants.LATEST_TIMESTAMP.
 217 //        // (Key does not match content if we 'put' with LATEST_TIMESTAMP).
 218 //        long l = Bytes.toLong(value.get(COLUMN).getValue());
 219 //        assertTrue(key.getTimestamp() == l ||
 220 //          HConstants.LATEST_TIMESTAMP == l);
 221 //        count++;
 222 //        value.clear();
 223 //      }
 224     } finally {
 225       scanner.close();
 226     }
 227     return count;
 228   }
 229
 230   public static void put(final Table loader, final long ts)
 231   throws IOException {
 232     put(loader, Bytes.toBytes(ts), ts);
 233   }
 234
 235   public static void put(final Table loader)
 236   throws IOException {
 237     long ts = HConstants.LATEST_TIMESTAMP;
 238     put(loader, Bytes.toBytes(ts), ts);
 239   }
 240
 241   /*
 242    * Put values.
 243    * @param loader
 244    * @param bytes
 245    * @param ts
 246    * @throws IOException
 247    */
 248   public static void put(final Table loader, final byte [] bytes,
 249     final long ts)
 250   throws IOException {
 251     Put put = new Put(ROW, ts);
 252     put.setDurability(Durability.SKIP_WAL);
 253     put.addColumn(FAMILY_NAME, QUALIFIER_NAME, bytes);
 254     loader.put(put);
 255   }
 256
 257   public static void delete(final Table loader) throws IOException {
 258     delete(loader, null);
 259   }
 260
 261   public static void delete(final Table loader, final byte [] column)
 262   throws IOException {
 263     delete(loader, column, HConstants.LATEST_TIMESTAMP);
 264   }
 265
 266   public static void delete(final Table loader, final long ts)
 267   throws IOException {
 268     delete(loader, null, ts);
 269   }
 270
 271   public static void delete(final Table loader, final byte [] column,
 272       final long ts)
 273   throws IOException {
 274     Delete delete = ts == HConstants.LATEST_TIMESTAMP?
 275       new Delete(ROW): new Delete(ROW, ts);
 276     delete.addColumn(FAMILY_NAME, QUALIFIER_NAME, ts);
 277     loader.delete(delete);
 278   }
 279
 280   public static Result get(final Table loader) throws IOException {
 281     return loader.get(new Get(ROW));
 282   }
 283 }