hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALMethods.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.wal;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertNotNull;
  22 import static org.junit.Assert.assertNotSame;
  23 import static org.junit.Assert.assertNull;
  24 import static org.junit.Assert.assertTrue;
  25
  26 import java.io.IOException;
  27 import java.util.NavigableSet;
  28 import org.apache.hadoop.conf.Configuration;
  29 import org.apache.hadoop.fs.FSDataOutputStream;
  30 import org.apache.hadoop.fs.FileSystem;
  31 import org.apache.hadoop.fs.Path;
  32 import org.apache.hadoop.hbase.HBaseClassTestRule;
  33 import org.apache.hadoop.hbase.HBaseTestingUtil;
  34 import org.apache.hadoop.hbase.HConstants;
  35 import org.apache.hadoop.hbase.KeyValueTestUtil;
  36 import org.apache.hadoop.hbase.ServerName;
  37 import org.apache.hadoop.hbase.TableName;
  38 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
  39 import org.apache.hadoop.hbase.testclassification.SmallTests;
  40 import org.apache.hadoop.hbase.util.Bytes;
  41 import org.apache.hadoop.hbase.util.CommonFSUtils;
  42 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
  43 import org.apache.hadoop.hbase.wal.WALSplitter.PipelineController;
  44 import org.junit.ClassRule;
  45 import org.junit.Test;
  46 import org.junit.experimental.categories.Category;
  47
  48 /**
  49  * Simple testing of a few WAL methods.
  50  */
  51 @Category({RegionServerTests.class, SmallTests.class})
  52 public class TestWALMethods {
  53
  54   @ClassRule
  55   public static final HBaseClassTestRule CLASS_RULE =
  56       HBaseClassTestRule.forClass(TestWALMethods.class);
  57
  58   private static final byte[] TEST_REGION = Bytes.toBytes("test_region");
  59   private static final TableName TEST_TABLE =
  60       TableName.valueOf("test_table");
  61
  62   private final HBaseTestingUtil util = new HBaseTestingUtil();
  63
  64   @Test
  65   public void testServerNameFromWAL() throws Exception {
  66     Path walPath = new Path("/hbase/WALs/regionserver-2.example.com,22101,1487767381290",
  67         "regionserver-2.example.com%2C22101%2C1487767381290.null0.1487785392316");
  68     ServerName name = AbstractFSWALProvider.getServerNameFromWALDirectoryName(walPath);
  69     assertEquals(ServerName.valueOf("regionserver-2.example.com", 22101, 1487767381290L), name);
  70   }
  71
  72   @Test
  73   public void testServerNameFromTestWAL() throws Exception {
  74     Path walPath = new Path(
  75         "/user/example/test-data/12ff1404-68c6-4715-a4b9-775e763842bc/WALs/TestWALRecordReader",
  76         "TestWALRecordReader.default.1487787939118");
  77     ServerName name = AbstractFSWALProvider.getServerNameFromWALDirectoryName(walPath);
  78     assertNull(name);
  79   }
  80
  81   /**
  82    * Assert that getSplitEditFilesSorted returns files in expected order and
  83    * that it skips moved-aside files.
  84    * @throws IOException
  85    */
  86   @Test public void testGetSplitEditFilesSorted() throws IOException {
  87     FileSystem fs = FileSystem.get(util.getConfiguration());
  88     Path regiondir = util.getDataTestDir("regiondir");
  89     fs.delete(regiondir, true);
  90     fs.mkdirs(regiondir);
  91     Path recoverededits = WALSplitUtil.getRegionDirRecoveredEditsDir(regiondir);
  92     String first = WALSplitUtil.formatRecoveredEditsFileName(-1);
  93     createFile(fs, recoverededits, first);
  94     createFile(fs, recoverededits, WALSplitUtil.formatRecoveredEditsFileName(0));
  95     createFile(fs, recoverededits, WALSplitUtil.formatRecoveredEditsFileName(1));
  96     createFile(fs, recoverededits, WALSplitUtil
  97         .formatRecoveredEditsFileName(11));
  98     createFile(fs, recoverededits, WALSplitUtil.formatRecoveredEditsFileName(2));
  99     createFile(fs, recoverededits, WALSplitUtil
 100         .formatRecoveredEditsFileName(50));
 101     String last = WALSplitUtil.formatRecoveredEditsFileName(Long.MAX_VALUE);
 102     createFile(fs, recoverededits, last);
 103     createFile(fs, recoverededits,
 104       Long.toString(Long.MAX_VALUE) + "." + EnvironmentEdgeManager.currentTime());
 105
 106     final Configuration walConf = new Configuration(util.getConfiguration());
 107     CommonFSUtils.setRootDir(walConf, regiondir);
 108     (new WALFactory(walConf, "dummyLogName")).getWAL(null);
 109
 110     NavigableSet<Path> files = WALSplitUtil.getSplitEditFilesSorted(fs, regiondir);
 111     assertEquals(7, files.size());
 112     assertEquals(files.pollFirst().getName(), first);
 113     assertEquals(files.pollLast().getName(), last);
 114     assertEquals(files.pollFirst().getName(),
 115         WALSplitUtil
 116         .formatRecoveredEditsFileName(0));
 117     assertEquals(files.pollFirst().getName(),
 118         WALSplitUtil
 119         .formatRecoveredEditsFileName(1));
 120     assertEquals(files.pollFirst().getName(),
 121         WALSplitUtil
 122         .formatRecoveredEditsFileName(2));
 123     assertEquals(files.pollFirst().getName(),
 124         WALSplitUtil
 125         .formatRecoveredEditsFileName(11));
 126   }
 127
 128   private void createFile(final FileSystem fs, final Path testdir,
 129       final String name)
 130   throws IOException {
 131     FSDataOutputStream fdos = fs.create(new Path(testdir, name), true);
 132     fdos.close();
 133   }
 134
 135   @Test
 136   public void testRegionEntryBuffer() throws Exception {
 137     EntryBuffers.RegionEntryBuffer reb = new EntryBuffers.RegionEntryBuffer(
 138         TEST_TABLE, TEST_REGION);
 139     assertEquals(0, reb.heapSize());
 140
 141     reb.appendEntry(createTestLogEntry(1));
 142     assertTrue(reb.heapSize() > 0);
 143   }
 144
 145   @Test
 146   public void testEntrySink() throws Exception {
 147     EntryBuffers sink = new EntryBuffers(new PipelineController(), 1*1024*1024);
 148     for (int i = 0; i < 1000; i++) {
 149       WAL.Entry entry = createTestLogEntry(i);
 150       sink.appendEntry(entry);
 151     }
 152
 153     assertTrue(sink.totalBuffered > 0);
 154     long amountInChunk = sink.totalBuffered;
 155     // Get a chunk
 156     EntryBuffers.RegionEntryBuffer chunk = sink.getChunkToWrite();
 157     assertEquals(chunk.heapSize(), amountInChunk);
 158
 159     // Make sure it got marked that a thread is "working on this"
 160     assertTrue(sink.isRegionCurrentlyWriting(TEST_REGION));
 161
 162     // Insert some more entries
 163     for (int i = 0; i < 500; i++) {
 164       WAL.Entry entry = createTestLogEntry(i);
 165       sink.appendEntry(entry);
 166     }
 167     // Asking for another chunk shouldn't work since the first one
 168     // is still writing
 169     assertNull(sink.getChunkToWrite());
 170
 171     // If we say we're done writing the first chunk, then we should be able
 172     // to get the second
 173     sink.doneWriting(chunk);
 174
 175     EntryBuffers.RegionEntryBuffer chunk2 = sink.getChunkToWrite();
 176     assertNotNull(chunk2);
 177     assertNotSame(chunk, chunk2);
 178     long amountInChunk2 = sink.totalBuffered;
 179     // The second chunk had fewer rows than the first
 180     assertTrue(amountInChunk2 < amountInChunk);
 181
 182     sink.doneWriting(chunk2);
 183     assertEquals(0, sink.totalBuffered);
 184   }
 185
 186   private WAL.Entry createTestLogEntry(int i) {
 187     long seq = i;
 188     long now = i * 1000;
 189
 190     WALEdit edit = new WALEdit();
 191     edit.add(KeyValueTestUtil.create("row", "fam", "qual", 1234, "val"));
 192     WALKeyImpl key = new WALKeyImpl(TEST_REGION, TEST_TABLE, seq, now,
 193         HConstants.DEFAULT_CLUSTER_ID);
 194     WAL.Entry entry = new WAL.Entry(key, edit);
 195     return entry;
 196   }
 197
 198 }
 199