hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALMethods.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.wal;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertNotNull;
  22 import static org.junit.Assert.assertNotSame;
  23 import static org.junit.Assert.assertNull;
  24 import static org.junit.Assert.assertTrue;
  25
  26 import java.io.IOException;
  27 import java.util.NavigableSet;
  28 import org.apache.hadoop.conf.Configuration;
  29 import org.apache.hadoop.fs.FSDataOutputStream;
  30 import org.apache.hadoop.fs.FileSystem;
  31 import org.apache.hadoop.fs.Path;
  32 import org.apache.hadoop.hbase.HBaseClassTestRule;
  33 import org.apache.hadoop.hbase.HBaseTestingUtility;
  34 import org.apache.hadoop.hbase.HConstants;
  35 import org.apache.hadoop.hbase.KeyValueTestUtil;
  36 import org.apache.hadoop.hbase.ServerName;
  37 import org.apache.hadoop.hbase.TableName;
  38 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
  39 import org.apache.hadoop.hbase.testclassification.SmallTests;
  40 import org.apache.hadoop.hbase.util.Bytes;
  41 import org.apache.hadoop.hbase.util.FSUtils;
  42 import org.apache.hadoop.hbase.wal.WALSplitter.EntryBuffers;
  43 import org.apache.hadoop.hbase.wal.WALSplitter.PipelineController;
  44 import org.apache.hadoop.hbase.wal.WALSplitter.RegionEntryBuffer;
  45 import org.junit.ClassRule;
  46 import org.junit.Test;
  47 import org.junit.experimental.categories.Category;
  48
  49 /**
  50  * Simple testing of a few WAL methods.
  51  */
  52 @Category({RegionServerTests.class, SmallTests.class})
  53 public class TestWALMethods {
  54
  55   @ClassRule
  56   public static final HBaseClassTestRule CLASS_RULE =
  57       HBaseClassTestRule.forClass(TestWALMethods.class);
  58
  59   private static final byte[] TEST_REGION = Bytes.toBytes("test_region");
  60   private static final TableName TEST_TABLE =
  61       TableName.valueOf("test_table");
  62
  63   private final HBaseTestingUtility util = new HBaseTestingUtility();
  64
  65   @Test
  66   public void testServerNameFromWAL() throws Exception {
  67     Path walPath = new Path("/hbase/WALs/regionserver-2.example.com,22101,1487767381290",
  68         "regionserver-2.example.com%2C22101%2C1487767381290.null0.1487785392316");
  69     ServerName name = AbstractFSWALProvider.getServerNameFromWALDirectoryName(walPath);
  70     assertEquals(ServerName.valueOf("regionserver-2.example.com", 22101, 1487767381290L), name);
  71   }
  72
  73   @Test
  74   public void testServerNameFromTestWAL() throws Exception {
  75     Path walPath = new Path(
  76         "/user/example/test-data/12ff1404-68c6-4715-a4b9-775e763842bc/WALs/TestWALRecordReader",
  77         "TestWALRecordReader.default.1487787939118");
  78     ServerName name = AbstractFSWALProvider.getServerNameFromWALDirectoryName(walPath);
  79     assertNull(name);
  80   }
  81
  82   /**
  83    * Assert that getSplitEditFilesSorted returns files in expected order and
  84    * that it skips moved-aside files.
  85    * @throws IOException
  86    */
  87   @Test public void testGetSplitEditFilesSorted() throws IOException {
  88     FileSystem fs = FileSystem.get(util.getConfiguration());
  89     Path regiondir = util.getDataTestDir("regiondir");
  90     fs.delete(regiondir, true);
  91     fs.mkdirs(regiondir);
  92     Path recoverededits = WALSplitter.getRegionDirRecoveredEditsDir(regiondir);
  93     String first = WALSplitter.formatRecoveredEditsFileName(-1);
  94     createFile(fs, recoverededits, first);
  95     createFile(fs, recoverededits, WALSplitter.formatRecoveredEditsFileName(0));
  96     createFile(fs, recoverededits, WALSplitter.formatRecoveredEditsFileName(1));
  97     createFile(fs, recoverededits, WALSplitter
  98         .formatRecoveredEditsFileName(11));
  99     createFile(fs, recoverededits, WALSplitter.formatRecoveredEditsFileName(2));
 100     createFile(fs, recoverededits, WALSplitter
 101         .formatRecoveredEditsFileName(50));
 102     String last = WALSplitter.formatRecoveredEditsFileName(Long.MAX_VALUE);
 103     createFile(fs, recoverededits, last);
 104     createFile(fs, recoverededits,
 105       Long.toString(Long.MAX_VALUE) + "." + System.currentTimeMillis());
 106
 107     final Configuration walConf = new Configuration(util.getConfiguration());
 108     FSUtils.setRootDir(walConf, regiondir);
 109     (new WALFactory(walConf, "dummyLogName")).getWAL(null);
 110
 111     NavigableSet<Path> files = WALSplitter.getSplitEditFilesSorted(fs, regiondir);
 112     assertEquals(7, files.size());
 113     assertEquals(files.pollFirst().getName(), first);
 114     assertEquals(files.pollLast().getName(), last);
 115     assertEquals(files.pollFirst().getName(),
 116       WALSplitter
 117         .formatRecoveredEditsFileName(0));
 118     assertEquals(files.pollFirst().getName(),
 119       WALSplitter
 120         .formatRecoveredEditsFileName(1));
 121     assertEquals(files.pollFirst().getName(),
 122       WALSplitter
 123         .formatRecoveredEditsFileName(2));
 124     assertEquals(files.pollFirst().getName(),
 125       WALSplitter
 126         .formatRecoveredEditsFileName(11));
 127   }
 128
 129   private void createFile(final FileSystem fs, final Path testdir,
 130       final String name)
 131   throws IOException {
 132     FSDataOutputStream fdos = fs.create(new Path(testdir, name), true);
 133     fdos.close();
 134   }
 135
 136   @Test
 137   public void testRegionEntryBuffer() throws Exception {
 138     WALSplitter.RegionEntryBuffer reb = new WALSplitter.RegionEntryBuffer(
 139         TEST_TABLE, TEST_REGION);
 140     assertEquals(0, reb.heapSize());
 141
 142     reb.appendEntry(createTestLogEntry(1));
 143     assertTrue(reb.heapSize() > 0);
 144   }
 145
 146   @Test
 147   public void testEntrySink() throws Exception {
 148     EntryBuffers sink = new EntryBuffers(new PipelineController(), 1*1024*1024);
 149     for (int i = 0; i < 1000; i++) {
 150       WAL.Entry entry = createTestLogEntry(i);
 151       sink.appendEntry(entry);
 152     }
 153
 154     assertTrue(sink.totalBuffered > 0);
 155     long amountInChunk = sink.totalBuffered;
 156     // Get a chunk
 157     RegionEntryBuffer chunk = sink.getChunkToWrite();
 158     assertEquals(chunk.heapSize(), amountInChunk);
 159
 160     // Make sure it got marked that a thread is "working on this"
 161     assertTrue(sink.isRegionCurrentlyWriting(TEST_REGION));
 162
 163     // Insert some more entries
 164     for (int i = 0; i < 500; i++) {
 165       WAL.Entry entry = createTestLogEntry(i);
 166       sink.appendEntry(entry);
 167     }
 168     // Asking for another chunk shouldn't work since the first one
 169     // is still writing
 170     assertNull(sink.getChunkToWrite());
 171
 172     // If we say we're done writing the first chunk, then we should be able
 173     // to get the second
 174     sink.doneWriting(chunk);
 175
 176     RegionEntryBuffer chunk2 = sink.getChunkToWrite();
 177     assertNotNull(chunk2);
 178     assertNotSame(chunk, chunk2);
 179     long amountInChunk2 = sink.totalBuffered;
 180     // The second chunk had fewer rows than the first
 181     assertTrue(amountInChunk2 < amountInChunk);
 182
 183     sink.doneWriting(chunk2);
 184     assertEquals(0, sink.totalBuffered);
 185   }
 186
 187   private WAL.Entry createTestLogEntry(int i) {
 188     long seq = i;
 189     long now = i * 1000;
 190
 191     WALEdit edit = new WALEdit();
 192     edit.add(KeyValueTestUtil.create("row", "fam", "qual", 1234, "val"));
 193     WALKeyImpl key = new WALKeyImpl(TEST_REGION, TEST_TABLE, seq, now,
 194         HConstants.DEFAULT_CLUSTER_ID);
 195     WAL.Entry entry = new WAL.Entry(key, edit);
 196     return entry;
 197   }
 198
 199 }
 200