hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestSecureWAL.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.wal;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertFalse;
  22 import static org.junit.Assert.assertTrue;
  23
  24 import java.util.Arrays;
  25 import java.util.List;
  26 import java.util.NavigableMap;
  27 import java.util.TreeMap;
  28 import org.apache.commons.io.IOUtils;
  29 import org.apache.hadoop.conf.Configuration;
  30 import org.apache.hadoop.fs.FSDataInputStream;
  31 import org.apache.hadoop.fs.FileSystem;
  32 import org.apache.hadoop.fs.Path;
  33 import org.apache.hadoop.hbase.Cell;
  34 import org.apache.hadoop.hbase.HBaseClassTestRule;
  35 import org.apache.hadoop.hbase.HBaseTestingUtil;
  36 import org.apache.hadoop.hbase.HConstants;
  37 import org.apache.hadoop.hbase.KeyValue;
  38 import org.apache.hadoop.hbase.TableName;
  39 import org.apache.hadoop.hbase.client.RegionInfo;
  40 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
  41 import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting;
  42 import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
  43 import org.apache.hadoop.hbase.regionserver.wal.SecureAsyncProtobufLogWriter;
  44 import org.apache.hadoop.hbase.regionserver.wal.SecureProtobufLogReader;
  45 import org.apache.hadoop.hbase.regionserver.wal.SecureProtobufLogWriter;
  46 import org.apache.hadoop.hbase.testclassification.MediumTests;
  47 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
  48 import org.apache.hadoop.hbase.util.Bytes;
  49 import org.apache.hadoop.hbase.util.CommonFSUtils;
  50 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
  51 import org.junit.AfterClass;
  52 import org.junit.Before;
  53 import org.junit.BeforeClass;
  54 import org.junit.ClassRule;
  55 import org.junit.Rule;
  56 import org.junit.Test;
  57 import org.junit.experimental.categories.Category;
  58 import org.junit.rules.TestName;
  59 import org.junit.runner.RunWith;
  60 import org.junit.runners.Parameterized;
  61 import org.junit.runners.Parameterized.Parameter;
  62 import org.junit.runners.Parameterized.Parameters;
  63
  64 @RunWith(Parameterized.class)
  65 @Category({ RegionServerTests.class, MediumTests.class })
  66 public class TestSecureWAL {
  67
  68   @ClassRule
  69   public static final HBaseClassTestRule CLASS_RULE =
  70       HBaseClassTestRule.forClass(TestSecureWAL.class);
  71
  72   static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
  73
  74   @Rule
  75   public TestName name = new TestName();
  76
  77   @Parameter
  78   public String walProvider;
  79
  80   @Parameters(name = "{index}: provider={0}")
  81   public static Iterable<Object[]> data() {
  82     return Arrays.asList(new Object[] { "defaultProvider" }, new Object[] { "asyncfs" });
  83   }
  84
  85   @BeforeClass
  86   public static void setUpBeforeClass() throws Exception {
  87     Configuration conf = TEST_UTIL.getConfiguration();
  88     conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
  89     conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
  90     conf.setClass("hbase.regionserver.hlog.reader.impl", SecureProtobufLogReader.class,
  91       WAL.Reader.class);
  92     conf.setClass("hbase.regionserver.hlog.writer.impl", SecureProtobufLogWriter.class,
  93       WALProvider.Writer.class);
  94     conf.setClass("hbase.regionserver.hlog.async.writer.impl", SecureAsyncProtobufLogWriter.class,
  95       WALProvider.AsyncWriter.class);
  96     conf.setBoolean(HConstants.ENABLE_WAL_ENCRYPTION, true);
  97     CommonFSUtils.setRootDir(conf, TEST_UTIL.getDataTestDirOnTestFS());
  98     TEST_UTIL.startMiniDFSCluster(3);
  99   }
 100
 101   @AfterClass
 102   public static void tearDownAfterClass() throws Exception {
 103     TEST_UTIL.shutdownMiniCluster();
 104   }
 105
 106   @Before
 107   public void setUp() {
 108     TEST_UTIL.getConfiguration().set(WALFactory.WAL_PROVIDER, walProvider);
 109   }
 110
 111   @Test
 112   public void testSecureWAL() throws Exception {
 113     TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
 114     NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
 115     scopes.put(tableName.getName(), 0);
 116     RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).build();
 117     final int total = 10;
 118     final byte[] row = Bytes.toBytes("row");
 119     final byte[] family = Bytes.toBytes("family");
 120     final byte[] value = Bytes.toBytes("Test value");
 121     FileSystem fs = TEST_UTIL.getDFSCluster().getFileSystem();
 122     final WALFactory wals =
 123         new WALFactory(TEST_UTIL.getConfiguration(), tableName.getNameAsString());
 124
 125     // Write the WAL
 126     final WAL wal = wals.getWAL(regionInfo);
 127
 128     MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
 129
 130     for (int i = 0; i < total; i++) {
 131       WALEdit kvs = new WALEdit();
 132       kvs.add(new KeyValue(row, family, Bytes.toBytes(i), value));
 133       wal.appendData(regionInfo, new WALKeyImpl(regionInfo.getEncodedNameAsBytes(), tableName,
 134         EnvironmentEdgeManager.currentTime(), mvcc, scopes), kvs);
 135     }
 136     wal.sync();
 137     final Path walPath = AbstractFSWALProvider.getCurrentFileName(wal);
 138     wals.shutdown();
 139
 140     // Insure edits are not plaintext
 141     long length = fs.getFileStatus(walPath).getLen();
 142     FSDataInputStream in = fs.open(walPath);
 143     byte[] fileData = new byte[(int)length];
 144     IOUtils.readFully(in, fileData);
 145     in.close();
 146     assertFalse("Cells appear to be plaintext", Bytes.contains(fileData, value));
 147
 148     // Confirm the WAL can be read back
 149     WAL.Reader reader = wals.createReader(TEST_UTIL.getTestFileSystem(), walPath);
 150     int count = 0;
 151     WAL.Entry entry = new WAL.Entry();
 152     while (reader.next(entry) != null) {
 153       count++;
 154       List<Cell> cells = entry.getEdit().getCells();
 155       assertTrue("Should be one KV per WALEdit", cells.size() == 1);
 156       for (Cell cell: cells) {
 157         assertTrue("Incorrect row", Bytes.equals(cell.getRowArray(), cell.getRowOffset(),
 158           cell.getRowLength(), row, 0, row.length));
 159         assertTrue("Incorrect family", Bytes.equals(cell.getFamilyArray(), cell.getFamilyOffset(),
 160           cell.getFamilyLength(), family, 0, family.length));
 161         assertTrue("Incorrect value", Bytes.equals(cell.getValueArray(), cell.getValueOffset(),
 162           cell.getValueLength(), value, 0, value.length));
 163       }
 164     }
 165     assertEquals("Should have read back as many KVs as written", total, count);
 166     reader.close();
 167   }
 168 }