hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestSecureWAL.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.wal;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertFalse;
  22 import static org.junit.Assert.assertTrue;
  23
  24 import java.util.Arrays;
  25 import java.util.List;
  26 import java.util.NavigableMap;
  27 import java.util.TreeMap;
  28 import org.apache.commons.io.IOUtils;
  29 import org.apache.hadoop.conf.Configuration;
  30 import org.apache.hadoop.fs.FSDataInputStream;
  31 import org.apache.hadoop.fs.FileSystem;
  32 import org.apache.hadoop.fs.Path;
  33 import org.apache.hadoop.hbase.Cell;
  34 import org.apache.hadoop.hbase.HBaseClassTestRule;
  35 import org.apache.hadoop.hbase.HBaseTestingUtility;
  36 import org.apache.hadoop.hbase.HConstants;
  37 import org.apache.hadoop.hbase.KeyValue;
  38 import org.apache.hadoop.hbase.TableName;
  39 import org.apache.hadoop.hbase.client.RegionInfo;
  40 import org.apache.hadoop.hbase.client.RegionInfoBuilder;
  41 import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting;
  42 import org.apache.hadoop.hbase.regionserver.MultiVersionConcurrencyControl;
  43 import org.apache.hadoop.hbase.regionserver.wal.SecureAsyncProtobufLogWriter;
  44 import org.apache.hadoop.hbase.regionserver.wal.SecureProtobufLogReader;
  45 import org.apache.hadoop.hbase.regionserver.wal.SecureProtobufLogWriter;
  46 import org.apache.hadoop.hbase.testclassification.MediumTests;
  47 import org.apache.hadoop.hbase.testclassification.RegionServerTests;
  48 import org.apache.hadoop.hbase.util.Bytes;
  49 import org.apache.hadoop.hbase.util.FSUtils;
  50 import org.junit.AfterClass;
  51 import org.junit.Before;
  52 import org.junit.BeforeClass;
  53 import org.junit.ClassRule;
  54 import org.junit.Rule;
  55 import org.junit.Test;
  56 import org.junit.experimental.categories.Category;
  57 import org.junit.rules.TestName;
  58 import org.junit.runner.RunWith;
  59 import org.junit.runners.Parameterized;
  60 import org.junit.runners.Parameterized.Parameter;
  61 import org.junit.runners.Parameterized.Parameters;
  62
  63 @RunWith(Parameterized.class)
  64 @Category({ RegionServerTests.class, MediumTests.class })
  65 public class TestSecureWAL {
  66
  67   @ClassRule
  68   public static final HBaseClassTestRule CLASS_RULE =
  69       HBaseClassTestRule.forClass(TestSecureWAL.class);
  70
  71   static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
  72
  73   @Rule
  74   public TestName name = new TestName();
  75
  76   @Parameter
  77   public String walProvider;
  78
  79   @Parameters(name = "{index}: provider={0}")
  80   public static Iterable<Object[]> data() {
  81     return Arrays.asList(new Object[] { "defaultProvider" }, new Object[] { "asyncfs" });
  82   }
  83
  84   @BeforeClass
  85   public static void setUpBeforeClass() throws Exception {
  86     Configuration conf = TEST_UTIL.getConfiguration();
  87     conf.set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY, KeyProviderForTesting.class.getName());
  88     conf.set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
  89     conf.setClass("hbase.regionserver.hlog.reader.impl", SecureProtobufLogReader.class,
  90       WAL.Reader.class);
  91     conf.setClass("hbase.regionserver.hlog.writer.impl", SecureProtobufLogWriter.class,
  92       WALProvider.Writer.class);
  93     conf.setClass("hbase.regionserver.hlog.async.writer.impl", SecureAsyncProtobufLogWriter.class,
  94       WALProvider.AsyncWriter.class);
  95     conf.setBoolean(HConstants.ENABLE_WAL_ENCRYPTION, true);
  96     FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDirOnTestFS());
  97     TEST_UTIL.startMiniDFSCluster(3);
  98   }
  99
 100   @AfterClass
 101   public static void tearDownAfterClass() throws Exception {
 102     TEST_UTIL.shutdownMiniCluster();
 103   }
 104
 105   @Before
 106   public void setUp() {
 107     TEST_UTIL.getConfiguration().set(WALFactory.WAL_PROVIDER, walProvider);
 108   }
 109
 110   @Test
 111   public void testSecureWAL() throws Exception {
 112     TableName tableName = TableName.valueOf(name.getMethodName().replaceAll("[^a-zA-Z0-9]", "_"));
 113     NavigableMap<byte[], Integer> scopes = new TreeMap<>(Bytes.BYTES_COMPARATOR);
 114     scopes.put(tableName.getName(), 0);
 115     RegionInfo regionInfo = RegionInfoBuilder.newBuilder(tableName).build();
 116     final int total = 10;
 117     final byte[] row = Bytes.toBytes("row");
 118     final byte[] family = Bytes.toBytes("family");
 119     final byte[] value = Bytes.toBytes("Test value");
 120     FileSystem fs = TEST_UTIL.getDFSCluster().getFileSystem();
 121     final WALFactory wals =
 122         new WALFactory(TEST_UTIL.getConfiguration(), tableName.getNameAsString());
 123
 124     // Write the WAL
 125     final WAL wal = wals.getWAL(regionInfo);
 126
 127     MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
 128
 129     for (int i = 0; i < total; i++) {
 130       WALEdit kvs = new WALEdit();
 131       kvs.add(new KeyValue(row, family, Bytes.toBytes(i), value));
 132       wal.append(regionInfo, new WALKeyImpl(regionInfo.getEncodedNameAsBytes(), tableName,
 133           System.currentTimeMillis(), mvcc, scopes), kvs, true);
 134     }
 135     wal.sync();
 136     final Path walPath = AbstractFSWALProvider.getCurrentFileName(wal);
 137     wals.shutdown();
 138
 139     // Insure edits are not plaintext
 140     long length = fs.getFileStatus(walPath).getLen();
 141     FSDataInputStream in = fs.open(walPath);
 142     byte[] fileData = new byte[(int)length];
 143     IOUtils.readFully(in, fileData);
 144     in.close();
 145     assertFalse("Cells appear to be plaintext", Bytes.contains(fileData, value));
 146
 147     // Confirm the WAL can be read back
 148     WAL.Reader reader = wals.createReader(TEST_UTIL.getTestFileSystem(), walPath);
 149     int count = 0;
 150     WAL.Entry entry = new WAL.Entry();
 151     while (reader.next(entry) != null) {
 152       count++;
 153       List<Cell> cells = entry.getEdit().getCells();
 154       assertTrue("Should be one KV per WALEdit", cells.size() == 1);
 155       for (Cell cell: cells) {
 156         assertTrue("Incorrect row", Bytes.equals(cell.getRowArray(), cell.getRowOffset(),
 157           cell.getRowLength(), row, 0, row.length));
 158         assertTrue("Incorrect family", Bytes.equals(cell.getFamilyArray(), cell.getFamilyOffset(),
 159           cell.getFamilyLength(), family, 0, family.length));
 160         assertTrue("Incorrect value", Bytes.equals(cell.getValueArray(), cell.getValueOffset(),
 161           cell.getValueLength(), value, 0, value.length));
 162       }
 163     }
 164     assertEquals("Should have read back as many KVs as written", total, count);
 165     reader.close();
 166   }
 167 }