hbase-server/src/test/java/org/apache/hadoop/hbase/util/HFileArchiveTestingUtil.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.util;
  19
  20 import static org.junit.Assert.assertEquals;
  21 import static org.junit.Assert.assertNull;
  22
  23 import java.io.IOException;
  24 import java.util.ArrayList;
  25 import java.util.Arrays;
  26 import java.util.Collections;
  27 import java.util.List;
  28
  29 import org.apache.hadoop.conf.Configuration;
  30 import org.apache.hadoop.fs.FileStatus;
  31 import org.apache.hadoop.fs.FileSystem;
  32 import org.apache.hadoop.fs.Path;
  33 import org.apache.hadoop.hbase.HBaseTestingUtility;
  34 import org.apache.hadoop.hbase.regionserver.HRegion;
  35 import org.apache.hadoop.hbase.regionserver.Store;
  36 import org.slf4j.Logger;
  37 import org.slf4j.LoggerFactory;
  38
  39 /**
  40  * Test helper for testing archiving of HFiles
  41  */
  42 public class HFileArchiveTestingUtil {
  43
  44   private static final Logger LOG = LoggerFactory.getLogger(HFileArchiveTestingUtil.class);
  45
  46   private HFileArchiveTestingUtil() {
  47     // NOOP private ctor since this is just a utility class
  48   }
  49
  50   public static boolean compareArchiveToOriginal(FileStatus[] previous, FileStatus[] archived,
  51       FileSystem fs, boolean hasTimedBackup) {
  52
  53     List<List<String>> lists = getFileLists(previous, archived);
  54     List<String> original = lists.get(0);
  55     Collections.sort(original);
  56
  57     List<String> currentFiles = lists.get(1);
  58     Collections.sort(currentFiles);
  59
  60     List<String> backedup = lists.get(2);
  61     Collections.sort(backedup);
  62
  63     // check the backed up files versus the current (should match up, less the
  64     // backup time in the name)
  65     if (!hasTimedBackup == (backedup.size() > 0)) {
  66       LOG.debug("backedup files doesn't match expected.");
  67       return false;
  68     }
  69     String msg = null;
  70     if (hasTimedBackup) {
  71       msg = assertArchiveEquality(original, backedup);
  72       if (msg != null) {
  73         LOG.debug(msg);
  74         return false;
  75       }
  76     }
  77     msg = assertArchiveEquality(original, currentFiles);
  78     if (msg != null) {
  79       LOG.debug(msg);
  80       return false;
  81     }
  82     return true;
  83   }
  84
  85   /**
  86    * Compare the archived files to the files in the original directory
  87    * @param expected original files that should have been archived
  88    * @param actual files that were archived
  89    * @param fs filessystem on which the archiving took place
  90    * @throws IOException
  91    */
  92   public static void assertArchiveEqualToOriginal(FileStatus[] expected, FileStatus[] actual,
  93       FileSystem fs) throws IOException {
  94     assertArchiveEqualToOriginal(expected, actual, fs, false);
  95   }
  96
  97   /**
  98    * Compare the archived files to the files in the original directory
  99    * @param expected original files that should have been archived
 100    * @param actual files that were archived
 101    * @param fs {@link FileSystem} on which the archiving took place
 102    * @param hasTimedBackup <tt>true</tt> if we expect to find an archive backup directory with a
 103    *          copy of the files in the archive directory (and the original files).
 104    * @throws IOException
 105    */
 106   public static void assertArchiveEqualToOriginal(FileStatus[] expected, FileStatus[] actual,
 107       FileSystem fs, boolean hasTimedBackup) throws IOException {
 108
 109     List<List<String>> lists = getFileLists(expected, actual);
 110     List<String> original = lists.get(0);
 111     Collections.sort(original);
 112
 113     List<String> currentFiles = lists.get(1);
 114     Collections.sort(currentFiles);
 115
 116     List<String> backedup = lists.get(2);
 117     Collections.sort(backedup);
 118
 119     // check the backed up files versus the current (should match up, less the
 120     // backup time in the name)
 121     assertEquals("Didn't expect any backup files, but got: " + backedup, hasTimedBackup,
 122       backedup.size() > 0);
 123     String msg = null;
 124     if (hasTimedBackup) {
 125       assertArchiveEquality(original, backedup);
 126       assertNull(msg, msg);
 127     }
 128
 129     // do the rest of the comparison
 130     msg = assertArchiveEquality(original, currentFiles);
 131     assertNull(msg, msg);
 132   }
 133
 134   private static String assertArchiveEquality(List<String> expected, List<String> archived) {
 135     String compare = compareFileLists(expected, archived);
 136     if (!(expected.size() == archived.size())) return "Not the same number of current files\n"
 137         + compare;
 138     if (!expected.equals(archived)) return "Different backup files, but same amount\n" + compare;
 139     return null;
 140   }
 141
 142   /**
 143    * @return &lt;expected, gotten, backup&gt;, where each is sorted
 144    */
 145   private static List<List<String>> getFileLists(FileStatus[] previous, FileStatus[] archived) {
 146     List<List<String>> files = new ArrayList<>(3);
 147
 148     // copy over the original files
 149     List<String> originalFileNames = convertToString(previous);
 150     files.add(originalFileNames);
 151
 152     List<String> currentFiles = new ArrayList<>(previous.length);
 153     List<FileStatus> backedupFiles = new ArrayList<>(previous.length);
 154     for (FileStatus f : archived) {
 155       String name = f.getPath().getName();
 156       // if the file has been backed up
 157       if (name.contains(".")) {
 158         Path parent = f.getPath().getParent();
 159         String shortName = name.split("[.]")[0];
 160         Path modPath = new Path(parent, shortName);
 161         FileStatus file = new FileStatus(f.getLen(), f.isDirectory(), f.getReplication(),
 162             f.getBlockSize(), f.getModificationTime(), modPath);
 163         backedupFiles.add(file);
 164       } else {
 165         // otherwise, add it to the list to compare to the original store files
 166         currentFiles.add(name);
 167       }
 168     }
 169
 170     files.add(currentFiles);
 171     files.add(convertToString(backedupFiles));
 172     return files;
 173   }
 174
 175   private static List<String> convertToString(FileStatus[] files) {
 176     return convertToString(Arrays.asList(files));
 177   }
 178
 179   private static List<String> convertToString(List<FileStatus> files) {
 180     List<String> originalFileNames = new ArrayList<>(files.size());
 181     for (FileStatus f : files) {
 182       originalFileNames.add(f.getPath().getName());
 183     }
 184     return originalFileNames;
 185   }
 186
 187   /* Get a pretty representation of the differences */
 188   private static String compareFileLists(List<String> expected, List<String> gotten) {
 189     StringBuilder sb = new StringBuilder("Expected (" + expected.size() + "): \t\t Gotten ("
 190         + gotten.size() + "):\n");
 191     List<String> notFound = new ArrayList<>();
 192     for (String s : expected) {
 193       if (gotten.contains(s)) sb.append(s + "\t\t" + s + "\n");
 194       else notFound.add(s);
 195     }
 196     sb.append("Not Found:\n");
 197     for (String s : notFound) {
 198       sb.append(s + "\n");
 199     }
 200     sb.append("\nExtra:\n");
 201     for (String s : gotten) {
 202       if (!expected.contains(s)) sb.append(s + "\n");
 203     }
 204     return sb.toString();
 205   }
 206
 207   /**
 208    * Helper method to get the archive directory for the specified region
 209    * @param conf {@link Configuration} to check for the name of the archive directory
 210    * @param region region that is being archived
 211    * @return {@link Path} to the archive directory for the given region
 212    */
 213   public static Path getRegionArchiveDir(Configuration conf, HRegion region) throws IOException {
 214     return HFileArchiveUtil.getRegionArchiveDir(
 215         FSUtils.getRootDir(conf),
 216         region.getTableDescriptor().getTableName(),
 217         region.getRegionInfo().getEncodedName());
 218   }
 219
 220   /**
 221    * Helper method to get the store archive directory for the specified region
 222    * @param conf {@link Configuration} to check for the name of the archive directory
 223    * @param region region that is being archived
 224    * @param store store that is archiving files
 225    * @return {@link Path} to the store archive directory for the given region
 226    */
 227   public static Path getStoreArchivePath(Configuration conf, HRegion region, Store store)
 228       throws IOException {
 229     return HFileArchiveUtil.getStoreArchivePath(conf, region.getRegionInfo(),
 230         region.getRegionFileSystem().getTableDir(), store.getColumnFamilyDescriptor().getName());
 231   }
 232
 233   public static Path getStoreArchivePath(HBaseTestingUtility util, String tableName,
 234       byte[] storeName) throws IOException {
 235     byte[] table = Bytes.toBytes(tableName);
 236     // get the RS and region serving our table
 237     List<HRegion> servingRegions = util.getHBaseCluster().getRegions(table);
 238     HRegion region = servingRegions.get(0);
 239
 240     // check that we actually have some store files that were archived
 241     Store store = region.getStore(storeName);
 242     return HFileArchiveTestingUtil.getStoreArchivePath(util.getConfiguration(), region, store);
 243   }
 244 }