hbase-server/src/test/java/org/apache/hadoop/hbase/quotas/TestFileArchiverNotifierImpl.java

   1 /*
   2  * Licensed to the Apache Software Foundation (ASF) under one or more
   3  * contributor license agreements.  See the NOTICE file distributed with
   4  * this work for additional information regarding copyright ownership.
   5  * The ASF licenses this file to you under the Apache License, Version 2.0
   6  * (the "License"); you may not use this file except in compliance with
   7  * the License.  You may obtain a copy of the License at
   8  *
   9  * http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17 package org.apache.hadoop.hbase.quotas;
  18
  19 import static org.junit.Assert.assertEquals;
  20 import static org.junit.Assert.assertNotNull;
  21 import static org.junit.Assert.assertTrue;
  22
  23 import java.io.IOException;
  24 import java.util.ArrayList;
  25 import java.util.Arrays;
  26 import java.util.Collections;
  27 import java.util.HashSet;
  28 import java.util.List;
  29 import java.util.Map.Entry;
  30 import java.util.Set;
  31 import java.util.concurrent.atomic.AtomicLong;
  32 import org.apache.hadoop.conf.Configuration;
  33 import org.apache.hadoop.fs.FileSystem;
  34 import org.apache.hadoop.fs.Path;
  35 import org.apache.hadoop.hbase.Cell;
  36 import org.apache.hadoop.hbase.CellScanner;
  37 import org.apache.hadoop.hbase.HBaseClassTestRule;
  38 import org.apache.hadoop.hbase.HBaseTestingUtil;
  39 import org.apache.hadoop.hbase.TableName;
  40 import org.apache.hadoop.hbase.client.Admin;
  41 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
  42 import org.apache.hadoop.hbase.client.Connection;
  43 import org.apache.hadoop.hbase.client.Get;
  44 import org.apache.hadoop.hbase.client.Result;
  45 import org.apache.hadoop.hbase.client.ResultScanner;
  46 import org.apache.hadoop.hbase.client.Scan;
  47 import org.apache.hadoop.hbase.client.SnapshotDescription;
  48 import org.apache.hadoop.hbase.client.SnapshotType;
  49 import org.apache.hadoop.hbase.client.Table;
  50 import org.apache.hadoop.hbase.client.TableDescriptor;
  51 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
  52 import org.apache.hadoop.hbase.quotas.FileArchiverNotifierImpl.SnapshotWithSize;
  53 import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
  54 import org.apache.hadoop.hbase.snapshot.SnapshotManifest;
  55 import org.apache.hadoop.hbase.testclassification.MediumTests;
  56 import org.apache.hadoop.hbase.util.CommonFSUtils;
  57 import org.junit.AfterClass;
  58 import org.junit.Before;
  59 import org.junit.BeforeClass;
  60 import org.junit.ClassRule;
  61 import org.junit.Rule;
  62 import org.junit.Test;
  63 import org.junit.experimental.categories.Category;
  64 import org.junit.rules.TestName;
  65
  66 import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet;
  67 import org.apache.hbase.thirdparty.com.google.common.collect.Iterables;
  68 import org.apache.hbase.thirdparty.com.google.common.collect.Maps;
  69
  70 import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos;
  71 import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
  72 import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.FamilyFiles;
  73 import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest.StoreFile;
  74
  75 /**
  76  * Test class for {@link FileArchiverNotifierImpl}.
  77  */
  78 @Category(MediumTests.class)
  79 public class TestFileArchiverNotifierImpl {
  80   @ClassRule
  81   public static final HBaseClassTestRule CLASS_RULE =
  82       HBaseClassTestRule.forClass(TestFileArchiverNotifierImpl.class);
  83
  84   private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil();
  85   private static final AtomicLong COUNTER = new AtomicLong();
  86
  87   @Rule
  88   public TestName testName = new TestName();
  89
  90   private Connection conn;
  91   private Admin admin;
  92   private SpaceQuotaHelperForTests helper;
  93   private FileSystem fs;
  94   private Configuration conf;
  95
  96   @BeforeClass
  97   public static void setUp() throws Exception {
  98     Configuration conf = TEST_UTIL.getConfiguration();
  99     SpaceQuotaHelperForTests.updateConfigForQuotas(conf);
 100     // Clean up the compacted files faster than normal (15s instead of 2mins)
 101     conf.setInt("hbase.hfile.compaction.discharger.interval", 15 * 1000);
 102     // Prevent the SnapshotQuotaObserverChore from running
 103     conf.setInt(SnapshotQuotaObserverChore.SNAPSHOT_QUOTA_CHORE_DELAY_KEY, 60 * 60 * 1000);
 104     conf.setInt(SnapshotQuotaObserverChore.SNAPSHOT_QUOTA_CHORE_PERIOD_KEY, 60 * 60 * 1000);
 105     TEST_UTIL.startMiniCluster(1);
 106   }
 107
 108   @AfterClass
 109   public static void tearDown() throws Exception {
 110     TEST_UTIL.shutdownMiniCluster();
 111   }
 112
 113   @Before
 114   public void setup() throws Exception {
 115     conn = TEST_UTIL.getConnection();
 116     admin = TEST_UTIL.getAdmin();
 117     helper = new SpaceQuotaHelperForTests(TEST_UTIL, testName, COUNTER);
 118     helper.removeAllQuotas(conn);
 119     fs = TEST_UTIL.getTestFileSystem();
 120     conf = TEST_UTIL.getConfiguration();
 121   }
 122
 123   @Test
 124   public void testSnapshotSizePersistence() throws IOException {
 125     final Admin admin = TEST_UTIL.getAdmin();
 126     final TableName tn = TableName.valueOf(testName.getMethodName());
 127     if (admin.tableExists(tn)) {
 128       admin.disableTable(tn);
 129       admin.deleteTable(tn);
 130     }
 131     TableDescriptor desc = TableDescriptorBuilder.newBuilder(tn).setColumnFamily(
 132         ColumnFamilyDescriptorBuilder.of(QuotaTableUtil.QUOTA_FAMILY_USAGE)).build();
 133     admin.createTable(desc);
 134
 135     FileArchiverNotifierImpl notifier = new FileArchiverNotifierImpl(conn, conf, fs, tn);
 136     List<SnapshotWithSize> snapshotsWithSizes = new ArrayList<>();
 137     try (Table table = conn.getTable(tn)) {
 138       // Writing no values will result in no records written.
 139       verify(table, () -> {
 140         notifier.persistSnapshotSizes(table, snapshotsWithSizes);
 141         assertEquals(0, count(table));
 142       });
 143
 144       verify(table, () -> {
 145         snapshotsWithSizes.add(new SnapshotWithSize("ss1", 1024L));
 146         snapshotsWithSizes.add(new SnapshotWithSize("ss2", 4096L));
 147         notifier.persistSnapshotSizes(table, snapshotsWithSizes);
 148         assertEquals(2, count(table));
 149         assertEquals(1024L, extractSnapshotSize(table, tn, "ss1"));
 150         assertEquals(4096L, extractSnapshotSize(table, tn, "ss2"));
 151       });
 152     }
 153   }
 154
 155   @Test
 156   public void testIncrementalFileArchiving() throws Exception {
 157     final Admin admin = TEST_UTIL.getAdmin();
 158     final TableName tn = TableName.valueOf(testName.getMethodName());
 159     if (admin.tableExists(tn)) {
 160       admin.disableTable(tn);
 161       admin.deleteTable(tn);
 162     }
 163     final Table quotaTable = conn.getTable(QuotaUtil.QUOTA_TABLE_NAME);
 164     final TableName tn1 = helper.createTableWithRegions(1);
 165     admin.setQuota(QuotaSettingsFactory.limitTableSpace(
 166         tn1, SpaceQuotaHelperForTests.ONE_GIGABYTE, SpaceViolationPolicy.NO_INSERTS));
 167
 168     // Write some data and flush it
 169     helper.writeData(tn1, 256L * SpaceQuotaHelperForTests.ONE_KILOBYTE);
 170     admin.flush(tn1);
 171
 172     // Create a snapshot on the table
 173     final String snapshotName1 = tn1 + "snapshot1";
 174     admin.snapshot(new SnapshotDescription(snapshotName1, tn1, SnapshotType.SKIPFLUSH));
 175
 176     FileArchiverNotifierImpl notifier = new FileArchiverNotifierImpl(conn, conf, fs, tn);
 177     long t1 = notifier.getLastFullCompute();
 178     long snapshotSize = notifier.computeAndStoreSnapshotSizes(Arrays.asList(snapshotName1));
 179     assertEquals("The size of the snapshots should be zero", 0, snapshotSize);
 180     assertTrue("Last compute time was not less than current compute time",
 181         t1 < notifier.getLastFullCompute());
 182
 183     // No recently archived files and the snapshot should have no size
 184     assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
 185
 186     // Invoke the addArchivedFiles method with no files
 187     notifier.addArchivedFiles(Collections.emptySet());
 188
 189     // The size should not have changed
 190     assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
 191
 192     notifier.addArchivedFiles(ImmutableSet.of(entry("a", 1024L), entry("b", 1024L)));
 193
 194     // The size should not have changed
 195     assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
 196
 197     // Pull one file referenced by the snapshot out of the manifest
 198     Set<String> referencedFiles = getFilesReferencedBySnapshot(snapshotName1);
 199     assertTrue("Found snapshot referenced files: " + referencedFiles, referencedFiles.size() >= 1);
 200     String referencedFile = Iterables.getFirst(referencedFiles, null);
 201     assertNotNull(referencedFile);
 202
 203     // Report that a file this snapshot referenced was moved to the archive. This is a sign
 204     // that the snapshot should now "own" the size of this file
 205     final long fakeFileSize = 2048L;
 206     notifier.addArchivedFiles(ImmutableSet.of(entry(referencedFile, fakeFileSize)));
 207
 208     // Verify that the snapshot owns this file.
 209     assertEquals(fakeFileSize, extractSnapshotSize(quotaTable, tn, snapshotName1));
 210
 211     // In reality, we did not actually move the file, so a "full" computation should re-set the
 212     // size of the snapshot back to 0.
 213     long t2 = notifier.getLastFullCompute();
 214     snapshotSize = notifier.computeAndStoreSnapshotSizes(Arrays.asList(snapshotName1));
 215     assertEquals(0, snapshotSize);
 216     assertEquals(0, extractSnapshotSize(quotaTable, tn, snapshotName1));
 217     // We should also have no recently archived files after a re-computation
 218     assertTrue("Last compute time was not less than current compute time",
 219         t2 < notifier.getLastFullCompute());
 220   }
 221
 222   @Test
 223   public void testParseOldNamespaceSnapshotSize() throws Exception {
 224     final Admin admin = TEST_UTIL.getAdmin();
 225     final TableName fakeQuotaTableName = TableName.valueOf(testName.getMethodName());
 226     final TableName tn = TableName.valueOf(testName.getMethodName() + "1");
 227     if (admin.tableExists(fakeQuotaTableName)) {
 228       admin.disableTable(fakeQuotaTableName);
 229       admin.deleteTable(fakeQuotaTableName);
 230     }
 231     TableDescriptor desc = TableDescriptorBuilder.newBuilder(fakeQuotaTableName).setColumnFamily(
 232         ColumnFamilyDescriptorBuilder.of(QuotaTableUtil.QUOTA_FAMILY_USAGE))
 233         .setColumnFamily(ColumnFamilyDescriptorBuilder.of(QuotaUtil.QUOTA_FAMILY_INFO)).build();
 234     admin.createTable(desc);
 235
 236     final String ns = "";
 237     try (Table fakeQuotaTable = conn.getTable(fakeQuotaTableName)) {
 238       FileArchiverNotifierImpl notifier = new FileArchiverNotifierImpl(conn, conf, fs, tn);
 239       // Verify no record is treated as zero
 240       assertEquals(0, notifier.getPreviousNamespaceSnapshotSize(fakeQuotaTable, ns));
 241
 242       // Set an explicit value of zero
 243       fakeQuotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns, 0L));
 244       assertEquals(0, notifier.getPreviousNamespaceSnapshotSize(fakeQuotaTable, ns));
 245
 246       // Set a non-zero value
 247       fakeQuotaTable.put(QuotaTableUtil.createPutForNamespaceSnapshotSize(ns, 1024L));
 248       assertEquals(1024L, notifier.getPreviousNamespaceSnapshotSize(fakeQuotaTable, ns));
 249     }
 250   }
 251
 252   private long count(Table t) throws IOException {
 253     try (ResultScanner rs = t.getScanner(new Scan())) {
 254       long sum = 0;
 255       for (Result r : rs) {
 256         while (r.advance()) {
 257           sum++;
 258         }
 259       }
 260       return sum;
 261     }
 262   }
 263
 264   private long extractSnapshotSize(
 265       Table quotaTable, TableName tn, String snapshot) throws IOException {
 266     Get g = QuotaTableUtil.makeGetForSnapshotSize(tn, snapshot);
 267     Result r = quotaTable.get(g);
 268     assertNotNull(r);
 269     CellScanner cs = r.cellScanner();
 270     assertTrue(cs.advance());
 271     Cell c = cs.current();
 272     assertNotNull(c);
 273     return QuotaTableUtil.extractSnapshotSize(
 274         c.getValueArray(), c.getValueOffset(), c.getValueLength());
 275   }
 276
 277   private void verify(Table t, IOThrowingRunnable test) throws IOException {
 278     admin.disableTable(t.getName());
 279     admin.truncateTable(t.getName(), false);
 280     test.run();
 281   }
 282
 283   @FunctionalInterface
 284   private interface IOThrowingRunnable {
 285     void run() throws IOException;
 286   }
 287
 288   private Set<String> getFilesReferencedBySnapshot(String snapshotName) throws IOException {
 289     HashSet<String> files = new HashSet<>();
 290     Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(
 291         snapshotName, CommonFSUtils.getRootDir(conf));
 292     SnapshotProtos.SnapshotDescription sd = SnapshotDescriptionUtils.readSnapshotInfo(
 293         fs, snapshotDir);
 294     SnapshotManifest manifest = SnapshotManifest.open(conf, fs, snapshotDir, sd);
 295     // For each region referenced by the snapshot
 296     for (SnapshotRegionManifest rm : manifest.getRegionManifests()) {
 297       // For each column family in this region
 298       for (FamilyFiles ff : rm.getFamilyFilesList()) {
 299         // And each store file in that family
 300         for (StoreFile sf : ff.getStoreFilesList()) {
 301           files.add(sf.getName());
 302         }
 303       }
 304     }
 305     return files;
 306   }
 307
 308   private <K,V> Entry<K,V> entry(K k, V v) {
 309     return Maps.immutableEntry(k, v);
 310   }
 311 }