hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaCache.java

   1 /**
   2  * Licensed to the Apache Software Foundation (ASF) under one
   3  * or more contributor license agreements.  See the NOTICE file
   4  * distributed with this work for additional information
   5  * regarding copyright ownership.  The ASF licenses this file
   6  * to you under the Apache License, Version 2.0 (the
   7  * "License"); you may not use this file except in compliance
   8  * with the License.  You may obtain a copy of the License at
   9  *
  10  *     http://www.apache.org/licenses/LICENSE-2.0
  11  *
  12  * Unless required by applicable law or agreed to in writing, software
  13  * distributed under the License is distributed on an "AS IS" BASIS,
  14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  15  * See the License for the specific language governing permissions and
  16  * limitations under the License.
  17  */
  18 package org.apache.hadoop.hbase.client;
  19
  20 import static junit.framework.Assert.assertEquals;
  21 import static org.junit.Assert.assertNotNull;
  22 import static org.junit.Assert.assertNull;
  23 import static org.junit.Assert.assertTrue;
  24 import static org.junit.Assert.fail;
  25
  26 import java.io.IOException;
  27 import java.util.ArrayList;
  28 import java.util.List;
  29 import org.apache.hadoop.conf.Configuration;
  30 import org.apache.hadoop.hbase.*;
  31 import org.apache.hadoop.hbase.HBaseClassTestRule;
  32 import org.apache.hadoop.hbase.exceptions.ClientExceptionsUtil;
  33 import org.apache.hadoop.hbase.exceptions.RegionOpeningException;
  34 import org.apache.hadoop.hbase.quotas.RpcThrottlingException;
  35 import org.apache.hadoop.hbase.regionserver.HRegionServer;
  36 import org.apache.hadoop.hbase.regionserver.RSRpcServices;
  37 import org.apache.hadoop.hbase.testclassification.ClientTests;
  38 import org.apache.hadoop.hbase.testclassification.MediumTests;
  39 import org.apache.hadoop.hbase.util.Bytes;
  40 import org.junit.AfterClass;
  41 import org.junit.BeforeClass;
  42 import org.junit.ClassRule;
  43 import org.junit.Test;
  44 import org.junit.experimental.categories.Category;
  45
  46 import org.apache.hbase.thirdparty.com.google.protobuf.RpcController;
  47 import org.apache.hbase.thirdparty.com.google.protobuf.ServiceException;
  48
  49 import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
  50 import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.GetResponse;
  51 import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
  52
  53 @Category({MediumTests.class, ClientTests.class})
  54 public class TestMetaCache {
  55
  56   @ClassRule
  57   public static final HBaseClassTestRule CLASS_RULE =
  58       HBaseClassTestRule.forClass(TestMetaCache.class);
  59
  60   private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
  61   private static final TableName TABLE_NAME = TableName.valueOf("test_table");
  62   private static final byte[] FAMILY = Bytes.toBytes("fam1");
  63   private static final byte[] QUALIFIER = Bytes.toBytes("qual");
  64
  65   private static HRegionServer badRS;
  66
  67   /**
  68    * @throws java.lang.Exception
  69    */
  70   @BeforeClass
  71   public static void setUpBeforeClass() throws Exception {
  72     Configuration conf = TEST_UTIL.getConfiguration();
  73     conf.setStrings(HConstants.REGION_SERVER_IMPL,
  74         RegionServerWithFakeRpcServices.class.getName());
  75     TEST_UTIL.startMiniCluster(1);
  76     TEST_UTIL.getHBaseCluster().waitForActiveAndReadyMaster();
  77     TEST_UTIL.waitUntilAllRegionsAssigned(TABLE_NAME.META_TABLE_NAME);
  78     badRS = TEST_UTIL.getHBaseCluster().getRegionServer(0);
  79     assertTrue(badRS.getRSRpcServices() instanceof FakeRSRpcServices);
  80     HTableDescriptor table = new HTableDescriptor(TABLE_NAME);
  81     HColumnDescriptor fam = new HColumnDescriptor(FAMILY);
  82     fam.setMaxVersions(2);
  83     table.addFamily(fam);
  84     TEST_UTIL.createTable(table, null);
  85   }
  86
  87
  88   /**
  89    * @throws java.lang.Exception
  90    */
  91   @AfterClass
  92   public static void tearDownAfterClass() throws Exception {
  93     TEST_UTIL.shutdownMiniCluster();
  94   }
  95
  96   @Test
  97   public void testPreserveMetaCacheOnException() throws Exception {
  98     ((FakeRSRpcServices)badRS.getRSRpcServices()).setExceptionInjector(
  99         new RoundRobinExceptionInjector());
 100     Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
 101     conf.set("hbase.client.retries.number", "1");
 102     ConnectionImplementation conn =
 103         (ConnectionImplementation) ConnectionFactory.createConnection(conf);
 104     try {
 105       Table table = conn.getTable(TABLE_NAME);
 106       byte[] row = Bytes.toBytes("row1");
 107
 108       Put put = new Put(row);
 109       put.addColumn(FAMILY, QUALIFIER, Bytes.toBytes(10));
 110       Get get = new Get(row);
 111       Append append = new Append(row);
 112       append.addColumn(FAMILY, QUALIFIER, Bytes.toBytes(11));
 113       Increment increment = new Increment(row);
 114       increment.addColumn(FAMILY, QUALIFIER, 10);
 115       Delete delete = new Delete(row);
 116       delete.addColumn(FAMILY, QUALIFIER);
 117       RowMutations mutations = new RowMutations(row);
 118       mutations.add(put);
 119       mutations.add(delete);
 120
 121       Exception exp;
 122       boolean success;
 123       for (int i = 0; i < 50; i++) {
 124         exp = null;
 125         success = false;
 126         try {
 127           table.put(put);
 128           // If at least one operation succeeded, we should have cached the region location.
 129           success = true;
 130           table.get(get);
 131           table.append(append);
 132           table.increment(increment);
 133           table.delete(delete);
 134           table.mutateRow(mutations);
 135         } catch (IOException ex) {
 136           // Only keep track of the last exception that updated the meta cache
 137           if (ClientExceptionsUtil.isMetaClearingException(ex) || success) {
 138             exp = ex;
 139           }
 140         }
 141         // Do not test if we did not touch the meta cache in this iteration.
 142         if (exp != null && ClientExceptionsUtil.isMetaClearingException(exp)) {
 143           assertNull(conn.getCachedLocation(TABLE_NAME, row));
 144         } else if (success) {
 145           assertNotNull(conn.getCachedLocation(TABLE_NAME, row));
 146         }
 147       }
 148     } finally {
 149       conn.close();
 150     }
 151   }
 152
 153   @Test
 154   public void testCacheClearingOnCallQueueTooBig() throws Exception {
 155     ((FakeRSRpcServices)badRS.getRSRpcServices()).setExceptionInjector(
 156         new CallQueueTooBigExceptionInjector());
 157     Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
 158     conf.set("hbase.client.retries.number", "2");
 159     conf.set(MetricsConnection.CLIENT_SIDE_METRICS_ENABLED_KEY, "true");
 160     ConnectionImplementation conn =
 161         (ConnectionImplementation) ConnectionFactory.createConnection(conf);
 162     try {
 163       Table table = conn.getTable(TABLE_NAME);
 164       byte[] row = Bytes.toBytes("row1");
 165
 166       Put put = new Put(row);
 167       put.addColumn(FAMILY, QUALIFIER, Bytes.toBytes(10));
 168       table.put(put);
 169
 170       // obtain the client metrics
 171       MetricsConnection metrics = conn.getConnectionMetrics();
 172       long preGetRegionClears = metrics.metaCacheNumClearRegion.getCount();
 173       long preGetServerClears = metrics.metaCacheNumClearServer.getCount();
 174
 175       // attempt a get on the test table
 176       Get get = new Get(row);
 177       try {
 178         table.get(get);
 179         fail("Expected CallQueueTooBigException");
 180       } catch (RetriesExhaustedException ree) {
 181         // expected
 182       }
 183
 184       // verify that no cache clearing took place
 185       long postGetRegionClears = metrics.metaCacheNumClearRegion.getCount();
 186       long postGetServerClears = metrics.metaCacheNumClearServer.getCount();
 187       assertEquals(preGetRegionClears, postGetRegionClears);
 188       assertEquals(preGetServerClears, postGetServerClears);
 189     } finally {
 190       conn.close();
 191     }
 192   }
 193
 194   public static List<Throwable> metaCachePreservingExceptions() {
 195     return new ArrayList<Throwable>() {{
 196         add(new RegionOpeningException(" "));
 197         add(new RegionTooBusyException("Some old message"));
 198         add(new RpcThrottlingException(" "));
 199         add(new MultiActionResultTooLarge(" "));
 200         add(new RetryImmediatelyException(" "));
 201         add(new CallQueueTooBigException());
 202     }};
 203   }
 204
 205   public static class RegionServerWithFakeRpcServices extends HRegionServer {
 206     private FakeRSRpcServices rsRpcServices;
 207
 208     public RegionServerWithFakeRpcServices(Configuration conf)
 209       throws IOException, InterruptedException {
 210       super(conf);
 211     }
 212
 213     @Override
 214     protected RSRpcServices createRpcServices() throws IOException {
 215       this.rsRpcServices = new FakeRSRpcServices(this);
 216       return rsRpcServices;
 217     }
 218
 219     public void setExceptionInjector(ExceptionInjector injector) {
 220       rsRpcServices.setExceptionInjector(injector);
 221     }
 222   }
 223
 224   public static class FakeRSRpcServices extends RSRpcServices {
 225
 226     private ExceptionInjector exceptions;
 227
 228     public FakeRSRpcServices(HRegionServer rs) throws IOException {
 229       super(rs);
 230       exceptions = new RoundRobinExceptionInjector();
 231     }
 232
 233     public void setExceptionInjector(ExceptionInjector injector) {
 234       this.exceptions = injector;
 235     }
 236
 237     @Override
 238     public GetResponse get(final RpcController controller,
 239                            final ClientProtos.GetRequest request) throws ServiceException {
 240       exceptions.throwOnGet(this, request);
 241       return super.get(controller, request);
 242     }
 243
 244     @Override
 245     public ClientProtos.MutateResponse mutate(final RpcController controller,
 246         final ClientProtos.MutateRequest request) throws ServiceException {
 247       exceptions.throwOnMutate(this, request);
 248       return super.mutate(controller, request);
 249     }
 250
 251     @Override
 252     public ClientProtos.ScanResponse scan(final RpcController controller,
 253         final ClientProtos.ScanRequest request) throws ServiceException {
 254       exceptions.throwOnScan(this, request);
 255       return super.scan(controller, request);
 256     }
 257   }
 258
 259   public static abstract class ExceptionInjector {
 260     protected boolean isTestTable(FakeRSRpcServices rpcServices,
 261                                   HBaseProtos.RegionSpecifier regionSpec) throws ServiceException {
 262       try {
 263         return TABLE_NAME.equals(
 264             rpcServices.getRegion(regionSpec).getTableDescriptor().getTableName());
 265       } catch (IOException ioe) {
 266         throw new ServiceException(ioe);
 267       }
 268     }
 269
 270     public abstract void throwOnGet(FakeRSRpcServices rpcServices, ClientProtos.GetRequest request)
 271         throws ServiceException;
 272
 273     public abstract void throwOnMutate(FakeRSRpcServices rpcServices, ClientProtos.MutateRequest request)
 274         throws ServiceException;
 275
 276     public abstract void throwOnScan(FakeRSRpcServices rpcServices, ClientProtos.ScanRequest request)
 277         throws ServiceException;
 278   }
 279
 280   /**
 281    * Rotates through the possible cache clearing and non-cache clearing exceptions
 282    * for requests.
 283    */
 284   public static class RoundRobinExceptionInjector extends ExceptionInjector {
 285     private int numReqs = -1;
 286     private int expCount = -1;
 287     private List<Throwable> metaCachePreservingExceptions = metaCachePreservingExceptions();
 288
 289     @Override
 290     public void throwOnGet(FakeRSRpcServices rpcServices, ClientProtos.GetRequest request)
 291         throws ServiceException {
 292       throwSomeExceptions(rpcServices, request.getRegion());
 293     }
 294
 295     @Override
 296     public void throwOnMutate(FakeRSRpcServices rpcServices, ClientProtos.MutateRequest request)
 297         throws ServiceException {
 298       throwSomeExceptions(rpcServices, request.getRegion());
 299     }
 300
 301     @Override
 302     public void throwOnScan(FakeRSRpcServices rpcServices, ClientProtos.ScanRequest request)
 303         throws ServiceException {
 304       if (!request.hasScannerId()) {
 305         // only handle initial scan requests
 306         throwSomeExceptions(rpcServices, request.getRegion());
 307       }
 308     }
 309
 310     /**
 311      * Throw some exceptions. Mostly throw exceptions which do not clear meta cache.
 312      * Periodically throw NotSevingRegionException which clears the meta cache.
 313      * @throws ServiceException
 314      */
 315     private void throwSomeExceptions(FakeRSRpcServices rpcServices,
 316                                      HBaseProtos.RegionSpecifier regionSpec)
 317         throws ServiceException {
 318       if (!isTestTable(rpcServices, regionSpec)) {
 319         return;
 320       }
 321
 322       numReqs++;
 323       // Succeed every 5 request, throw cache clearing exceptions twice every 5 requests and throw
 324       // meta cache preserving exceptions otherwise.
 325       if (numReqs % 5 ==0) {
 326         return;
 327       } else if (numReqs % 5 == 1 || numReqs % 5 == 2) {
 328         throw new ServiceException(new NotServingRegionException());
 329       }
 330       // Round robin between different special exceptions.
 331       // This is not ideal since exception types are not tied to the operation performed here,
 332       // But, we don't really care here if we throw MultiActionTooLargeException while doing
 333       // single Gets.
 334       expCount++;
 335       Throwable t = metaCachePreservingExceptions.get(
 336           expCount % metaCachePreservingExceptions.size());
 337       throw new ServiceException(t);
 338     }
 339   }
 340
 341   /**
 342    * Throws CallQueueTooBigException for all gets.
 343    */
 344   public static class CallQueueTooBigExceptionInjector extends ExceptionInjector {
 345     @Override
 346     public void throwOnGet(FakeRSRpcServices rpcServices, ClientProtos.GetRequest request)
 347         throws ServiceException {
 348       if (isTestTable(rpcServices, request.getRegion())) {
 349         throw new ServiceException(new CallQueueTooBigException());
 350       }
 351     }
 352
 353     @Override
 354     public void throwOnMutate(FakeRSRpcServices rpcServices, ClientProtos.MutateRequest request)
 355         throws ServiceException {
 356     }
 357
 358     @Override
 359     public void throwOnScan(FakeRSRpcServices rpcServices, ClientProtos.ScanRequest request)
 360         throws ServiceException {
 361     }
 362   }
 363 }