chrome/common/extensions/docs/server2/url_fetcher_appengine.py

   1 # Copyright 2015 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import google.appengine.api.urlfetch as urlfetch
   6 import logging
   7 import time
   8 import traceback
   9
  10 from future import Future
  11 from url_fetcher import UrlFetcher
  12
  13
  14 _MAX_RETRIES = 5
  15 _RETRY_DELAY_SECONDS = 30
  16
  17
  18 class UrlFetcherAppengine(UrlFetcher):
  19   """An implementation of UrlFetcher which uses the GAE urlfetch API to
  20   execute URL fetches.
  21   """
  22   def __init__(self):
  23     self._retries_left = _MAX_RETRIES
  24
  25   def FetchImpl(self, url, headers):
  26     return urlfetch.fetch(url, deadline=20, headers=headers)
  27
  28   def FetchAsyncImpl(self, url, headers):
  29     def process_result(result):
  30       if result.status_code == 429:
  31         if self._retries_left == 0:
  32           logging.error('Still throttled. Giving up.')
  33           return result
  34         self._retries_left -= 1
  35         logging.info('Throttled. Trying again in %s seconds.' %
  36                      _RETRY_DELAY_SECONDS)
  37         time.sleep(_RETRY_DELAY_SECONDS)
  38         return self.FetchAsync(url, username, password, access_token).Get()
  39       return result
  40
  41     rpc = urlfetch.create_rpc(deadline=20)
  42     urlfetch.make_fetch_call(rpc, url, headers=headers)
  43     return Future(callback=lambda: process_result(rpc.get_result()))