mediagoblin/init/celery/__init__.py

   1 # GNU MediaGoblin -- federated, autonomous media hosting
   2 # Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published by
   6 # the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  16
  17 import os
  18 import sys
  19 import datetime
  20 import logging
  21
  22 import six
  23
  24 from celery import Celery
  25 from mediagoblin.tools.pluginapi import hook_runall
  26
  27
  28 _log = logging.getLogger(__name__)
  29
  30
  31 MANDATORY_CELERY_IMPORTS = [
  32     'mediagoblin.processing.task',
  33     'mediagoblin.notifications.task',
  34     'mediagoblin.submit.task',
  35 ]
  36
  37 DEFAULT_SETTINGS_MODULE = 'mediagoblin.init.celery.dummy_settings_module'
  38
  39
  40 def get_celery_settings_dict(app_config, global_config,
  41                              force_celery_always_eager=False):
  42     """
  43     Get a celery settings dictionary from reading the config
  44     """
  45     if 'celery' in global_config:
  46         celery_conf = global_config['celery']
  47     else:
  48         celery_conf = {}
  49
  50     celery_settings = {}
  51
  52     # Add all celery settings from config
  53     for key, value in six.iteritems(celery_conf):
  54         celery_settings[key] = value
  55
  56     # TODO: use default result stuff here if it exists
  57
  58     # add mandatory celery imports
  59     celery_imports = celery_settings.setdefault('CELERY_IMPORTS', [])
  60     celery_imports.extend(MANDATORY_CELERY_IMPORTS)
  61
  62     if force_celery_always_eager:
  63         celery_settings['CELERY_ALWAYS_EAGER'] = True
  64         celery_settings['CELERY_EAGER_PROPAGATES_EXCEPTIONS'] = True
  65
  66     # Garbage collection periodic task
  67     frequency = app_config.get('garbage_collection', 60)
  68     if frequency:
  69         frequency = int(frequency)
  70         celery_settings['CELERYBEAT_SCHEDULE'] = {
  71             'garbage-collection': {
  72                 'task': 'mediagoblin.submit.task.collect_garbage',
  73                 'schedule': datetime.timedelta(minutes=frequency),
  74             }
  75         }
  76         celery_settings['BROKER_HEARTBEAT'] = 1
  77
  78     return celery_settings
  79
  80
  81 def setup_celery_app(app_config, global_config,
  82                      settings_module=DEFAULT_SETTINGS_MODULE,
  83                      force_celery_always_eager=False):
  84     """
  85     Setup celery without using terrible setup-celery-module hacks.
  86     """
  87     celery_settings = get_celery_settings_dict(
  88         app_config, global_config, force_celery_always_eager)
  89     celery_app = Celery()
  90     celery_app.config_from_object(celery_settings)
  91
  92     hook_runall('celery_setup', celery_app)
  93
  94
  95 def setup_celery_from_config(app_config, global_config,
  96                              settings_module=DEFAULT_SETTINGS_MODULE,
  97                              force_celery_always_eager=False,
  98                              set_environ=True):
  99     """
 100     Take a mediagoblin app config and try to set up a celery settings
 101     module from this.
 102
 103     Args:
 104     - app_config: the application config section
 105     - global_config: the entire ConfigObj loaded config, all sections
 106     - settings_module: the module to populate, as a string
 107     - force_celery_always_eager: whether or not to force celery into
 108       always eager mode; good for development and small installs
 109     - set_environ: if set, this will CELERY_CONFIG_MODULE to the
 110       settings_module
 111     """
 112     celery_settings = get_celery_settings_dict(
 113         app_config, global_config, force_celery_always_eager)
 114
 115     __import__(settings_module)
 116     this_module = sys.modules[settings_module]
 117
 118     for key, value in six.iteritems(celery_settings):
 119         setattr(this_module, key, value)
 120
 121     if set_environ:
 122         os.environ['CELERY_CONFIG_MODULE'] = settings_module
 123
 124     # Replace the default celery.current_app.conf if celery has already been
 125     # initiated
 126     from celery import current_app
 127
 128     _log.info('Setting celery configuration from object "{0}"'.format(
 129         settings_module))
 130     current_app.config_from_object(this_module)
 131
 132     _log.debug('Celery broker host: {0}'.format(current_app.conf['BROKER_HOST']))