diff options
Diffstat (limited to 'mediagoblin/celery_setup')
| -rw-r--r-- | mediagoblin/celery_setup/__init__.py | 81 | ||||
| -rw-r--r-- | mediagoblin/celery_setup/from_celery.py | 70 | ||||
| -rw-r--r-- | mediagoblin/celery_setup/from_tests.py | 43 |
3 files changed, 25 insertions, 169 deletions
diff --git a/mediagoblin/celery_setup/__init__.py b/mediagoblin/celery_setup/__init__.py index d4f25b07..b6e35e99 100644 --- a/mediagoblin/celery_setup/__init__.py +++ b/mediagoblin/celery_setup/__init__.py @@ -17,86 +17,35 @@ import os import sys -from paste.deploy.converters import asbool, asint, aslist - - -KNOWN_CONFIG_BOOLS = [ - 'CELERY_RESULT_PERSISTENT', - 'CELERY_CREATE_MISSING_QUEUES', - 'BROKER_USE_SSL', 'BROKER_CONNECTION_RETRY', - 'CELERY_ALWAYS_EAGER', 'CELERY_EAGER_PROPAGATES_EXCEPTIONS', - 'CELERY_IGNORE_RESULT', 'CELERY_TRACK_STARTED', - 'CELERY_DISABLE_RATE_LIMITS', 'CELERY_ACKS_LATE', - 'CELERY_STORE_ERRORS_EVEN_IF_IGNORED', - 'CELERY_SEND_TASK_ERROR_EMAILS', - 'CELERY_SEND_EVENTS', 'CELERY_SEND_TASK_SENT_EVENT', - 'CELERYD_LOG_COLOR', 'CELERY_REDIRECT_STDOUTS', - ] - -KNOWN_CONFIG_INTS = [ - 'CELERYD_CONCURRENCY', - 'CELERYD_PREFETCH_MULTIPLIER', - 'CELERY_AMQP_TASK_RESULT_EXPIRES', - 'CELERY_AMQP_TASK_RESULT_CONNECTION_MAX', - 'REDIS_PORT', 'REDIS_DB', - 'BROKER_PORT', 'BROKER_CONNECTION_TIMEOUT', - 'CELERY_BROKER_CONNECTION_MAX_RETRIES', - 'CELERY_TASK_RESULT_EXPIRES', 'CELERY_MAX_CACHED_RESULTS', - 'CELERY_DEFAULT_RATE_LIMIT', # ?? - 'CELERYD_MAX_TASKS_PER_CHILD', 'CELERYD_TASK_TIME_LIMIT', - 'CELERYD_TASK_SOFT_TIME_LIMIT', - 'MAIL_PORT', 'CELERYBEAT_MAX_LOOP_INTERVAL', - ] - -KNOWN_CONFIG_FLOATS = [ - 'CELERYD_ETA_SCHEDULER_PRECISION', - ] - -KNOWN_CONFIG_LISTS = [ - 'CELERY_ROUTES', 'CELERY_IMPORTS', - ] - - -## Needs special processing: -# ADMINS, ??? -# there are a lot more; we should list here or process specially. - - -def asfloat(obj): - try: - return float(obj) - except (TypeError, ValueError), e: - raise ValueError( - "Bad float value: %r" % obj) - MANDATORY_CELERY_IMPORTS = ['mediagoblin.process_media'] DEFAULT_SETTINGS_MODULE = 'mediagoblin.celery_setup.dummy_settings_module' + def setup_celery_from_config(app_config, global_config, settings_module=DEFAULT_SETTINGS_MODULE, force_celery_always_eager=False, set_environ=True): """ - Take a mediagoblin app config and the global config from a paste - factory and try to set up a celery settings module from this. + Take a mediagoblin app config and try to set up a celery settings + module from this. Args: - app_config: the application config section - - global_config: the entire paste config, all sections + - global_config: the entire ConfigObj loaded config, all sections - settings_module: the module to populate, as a string - - + - force_celery_always_eager: whether or not to force celery into + always eager mode; good for development and small installs - set_environ: if set, this will CELERY_CONFIG_MODULE to the settings_module """ - if asbool(app_config.get('use_celery_environment_var')) == True: + if app_config.get('celery_setup_elsewhere') == True: # Don't setup celery based on our config file. return - celery_conf_section = app_config.get('celery_section', 'celery') - if global_config.has_key(celery_conf_section): - celery_conf = global_config[celery_conf_section] + if global_config.has_key('celery'): + celery_conf = global_config['celery'] else: celery_conf = {} @@ -114,9 +63,9 @@ def setup_celery_from_config(app_config, global_config, if celery_settings['BROKER_BACKEND'] == 'mongodb': celery_settings['BROKER_HOST'] = app_config['db_host'] if app_config.has_key('db_port'): - celery_mongo_settings['port'] = asint(app_config['db_port']) + celery_mongo_settings['port'] = app_config['db_port'] if celery_settings['BROKER_BACKEND'] == 'mongodb': - celery_settings['BROKER_PORT'] = asint(app_config['db_port']) + celery_settings['BROKER_PORT'] = app_config['db_port'] celery_mongo_settings['database'] = app_config.get('db_name', 'mediagoblin') celery_settings['CELERY_MONGODB_BACKEND_SETTINGS'] = celery_mongo_settings @@ -124,14 +73,6 @@ def setup_celery_from_config(app_config, global_config, # Add anything else for key, value in celery_conf.iteritems(): key = key.upper() - if key in KNOWN_CONFIG_BOOLS: - value = asbool(value) - elif key in KNOWN_CONFIG_INTS: - value = asint(value) - elif key in KNOWN_CONFIG_FLOATS: - value = asfloat(value) - elif key in KNOWN_CONFIG_LISTS: - value = aslist(value) celery_settings[key] = value # add mandatory celery imports diff --git a/mediagoblin/celery_setup/from_celery.py b/mediagoblin/celery_setup/from_celery.py index 5fa9ba76..45e65e52 100644 --- a/mediagoblin/celery_setup/from_celery.py +++ b/mediagoblin/celery_setup/from_celery.py @@ -16,81 +16,39 @@ import os -from paste.deploy.loadwsgi import NicerConfigParser -from paste.deploy.converters import asbool - -from mediagoblin import storage -from mediagoblin.db.open import setup_connection_and_db_from_config +from mediagoblin import app, mg_globals from mediagoblin.celery_setup import setup_celery_from_config -from mediagoblin.globals import setup_globals -from mediagoblin.workbench import WorkbenchManager, DEFAULT_WORKBENCH_DIR -OUR_MODULENAME = 'mediagoblin.celery_setup.from_celery' +OUR_MODULENAME = __name__ -def setup_self(setup_globals_func=setup_globals): +def setup_self(): """ Transform this module into a celery config module by reading the mediagoblin config file. Set the environment variable - MEDIAGOBLIN_CONFIG to specify where this config file is at and - what section it uses. - - By default it defaults to 'mediagoblin.ini:app:mediagoblin'. + MEDIAGOBLIN_CONFIG to specify where this config file is. - The first colon ":" is a delimiter between the filename and the - config section, so in this case the filename is 'mediagoblin.ini' - and the section where mediagoblin is defined is 'app:mediagoblin'. + By default it defaults to 'mediagoblin.ini'. - Args: - - 'setup_globals_func': this is for testing purposes only. Don't - set this! + Note that if celery_setup_elsewhere is set in your config file, + this simply won't work. """ - mgoblin_conf_file, mgoblin_section = os.environ.get( - 'MEDIAGOBLIN_CONFIG', 'mediagoblin.ini:app:mediagoblin').split(':', 1) + mgoblin_conf_file = os.path.abspath( + os.environ.get('MEDIAGOBLIN_CONFIG', 'mediagoblin.ini')) if not os.path.exists(mgoblin_conf_file): raise IOError( "MEDIAGOBLIN_CONFIG not set or file does not exist") - parser = NicerConfigParser(mgoblin_conf_file) - parser.read(mgoblin_conf_file) - parser._defaults.setdefault( - 'here', os.path.dirname(os.path.abspath(mgoblin_conf_file))) - parser._defaults.setdefault( - '__file__', os.path.abspath(mgoblin_conf_file)) - - mgoblin_section = dict(parser.items(mgoblin_section)) - mgoblin_conf = dict( - [(section_name, dict(parser.items(section_name))) - for section_name in parser.sections()]) + # By setting the environment variable here we should ensure that + # this is the module that gets set up. + os.environ['CELERY_CONFIG_MODULE'] = OUR_MODULENAME + app.MediaGoblinApp(mgoblin_conf_file, setup_celery=False) setup_celery_from_config( - mgoblin_section, mgoblin_conf, + mg_globals.app_config, mg_globals.global_config, settings_module=OUR_MODULENAME, set_environ=False) - connection, db = setup_connection_and_db_from_config(mgoblin_section) - - # Set up the storage systems. - public_store = storage.storage_system_from_paste_config( - mgoblin_section, 'publicstore') - queue_store = storage.storage_system_from_paste_config( - mgoblin_section, 'queuestore') - - workbench_manager = WorkbenchManager( - mgoblin_section.get( - 'workbench_path', DEFAULT_WORKBENCH_DIR)) - - setup_globals_func( - db_connection=connection, - database=db, - public_store=public_store, - email_debug_mode=asbool(mgoblin_section.get('email_debug_mode')), - email_sender_address=mgoblin_section.get( - 'email_sender_address', - 'notice@mediagoblin.example.org'), - queue_store=queue_store, - workbench_manager=workbench_manager) - if os.environ['CELERY_CONFIG_MODULE'] == OUR_MODULENAME: setup_self() diff --git a/mediagoblin/celery_setup/from_tests.py b/mediagoblin/celery_setup/from_tests.py deleted file mode 100644 index fe7d7314..00000000 --- a/mediagoblin/celery_setup/from_tests.py +++ /dev/null @@ -1,43 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011 Free Software Foundation, Inc -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -import os - -from mediagoblin.tests.tools import TEST_APP_CONFIG -from mediagoblin import util -from mediagoblin.celery_setup import setup_celery_from_config -from mediagoblin.globals import setup_globals - - -OUR_MODULENAME = 'mediagoblin.celery_setup.from_tests' - - -def setup_self(setup_globals_func=setup_globals): - """ - Set up celery for testing's sake, which just needs to set up - celery and celery only. - """ - mgoblin_conf = util.read_config_file(TEST_APP_CONFIG) - mgoblin_section = mgoblin_conf['app:mediagoblin'] - - setup_celery_from_config( - mgoblin_section, mgoblin_conf, - settings_module=OUR_MODULENAME, - set_environ=False) - - -if os.environ.get('CELERY_CONFIG_MODULE') == OUR_MODULENAME: - setup_self() |