aboutsummaryrefslogtreecommitdiffstats
path: root/mediagoblin/celery_setup
diff options
context:
space:
mode:
Diffstat (limited to 'mediagoblin/celery_setup')
-rw-r--r--mediagoblin/celery_setup/__init__.py81
-rw-r--r--mediagoblin/celery_setup/from_celery.py74
-rw-r--r--mediagoblin/celery_setup/from_tests.py20
3 files changed, 31 insertions, 144 deletions
diff --git a/mediagoblin/celery_setup/__init__.py b/mediagoblin/celery_setup/__init__.py
index d4f25b07..b6e35e99 100644
--- a/mediagoblin/celery_setup/__init__.py
+++ b/mediagoblin/celery_setup/__init__.py
@@ -17,86 +17,35 @@
import os
import sys
-from paste.deploy.converters import asbool, asint, aslist
-
-
-KNOWN_CONFIG_BOOLS = [
- 'CELERY_RESULT_PERSISTENT',
- 'CELERY_CREATE_MISSING_QUEUES',
- 'BROKER_USE_SSL', 'BROKER_CONNECTION_RETRY',
- 'CELERY_ALWAYS_EAGER', 'CELERY_EAGER_PROPAGATES_EXCEPTIONS',
- 'CELERY_IGNORE_RESULT', 'CELERY_TRACK_STARTED',
- 'CELERY_DISABLE_RATE_LIMITS', 'CELERY_ACKS_LATE',
- 'CELERY_STORE_ERRORS_EVEN_IF_IGNORED',
- 'CELERY_SEND_TASK_ERROR_EMAILS',
- 'CELERY_SEND_EVENTS', 'CELERY_SEND_TASK_SENT_EVENT',
- 'CELERYD_LOG_COLOR', 'CELERY_REDIRECT_STDOUTS',
- ]
-
-KNOWN_CONFIG_INTS = [
- 'CELERYD_CONCURRENCY',
- 'CELERYD_PREFETCH_MULTIPLIER',
- 'CELERY_AMQP_TASK_RESULT_EXPIRES',
- 'CELERY_AMQP_TASK_RESULT_CONNECTION_MAX',
- 'REDIS_PORT', 'REDIS_DB',
- 'BROKER_PORT', 'BROKER_CONNECTION_TIMEOUT',
- 'CELERY_BROKER_CONNECTION_MAX_RETRIES',
- 'CELERY_TASK_RESULT_EXPIRES', 'CELERY_MAX_CACHED_RESULTS',
- 'CELERY_DEFAULT_RATE_LIMIT', # ??
- 'CELERYD_MAX_TASKS_PER_CHILD', 'CELERYD_TASK_TIME_LIMIT',
- 'CELERYD_TASK_SOFT_TIME_LIMIT',
- 'MAIL_PORT', 'CELERYBEAT_MAX_LOOP_INTERVAL',
- ]
-
-KNOWN_CONFIG_FLOATS = [
- 'CELERYD_ETA_SCHEDULER_PRECISION',
- ]
-
-KNOWN_CONFIG_LISTS = [
- 'CELERY_ROUTES', 'CELERY_IMPORTS',
- ]
-
-
-## Needs special processing:
-# ADMINS, ???
-# there are a lot more; we should list here or process specially.
-
-
-def asfloat(obj):
- try:
- return float(obj)
- except (TypeError, ValueError), e:
- raise ValueError(
- "Bad float value: %r" % obj)
-
MANDATORY_CELERY_IMPORTS = ['mediagoblin.process_media']
DEFAULT_SETTINGS_MODULE = 'mediagoblin.celery_setup.dummy_settings_module'
+
def setup_celery_from_config(app_config, global_config,
settings_module=DEFAULT_SETTINGS_MODULE,
force_celery_always_eager=False,
set_environ=True):
"""
- Take a mediagoblin app config and the global config from a paste
- factory and try to set up a celery settings module from this.
+ Take a mediagoblin app config and try to set up a celery settings
+ module from this.
Args:
- app_config: the application config section
- - global_config: the entire paste config, all sections
+ - global_config: the entire ConfigObj loaded config, all sections
- settings_module: the module to populate, as a string
- -
+ - force_celery_always_eager: whether or not to force celery into
+ always eager mode; good for development and small installs
- set_environ: if set, this will CELERY_CONFIG_MODULE to the
settings_module
"""
- if asbool(app_config.get('use_celery_environment_var')) == True:
+ if app_config.get('celery_setup_elsewhere') == True:
# Don't setup celery based on our config file.
return
- celery_conf_section = app_config.get('celery_section', 'celery')
- if global_config.has_key(celery_conf_section):
- celery_conf = global_config[celery_conf_section]
+ if global_config.has_key('celery'):
+ celery_conf = global_config['celery']
else:
celery_conf = {}
@@ -114,9 +63,9 @@ def setup_celery_from_config(app_config, global_config,
if celery_settings['BROKER_BACKEND'] == 'mongodb':
celery_settings['BROKER_HOST'] = app_config['db_host']
if app_config.has_key('db_port'):
- celery_mongo_settings['port'] = asint(app_config['db_port'])
+ celery_mongo_settings['port'] = app_config['db_port']
if celery_settings['BROKER_BACKEND'] == 'mongodb':
- celery_settings['BROKER_PORT'] = asint(app_config['db_port'])
+ celery_settings['BROKER_PORT'] = app_config['db_port']
celery_mongo_settings['database'] = app_config.get('db_name', 'mediagoblin')
celery_settings['CELERY_MONGODB_BACKEND_SETTINGS'] = celery_mongo_settings
@@ -124,14 +73,6 @@ def setup_celery_from_config(app_config, global_config,
# Add anything else
for key, value in celery_conf.iteritems():
key = key.upper()
- if key in KNOWN_CONFIG_BOOLS:
- value = asbool(value)
- elif key in KNOWN_CONFIG_INTS:
- value = asint(value)
- elif key in KNOWN_CONFIG_FLOATS:
- value = asfloat(value)
- elif key in KNOWN_CONFIG_LISTS:
- value = aslist(value)
celery_settings[key] = value
# add mandatory celery imports
diff --git a/mediagoblin/celery_setup/from_celery.py b/mediagoblin/celery_setup/from_celery.py
index c8ccebc8..046aaa50 100644
--- a/mediagoblin/celery_setup/from_celery.py
+++ b/mediagoblin/celery_setup/from_celery.py
@@ -16,81 +16,43 @@
import os
-from paste.deploy.loadwsgi import NicerConfigParser
-from paste.deploy.converters import asbool
-
-from mediagoblin import storage
-from mediagoblin.db.open import setup_connection_and_db_from_config
+from mediagoblin import app, mg_globals
from mediagoblin.celery_setup import setup_celery_from_config
-from mediagoblin.mg_globals import setup_globals
-from mediagoblin.workbench import WorkbenchManager, DEFAULT_WORKBENCH_DIR
OUR_MODULENAME = __name__
-def setup_self():
+def setup_self(check_environ_for_conf=True, module_name=OUR_MODULENAME):
"""
Transform this module into a celery config module by reading the
mediagoblin config file. Set the environment variable
- MEDIAGOBLIN_CONFIG to specify where this config file is at and
- what section it uses.
-
- By default it defaults to 'mediagoblin.ini:app:mediagoblin'.
+ MEDIAGOBLIN_CONFIG to specify where this config file is.
- The first colon ":" is a delimiter between the filename and the
- config section, so in this case the filename is 'mediagoblin.ini'
- and the section where mediagoblin is defined is 'app:mediagoblin'.
+ By default it defaults to 'mediagoblin.ini'.
- Args:
- - 'setup_globals_func': this is for testing purposes only. Don't
- set this!
+ Note that if celery_setup_elsewhere is set in your config file,
+ this simply won't work.
"""
- mgoblin_conf_file, mgoblin_section = os.environ.get(
- 'MEDIAGOBLIN_CONFIG', 'mediagoblin.ini:app:mediagoblin').split(':', 1)
+ if check_environ_for_conf:
+ mgoblin_conf_file = os.path.abspath(
+ os.environ.get('MEDIAGOBLIN_CONFIG', 'mediagoblin.ini'))
+ else:
+ mgoblin_conf_file = 'mediagoblin.ini'
+
if not os.path.exists(mgoblin_conf_file):
raise IOError(
"MEDIAGOBLIN_CONFIG not set or file does not exist")
- parser = NicerConfigParser(mgoblin_conf_file)
- parser.read(mgoblin_conf_file)
- parser._defaults.setdefault(
- 'here', os.path.dirname(os.path.abspath(mgoblin_conf_file)))
- parser._defaults.setdefault(
- '__file__', os.path.abspath(mgoblin_conf_file))
-
- mgoblin_section = dict(parser.items(mgoblin_section))
- mgoblin_conf = dict(
- [(section_name, dict(parser.items(section_name)))
- for section_name in parser.sections()])
+ # By setting the environment variable here we should ensure that
+ # this is the module that gets set up.
+ os.environ['CELERY_CONFIG_MODULE'] = module_name
+ app.MediaGoblinApp(mgoblin_conf_file, setup_celery=False)
setup_celery_from_config(
- mgoblin_section, mgoblin_conf,
- settings_module=OUR_MODULENAME,
+ mg_globals.app_config, mg_globals.global_config,
+ settings_module=module_name,
set_environ=False)
- connection, db = setup_connection_and_db_from_config(mgoblin_section)
-
- # Set up the storage systems.
- public_store = storage.storage_system_from_paste_config(
- mgoblin_section, 'publicstore')
- queue_store = storage.storage_system_from_paste_config(
- mgoblin_section, 'queuestore')
-
- workbench_manager = WorkbenchManager(
- mgoblin_section.get(
- 'workbench_path', DEFAULT_WORKBENCH_DIR))
-
- setup_globals(
- db_connection=connection,
- database=db,
- public_store=public_store,
- email_debug_mode=asbool(mgoblin_section.get('email_debug_mode')),
- email_sender_address=mgoblin_section.get(
- 'email_sender_address',
- 'notice@mediagoblin.example.org'),
- queue_store=queue_store,
- workbench_manager=workbench_manager)
-
if os.environ['CELERY_CONFIG_MODULE'] == OUR_MODULENAME:
setup_self()
diff --git a/mediagoblin/celery_setup/from_tests.py b/mediagoblin/celery_setup/from_tests.py
index 70814075..43032f41 100644
--- a/mediagoblin/celery_setup/from_tests.py
+++ b/mediagoblin/celery_setup/from_tests.py
@@ -16,27 +16,11 @@
import os
-from mediagoblin.tests.tools import TEST_APP_CONFIG
-from mediagoblin import util
-from mediagoblin.celery_setup import setup_celery_from_config
+from mediagoblin.celery_setup.from_celery import setup_self
OUR_MODULENAME = __name__
-def setup_self():
- """
- Set up celery for testing's sake, which just needs to set up
- celery and celery only.
- """
- mgoblin_conf = util.read_config_file(TEST_APP_CONFIG)
- mgoblin_section = mgoblin_conf['app:mediagoblin']
-
- setup_celery_from_config(
- mgoblin_section, mgoblin_conf,
- settings_module=OUR_MODULENAME,
- set_environ=False)
-
-
if os.environ.get('CELERY_CONFIG_MODULE') == OUR_MODULENAME:
- setup_self()
+ setup_self(check_environ_for_conf=False, module_name=OUR_MODULENAME)
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-29 23:59:34 +0000