diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | mediagoblin/config_spec.ini | 8 | ||||
| -rw-r--r-- | mediagoblin/init/celery/__init__.py | 23 | ||||
| -rw-r--r-- | mediagoblin/tests/fake_celery_conf_mgdb.ini | 14 | ||||
| -rw-r--r-- | mediagoblin/tests/test_celery_setup.py | 43 | ||||
| -rw-r--r-- | setup.py | 1 |
6 files changed, 23 insertions, 68 deletions
@@ -16,6 +16,8 @@ /paste_local.ini /mediagoblin_local.ini /mediagoblin.db +/celery.db +/kombu.db /server-log.txt # Tests diff --git a/mediagoblin/config_spec.ini b/mediagoblin/config_spec.ini index 2b4ba2f9..94e90c44 100644 --- a/mediagoblin/config_spec.ini +++ b/mediagoblin/config_spec.ini @@ -79,6 +79,14 @@ lock_dir = string(default="%(here)s/user_dev/beaker/cache/lock") [celery] +# default result stuff +celery_result_backend = string(default="database") +celery_result_dburi = string(default="sqlite:///%(here)s/celery.db") + +# default kombu stuff +broker_transport = string(default="sqlalchemy") +broker_host = string(default="sqlite:///%(here)s/kombu.db") + # known booleans celery_result_persistent = boolean() celery_create_missing_queues = boolean() diff --git a/mediagoblin/init/celery/__init__.py b/mediagoblin/init/celery/__init__.py index fb958909..6dcea239 100644 --- a/mediagoblin/init/celery/__init__.py +++ b/mediagoblin/init/celery/__init__.py @@ -47,30 +47,13 @@ def setup_celery_from_config(app_config, global_config, celery_settings = {} - # set up mongodb stuff - celery_settings['CELERY_RESULT_BACKEND'] = 'mongodb' - if 'BROKER_BACKEND' not in celery_settings: - celery_settings['BROKER_BACKEND'] = 'mongodb' - - celery_mongo_settings = {} - - if 'db_host' in app_config: - celery_mongo_settings['host'] = app_config['db_host'] - if celery_settings['BROKER_BACKEND'] == 'mongodb': - celery_settings['BROKER_HOST'] = app_config['db_host'] - if 'db_port' in app_config: - celery_mongo_settings['port'] = app_config['db_port'] - if celery_settings['BROKER_BACKEND'] == 'mongodb': - celery_settings['BROKER_PORT'] = app_config['db_port'] - celery_mongo_settings['database'] = app_config['db_name'] - - celery_settings['CELERY_MONGODB_BACKEND_SETTINGS'] = celery_mongo_settings - - # Add anything else + # Add all celery settings from config for key, value in celery_conf.iteritems(): key = key.upper() celery_settings[key] = value + # TODO: use default result stuff here if it exists + # add mandatory celery imports celery_imports = celery_settings.setdefault('CELERY_IMPORTS', []) celery_imports.extend(MANDATORY_CELERY_IMPORTS) diff --git a/mediagoblin/tests/fake_celery_conf_mgdb.ini b/mediagoblin/tests/fake_celery_conf_mgdb.ini deleted file mode 100644 index 52671c14..00000000 --- a/mediagoblin/tests/fake_celery_conf_mgdb.ini +++ /dev/null @@ -1,14 +0,0 @@ -['mediagoblin'] -db_host = mongodb.example.org -db_port = 8080 -db_name = captain_lollerskates - -['something'] -or = other - -['celery'] -some_variable = poolf -mail_port = 2020 -celeryd_eta_scheduler_precision = 3.1 -celery_result_persistent = false -celery_imports = baz.bar.foo, import.is.a.this diff --git a/mediagoblin/tests/test_celery_setup.py b/mediagoblin/tests/test_celery_setup.py index c9c77821..fd600f56 100644 --- a/mediagoblin/tests/test_celery_setup.py +++ b/mediagoblin/tests/test_celery_setup.py @@ -22,8 +22,6 @@ from mediagoblin.init.config import read_mediagoblin_config TEST_CELERY_CONF_NOSPECIALDB = pkg_resources.resource_filename( 'mediagoblin.tests', 'fake_celery_conf.ini') -TEST_CELERY_CONF_MGSPECIALDB = pkg_resources.resource_filename( - 'mediagoblin.tests', 'fake_celery_conf_mgdb.ini') def test_setup_celery_from_config(): @@ -51,35 +49,12 @@ def test_setup_celery_from_config(): assert fake_celery_module.CELERY_RESULT_PERSISTENT is True assert fake_celery_module.CELERY_IMPORTS == [ 'foo.bar.baz', 'this.is.an.import', 'mediagoblin.processing'] - assert fake_celery_module.CELERY_MONGODB_BACKEND_SETTINGS == { - 'database': 'mediagoblin'} - assert fake_celery_module.CELERY_RESULT_BACKEND == 'mongodb' - assert fake_celery_module.BROKER_BACKEND == 'mongodb' - - _wipe_testmodule_clean(fake_celery_module) - - global_config, validation_result = read_mediagoblin_config( - TEST_CELERY_CONF_MGSPECIALDB) - app_config = global_config['mediagoblin'] - - celery_setup.setup_celery_from_config( - app_config, global_config, - 'mediagoblin.tests.fake_celery_module', set_environ=False) - - from mediagoblin.tests import fake_celery_module - assert fake_celery_module.SOME_VARIABLE == 'poolf' - assert fake_celery_module.MAIL_PORT == 2020 - assert isinstance(fake_celery_module.MAIL_PORT, int) - assert fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION == 3.1 - assert isinstance(fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION, float) - assert fake_celery_module.CELERY_RESULT_PERSISTENT is False - assert fake_celery_module.CELERY_IMPORTS == [ - 'baz.bar.foo', 'import.is.a.this', 'mediagoblin.processing'] - assert fake_celery_module.CELERY_MONGODB_BACKEND_SETTINGS == { - 'database': 'captain_lollerskates', - 'host': 'mongodb.example.org', - 'port': 8080} - assert fake_celery_module.CELERY_RESULT_BACKEND == 'mongodb' - assert fake_celery_module.BROKER_BACKEND == 'mongodb' - assert fake_celery_module.BROKER_HOST == 'mongodb.example.org' - assert fake_celery_module.BROKER_PORT == 8080 + assert fake_celery_module.CELERY_RESULT_BACKEND == 'database' + assert fake_celery_module.CELERY_RESULT_DBURI == ( + 'sqlite:///' + + pkg_resources.resource_filename('mediagoblin.tests', 'celery.db')) + + assert fake_celery_module.BROKER_TRANSPORT == 'sqlalchemy' + assert fake_celery_module.BROKER_HOST == ( + 'sqlite:///' + + pkg_resources.resource_filename('mediagoblin.tests', 'kombu.db')) @@ -64,6 +64,7 @@ setup( 'Markdown', 'sqlalchemy', 'sqlalchemy-migrate', + 'kombu-sqlalchemy', ## For now we're expecting that users will install this from ## their package managers. # 'lxml', |