From 073b61fe53d4e499d7034bb8de2404cd87dc9095 Mon Sep 17 00:00:00 2001 From: Elrond Date: Sun, 19 Jun 2011 23:09:35 +0200 Subject: Move mediagoblin.celery_setup -> mediagoblin.init.celery As the first target of the new .init. submodule move celery_setup there. Quite straight forward, just a lot of places to change. --- docs/hackinghowto.rst | 2 +- mediagoblin/app.py | 2 +- mediagoblin/celery_setup/__init__.py | 89 ----------------------- mediagoblin/celery_setup/dummy_settings_module.py | 0 mediagoblin/celery_setup/from_celery.py | 60 --------------- mediagoblin/celery_setup/from_tests.py | 33 --------- mediagoblin/init/celery/__init__.py | 89 +++++++++++++++++++++++ mediagoblin/init/celery/dummy_settings_module.py | 0 mediagoblin/init/celery/from_celery.py | 60 +++++++++++++++ mediagoblin/init/celery/from_tests.py | 33 +++++++++ mediagoblin/tests/test_celery_setup.py | 2 +- mediagoblin/tests/test_mgoblin_app.ini | 2 +- mediagoblin/tests/tools.py | 8 +- runtests.sh | 2 +- 14 files changed, 191 insertions(+), 191 deletions(-) delete mode 100644 mediagoblin/celery_setup/__init__.py delete mode 100644 mediagoblin/celery_setup/dummy_settings_module.py delete mode 100644 mediagoblin/celery_setup/from_celery.py delete mode 100644 mediagoblin/celery_setup/from_tests.py create mode 100644 mediagoblin/init/celery/__init__.py create mode 100644 mediagoblin/init/celery/dummy_settings_module.py create mode 100644 mediagoblin/init/celery/from_celery.py create mode 100644 mediagoblin/init/celery/from_tests.py diff --git a/docs/hackinghowto.rst b/docs/hackinghowto.rst index 911f2340..08b228f1 100644 --- a/docs/hackinghowto.rst +++ b/docs/hackinghowto.rst @@ -150,7 +150,7 @@ celeryd in another window. Run:: - CELERY_CONFIG_MODULE=mediagoblin.celery_setup.from_celery ./bin/celeryd + CELERY_CONFIG_MODULE=mediagoblin.init.celery.from_celery ./bin/celeryd Running the test suite diff --git a/mediagoblin/app.py b/mediagoblin/app.py index b27b5761..080c8e3a 100644 --- a/mediagoblin/app.py +++ b/mediagoblin/app.py @@ -25,7 +25,7 @@ from mediagoblin.config import ( read_mediagoblin_config, generate_validation_report) from mediagoblin.db.open import setup_connection_and_db_from_config from mediagoblin.mg_globals import setup_globals -from mediagoblin.celery_setup import setup_celery_from_config +from mediagoblin.init.celery import setup_celery_from_config from mediagoblin.workbench import WorkbenchManager diff --git a/mediagoblin/celery_setup/__init__.py b/mediagoblin/celery_setup/__init__.py deleted file mode 100644 index e35dbce2..00000000 --- a/mediagoblin/celery_setup/__init__.py +++ /dev/null @@ -1,89 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011 Free Software Foundation, Inc -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import os -import sys - - -MANDATORY_CELERY_IMPORTS = ['mediagoblin.process_media'] - -DEFAULT_SETTINGS_MODULE = 'mediagoblin.celery_setup.dummy_settings_module' - - -def setup_celery_from_config(app_config, global_config, - settings_module=DEFAULT_SETTINGS_MODULE, - force_celery_always_eager=False, - set_environ=True): - """ - Take a mediagoblin app config and try to set up a celery settings - module from this. - - Args: - - app_config: the application config section - - global_config: the entire ConfigObj loaded config, all sections - - settings_module: the module to populate, as a string - - force_celery_always_eager: whether or not to force celery into - always eager mode; good for development and small installs - - set_environ: if set, this will CELERY_CONFIG_MODULE to the - settings_module - """ - if global_config.has_key('celery'): - celery_conf = global_config['celery'] - else: - celery_conf = {} - - celery_settings = {} - - # set up mongodb stuff - celery_settings['CELERY_RESULT_BACKEND'] = 'mongodb' - if not celery_settings.has_key('BROKER_BACKEND'): - celery_settings['BROKER_BACKEND'] = 'mongodb' - - celery_mongo_settings = {} - - if app_config.has_key('db_host'): - celery_mongo_settings['host'] = app_config['db_host'] - if celery_settings['BROKER_BACKEND'] == 'mongodb': - celery_settings['BROKER_HOST'] = app_config['db_host'] - if app_config.has_key('db_port'): - celery_mongo_settings['port'] = app_config['db_port'] - if celery_settings['BROKER_BACKEND'] == 'mongodb': - celery_settings['BROKER_PORT'] = app_config['db_port'] - celery_mongo_settings['database'] = app_config.get('db_name', 'mediagoblin') - - celery_settings['CELERY_MONGODB_BACKEND_SETTINGS'] = celery_mongo_settings - - # Add anything else - for key, value in celery_conf.iteritems(): - key = key.upper() - celery_settings[key] = value - - # add mandatory celery imports - celery_imports = celery_settings.setdefault('CELERY_IMPORTS', []) - celery_imports.extend(MANDATORY_CELERY_IMPORTS) - - if force_celery_always_eager: - celery_settings['CELERY_ALWAYS_EAGER'] = True - celery_settings['CELERY_EAGER_PROPAGATES_EXCEPTIONS'] = True - - __import__(settings_module) - this_module = sys.modules[settings_module] - - for key, value in celery_settings.iteritems(): - setattr(this_module, key, value) - - if set_environ: - os.environ['CELERY_CONFIG_MODULE'] = settings_module diff --git a/mediagoblin/celery_setup/dummy_settings_module.py b/mediagoblin/celery_setup/dummy_settings_module.py deleted file mode 100644 index e69de29b..00000000 diff --git a/mediagoblin/celery_setup/from_celery.py b/mediagoblin/celery_setup/from_celery.py deleted file mode 100644 index ed0a409e..00000000 --- a/mediagoblin/celery_setup/from_celery.py +++ /dev/null @@ -1,60 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011 Free Software Foundation, Inc -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import os - -from mediagoblin import app, mg_globals -from mediagoblin.celery_setup import setup_celery_from_config - - -OUR_MODULENAME = __name__ - - -def setup_self(check_environ_for_conf=True, module_name=OUR_MODULENAME, - default_conf_file='mediagoblin.ini'): - """ - Transform this module into a celery config module by reading the - mediagoblin config file. Set the environment variable - MEDIAGOBLIN_CONFIG to specify where this config file is. - - By default it defaults to 'mediagoblin.ini'. - - Note that if celery_setup_elsewhere is set in your config file, - this simply won't work. - """ - if check_environ_for_conf: - mgoblin_conf_file = os.path.abspath( - os.environ.get('MEDIAGOBLIN_CONFIG', default_conf_file)) - else: - mgoblin_conf_file = default_conf_file - - if not os.path.exists(mgoblin_conf_file): - raise IOError( - "MEDIAGOBLIN_CONFIG not set or file does not exist") - - # By setting the environment variable here we should ensure that - # this is the module that gets set up. - os.environ['CELERY_CONFIG_MODULE'] = module_name - app.MediaGoblinApp(mgoblin_conf_file, setup_celery=False) - - setup_celery_from_config( - mg_globals.app_config, mg_globals.global_config, - settings_module=module_name, - set_environ=False) - - -if os.environ['CELERY_CONFIG_MODULE'] == OUR_MODULENAME: - setup_self() diff --git a/mediagoblin/celery_setup/from_tests.py b/mediagoblin/celery_setup/from_tests.py deleted file mode 100644 index 779ecd65..00000000 --- a/mediagoblin/celery_setup/from_tests.py +++ /dev/null @@ -1,33 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011 Free Software Foundation, Inc -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -import os - -from mediagoblin.tests.tools import TEST_APP_CONFIG -from mediagoblin.celery_setup.from_celery import setup_self - - -OUR_MODULENAME = __name__ -CELERY_SETUP = False - - -if os.environ.get('CELERY_CONFIG_MODULE') == OUR_MODULENAME: - if CELERY_SETUP: - pass - else: - setup_self(check_environ_for_conf=False, module_name=OUR_MODULENAME, - default_conf_file=TEST_APP_CONFIG) - CELERY_SETUP = True diff --git a/mediagoblin/init/celery/__init__.py b/mediagoblin/init/celery/__init__.py new file mode 100644 index 00000000..67c3dfa0 --- /dev/null +++ b/mediagoblin/init/celery/__init__.py @@ -0,0 +1,89 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os +import sys + + +MANDATORY_CELERY_IMPORTS = ['mediagoblin.process_media'] + +DEFAULT_SETTINGS_MODULE = 'mediagoblin.init.celery.dummy_settings_module' + + +def setup_celery_from_config(app_config, global_config, + settings_module=DEFAULT_SETTINGS_MODULE, + force_celery_always_eager=False, + set_environ=True): + """ + Take a mediagoblin app config and try to set up a celery settings + module from this. + + Args: + - app_config: the application config section + - global_config: the entire ConfigObj loaded config, all sections + - settings_module: the module to populate, as a string + - force_celery_always_eager: whether or not to force celery into + always eager mode; good for development and small installs + - set_environ: if set, this will CELERY_CONFIG_MODULE to the + settings_module + """ + if global_config.has_key('celery'): + celery_conf = global_config['celery'] + else: + celery_conf = {} + + celery_settings = {} + + # set up mongodb stuff + celery_settings['CELERY_RESULT_BACKEND'] = 'mongodb' + if not celery_settings.has_key('BROKER_BACKEND'): + celery_settings['BROKER_BACKEND'] = 'mongodb' + + celery_mongo_settings = {} + + if app_config.has_key('db_host'): + celery_mongo_settings['host'] = app_config['db_host'] + if celery_settings['BROKER_BACKEND'] == 'mongodb': + celery_settings['BROKER_HOST'] = app_config['db_host'] + if app_config.has_key('db_port'): + celery_mongo_settings['port'] = app_config['db_port'] + if celery_settings['BROKER_BACKEND'] == 'mongodb': + celery_settings['BROKER_PORT'] = app_config['db_port'] + celery_mongo_settings['database'] = app_config.get('db_name', 'mediagoblin') + + celery_settings['CELERY_MONGODB_BACKEND_SETTINGS'] = celery_mongo_settings + + # Add anything else + for key, value in celery_conf.iteritems(): + key = key.upper() + celery_settings[key] = value + + # add mandatory celery imports + celery_imports = celery_settings.setdefault('CELERY_IMPORTS', []) + celery_imports.extend(MANDATORY_CELERY_IMPORTS) + + if force_celery_always_eager: + celery_settings['CELERY_ALWAYS_EAGER'] = True + celery_settings['CELERY_EAGER_PROPAGATES_EXCEPTIONS'] = True + + __import__(settings_module) + this_module = sys.modules[settings_module] + + for key, value in celery_settings.iteritems(): + setattr(this_module, key, value) + + if set_environ: + os.environ['CELERY_CONFIG_MODULE'] = settings_module diff --git a/mediagoblin/init/celery/dummy_settings_module.py b/mediagoblin/init/celery/dummy_settings_module.py new file mode 100644 index 00000000..e69de29b diff --git a/mediagoblin/init/celery/from_celery.py b/mediagoblin/init/celery/from_celery.py new file mode 100644 index 00000000..c053591b --- /dev/null +++ b/mediagoblin/init/celery/from_celery.py @@ -0,0 +1,60 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os + +from mediagoblin import app, mg_globals +from mediagoblin.init.celery import setup_celery_from_config + + +OUR_MODULENAME = __name__ + + +def setup_self(check_environ_for_conf=True, module_name=OUR_MODULENAME, + default_conf_file='mediagoblin.ini'): + """ + Transform this module into a celery config module by reading the + mediagoblin config file. Set the environment variable + MEDIAGOBLIN_CONFIG to specify where this config file is. + + By default it defaults to 'mediagoblin.ini'. + + Note that if celery_setup_elsewhere is set in your config file, + this simply won't work. + """ + if check_environ_for_conf: + mgoblin_conf_file = os.path.abspath( + os.environ.get('MEDIAGOBLIN_CONFIG', default_conf_file)) + else: + mgoblin_conf_file = default_conf_file + + if not os.path.exists(mgoblin_conf_file): + raise IOError( + "MEDIAGOBLIN_CONFIG not set or file does not exist") + + # By setting the environment variable here we should ensure that + # this is the module that gets set up. + os.environ['CELERY_CONFIG_MODULE'] = module_name + app.MediaGoblinApp(mgoblin_conf_file, setup_celery=False) + + setup_celery_from_config( + mg_globals.app_config, mg_globals.global_config, + settings_module=module_name, + set_environ=False) + + +if os.environ['CELERY_CONFIG_MODULE'] == OUR_MODULENAME: + setup_self() diff --git a/mediagoblin/init/celery/from_tests.py b/mediagoblin/init/celery/from_tests.py new file mode 100644 index 00000000..b2293e2c --- /dev/null +++ b/mediagoblin/init/celery/from_tests.py @@ -0,0 +1,33 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os + +from mediagoblin.tests.tools import TEST_APP_CONFIG +from mediagoblin.init.celery.from_celery import setup_self + + +OUR_MODULENAME = __name__ +CELERY_SETUP = False + + +if os.environ.get('CELERY_CONFIG_MODULE') == OUR_MODULENAME: + if CELERY_SETUP: + pass + else: + setup_self(check_environ_for_conf=False, module_name=OUR_MODULENAME, + default_conf_file=TEST_APP_CONFIG) + CELERY_SETUP = True diff --git a/mediagoblin/tests/test_celery_setup.py b/mediagoblin/tests/test_celery_setup.py index 8bf97ae4..d9222620 100644 --- a/mediagoblin/tests/test_celery_setup.py +++ b/mediagoblin/tests/test_celery_setup.py @@ -16,7 +16,7 @@ import pkg_resources -from mediagoblin import celery_setup +from mediagoblin.init import celery as celery_setup from mediagoblin.config import read_mediagoblin_config diff --git a/mediagoblin/tests/test_mgoblin_app.ini b/mediagoblin/tests/test_mgoblin_app.ini index e022d47b..fd0f87a4 100644 --- a/mediagoblin/tests/test_mgoblin_app.ini +++ b/mediagoblin/tests/test_mgoblin_app.ini @@ -8,7 +8,7 @@ email_debug_mode = true db_name = __mediagoblin_tests__ # Celery shouldn't be set up by the application as it's setup via -# mediagoblin.celery_setup.from_celery +# mediagoblin.init.celery.from_celery celery_setup_elsewhere = true [celery] diff --git a/mediagoblin/tests/tools.py b/mediagoblin/tests/tools.py index 64f773f0..966515d2 100644 --- a/mediagoblin/tests/tools.py +++ b/mediagoblin/tests/tools.py @@ -42,8 +42,8 @@ USER_DEV_DIRECTORIES_TO_SETUP = [ BAD_CELERY_MESSAGE = """\ Sorry, you *absolutely* must run nosetests with the -mediagoblin.celery_setup.from_tests module. Like so: -$ CELERY_CONFIG_MODULE=mediagoblin.celery_setup.from_tests ./bin/nosetests""" +mediagoblin.init.celery.from_tests module. Like so: +$ CELERY_CONFIG_MODULE=mediagoblin.init.celery.from_tests ./bin/nosetests""" class BadCeleryEnviron(Exception): pass @@ -51,7 +51,7 @@ class BadCeleryEnviron(Exception): pass def suicide_if_bad_celery_environ(): if not os.environ.get('CELERY_CONFIG_MODULE') == \ - 'mediagoblin.celery_setup.from_tests': + 'mediagoblin.init.celery.from_tests': raise BadCeleryEnviron(BAD_CELERY_MESSAGE) @@ -59,7 +59,7 @@ def get_test_app(dump_old_app=True): suicide_if_bad_celery_environ() # Leave this imported as it sets up celery. - from mediagoblin.celery_setup import from_tests + from mediagoblin.init.celery import from_tests global MGOBLIN_APP diff --git a/runtests.sh b/runtests.sh index 9b96b17c..1dfbf093 100755 --- a/runtests.sh +++ b/runtests.sh @@ -27,4 +27,4 @@ else exit 1 fi -CELERY_CONFIG_MODULE=mediagoblin.celery_setup.from_tests $NOSETESTS $@ +CELERY_CONFIG_MODULE=mediagoblin.init.celery.from_tests $NOSETESTS $@ -- cgit v1.2.3