diff options
| -rw-r--r-- | buildout.cfg | 1 | ||||
| -rw-r--r-- | docs/designdecisions.rst | 40 | ||||
| -rw-r--r-- | mediagoblin/app.py | 37 | ||||
| -rw-r--r-- | mediagoblin/celery_setup/__init__.py | 140 | ||||
| -rw-r--r-- | mediagoblin/celery_setup/dummy_settings_module.py | 0 | ||||
| -rw-r--r-- | mediagoblin/celery_setup/from_celery.py | 87 | ||||
| -rw-r--r-- | mediagoblin/globals.py | 24 | ||||
| -rw-r--r-- | mediagoblin/tests/fake_celery_module.py | 15 | ||||
| -rw-r--r-- | mediagoblin/tests/test_celery_setup.py | 85 | ||||
| -rw-r--r-- | mediagoblin/tests/test_globals.py | 29 |
10 files changed, 430 insertions, 28 deletions
diff --git a/buildout.cfg b/buildout.cfg index 2b36fb7c..520d5907 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -5,6 +5,7 @@ parts = mediagoblin make_user_dev_dirs [mediagoblin] recipe=zc.recipe.egg interpreter=python +dependent-scripts = true eggs=mediagoblin entry-points = nosetests=nose:run_exit diff --git a/docs/designdecisions.rst b/docs/designdecisions.rst index 3398c24b..8fe4d1f0 100644 --- a/docs/designdecisions.rst +++ b/docs/designdecisions.rst @@ -47,11 +47,10 @@ Why WSGI Minimalism Chris Webber on "Why WSGI Minimalism": - If you notice in the technology listI list a lot of - components that are very `Django Project`_, but not actually - Django components. What can I say, I really like a lot of the - ideas in Django! Which leads to the question: why not just use - Django? + If you notice in the technology list I list a lot of components + that are very "django-like", but not actually `Django`_ + components. What can I say, I really like a lot of the ideas in + Django! Which leads to the question: why not just use Django? While I really like Django's ideas and a lot of its components, I also feel that most of the best ideas in Django I want have been @@ -85,7 +84,7 @@ Chris Webber on "Why WSGI Minimalism": deployment-howto, especially in the former making some notes on how to make it easier for Django hackers to get started. -.. _Django Project: http://www.djangoproject.com/ +.. _Django: http://www.djangoproject.com/ .. _Pylons: http://pylonshq.com/ .. _Pyramid: http://docs.pylonsproject.org/projects/pyramid/dev/ .. _Flask: http://flask.pocoo.org/ @@ -244,17 +243,24 @@ everyone is the hero by Will on "Why AGPLv3 and CC0": .. _CC0 v1: http://creativecommons.org/publicdomain/zero/1.0/ -Why copyright assignment? -========================= +Why (non-mandatory) copyright assignment? +========================================= -Will Kahn-Greene on "Why copyright assignment?": +Chris Webber on "Why copyright assignment?": - GNU MediaGoblin is a GNU project with the copyrights held by the - FSF. Like other GNU projects, we require copyright assignment to - the FSF which gives the FSF the legal ability to defend the - AGPL-covered status of the software and distribute it. + GNU MediaGoblin is a GNU project with non-mandatory but heavily + encouraged copyright assignment to the FSF. Most, if not all, of + the core contributors to GNU MediaGoblin will have done a + copyright assignment, but unlike some other GNU projects, it isn't + required here. We think this is the best choice for GNU + MediaGoblin: it ensures that the Free Software Foundation may + protect the software by enforcing the AGPL if the FSF sees fit, + but it also means that we can immediately merge in changes from a + new contributor. It also means that some significant non-FSF + contributors might also be able to enforce the AGPL if seen fit. - This is important to us because it guarantees that this software - we're working so hard on will be available to everyone and will - survive us. As long as someone is interested in using it and/or - working on it, it will live on. + Again, assignment is not mandatory, but it is heavily encouraged, + even incentivized: significant contributors who do a copyright + assignment to the FSF are eligible to have a unique goblin drawing + produced for them by the project's main founder, Christopher Allan + Webber. See :ref:`contributinghowto` for details. diff --git a/mediagoblin/app.py b/mediagoblin/app.py index ae6db8f7..59b943dd 100644 --- a/mediagoblin/app.py +++ b/mediagoblin/app.py @@ -21,6 +21,8 @@ import mongokit from webob import Request, exc from mediagoblin import routing, util, models, storage, staticdirect +from mediagoblin.globals import setup_globals +from mediagoblin.celery_setup import setup_celery_from_config class Error(Exception): pass @@ -53,6 +55,15 @@ class MediaGoblinApp(object): # set up staticdirector tool self.staticdirector = staticdirector + # certain properties need to be accessed globally eg from + # validators, etc, which might not access to the request + # object. + setup_globals( + db_connection=connection, + database=self.db, + public_store=self.public_store, + queue_store=self.queue_store) + def __call__(self, environ, start_response): request = Request(environ) path_info = request.path_info @@ -71,7 +82,7 @@ class MediaGoblinApp(object): if request.GET: new_path_info = '%s?%s' % ( new_path_info, urllib.urlencode(request.GET)) - redirect = exc.HTTPTemporaryRedirect(location=new_path_info) + redirect = exc.HTTPFound(location=new_path_info) return request.get_response(redirect)(environ, start_response) # Okay, no matches. 404 time! @@ -97,33 +108,37 @@ class MediaGoblinApp(object): return controller(request)(environ, start_response) -def paste_app_factory(global_config, **kw): +def paste_app_factory(global_config, **app_config): # Get the database connection connection = mongokit.Connection( - kw.get('db_host'), kw.get('db_port')) + app_config.get('db_host'), app_config.get('db_port')) # Set up the storage systems. public_store = storage.storage_system_from_paste_config( - kw, 'publicstore') + app_config, 'publicstore') queue_store = storage.storage_system_from_paste_config( - kw, 'queuestore') + app_config, 'queuestore') # Set up the staticdirect system - if kw.has_key('direct_remote_path'): + if app_config.has_key('direct_remote_path'): staticdirector = staticdirect.RemoteStaticDirect( - kw['direct_remote_path'].strip()) - elif kw.has_key('direct_remote_paths'): + app_config['direct_remote_path'].strip()) + elif app_config.has_key('direct_remote_paths'): + direct_remote_path_lines = app_config[ + 'direct_remote_paths'].strip().splitlines() staticdirector = staticdirect.MultiRemoteStaticDirect( dict([line.strip().split(' ', 1) - for line in kw['direct_remote_paths'].strip().splitlines()])) + for line in direct_remote_path_lines])) else: raise ImproperlyConfigured( "One of direct_remote_path or direct_remote_paths must be provided") + setup_celery_from_config(app_config, global_config) + mgoblin_app = MediaGoblinApp( - connection, kw.get('db_name', 'mediagoblin'), + connection, app_config.get('db_name', 'mediagoblin'), public_store=public_store, queue_store=queue_store, staticdirector=staticdirector, - user_template_path=kw.get('local_templates')) + user_template_path=app_config.get('local_templates')) return mgoblin_app diff --git a/mediagoblin/celery_setup/__init__.py b/mediagoblin/celery_setup/__init__.py new file mode 100644 index 00000000..3a7f2a5d --- /dev/null +++ b/mediagoblin/celery_setup/__init__.py @@ -0,0 +1,140 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import sys + +from paste.deploy.converters import asbool, asint, aslist + + +KNOWN_CONFIG_BOOLS = [ + 'CELERY_RESULT_PERSISTENT', + 'CELERY_CREATE_MISSING_QUEUES', + 'BROKER_USE_SSL', 'BROKER_CONNECTION_RETRY', + 'CELERY_ALWAYS_EAGER', 'CELERY_EAGER_PROPAGATES_EXCEPTIONS', + 'CELERY_IGNORE_RESULT', 'CELERY_TRACK_STARTED', + 'CELERY_DISABLE_RATE_LIMITS', 'CELERY_ACKS_LATE', + 'CELERY_STORE_ERRORS_EVEN_IF_IGNORED', + 'CELERY_SEND_TASK_ERROR_EMAILS', + 'CELERY_SEND_EVENTS', 'CELERY_SEND_TASK_SENT_EVENT', + 'CELERYD_LOG_COLOR', 'CELERY_REDIRECT_STDOUTS', + ] + +KNOWN_CONFIG_INTS = [ + 'CELERYD_CONCURRENCY', + 'CELERYD_PREFETCH_MULTIPLIER', + 'CELERY_AMQP_TASK_RESULT_EXPIRES', + 'CELERY_AMQP_TASK_RESULT_CONNECTION_MAX', + 'REDIS_PORT', 'REDIS_DB', + 'BROKER_PORT', 'BROKER_CONNECTION_TIMEOUT', + 'CELERY_BROKER_CONNECTION_MAX_RETRIES', + 'CELERY_TASK_RESULT_EXPIRES', 'CELERY_MAX_CACHED_RESULTS', + 'CELERY_DEFAULT_RATE_LIMIT', # ?? + 'CELERYD_MAX_TASKS_PER_CHILD', 'CELERYD_TASK_TIME_LIMIT', + 'CELERYD_TASK_SOFT_TIME_LIMIT', + 'MAIL_PORT', 'CELERYBEAT_MAX_LOOP_INTERVAL', + ] + +KNOWN_CONFIG_FLOATS = [ + 'CELERYD_ETA_SCHEDULER_PRECISION', + ] + +KNOWN_CONFIG_LISTS = [ + 'CELERY_ROUTES', 'CELERY_IMPORTS', + ] + + +## Needs special processing: +# ADMINS, ??? +# there are a lot more; we should list here or process specially. + + +def asfloat(obj): + try: + return float(obj) + except (TypeError, ValueError), e: + raise ValueError( + "Bad float value: %r" % obj) + + +DEFAULT_SETTINGS_MODULE = 'mediagoblin.celery_setup.dummy_settings_module' + +def setup_celery_from_config(app_config, global_config, + settings_module=DEFAULT_SETTINGS_MODULE, + set_environ=True): + """ + Take a mediagoblin app config and the global config from a paste + factory and try to set up a celery settings module from this. + + Args: + - app_config: the application config section + - global_config: the entire paste config, all sections + - settings_module: the module to populate, as a string + - set_environ: if set, this will CELERY_CONFIG_MODULE to the + settings_module + """ + if asbool(app_config.get('use_celery_environment_var')) == True: + # Don't setup celery based on our config file. + return + + celery_conf_section = app_config.get('celery_section', 'celery') + if global_config.has_key(celery_conf_section): + celery_conf = global_config[celery_conf_section] + else: + celery_conf = {} + + celery_settings = {} + + # set up mongodb stuff + celery_settings['CELERY_RESULT_BACKEND'] = 'mongodb' + if not celery_settings.has_key('BROKER_BACKEND'): + celery_settings['BROKER_BACKEND'] = 'mongodb' + + celery_mongo_settings = {} + + if app_config.has_key('db_host'): + celery_mongo_settings['host'] = app_config['db_host'] + if celery_settings['BROKER_BACKEND'] == 'mongodb': + celery_settings['BROKER_HOST'] = app_config['db_host'] + if app_config.has_key('db_port'): + celery_mongo_settings['port'] = asint(app_config['db_port']) + if celery_settings['BROKER_BACKEND'] == 'mongodb': + celery_settings['BROKER_PORT'] = asint(app_config['db_port']) + celery_mongo_settings['database'] = app_config.get('db_name', 'mediagoblin') + + celery_settings['CELERY_MONGODB_BACKEND_SETTINGS'] = celery_mongo_settings + + # Add anything else + for key, value in celery_conf.iteritems(): + key = key.upper() + if key in KNOWN_CONFIG_BOOLS: + value = asbool(value) + elif key in KNOWN_CONFIG_INTS: + value = asint(value) + elif key in KNOWN_CONFIG_FLOATS: + value = asfloat(value) + elif key in KNOWN_CONFIG_LISTS: + value = aslist(value) + celery_settings[key] = value + + __import__(settings_module) + this_module = sys.modules[settings_module] + + for key, value in celery_settings.iteritems(): + setattr(this_module, key, value) + + if set_environ: + os.environ['CELERY_CONFIG_MODULE'] = settings_module diff --git a/mediagoblin/celery_setup/dummy_settings_module.py b/mediagoblin/celery_setup/dummy_settings_module.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/mediagoblin/celery_setup/dummy_settings_module.py diff --git a/mediagoblin/celery_setup/from_celery.py b/mediagoblin/celery_setup/from_celery.py new file mode 100644 index 00000000..851cbaa1 --- /dev/null +++ b/mediagoblin/celery_setup/from_celery.py @@ -0,0 +1,87 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os + +import mongokit +from paste.deploy.loadwsgi import NicerConfigParser + +from mediagoblin import storage +from mediagoblin.celery_setup import setup_celery_from_config +from mediagoblin.globals import setup_globals + + +OUR_MODULENAME = 'mediagoblin.celery_setup.from_celery' + + +def setup_self(setup_globals_func=setup_globals): + """ + Transform this module into a celery config module by reading the + mediagoblin config file. Set the environment variable + MEDIAGOBLIN_CONFIG to specify where this config file is at and + what section it uses. + + By default it defaults to 'mediagoblin.ini:app:mediagoblin'. + + The first colon ":" is a delimiter between the filename and the + config section, so in this case the filename is 'mediagoblin.ini' + and the section where mediagoblin is defined is 'app:mediagoblin'. + + Args: + - 'setup_globals_func': this is for testing purposes only. Don't + set this! + """ + mgoblin_conf_file, mgoblin_section = os.environ.get( + 'MEDIAGOBLIN_CONFIG', 'mediagoblin.ini:app:mediagoblin').split(':', 1) + if not os.path.exists(mgoblin_conf_file): + raise IOError( + "MEDIAGOBLIN_CONFIG not set or file does not exist") + + parser = NicerConfigParser(mgoblin_conf_file) + parser.read(mgoblin_conf_file) + parser._defaults.setdefault( + 'here', os.path.dirname(os.path.abspath(mgoblin_conf_file))) + parser._defaults.setdefault( + '__file__', os.path.abspath(mgoblin_conf_file)) + + mgoblin_section = dict(parser.items(mgoblin_section)) + mgoblin_conf = dict( + [(section_name, dict(parser.items(section_name))) + for section_name in parser.sections()]) + setup_celery_from_config( + mgoblin_section, mgoblin_conf, + settings_module=OUR_MODULENAME, + set_environ=False) + + connection = mongokit.Connection( + mgoblin_section.get('db_host'), mgoblin_section.get('db_port')) + db = connection[mgoblin_section.get('db_name', 'mediagoblin')] + + # Set up the storage systems. + public_store = storage.storage_system_from_paste_config( + mgoblin_section, 'publicstore') + queue_store = storage.storage_system_from_paste_config( + mgoblin_section, 'queuestore') + + setup_globals_func( + db_connection=connection, + database=db, + public_store=public_store, + queue_store=queue_store) + + +if os.environ['CELERY_CONFIG_MODULE'] == OUR_MODULENAME: + setup_self() diff --git a/mediagoblin/globals.py b/mediagoblin/globals.py new file mode 100644 index 00000000..59a94558 --- /dev/null +++ b/mediagoblin/globals.py @@ -0,0 +1,24 @@ +""" +In some places, we need to access the database, public_store, queue_store +""" + +############################# +# General mediagoblin globals +############################# + +# mongokit.Connection +db_connection = None + +# mongokit.Connection +database = None + +# should be the same as the +public_store = None +queue_store = None + + +def setup_globals(**kwargs): + from mediagoblin import globals as mg_globals + + for key, value in kwargs.iteritems(): + setattr(mg_globals, key, value) diff --git a/mediagoblin/tests/fake_celery_module.py b/mediagoblin/tests/fake_celery_module.py new file mode 100644 index 00000000..c129cbf8 --- /dev/null +++ b/mediagoblin/tests/fake_celery_module.py @@ -0,0 +1,15 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. diff --git a/mediagoblin/tests/test_celery_setup.py b/mediagoblin/tests/test_celery_setup.py new file mode 100644 index 00000000..da18b0ef --- /dev/null +++ b/mediagoblin/tests/test_celery_setup.py @@ -0,0 +1,85 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import pkg_resources + +from mediagoblin import celery_setup + + +def test_setup_celery_from_config(): + def _wipe_testmodule_clean(module): + vars_to_wipe = [ + var for var in dir(module) + if not var.startswith('__') and not var.endswith('__')] + for var in vars_to_wipe: + delattr(module, var) + + celery_setup.setup_celery_from_config( + {}, + {'something': {'or': 'other'}, + 'celery': {'some_variable': 'floop', + 'mail_port': '2000', + 'CELERYD_ETA_SCHEDULER_PRECISION': '1.3', + 'celery_result_persistent': 'true', + 'celery_imports': 'foo.bar.baz this.is.an.import'}}, + 'mediagoblin.tests.fake_celery_module', set_environ=False) + + from mediagoblin.tests import fake_celery_module + assert fake_celery_module.SOME_VARIABLE == 'floop' + assert fake_celery_module.MAIL_PORT == 2000 + assert isinstance(fake_celery_module.MAIL_PORT, int) + assert fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION == 1.3 + assert isinstance(fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION, float) + assert fake_celery_module.CELERY_RESULT_PERSISTENT is True + assert fake_celery_module.CELERY_IMPORTS == [ + 'foo.bar.baz', 'this.is.an.import'] + assert fake_celery_module.CELERY_MONGODB_BACKEND_SETTINGS == { + 'database': 'mediagoblin'} + assert fake_celery_module.CELERY_RESULT_BACKEND == 'mongodb' + assert fake_celery_module.BROKER_BACKEND == 'mongodb' + + _wipe_testmodule_clean(fake_celery_module) + + celery_setup.setup_celery_from_config( + {'db_host': 'mongodb.example.org', + 'db_port': '8080', + 'db_name': 'captain_lollerskates', + 'celery_section': 'vegetable'}, + {'something': {'or': 'other'}, + 'vegetable': {'some_variable': 'poolf', + 'mail_port': '2020', + 'CELERYD_ETA_SCHEDULER_PRECISION': '3.1', + 'celery_result_persistent': 'false', + 'celery_imports': 'baz.bar.foo import.is.a.this'}}, + 'mediagoblin.tests.fake_celery_module', set_environ=False) + + from mediagoblin.tests import fake_celery_module + assert fake_celery_module.SOME_VARIABLE == 'poolf' + assert fake_celery_module.MAIL_PORT == 2020 + assert isinstance(fake_celery_module.MAIL_PORT, int) + assert fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION == 3.1 + assert isinstance(fake_celery_module.CELERYD_ETA_SCHEDULER_PRECISION, float) + assert fake_celery_module.CELERY_RESULT_PERSISTENT is False + assert fake_celery_module.CELERY_IMPORTS == [ + 'baz.bar.foo', 'import.is.a.this'] + assert fake_celery_module.CELERY_MONGODB_BACKEND_SETTINGS == { + 'database': 'captain_lollerskates', + 'host': 'mongodb.example.org', + 'port': 8080} + assert fake_celery_module.CELERY_RESULT_BACKEND == 'mongodb' + assert fake_celery_module.BROKER_BACKEND == 'mongodb' + assert fake_celery_module.BROKER_HOST == 'mongodb.example.org' + assert fake_celery_module.BROKER_PORT == 8080 diff --git a/mediagoblin/tests/test_globals.py b/mediagoblin/tests/test_globals.py new file mode 100644 index 00000000..6d2e01da --- /dev/null +++ b/mediagoblin/tests/test_globals.py @@ -0,0 +1,29 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from mediagoblin import globals as mg_globals + +def test_setup_globals(): + mg_globals.setup_globals( + db_connection='my favorite db_connection!', + database='my favorite database!', + public_store='my favorite public_store!', + queue_store='my favorite queue_store!') + + assert mg_globals.db_connection == 'my favorite db_connection!' + assert mg_globals.database == 'my favorite database!' + assert mg_globals.public_store == 'my favorite public_store!' + assert mg_globals.queue_store == 'my favorite queue_store!' |