aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--mediagoblin/app.py37
-rw-r--r--mediagoblin/celery_setup/__init__.py121
-rw-r--r--mediagoblin/celery_setup/dummy_settings_module.py0
-rw-r--r--mediagoblin/globals.py24
-rw-r--r--mediagoblin/tests/test_globals.py29
5 files changed, 200 insertions, 11 deletions
diff --git a/mediagoblin/app.py b/mediagoblin/app.py
index ae6db8f7..59b943dd 100644
--- a/mediagoblin/app.py
+++ b/mediagoblin/app.py
@@ -21,6 +21,8 @@ import mongokit
from webob import Request, exc
from mediagoblin import routing, util, models, storage, staticdirect
+from mediagoblin.globals import setup_globals
+from mediagoblin.celery_setup import setup_celery_from_config
class Error(Exception): pass
@@ -53,6 +55,15 @@ class MediaGoblinApp(object):
# set up staticdirector tool
self.staticdirector = staticdirector
+ # certain properties need to be accessed globally eg from
+ # validators, etc, which might not access to the request
+ # object.
+ setup_globals(
+ db_connection=connection,
+ database=self.db,
+ public_store=self.public_store,
+ queue_store=self.queue_store)
+
def __call__(self, environ, start_response):
request = Request(environ)
path_info = request.path_info
@@ -71,7 +82,7 @@ class MediaGoblinApp(object):
if request.GET:
new_path_info = '%s?%s' % (
new_path_info, urllib.urlencode(request.GET))
- redirect = exc.HTTPTemporaryRedirect(location=new_path_info)
+ redirect = exc.HTTPFound(location=new_path_info)
return request.get_response(redirect)(environ, start_response)
# Okay, no matches. 404 time!
@@ -97,33 +108,37 @@ class MediaGoblinApp(object):
return controller(request)(environ, start_response)
-def paste_app_factory(global_config, **kw):
+def paste_app_factory(global_config, **app_config):
# Get the database connection
connection = mongokit.Connection(
- kw.get('db_host'), kw.get('db_port'))
+ app_config.get('db_host'), app_config.get('db_port'))
# Set up the storage systems.
public_store = storage.storage_system_from_paste_config(
- kw, 'publicstore')
+ app_config, 'publicstore')
queue_store = storage.storage_system_from_paste_config(
- kw, 'queuestore')
+ app_config, 'queuestore')
# Set up the staticdirect system
- if kw.has_key('direct_remote_path'):
+ if app_config.has_key('direct_remote_path'):
staticdirector = staticdirect.RemoteStaticDirect(
- kw['direct_remote_path'].strip())
- elif kw.has_key('direct_remote_paths'):
+ app_config['direct_remote_path'].strip())
+ elif app_config.has_key('direct_remote_paths'):
+ direct_remote_path_lines = app_config[
+ 'direct_remote_paths'].strip().splitlines()
staticdirector = staticdirect.MultiRemoteStaticDirect(
dict([line.strip().split(' ', 1)
- for line in kw['direct_remote_paths'].strip().splitlines()]))
+ for line in direct_remote_path_lines]))
else:
raise ImproperlyConfigured(
"One of direct_remote_path or direct_remote_paths must be provided")
+ setup_celery_from_config(app_config, global_config)
+
mgoblin_app = MediaGoblinApp(
- connection, kw.get('db_name', 'mediagoblin'),
+ connection, app_config.get('db_name', 'mediagoblin'),
public_store=public_store, queue_store=queue_store,
staticdirector=staticdirector,
- user_template_path=kw.get('local_templates'))
+ user_template_path=app_config.get('local_templates'))
return mgoblin_app
diff --git a/mediagoblin/celery_setup/__init__.py b/mediagoblin/celery_setup/__init__.py
new file mode 100644
index 00000000..171b9a6f
--- /dev/null
+++ b/mediagoblin/celery_setup/__init__.py
@@ -0,0 +1,121 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011 Free Software Foundation, Inc
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+from paste.deploy.converters import asbool, asint, aslist
+
+
+KNOWN_CONFIG_BOOLS = [
+ 'CELERY_RESULT_PERSISTENT',
+ 'CELERY_CREATE_MISSING_QUEUES',
+ 'BROKER_USE_SSL', 'BROKER_CONNECTION_RETRY',
+ 'CELERY_ALWAYS_EAGER', 'CELERY_EAGER_PROPAGATES_EXCEPTIONS',
+ 'CELERY_IGNORE_RESULT', 'CELERY_TRACK_STARTED',
+ 'CELERY_DISABLE_RATE_LIMITS', 'CELERY_ACKS_LATE',
+ 'CELERY_STORE_ERRORS_EVEN_IF_IGNORED',
+ 'CELERY_SEND_TASK_ERROR_EMAILS',
+ 'CELERY_SEND_EVENTS', 'CELERY_SEND_TASK_SENT_EVENT',
+ 'CELERYD_LOG_COLOR', 'CELERY_REDIRECT_STDOUTS',
+ ]
+
+KNOWN_CONFIG_INTS = [
+ 'CELERYD_CONCURRENCY',
+ 'CELERYD_PREFETCH_MULTIPLIER',
+ 'CELERY_AMQP_TASK_RESULT_EXPIRES',
+ 'CELERY_AMQP_TASK_RESULT_CONNECTION_MAX',
+ 'REDIS_PORT', 'REDIS_DB',
+ 'BROKER_PORT', 'BROKER_CONNECTION_TIMEOUT',
+ 'CELERY_BROKER_CONNECTION_MAX_RETRIES',
+ 'CELERY_TASK_RESULT_EXPIRES', 'CELERY_MAX_CACHED_RESULTS',
+ 'CELERY_DEFAULT_RATE_LIMIT', # ??
+ 'CELERYD_MAX_TASKS_PER_CHILD', 'CELERYD_TASK_TIME_LIMIT',
+ 'CELERYD_TASK_SOFT_TIME_LIMIT',
+ 'MAIL_PORT', 'CELERYBEAT_MAX_LOOP_INTERVAL',
+ ]
+
+KNOWN_CONFIG_FLOATS = [
+ 'CELERYD_ETA_SCHEDULER_PRECISION',
+ ]
+
+KNOWN_CONFIG_LISTS = [
+ 'CELERY_ROUTES', 'CELERY_IMPORTS',
+ ]
+
+
+## Needs special processing:
+# ADMINS, ???
+# there are a lot more; we should list here or process specially.
+
+
+def asfloat(obj):
+ try:
+ return float(obj)
+ except (TypeError, ValueError), e:
+ raise ValueError(
+ "Bad float value: %r" % obj)
+
+
+def setup_celery_from_config(app_config, global_config):
+ """
+ Take a mediagoblin app config and the global config from a paste
+ factory and try to set up a celery settings module from this.
+ """
+ if asbool(app_config.get('use_celery_environment_var')) == True:
+ # Don't setup celery based on our config file.
+ return
+
+ celery_conf_section = app_config.get('celery_section', 'celery')
+ if global_config.has_key(celery_conf_section):
+ celery_conf = global_config[celery_conf_section]
+ else:
+ celery_conf = {}
+
+ celery_settings = {}
+
+ # set up mongodb stuff
+ celery_mongo_settings = {}
+ if app_config.has_key('db_host'):
+ celery_mongo_settings['host'] = app_config['db_host']
+ celery_settings['BROKER_HOST'] = app_config['db_host']
+ if app_config.has_key('db_port'):
+ celery_mongo_settings['port'] = asint(app_config['db_port'])
+ celery_settings['BROKER_PORT'] = asint(app_config['db_port'])
+ celery_mongo_settings['database'] = app_config.get('db_name', 'mediagoblin')
+
+ celery_settings['CELERY_MONGODB_BACKEND_SETTINGS'] = celery_mongo_settings
+ celery_settings['CELERY_RESULT_BACKEND'] = 'mongodb'
+
+ # Add anything else
+ for key, value in celery_conf.iteritems():
+ key = key.upper()
+ if key in KNOWN_CONFIG_BOOLS:
+ value = asbool(value)
+ elif value in KNOWN_CONFIG_INTS:
+ value = asint(value)
+ elif value in KNOWN_CONFIG_FLOATS:
+ value = asfloat(value)
+ elif value in KNOWN_CONFIG_LISTS:
+ value = aslist(value)
+ celery_settings[key] = value
+
+ from mediagoblin.celery_setup import dummy_settings_module
+
+ for key, value in celery_settings.iteritems():
+ setattr(dummy_settings_module, key, value)
+
+ os.environ['CELERY_CONFIG_MODULE'] = \
+ 'mediagoblin.celery_setup.dummy_settings_module'
diff --git a/mediagoblin/celery_setup/dummy_settings_module.py b/mediagoblin/celery_setup/dummy_settings_module.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/mediagoblin/celery_setup/dummy_settings_module.py
diff --git a/mediagoblin/globals.py b/mediagoblin/globals.py
new file mode 100644
index 00000000..59a94558
--- /dev/null
+++ b/mediagoblin/globals.py
@@ -0,0 +1,24 @@
+"""
+In some places, we need to access the database, public_store, queue_store
+"""
+
+#############################
+# General mediagoblin globals
+#############################
+
+# mongokit.Connection
+db_connection = None
+
+# mongokit.Connection
+database = None
+
+# should be the same as the
+public_store = None
+queue_store = None
+
+
+def setup_globals(**kwargs):
+ from mediagoblin import globals as mg_globals
+
+ for key, value in kwargs.iteritems():
+ setattr(mg_globals, key, value)
diff --git a/mediagoblin/tests/test_globals.py b/mediagoblin/tests/test_globals.py
new file mode 100644
index 00000000..6d2e01da
--- /dev/null
+++ b/mediagoblin/tests/test_globals.py
@@ -0,0 +1,29 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011 Free Software Foundation, Inc
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from mediagoblin import globals as mg_globals
+
+def test_setup_globals():
+ mg_globals.setup_globals(
+ db_connection='my favorite db_connection!',
+ database='my favorite database!',
+ public_store='my favorite public_store!',
+ queue_store='my favorite queue_store!')
+
+ assert mg_globals.db_connection == 'my favorite db_connection!'
+ assert mg_globals.database == 'my favorite database!'
+ assert mg_globals.public_store == 'my favorite public_store!'
+ assert mg_globals.queue_store == 'my favorite queue_store!'