diff options
Diffstat (limited to 'mediagoblin/init')
| -rw-r--r-- | mediagoblin/init/__init__.py | 33 | ||||
| -rw-r--r-- | mediagoblin/init/celery/__init__.py | 89 | ||||
| -rw-r--r-- | mediagoblin/init/celery/dummy_settings_module.py | 0 | ||||
| -rw-r--r-- | mediagoblin/init/celery/from_celery.py | 60 | ||||
| -rw-r--r-- | mediagoblin/init/celery/from_tests.py | 33 | ||||
| -rw-r--r-- | mediagoblin/init/config.py | 122 |
6 files changed, 337 insertions, 0 deletions
diff --git a/mediagoblin/init/__init__.py b/mediagoblin/init/__init__.py new file mode 100644 index 00000000..b8ed2456 --- /dev/null +++ b/mediagoblin/init/__init__.py @@ -0,0 +1,33 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import jinja2 + + +def get_jinja_loader(user_template_path=None): + """ + Set up the Jinja template loaders, possibly allowing for user + overridden templates. + + (In the future we may have another system for providing theming; + for now this is good enough.) + """ + if user_template_path: + return jinja2.ChoiceLoader( + [jinja2.FileSystemLoader(user_template_path), + jinja2.PackageLoader('mediagoblin', 'templates')]) + else: + return jinja2.PackageLoader('mediagoblin', 'templates') diff --git a/mediagoblin/init/celery/__init__.py b/mediagoblin/init/celery/__init__.py new file mode 100644 index 00000000..67c3dfa0 --- /dev/null +++ b/mediagoblin/init/celery/__init__.py @@ -0,0 +1,89 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import sys + + +MANDATORY_CELERY_IMPORTS = ['mediagoblin.process_media'] + +DEFAULT_SETTINGS_MODULE = 'mediagoblin.init.celery.dummy_settings_module' + + +def setup_celery_from_config(app_config, global_config, + settings_module=DEFAULT_SETTINGS_MODULE, + force_celery_always_eager=False, + set_environ=True): + """ + Take a mediagoblin app config and try to set up a celery settings + module from this. + + Args: + - app_config: the application config section + - global_config: the entire ConfigObj loaded config, all sections + - settings_module: the module to populate, as a string + - force_celery_always_eager: whether or not to force celery into + always eager mode; good for development and small installs + - set_environ: if set, this will CELERY_CONFIG_MODULE to the + settings_module + """ + if global_config.has_key('celery'): + celery_conf = global_config['celery'] + else: + celery_conf = {} + + celery_settings = {} + + # set up mongodb stuff + celery_settings['CELERY_RESULT_BACKEND'] = 'mongodb' + if not celery_settings.has_key('BROKER_BACKEND'): + celery_settings['BROKER_BACKEND'] = 'mongodb' + + celery_mongo_settings = {} + + if app_config.has_key('db_host'): + celery_mongo_settings['host'] = app_config['db_host'] + if celery_settings['BROKER_BACKEND'] == 'mongodb': + celery_settings['BROKER_HOST'] = app_config['db_host'] + if app_config.has_key('db_port'): + celery_mongo_settings['port'] = app_config['db_port'] + if celery_settings['BROKER_BACKEND'] == 'mongodb': + celery_settings['BROKER_PORT'] = app_config['db_port'] + celery_mongo_settings['database'] = app_config.get('db_name', 'mediagoblin') + + celery_settings['CELERY_MONGODB_BACKEND_SETTINGS'] = celery_mongo_settings + + # Add anything else + for key, value in celery_conf.iteritems(): + key = key.upper() + celery_settings[key] = value + + # add mandatory celery imports + celery_imports = celery_settings.setdefault('CELERY_IMPORTS', []) + celery_imports.extend(MANDATORY_CELERY_IMPORTS) + + if force_celery_always_eager: + celery_settings['CELERY_ALWAYS_EAGER'] = True + celery_settings['CELERY_EAGER_PROPAGATES_EXCEPTIONS'] = True + + __import__(settings_module) + this_module = sys.modules[settings_module] + + for key, value in celery_settings.iteritems(): + setattr(this_module, key, value) + + if set_environ: + os.environ['CELERY_CONFIG_MODULE'] = settings_module diff --git a/mediagoblin/init/celery/dummy_settings_module.py b/mediagoblin/init/celery/dummy_settings_module.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/mediagoblin/init/celery/dummy_settings_module.py diff --git a/mediagoblin/init/celery/from_celery.py b/mediagoblin/init/celery/from_celery.py new file mode 100644 index 00000000..c053591b --- /dev/null +++ b/mediagoblin/init/celery/from_celery.py @@ -0,0 +1,60 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os + +from mediagoblin import app, mg_globals +from mediagoblin.init.celery import setup_celery_from_config + + +OUR_MODULENAME = __name__ + + +def setup_self(check_environ_for_conf=True, module_name=OUR_MODULENAME, + default_conf_file='mediagoblin.ini'): + """ + Transform this module into a celery config module by reading the + mediagoblin config file. Set the environment variable + MEDIAGOBLIN_CONFIG to specify where this config file is. + + By default it defaults to 'mediagoblin.ini'. + + Note that if celery_setup_elsewhere is set in your config file, + this simply won't work. + """ + if check_environ_for_conf: + mgoblin_conf_file = os.path.abspath( + os.environ.get('MEDIAGOBLIN_CONFIG', default_conf_file)) + else: + mgoblin_conf_file = default_conf_file + + if not os.path.exists(mgoblin_conf_file): + raise IOError( + "MEDIAGOBLIN_CONFIG not set or file does not exist") + + # By setting the environment variable here we should ensure that + # this is the module that gets set up. + os.environ['CELERY_CONFIG_MODULE'] = module_name + app.MediaGoblinApp(mgoblin_conf_file, setup_celery=False) + + setup_celery_from_config( + mg_globals.app_config, mg_globals.global_config, + settings_module=module_name, + set_environ=False) + + +if os.environ['CELERY_CONFIG_MODULE'] == OUR_MODULENAME: + setup_self() diff --git a/mediagoblin/init/celery/from_tests.py b/mediagoblin/init/celery/from_tests.py new file mode 100644 index 00000000..b2293e2c --- /dev/null +++ b/mediagoblin/init/celery/from_tests.py @@ -0,0 +1,33 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os + +from mediagoblin.tests.tools import TEST_APP_CONFIG +from mediagoblin.init.celery.from_celery import setup_self + + +OUR_MODULENAME = __name__ +CELERY_SETUP = False + + +if os.environ.get('CELERY_CONFIG_MODULE') == OUR_MODULENAME: + if CELERY_SETUP: + pass + else: + setup_self(check_environ_for_conf=False, module_name=OUR_MODULENAME, + default_conf_file=TEST_APP_CONFIG) + CELERY_SETUP = True diff --git a/mediagoblin/init/config.py b/mediagoblin/init/config.py new file mode 100644 index 00000000..2f93d32c --- /dev/null +++ b/mediagoblin/init/config.py @@ -0,0 +1,122 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import pkg_resources + +from configobj import ConfigObj, flatten_errors +from validate import Validator + + +CONFIG_SPEC_PATH = pkg_resources.resource_filename( + 'mediagoblin', 'config_spec.ini') + + +def _setup_defaults(config, config_path): + """ + Setup DEFAULTS in a config object from an (absolute) config_path. + """ + config.setdefault('DEFAULT', {}) + config['DEFAULT']['here'] = os.path.dirname(config_path) + config['DEFAULT']['__file__'] = config_path + + +def read_mediagoblin_config(config_path, config_spec=CONFIG_SPEC_PATH): + """ + Read a config object from config_path. + + Does automatic value transformation based on the config_spec. + Also provides %(__file__)s and %(here)s values of this file and + its directory respectively similar to paste deploy. + + This function doesn't itself raise any exceptions if validation + fails, you'll have to do something + + Args: + - config_path: path to the config file + - config_spec: config file that provides defaults and value types + for validation / conversion. Defaults to mediagoblin/config_spec.ini + + Returns: + A tuple like: (config, validation_result) + ... where 'conf' is the parsed config object and 'validation_result' + is the information from the validation process. + """ + config_path = os.path.abspath(config_path) + + config_spec = ConfigObj( + config_spec, + encoding='UTF8', list_values=False, _inspec=True) + + _setup_defaults(config_spec, config_path) + + config = ConfigObj( + config_path, + configspec=config_spec, + interpolation='ConfigParser') + + _setup_defaults(config, config_path) + + # For now the validator just works with the default functions, + # but in the future if we want to add additional validation/configuration + # functions we'd add them to validator.functions here. + # + # See also: + # http://www.voidspace.org.uk/python/validate.html#adding-functions + validator = Validator() + validation_result = config.validate(validator, preserve_errors=True) + + return config, validation_result + + +REPORT_HEADER = u"""\ +There were validation problems loading this config file: +-------------------------------------------------------- +""" + + +def generate_validation_report(config, validation_result): + """ + Generate a report if necessary of problems while validating. + + Returns: + Either a string describing for a user the problems validating + this config or None if there are no problems. + """ + report = [] + + # Organize the report + for entry in flatten_errors(config, validation_result): + # each entry is a tuple + section_list, key, error = entry + + if key is not None: + section_list.append(key) + else: + section_list.append(u'[missing section]') + + section_string = u':'.join(section_list) + + if error == False: + # We don't care about missing values for now. + continue + + report.append(u"%s = %s" % (section_string, error)) + + if report: + return REPORT_HEADER + u"\n".join(report) + else: + return None |