6 files changed, 574 insertions, 0 deletions
diff --git a/mediagoblin/init/__init__.py b/mediagoblin/init/__init__.py
new file mode 100644
index 00000000..444c624f
--- /dev/null
+++ b/mediagoblin/init/__init__.py
@@ -0,0 +1,153 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import jinja2
+
+from mediagoblin.tools import staticdirect
+from mediagoblin.tools.translate import set_available_locales
+from mediagoblin.init.config import (
+    read_mediagoblin_config, generate_validation_report)
+from mediagoblin import mg_globals
+from mediagoblin.mg_globals import setup_globals
+from mediagoblin.db.open import setup_connection_and_db_from_config, \
+    check_db_migrations_current, load_models
+from mediagoblin.tools.pluginapi import hook_runall
+from mediagoblin.tools.workbench import WorkbenchManager
+from mediagoblin.storage import storage_system_from_config
+
+
+class Error(Exception):
+    pass
+
+
+class ImproperlyConfigured(Error):
+    pass
+
+
+def setup_locales():
+    """Checks which language translations are available and sets them"""
+    set_available_locales()
+
+
+def setup_global_and_app_config(config_path):
+    global_config, validation_result = read_mediagoblin_config(config_path)
+    app_config = global_config['mediagoblin']
+    # report errors if necessary
+    validation_report = generate_validation_report(
+        global_config, validation_result)
+    if validation_report:
+        raise ImproperlyConfigured(validation_report)
+
+    setup_globals(
+        app_config=app_config,
+        global_config=global_config)
+
+    return global_config, app_config
+
+
+def setup_database():
+    app_config = mg_globals.app_config
+
+    # Load all models for media types (plugins, ...)
+    load_models(app_config)
+
+    # Set up the database
+    db = setup_connection_and_db_from_config(app_config)
+
+    check_db_migrations_current(db)
+
+    setup_globals(database=db)
+
+    return db
+
+
+def get_jinja_loader(user_template_path=None, current_theme=None,
+                     plugin_template_paths=None):
+    """
+    Set up the Jinja template loaders, possibly allowing for user
+    overridden templates.
+
+    (In the future we may have another system for providing theming;
+    for now this is good enough.)
+    """
+    path_list = []
+
+    # Add user path first--this takes precedence over everything.
+    if user_template_path is not None:
+        path_list.append(jinja2.FileSystemLoader(user_template_path))
+
+    # Any theme directories in the registry
+    if current_theme and current_theme.get('templates_dir'):
+        path_list.append(
+            jinja2.FileSystemLoader(
+                current_theme['templates_dir']))
+
+    # Add plugin template paths next--takes precedence over
+    # core templates.
+    if plugin_template_paths is not None:
+        path_list.extend((jinja2.FileSystemLoader(path)
+                          for path in plugin_template_paths))
+
+    # Add core templates last.
+    path_list.append(jinja2.PackageLoader('mediagoblin', 'templates'))
+
+    return jinja2.ChoiceLoader(path_list)
+
+
+def get_staticdirector(app_config):
+    # At minimum, we need the direct_remote_path
+    if not 'direct_remote_path' in app_config \
+            or not 'theme_web_path' in app_config:
+        raise ImproperlyConfigured(
+            "direct_remote_path and theme_web_path must be provided")
+
+    direct_domains = {None: app_config['direct_remote_path'].strip()}
+    direct_domains['theme'] = app_config['theme_web_path'].strip()
+
+    # Let plugins load additional paths
+    for plugin_static in hook_runall("static_setup"):
+        direct_domains[plugin_static.name] = "%s/%s" % (
+            app_config['plugin_web_path'].rstrip('/'),
+            plugin_static.name)
+
+    return staticdirect.StaticDirect(
+        direct_domains)
+
+
+def setup_storage():
+    global_config = mg_globals.global_config
+
+    key_short = 'publicstore'
+    key_long = "storage:" + key_short
+    public_store = storage_system_from_config(global_config[key_long])
+
+    key_short = 'queuestore'
+    key_long = "storage:" + key_short
+    queue_store = storage_system_from_config(global_config[key_long])
+
+    setup_globals(
+        public_store=public_store,
+        queue_store=queue_store)
+
+    return public_store, queue_store
+
+
+def setup_workbench():
+    app_config = mg_globals.app_config
+
+    workbench_manager = WorkbenchManager(app_config['workbench_path'])
+
+    setup_globals(workbench_manager=workbench_manager)
diff --git a/mediagoblin/init/celery/__init__.py b/mediagoblin/init/celery/__init__.py
new file mode 100644
index 00000000..169cc935
--- /dev/null
+++ b/mediagoblin/init/celery/__init__.py
@@ -0,0 +1,99 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import sys
+
+from celery import Celery
+from mediagoblin.tools.pluginapi import hook_runall
+
+
+MANDATORY_CELERY_IMPORTS = ['mediagoblin.processing.task']
+
+DEFAULT_SETTINGS_MODULE = 'mediagoblin.init.celery.dummy_settings_module'
+
+
+def get_celery_settings_dict(app_config, global_config,
+                             force_celery_always_eager=False):
+    """
+    Get a celery settings dictionary from reading the config
+    """
+    if 'celery' in global_config:
+        celery_conf = global_config['celery']
+    else:
+        celery_conf = {}
+
+    celery_settings = {}
+
+    # Add all celery settings from config
+    for key, value in celery_conf.iteritems():
+        celery_settings[key] = value
+
+    # TODO: use default result stuff here if it exists
+
+    # add mandatory celery imports
+    celery_imports = celery_settings.setdefault('CELERY_IMPORTS', [])
+    celery_imports.extend(MANDATORY_CELERY_IMPORTS)
+
+    if force_celery_always_eager:
+        celery_settings['CELERY_ALWAYS_EAGER'] = True
+        celery_settings['CELERY_EAGER_PROPAGATES_EXCEPTIONS'] = True
+
+    return celery_settings
+
+
+def setup_celery_app(app_config, global_config,
+                     settings_module=DEFAULT_SETTINGS_MODULE,
+                     force_celery_always_eager=False):
+    """
+    Setup celery without using terrible setup-celery-module hacks.
+    """
+    celery_settings = get_celery_settings_dict(
+        app_config, global_config, force_celery_always_eager)
+    celery_app = Celery()
+    celery_app.config_from_object(celery_settings)
+
+    hook_runall('celery_setup', celery_app)
+
+
+def setup_celery_from_config(app_config, global_config,
+                             settings_module=DEFAULT_SETTINGS_MODULE,
+                             force_celery_always_eager=False,
+                             set_environ=True):
+    """
+    Take a mediagoblin app config and try to set up a celery settings
+    module from this.
+
+    Args:
+    - app_config: the application config section
+    - global_config: the entire ConfigObj loaded config, all sections
+    - settings_module: the module to populate, as a string
+    - force_celery_always_eager: whether or not to force celery into
+      always eager mode; good for development and small installs
+    - set_environ: if set, this will CELERY_CONFIG_MODULE to the
+      settings_module
+    """
+    celery_settings = get_celery_settings_dict(
+        app_config, global_config, force_celery_always_eager)
+
+    __import__(settings_module)
+    this_module = sys.modules[settings_module]
+
+    for key, value in celery_settings.iteritems():
+        setattr(this_module, key, value)
+
+    if set_environ:
+        os.environ['CELERY_CONFIG_MODULE'] = settings_module
diff --git a/mediagoblin/init/celery/dummy_settings_module.py b/mediagoblin/init/celery/dummy_settings_module.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/mediagoblin/init/celery/dummy_settings_module.py
diff --git a/mediagoblin/init/celery/from_celery.py b/mediagoblin/init/celery/from_celery.py
new file mode 100644
index 00000000..b395a826
--- /dev/null
+++ b/mediagoblin/init/celery/from_celery.py
@@ -0,0 +1,96 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import logging
+import logging.config
+
+from celery.signals import setup_logging
+
+from mediagoblin import app, mg_globals
+from mediagoblin.init.celery import setup_celery_from_config
+from mediagoblin.tools.pluginapi import hook_runall
+
+
+OUR_MODULENAME = __name__
+
+_log = logging.getLogger(__name__)
+
+
+def setup_logging_from_paste_ini(loglevel, **kw):
+    if os.path.exists(os.path.abspath('paste_local.ini')):
+        logging_conf_file = 'paste_local.ini'
+    else:
+        logging_conf_file = 'paste.ini'
+
+    # allow users to set up explicitly which paste file to check via the
+    # PASTE_CONFIG environment variable
+    logging_conf_file = os.environ.get(
+        'PASTE_CONFIG', logging_conf_file)
+
+    if not os.path.exists(logging_conf_file):
+        raise IOError('{0} does not exist. Logging can not be set up.'.format(
+            logging_conf_file))
+
+    logging.config.fileConfig(logging_conf_file)
+
+    hook_runall('celery_logging_setup')
+
+
+setup_logging.connect(setup_logging_from_paste_ini)
+
+
+def setup_self(check_environ_for_conf=True, module_name=OUR_MODULENAME,
+               default_conf_file=None):
+    """
+    Transform this module into a celery config module by reading the
+    mediagoblin config file.  Set the environment variable
+    MEDIAGOBLIN_CONFIG to specify where this config file is.
+
+    By default it defaults to 'mediagoblin.ini'.
+
+    Note that if celery_setup_elsewhere is set in your config file,
+    this simply won't work.
+    """
+    if not default_conf_file:
+        if os.path.exists(os.path.abspath('mediagoblin_local.ini')):
+            default_conf_file = 'mediagoblin_local.ini'
+        else:
+            default_conf_file = 'mediagoblin.ini'
+
+    if check_environ_for_conf:
+        mgoblin_conf_file = os.path.abspath(
+            os.environ.get('MEDIAGOBLIN_CONFIG', default_conf_file))
+    else:
+        mgoblin_conf_file = default_conf_file
+
+    if not os.path.exists(mgoblin_conf_file):
+        raise IOError(
+            "MEDIAGOBLIN_CONFIG not set or file does not exist")
+
+    # By setting the environment variable here we should ensure that
+    # this is the module that gets set up.
+    os.environ['CELERY_CONFIG_MODULE'] = module_name
+    app.MediaGoblinApp(mgoblin_conf_file, setup_celery=False)
+
+    setup_celery_from_config(
+        mg_globals.app_config, mg_globals.global_config,
+        settings_module=module_name,
+        set_environ=False)
+
+
+if os.environ['CELERY_CONFIG_MODULE'] == OUR_MODULENAME:
+    setup_self()
diff --git a/mediagoblin/init/config.py b/mediagoblin/init/config.py
new file mode 100644
index 00000000..11a91cff
--- /dev/null
+++ b/mediagoblin/init/config.py
@@ -0,0 +1,164 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import os
+import pkg_resources
+
+from configobj import ConfigObj, flatten_errors
+from validate import Validator
+
+
+_log = logging.getLogger(__name__)
+
+
+CONFIG_SPEC_PATH = pkg_resources.resource_filename(
+    'mediagoblin', 'config_spec.ini')
+
+
+def _setup_defaults(config, config_path):
+    """
+    Setup DEFAULTS in a config object from an (absolute) config_path.
+    """
+    config.setdefault('DEFAULT', {})
+    config['DEFAULT']['here'] = os.path.dirname(config_path)
+    config['DEFAULT']['__file__'] = config_path
+
+
+def read_mediagoblin_config(config_path, config_spec=CONFIG_SPEC_PATH):
+    """
+    Read a config object from config_path.
+
+    Does automatic value transformation based on the config_spec.
+    Also provides %(__file__)s and %(here)s values of this file and
+    its directory respectively similar to paste deploy.
+
+    Also reads for [plugins] section, appends all config_spec.ini
+    files from said plugins into the general config_spec specification.
+
+    This function doesn't itself raise any exceptions if validation
+    fails, you'll have to do something
+
+    Args:
+     - config_path: path to the config file
+     - config_spec: config file that provides defaults and value types
+       for validation / conversion.  Defaults to mediagoblin/config_spec.ini
+
+    Returns:
+      A tuple like: (config, validation_result)
+      ... where 'conf' is the parsed config object and 'validation_result'
+      is the information from the validation process.
+    """
+    config_path = os.path.abspath(config_path)
+
+    # PRE-READ of config file.  This allows us to fetch the plugins so
+    # we can add their plugin specs to the general config_spec.
+    config = ConfigObj(
+        config_path,
+        interpolation='ConfigParser')
+
+    plugins = config.get("plugins", {}).keys()
+    plugin_configs = {}
+
+    for plugin in plugins:
+        try:
+            plugin_config_spec_path = pkg_resources.resource_filename(
+                plugin, "config_spec.ini")
+            if not os.path.exists(plugin_config_spec_path):
+                continue
+            
+            plugin_config_spec = ConfigObj(
+                plugin_config_spec_path,
+                encoding='UTF8', list_values=False, _inspec=True)
+            _setup_defaults(plugin_config_spec, config_path)
+
+            if not "plugin_spec" in plugin_config_spec:
+                continue
+
+            plugin_configs[plugin] = plugin_config_spec["plugin_spec"]
+
+        except ImportError:
+            _log.warning(
+                "When setting up config section, could not import '%s'" %
+                plugin)
+    
+    # Now load the main config spec
+    config_spec = ConfigObj(
+        config_spec,
+        encoding='UTF8', list_values=False, _inspec=True)
+
+    # append the plugin specific sections of the config spec
+    config_spec['plugins'] = plugin_configs
+
+    _setup_defaults(config_spec, config_path)
+
+    config = ConfigObj(
+        config_path,
+        configspec=config_spec,
+        interpolation='ConfigParser')
+
+    _setup_defaults(config, config_path)
+
+    # For now the validator just works with the default functions,
+    # but in the future if we want to add additional validation/configuration
+    # functions we'd add them to validator.functions here.
+    #
+    # See also:
+    #   http://www.voidspace.org.uk/python/validate.html#adding-functions
+    validator = Validator()
+    validation_result = config.validate(validator, preserve_errors=True)
+
+    return config, validation_result
+
+
+REPORT_HEADER = u"""\
+There were validation problems loading this config file:
+--------------------------------------------------------
+"""
+
+
+def generate_validation_report(config, validation_result):
+    """
+    Generate a report if necessary of problems while validating.
+
+    Returns:
+      Either a string describing for a user the problems validating
+      this config or None if there are no problems.
+    """
+    report = []
+
+    # Organize the report
+    for entry in flatten_errors(config, validation_result):
+        # each entry is a tuple
+        section_list, key, error = entry
+
+        if key is not None:
+            section_list.append(key)
+        else:
+            section_list.append(u'[missing section]')
+
+        section_string = u':'.join(section_list)
+
+        if error == False:
+            # We don't care about missing values for now.
+            continue
+
+        report.append(u"%s = %s" % (section_string, error))
+
+    if report:
+        return REPORT_HEADER + u"\n".join(report)
+    else:
+        return None
diff --git a/mediagoblin/init/plugins/__init__.py b/mediagoblin/init/plugins/__init__.py
new file mode 100644
index 00000000..0df4f381
--- /dev/null
+++ b/mediagoblin/init/plugins/__init__.py
@@ -0,0 +1,62 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011, 2012 MediaGoblin contributors.  See AUTHORS.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import sys
+
+from mediagoblin import mg_globals
+from mediagoblin.tools import pluginapi
+
+
+_log = logging.getLogger(__name__)
+
+
+def setup_plugins():
+    """This loads, configures and registers plugins
+
+    See plugin documentation for more details.
+    """
+
+    global_config = mg_globals.global_config
+    plugin_section = global_config.get('plugins', {})
+
+    if not plugin_section:
+        _log.info("No plugins to load")
+        return
+
+    pman = pluginapi.PluginManager()
+
+    # Go through and import all the modules that are subsections of
+    # the [plugins] section and read in the hooks.
+    for plugin_module, config in plugin_section.items():
+        # Skip any modules that start with -. This makes it easier for
+        # someone to tweak their configuration so as to not load a
+        # plugin without having to remove swaths of plugin
+        # configuration.
+        if plugin_module.startswith('-'):
+            continue
+
+        _log.info("Importing plugin module: %s" % plugin_module)
+        pman.register_plugin(plugin_module)
+        # If this throws errors, that's ok--it'll halt mediagoblin
+        # startup.
+        __import__(plugin_module)
+        plugin = sys.modules[plugin_module]
+        if hasattr(plugin, 'hooks'):
+            pman.register_hooks(plugin.hooks)
+
+    # Execute anything registered to the setup hook.
+    pluginapi.hook_runall('setup')