diff options
Diffstat (limited to 'mediagoblin/init')
| -rw-r--r-- | mediagoblin/init/__init__.py | 29 | ||||
| -rw-r--r-- | mediagoblin/init/celery/__init__.py | 23 | ||||
| -rw-r--r-- | mediagoblin/init/config.py | 47 |
3 files changed, 72 insertions, 27 deletions
diff --git a/mediagoblin/init/__init__.py b/mediagoblin/init/__init__.py index e0711416..05a26792 100644 --- a/mediagoblin/init/__init__.py +++ b/mediagoblin/init/__init__.py @@ -28,6 +28,8 @@ from mediagoblin.tools.pluginapi import hook_runall from mediagoblin.tools.workbench import WorkbenchManager from mediagoblin.storage import storage_system_from_config +from mediagoblin.tools.transition import DISABLE_GLOBALS + class Error(Exception): pass @@ -58,18 +60,26 @@ def setup_global_and_app_config(config_path): return global_config, app_config -def setup_database(run_migrations=False): - app_config = mg_globals.app_config - global_config = mg_globals.global_config +def setup_database(app): + app_config = app.app_config + global_config = app.global_config + run_migrations = app_config['run_migrations'] # Load all models for media types (plugins, ...) load_models(app_config) # Set up the database - db = setup_connection_and_db_from_config(app_config, run_migrations) + db = setup_connection_and_db_from_config( + app_config, run_migrations, app=app) + # run_migrations is used for tests if run_migrations: - #Run the migrations to initialize/update the database. - from mediagoblin.gmg_commands.dbupdate import run_all_migrations - run_all_migrations(db, app_config, global_config) + # Run the migrations to initialize/update the database. + # We only run the alembic migrations in the case of unit + # tests, in which case we don't need to run the legacy + # migrations. + from mediagoblin.gmg_commands.dbupdate import ( + run_alembic_migrations, run_foundations) + run_alembic_migrations(db, app_config, global_config) + run_foundations(db, global_config) else: check_db_migrations_current(db) @@ -154,4 +164,7 @@ def setup_workbench(): workbench_manager = WorkbenchManager(app_config['workbench_path']) - setup_globals(workbench_manager=workbench_manager) + if not DISABLE_GLOBALS: + setup_globals(workbench_manager=workbench_manager) + + return workbench_manager diff --git a/mediagoblin/init/celery/__init__.py b/mediagoblin/init/celery/__init__.py index 57242bf6..780e0055 100644 --- a/mediagoblin/init/celery/__init__.py +++ b/mediagoblin/init/celery/__init__.py @@ -16,8 +16,11 @@ import os import sys +import datetime import logging +import six + from celery import Celery from mediagoblin.tools.pluginapi import hook_runall @@ -27,7 +30,9 @@ _log = logging.getLogger(__name__) MANDATORY_CELERY_IMPORTS = [ 'mediagoblin.processing.task', - 'mediagoblin.notifications.task'] + 'mediagoblin.notifications.task', + 'mediagoblin.submit.task', +] DEFAULT_SETTINGS_MODULE = 'mediagoblin.init.celery.dummy_settings_module' @@ -45,7 +50,7 @@ def get_celery_settings_dict(app_config, global_config, celery_settings = {} # Add all celery settings from config - for key, value in celery_conf.iteritems(): + for key, value in six.iteritems(celery_conf): celery_settings[key] = value # TODO: use default result stuff here if it exists @@ -58,6 +63,18 @@ def get_celery_settings_dict(app_config, global_config, celery_settings['CELERY_ALWAYS_EAGER'] = True celery_settings['CELERY_EAGER_PROPAGATES_EXCEPTIONS'] = True + # Garbage collection periodic task + frequency = app_config.get('garbage_collection', 60) + if frequency: + frequency = int(frequency) + celery_settings['CELERYBEAT_SCHEDULE'] = { + 'garbage-collection': { + 'task': 'mediagoblin.submit.task.collect_garbage', + 'schedule': datetime.timedelta(minutes=frequency), + } + } + celery_settings['BROKER_HEARTBEAT'] = 1 + return celery_settings @@ -98,7 +115,7 @@ def setup_celery_from_config(app_config, global_config, __import__(settings_module) this_module = sys.modules[settings_module] - for key, value in celery_settings.iteritems(): + for key, value in six.iteritems(celery_settings): setattr(this_module, key, value) if set_environ: diff --git a/mediagoblin/init/config.py b/mediagoblin/init/config.py index 11a91cff..a9189e8d 100644 --- a/mediagoblin/init/config.py +++ b/mediagoblin/init/config.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +import copy import logging import os import pkg_resources @@ -29,16 +30,21 @@ CONFIG_SPEC_PATH = pkg_resources.resource_filename( 'mediagoblin', 'config_spec.ini') -def _setup_defaults(config, config_path): +def _setup_defaults(config, config_path, extra_defaults=None): """ Setup DEFAULTS in a config object from an (absolute) config_path. """ + extra_defaults = extra_defaults or {} + config.setdefault('DEFAULT', {}) config['DEFAULT']['here'] = os.path.dirname(config_path) config['DEFAULT']['__file__'] = config_path + for key, value in extra_defaults.items(): + config['DEFAULT'].setdefault(key, value) + -def read_mediagoblin_config(config_path, config_spec=CONFIG_SPEC_PATH): +def read_mediagoblin_config(config_path, config_spec_path=CONFIG_SPEC_PATH): """ Read a config object from config_path. @@ -54,7 +60,7 @@ def read_mediagoblin_config(config_path, config_spec=CONFIG_SPEC_PATH): Args: - config_path: path to the config file - - config_spec: config file that provides defaults and value types + - config_spec_path: config file that provides defaults and value types for validation / conversion. Defaults to mediagoblin/config_spec.ini Returns: @@ -68,7 +74,20 @@ def read_mediagoblin_config(config_path, config_spec=CONFIG_SPEC_PATH): # we can add their plugin specs to the general config_spec. config = ConfigObj( config_path, - interpolation='ConfigParser') + interpolation="ConfigParser") + + # temporary bootstrap, just setup here and __file__... we'll do this again + _setup_defaults(config, config_path) + + # Now load the main config spec + config_spec = ConfigObj( + config_spec_path, + encoding="UTF8", list_values=False, _inspec=True) + + # Set up extra defaults that will be pushed into the rest of the + # configs. This is a combined extrapolation of defaults based on + mainconfig_defaults = copy.copy(config_spec.get("DEFAULT", {})) + mainconfig_defaults.update(config["DEFAULT"]) plugins = config.get("plugins", {}).keys() plugin_configs = {} @@ -79,11 +98,12 @@ def read_mediagoblin_config(config_path, config_spec=CONFIG_SPEC_PATH): plugin, "config_spec.ini") if not os.path.exists(plugin_config_spec_path): continue - + plugin_config_spec = ConfigObj( plugin_config_spec_path, - encoding='UTF8', list_values=False, _inspec=True) - _setup_defaults(plugin_config_spec, config_path) + encoding="UTF8", list_values=False, _inspec=True) + _setup_defaults( + plugin_config_spec, config_path, mainconfig_defaults) if not "plugin_spec" in plugin_config_spec: continue @@ -94,23 +114,18 @@ def read_mediagoblin_config(config_path, config_spec=CONFIG_SPEC_PATH): _log.warning( "When setting up config section, could not import '%s'" % plugin) - - # Now load the main config spec - config_spec = ConfigObj( - config_spec, - encoding='UTF8', list_values=False, _inspec=True) # append the plugin specific sections of the config spec - config_spec['plugins'] = plugin_configs + config_spec["plugins"] = plugin_configs - _setup_defaults(config_spec, config_path) + _setup_defaults(config_spec, config_path, mainconfig_defaults) config = ConfigObj( config_path, configspec=config_spec, - interpolation='ConfigParser') + interpolation="ConfigParser") - _setup_defaults(config, config_path) + _setup_defaults(config, config_path, mainconfig_defaults) # For now the validator just works with the default functions, # but in the future if we want to add additional validation/configuration |