aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--mediagoblin/celery_setup/__init__.py4
-rw-r--r--mediagoblin/celery_setup/from_celery.py8
-rw-r--r--mediagoblin/celery_setup/from_tests.py6
-rw-r--r--mediagoblin/db/indexes.py118
-rw-r--r--mediagoblin/db/models.py5
-rw-r--r--mediagoblin/db/util.py80
-rw-r--r--mediagoblin/gmg_commands/migrate.py20
-rw-r--r--mediagoblin/tests/test_mgoblin_app.ini7
-rw-r--r--mediagoblin/tests/tools.py15
9 files changed, 235 insertions, 28 deletions
diff --git a/mediagoblin/celery_setup/__init__.py b/mediagoblin/celery_setup/__init__.py
index b6e35e99..e35dbce2 100644
--- a/mediagoblin/celery_setup/__init__.py
+++ b/mediagoblin/celery_setup/__init__.py
@@ -40,10 +40,6 @@ def setup_celery_from_config(app_config, global_config,
- set_environ: if set, this will CELERY_CONFIG_MODULE to the
settings_module
"""
- if app_config.get('celery_setup_elsewhere') == True:
- # Don't setup celery based on our config file.
- return
-
if global_config.has_key('celery'):
celery_conf = global_config['celery']
else:
diff --git a/mediagoblin/celery_setup/from_celery.py b/mediagoblin/celery_setup/from_celery.py
index 046aaa50..ed0a409e 100644
--- a/mediagoblin/celery_setup/from_celery.py
+++ b/mediagoblin/celery_setup/from_celery.py
@@ -23,7 +23,8 @@ from mediagoblin.celery_setup import setup_celery_from_config
OUR_MODULENAME = __name__
-def setup_self(check_environ_for_conf=True, module_name=OUR_MODULENAME):
+def setup_self(check_environ_for_conf=True, module_name=OUR_MODULENAME,
+ default_conf_file='mediagoblin.ini'):
"""
Transform this module into a celery config module by reading the
mediagoblin config file. Set the environment variable
@@ -36,9 +37,9 @@ def setup_self(check_environ_for_conf=True, module_name=OUR_MODULENAME):
"""
if check_environ_for_conf:
mgoblin_conf_file = os.path.abspath(
- os.environ.get('MEDIAGOBLIN_CONFIG', 'mediagoblin.ini'))
+ os.environ.get('MEDIAGOBLIN_CONFIG', default_conf_file))
else:
- mgoblin_conf_file = 'mediagoblin.ini'
+ mgoblin_conf_file = default_conf_file
if not os.path.exists(mgoblin_conf_file):
raise IOError(
@@ -48,6 +49,7 @@ def setup_self(check_environ_for_conf=True, module_name=OUR_MODULENAME):
# this is the module that gets set up.
os.environ['CELERY_CONFIG_MODULE'] = module_name
app.MediaGoblinApp(mgoblin_conf_file, setup_celery=False)
+
setup_celery_from_config(
mg_globals.app_config, mg_globals.global_config,
settings_module=module_name,
diff --git a/mediagoblin/celery_setup/from_tests.py b/mediagoblin/celery_setup/from_tests.py
index 43032f41..0f305df2 100644
--- a/mediagoblin/celery_setup/from_tests.py
+++ b/mediagoblin/celery_setup/from_tests.py
@@ -16,11 +16,15 @@
import os
+from mediagoblin.tests.tools import TEST_APP_CONFIG
from mediagoblin.celery_setup.from_celery import setup_self
OUR_MODULENAME = __name__
+CELERY_SETUP = False
if os.environ.get('CELERY_CONFIG_MODULE') == OUR_MODULENAME:
- setup_self(check_environ_for_conf=False, module_name=OUR_MODULENAME)
+ setup_self(check_environ_for_conf=False, module_name=OUR_MODULENAME,
+ default_conf_file=TEST_APP_CONFIG)
+ CELERY_SETUP = True
diff --git a/mediagoblin/db/indexes.py b/mediagoblin/db/indexes.py
new file mode 100644
index 00000000..bbcceb6d
--- /dev/null
+++ b/mediagoblin/db/indexes.py
@@ -0,0 +1,118 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011 Free Software Foundation, Inc
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""
+Indexes for the local database.
+
+To add new indexes
+------------------
+
+Indexes are recorded in the following format:
+
+ACTIVE_INDEXES = {
+ 'collection_name': {
+ 'identifier': { # key identifier used for possibly deprecating later
+ 'index': [index_foo_goes_here]}}
+
+... and anything else being parameters to the create_index function
+(including unique=True, etc)
+
+Current indexes must be registered in ACTIVE_INDEXES... deprecated
+indexes should be marked in DEPRECATED_INDEXES.
+
+Remember, ordering of compound indexes MATTERS. Read below for more.
+
+REQUIRED READING:
+ - http://kylebanker.com/blog/2010/09/21/the-joy-of-mongodb-indexes/
+
+ - http://www.mongodb.org/display/DOCS/Indexes
+ - http://www.mongodb.org/display/DOCS/Indexing+Advice+and+FAQ
+
+
+To remove deprecated indexes
+----------------------------
+
+Removing deprecated indexes is easier, just do:
+
+INACTIVE_INDEXES = {
+ 'collection_name': [
+ 'deprecated_index_identifier1', 'deprecated_index_identifier2']}
+
+... etc.
+
+If an index has been deprecated that identifier should NEVER BE USED
+AGAIN. Eg, if you previously had 'awesomepants_unique', you shouldn't
+use 'awesomepants_unique' again, you should create a totally new name
+or at worst use 'awesomepants_unique2'.
+"""
+
+from pymongo import ASCENDING, DESCENDING
+
+
+################
+# Active indexes
+################
+ACTIVE_INDEXES = {}
+
+# MediaEntry indexes
+# ------------------
+
+MEDIAENTRY_INDEXES = {
+ 'uploader_slug_unique': {
+ # Matching an object to an uploader + slug.
+ # MediaEntries are unique on these two combined, eg:
+ # /u/${myuser}/m/${myslugname}/
+ 'index': [('uploader', ASCENDING),
+ ('slug', ASCENDING)],
+ 'unique': True},
+
+ 'created': {
+ # A global index for all media entries created, in descending
+ # order. This is used for the site's frontpage.
+ 'index': [('created', DESCENDING)]},
+
+ 'uploader_created': {
+ # Indexing on uploaders and when media entries are created.
+ # Used for showing a user gallery, etc.
+ 'index': [('uploader', ASCENDING),
+ ('created', DESCENDING)]}}
+
+
+ACTIVE_INDEXES['media_entries'] = MEDIAENTRY_INDEXES
+
+
+# User indexes
+# ------------
+
+USER_INDEXES = {
+ 'username_unique': {
+ # Index usernames, and make sure they're unique.
+ # ... I guess we might need to adjust this once we're federated :)
+ 'index': 'username',
+ 'unique': True},
+ 'created': {
+ # All most recently created users
+ 'index': 'created'}}
+
+
+ACTIVE_INDEXES['users'] = USER_INDEXES
+
+
+####################
+# Deprecated indexes
+####################
+
+DEPRECATED_INDEXES = {}
diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py
index 600b79ff..8d06ae49 100644
--- a/mediagoblin/db/models.py
+++ b/mediagoblin/db/models.py
@@ -108,11 +108,6 @@ class MediaEntry(Document):
migration_handler = migrations.MediaEntryMigration
- indexes = [
- # Referene uniqueness of slugs by uploader
- {'fields': ['uploader', 'slug'],
- 'unique': True}]
-
def main_mediafile(self):
pass
diff --git a/mediagoblin/db/util.py b/mediagoblin/db/util.py
index 470da531..46f899f7 100644
--- a/mediagoblin/db/util.py
+++ b/mediagoblin/db/util.py
@@ -14,8 +14,88 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
+"""
+Utilities for database operations.
+
+Some note on migration and indexing tools:
+
+We store information about what the state of the database is in the
+'mediagoblin' document of the 'app_metadata' collection. Keys in that
+document relevant to here:
+
+ - 'migration_number': The integer representing the current state of
+ the migrations
+"""
+
+import copy
# Imports that other modules might use
from pymongo import DESCENDING
from pymongo.errors import InvalidId
from mongokit import ObjectId
+
+from mediagoblin.db.indexes import ACTIVE_INDEXES, DEPRECATED_INDEXES
+
+
+def add_new_indexes(database, active_indexes=ACTIVE_INDEXES):
+ """
+ Add any new indexes to the database.
+
+ Args:
+ - database: pymongo or mongokit database instance.
+ - active_indexes: indexes to possibly add in the pattern of:
+ {'collection_name': {
+ 'identifier': {
+ 'index': [index_foo_goes_here],
+ 'unique': True}}
+ where 'index' is the index to add and all other options are
+ arguments for collection.create_index.
+
+ Returns:
+ A list of indexes added in form ('collection', 'index_name')
+ """
+ indexes_added = []
+
+ for collection_name, indexes in active_indexes.iteritems():
+ collection = database[collection_name]
+ collection_indexes = collection.index_information().keys()
+
+ for index_name, index_data in indexes.iteritems():
+ if not index_name in collection_indexes:
+ # Get a copy actually so we don't modify the actual
+ # structure
+ index_data = copy.copy(index_data)
+ index = index_data.pop('index')
+ collection.create_index(
+ index, name=index_name, **index_data)
+
+ indexes_added.append((collection_name, index_name))
+
+ return indexes_added
+
+
+def remove_deprecated_indexes(database, deprecated_indexes=DEPRECATED_INDEXES):
+ """
+ Remove any deprecated indexes from the database.
+
+ Args:
+ - database: pymongo or mongokit database instance.
+ - deprecated_indexes: the indexes to deprecate in the pattern of:
+ {'collection': ['index_identifier1', 'index_identifier2']}
+
+ Returns:
+ A list of indexes removed in form ('collection', 'index_name')
+ """
+ indexes_removed = []
+
+ for collection_name, index_names in deprecated_indexes.iteritems():
+ collection = database[collection_name]
+ collection_indexes = collection.index_information().keys()
+
+ for index_name in index_names:
+ if index_name in collection_indexes:
+ collection.drop_index(index_name)
+
+ indexes_removed.append((collection_name, index_name))
+
+ return indexes_removed
diff --git a/mediagoblin/gmg_commands/migrate.py b/mediagoblin/gmg_commands/migrate.py
index 9e01d51c..ab1a267b 100644
--- a/mediagoblin/gmg_commands/migrate.py
+++ b/mediagoblin/gmg_commands/migrate.py
@@ -16,6 +16,7 @@
from mediagoblin.db import migrations
+from mediagoblin.db import util as db_util
from mediagoblin.gmg_commands import util as commands_util
@@ -27,8 +28,17 @@ def migrate_parser_setup(subparser):
def migrate(args):
mgoblin_app = commands_util.setup_app(args)
- print "Applying migrations..."
+ # Clear old indexes
+ print "== Clearing old indexes... =="
+ removed_indexes = db_util.remove_deprecated_indexes(mgoblin_app.db)
+
+ for collection, index_name in removed_indexes:
+ print "Removed index '%s' in collection '%s'" % (
+ index_name, collection)
+
+ # Migrate
+ print "== Applying migrations... =="
for model_name in migrations.MIGRATE_CLASSES:
model = getattr(mgoblin_app.db, model_name)
@@ -38,4 +48,10 @@ def migrate(args):
migration = model.migration_handler(model)
migration.migrate_all(collection=model.collection)
- print "... done."
+ # Add new indexes
+ print "== Adding new indexes... =="
+ new_indexes = db_util.add_new_indexes(mgoblin_app.db)
+
+ for collection, index_name in new_indexes:
+ print "Added index '%s' to collection '%s'" % (
+ index_name, collection)
diff --git a/mediagoblin/tests/test_mgoblin_app.ini b/mediagoblin/tests/test_mgoblin_app.ini
index 94eafb5a..e022d47b 100644
--- a/mediagoblin/tests/test_mgoblin_app.ini
+++ b/mediagoblin/tests/test_mgoblin_app.ini
@@ -7,6 +7,9 @@ email_sender_address = "notice@mediagoblin.example.org"
email_debug_mode = true
db_name = __mediagoblin_tests__
-# Celery shouldn't be set up by the paste app factory as it's set up
-# elsewhere
+# Celery shouldn't be set up by the application as it's setup via
+# mediagoblin.celery_setup.from_celery
celery_setup_elsewhere = true
+
+[celery]
+celery_always_eager = true
diff --git a/mediagoblin/tests/tools.py b/mediagoblin/tests/tools.py
index ebb5f1b5..64f773f0 100644
--- a/mediagoblin/tests/tools.py
+++ b/mediagoblin/tests/tools.py
@@ -21,9 +21,8 @@ import os, shutil
from paste.deploy import loadapp
from webtest import TestApp
-from mediagoblin import util, mg_globals
+from mediagoblin import util
from mediagoblin.config import read_mediagoblin_config
-from mediagoblin.celery_setup import setup_celery_from_config
from mediagoblin.decorators import _make_safe
from mediagoblin.db.open import setup_connection_and_db_from_config
@@ -36,7 +35,6 @@ TEST_APP_CONFIG = pkg_resources.resource_filename(
TEST_USER_DEV = pkg_resources.resource_filename(
'mediagoblin.tests', 'test_user_dev')
MGOBLIN_APP = None
-CELERY_SETUP = False
USER_DEV_DIRECTORIES_TO_SETUP = [
'media/public', 'media/queue',
@@ -60,8 +58,10 @@ def suicide_if_bad_celery_environ():
def get_test_app(dump_old_app=True):
suicide_if_bad_celery_environ()
+ # Leave this imported as it sets up celery.
+ from mediagoblin.celery_setup import from_tests
+
global MGOBLIN_APP
- global CELERY_SETUP
# Just return the old app if that exists and it's okay to set up
# and return
@@ -103,13 +103,6 @@ def get_test_app(dump_old_app=True):
app = TestApp(test_app)
MGOBLIN_APP = app
- # setup celery
- if not CELERY_SETUP:
- setup_celery_from_config(
- mg_globals.app_config, mg_globals.global_config,
- set_environ=True)
- CELERY_SETUP = True
-
return app