diff options
Diffstat (limited to 'mediagoblin/db')
| -rw-r--r-- | mediagoblin/db/__init__.py | 46 | ||||
| -rw-r--r-- | mediagoblin/db/indexes.py | 10 | ||||
| -rw-r--r-- | mediagoblin/db/migrations.py | 61 | ||||
| -rw-r--r-- | mediagoblin/db/models.py | 60 | ||||
| -rw-r--r-- | mediagoblin/db/open.py | 36 | ||||
| -rw-r--r-- | mediagoblin/db/util.py | 198 |
6 files changed, 326 insertions, 85 deletions
diff --git a/mediagoblin/db/__init__.py b/mediagoblin/db/__init__.py index c129cbf8..776025ca 100644 --- a/mediagoblin/db/__init__.py +++ b/mediagoblin/db/__init__.py @@ -13,3 +13,49 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + +""" +Database Abstraction/Wrapper Layer +================================== + + **NOTE from Chris Webber:** I asked Elrond to explain why he put + ASCENDING and DESCENDING in db/util.py when we could just import from + pymongo. Read beow for why, but note that nobody is actually doing + this and there's no proof that we'll ever support more than + MongoDB... it would be a huge amount of work to do so. + + If you really want to prove that possible, jump on IRC and talk to + us about making such a branch. In the meanwhile, it doesn't hurt to + have things as they are... if it ever makes it hard for us to + actually do things, we might revisit or remove this. But for more + information, read below. + +This submodule is for most of the db specific stuff. + +There are two main ideas here: + +1. Open up a small possibility to replace mongo by another + db. This means, that all direct mongo accesses should + happen in the db submodule. While all the rest uses an + API defined by this submodule. + + Currently this API happens to be basicly mongo. + Which means, that the abstraction/wrapper layer is + extremely thin. + +2. Give the rest of the app a simple and easy way to get most of + their db needs. Which often means some simple import + from db.util. + +What does that mean? + +* Never import mongo directly outside of this submodule. + +* Inside this submodule you can do whatever is needed. The + API border is exactly at the submodule layer. Nowhere + else. + +* helper functions can be moved in here. They become part + of the db.* API + +""" diff --git a/mediagoblin/db/indexes.py b/mediagoblin/db/indexes.py index d379a52b..a832e013 100644 --- a/mediagoblin/db/indexes.py +++ b/mediagoblin/db/indexes.py @@ -45,11 +45,13 @@ REQUIRED READING: To remove deprecated indexes ---------------------------- -Removing deprecated indexes is easier, just do: +Removing deprecated indexes is the same, just move the index into the +deprecated indexes mapping. -INACTIVE_INDEXES = { - 'collection_name': [ - 'deprecated_index_identifier1', 'deprecated_index_identifier2']} +DEPRECATED_INDEXES = { + 'collection_name': { + 'deprecated_index_identifier1': { + 'index': [index_foo_goes_here]}} ... etc. diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py index b888ad3e..f398f4b3 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/migrations.py @@ -14,56 +14,25 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +from mediagoblin.db.util import RegisterMigration from mediagoblin.util import cleaned_markdown_conversion -from mongokit import DocumentMigration +# Please see mediagoblin/tests/test_migrations.py for some examples of +# basic migrations. -class MediaEntryMigration(DocumentMigration): - def allmigration01_uploader_to_reference(self): - """ - Old MediaEntry['uploader'] accidentally embedded the User instead - of referencing it. Fix that! - """ - # uploader is an associative array - self.target = {'uploader': {'$type': 3}} - if not self.status: - for doc in self.collection.find(self.target): - self.update = { - '$set': { - 'uploader': doc['uploader']['_id']}} - self.collection.update( - self.target, self.update, multi=True, safe=True) - def allmigration02_add_description_html(self): - """ - Now that we can have rich descriptions via Markdown, we should - update all existing entries to record the rich description versions. - """ - self.target = {'description_html': {'$exists': False}, - 'description': {'$exists': True}} +@RegisterMigration(1) +def user_add_bio_html(database): + """ + Users now have richtext bios via Markdown, reflect appropriately. + """ + collection = database['users'] - if not self.status: - for doc in self.collection.find(self.target): - self.update = { - '$set': { - 'description_html': cleaned_markdown_conversion( - doc['description'])}} + target = collection.find( + {'bio_html': {'$exists': False}}) -class UserMigration(DocumentMigration): - def allmigration01_add_bio_and_url_profile(self): - """ - User can elaborate profile with home page and biography - """ - self.target = {'url': {'$exists': False}, - 'bio': {'$exists': False}} - if not self.status: - for doc in self.collection.find(self.target): - self.update = { - '$set': {'url': '', - 'bio': ''}} - self.collection.update( - self.target, self.update, multi=True, safe=True) - - -MIGRATE_CLASSES = ['MediaEntry', 'User'] + for document in target: + document['bio_html'] = cleaned_markdown_conversion( + document['bio']) + collection.save(document) diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 2d6a71f7..9d7bcf6b 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -16,16 +16,17 @@ import datetime, uuid -from mongokit import Document, Set +from mongokit import Document from mediagoblin import util from mediagoblin.auth import lib as auth_lib from mediagoblin import mg_globals from mediagoblin.db import migrations -from mediagoblin.db.util import DESCENDING, ObjectId +from mediagoblin.db.util import ASCENDING, DESCENDING, ObjectId from mediagoblin.util import Pagination from mediagoblin.util import DISPLAY_IMAGE_FETCHING_ORDER + ################### # Custom validators ################### @@ -49,7 +50,8 @@ class User(Document): 'verification_key': unicode, 'is_admin': bool, 'url' : unicode, - 'bio' : unicode + 'bio' : unicode, # May contain markdown + 'bio_html': unicode, # May contain plaintext, or HTML } required_fields = ['username', 'created', 'pw_hash', 'email'] @@ -61,8 +63,6 @@ class User(Document): 'verification_key': lambda: unicode(uuid.uuid4()), 'is_admin': False} - migration_handler = migrations.UserMigration - def check_login(self, password): """ See if a user can login with this password @@ -108,7 +108,9 @@ class MediaEntry(Document): 'created': datetime.datetime.utcnow, 'state': u'unprocessed'} - migration_handler = migrations.MediaEntryMigration + def get_comments(self): + return self.db.MediaComment.find({ + 'media_entry': self['_id']}).sort('created', DESCENDING) def get_display_media(self, media_map, fetch_order=DISPLAY_IMAGE_FETCHING_ORDER): """ @@ -130,22 +132,7 @@ class MediaEntry(Document): def main_mediafile(self): pass - - def get_comments(self, page): - cursor = self.db.MediaComment.find({ - 'media_entry': self['_id']}).sort('created', DESCENDING) - - pagination = Pagination(page, cursor) - comments = pagination() - - data = list() - for comment in comments: - comment['author'] = self.db.User.find_one({ - '_id': comment['author']}) - data.append(comment) - - return (data, pagination) - + def generate_slug(self): self['slug'] = util.slugify(self['title']) @@ -173,10 +160,38 @@ class MediaEntry(Document): 'mediagoblin.user_pages.media_home', user=uploader['username'], media=unicode(self['_id'])) + + def url_to_prev(self, urlgen): + """ + Provide a url to the previous entry from this user, if there is one + """ + cursor = self.db.MediaEntry.find({'_id' : {"$gt": self['_id']}, + 'uploader': self['uploader'], + 'state': 'processed'}).sort( + '_id', ASCENDING).limit(1) + if cursor.count(): + return urlgen('mediagoblin.user_pages.media_home', + user=self.uploader()['username'], + media=unicode(cursor[0]['slug'])) + + def url_to_next(self, urlgen): + """ + Provide a url to the next entry from this user, if there is one + """ + cursor = self.db.MediaEntry.find({'_id' : {"$lt": self['_id']}, + 'uploader': self['uploader'], + 'state': 'processed'}).sort( + '_id', DESCENDING).limit(1) + + if cursor.count(): + return urlgen('mediagoblin.user_pages.media_home', + user=self.uploader()['username'], + media=unicode(cursor[0]['slug'])) def uploader(self): return self.db.User.find_one({'_id': self['uploader']}) + class MediaComment(Document): __collection__ = 'media_comments' @@ -199,6 +214,7 @@ class MediaComment(Document): def author(self): return self.db.User.find_one({'_id': self['author']}) + REGISTER_MODELS = [ MediaEntry, User, diff --git a/mediagoblin/db/open.py b/mediagoblin/db/open.py index cae33394..e5fde6f9 100644 --- a/mediagoblin/db/open.py +++ b/mediagoblin/db/open.py @@ -14,24 +14,42 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +import pymongo import mongokit from paste.deploy.converters import asint from mediagoblin.db import models -def connect_database_from_config(app_config): - """Connect to the main database, take config from app_config""" +def connect_database_from_config(app_config, use_pymongo=False): + """ + Connect to the main database, take config from app_config + + Optionally use pymongo instead of mongokit for the connection. + """ port = app_config.get('db_port') if port: port = asint(port) - connection = mongokit.Connection( - app_config.get('db_host'), port) + + if use_pymongo: + connection = pymongo.Connection( + app_config.get('db_host'), port) + else: + connection = mongokit.Connection( + app_config.get('db_host'), port) return connection -def setup_connection_and_db_from_config(app_config): - connection = connect_database_from_config(app_config) - database_path = app_config.get('db_name', 'mediagoblin') + +def setup_connection_and_db_from_config(app_config, use_pymongo=False): + """ + Setup connection and database from config. + + Optionally use pymongo instead of mongokit. + """ + connection = connect_database_from_config(app_config, use_pymongo) + database_path = app_config['db_name'] db = connection[database_path] - models.register_models(connection) - # Could configure indexes here on db + + if not use_pymongo: + models.register_models(connection) + return (connection, db) diff --git a/mediagoblin/db/util.py b/mediagoblin/db/util.py index 46f899f7..0f3220d2 100644 --- a/mediagoblin/db/util.py +++ b/mediagoblin/db/util.py @@ -30,13 +30,18 @@ document relevant to here: import copy # Imports that other modules might use -from pymongo import DESCENDING +from pymongo import ASCENDING, DESCENDING from pymongo.errors import InvalidId from mongokit import ObjectId from mediagoblin.db.indexes import ACTIVE_INDEXES, DEPRECATED_INDEXES +################ +# Indexing tools +################ + + def add_new_indexes(database, active_indexes=ACTIVE_INDEXES): """ Add any new indexes to the database. @@ -81,21 +86,206 @@ def remove_deprecated_indexes(database, deprecated_indexes=DEPRECATED_INDEXES): Args: - database: pymongo or mongokit database instance. - deprecated_indexes: the indexes to deprecate in the pattern of: - {'collection': ['index_identifier1', 'index_identifier2']} + {'collection_name': { + 'identifier': { + 'index': [index_foo_goes_here], + 'unique': True}} + + (... although we really only need the 'identifier' here, as the + rest of the information isn't used in this case. But it's kept + around so we can remember what it was) Returns: A list of indexes removed in form ('collection', 'index_name') """ indexes_removed = [] - for collection_name, index_names in deprecated_indexes.iteritems(): + for collection_name, indexes in deprecated_indexes.iteritems(): collection = database[collection_name] collection_indexes = collection.index_information().keys() - for index_name in index_names: + for index_name, index_data in indexes.iteritems(): if index_name in collection_indexes: collection.drop_index(index_name) indexes_removed.append((collection_name, index_name)) return indexes_removed + + +################# +# Migration tools +################# + +# The default migration registry... +# +# Don't set this yourself! RegisterMigration will automatically fill +# this with stuff via decorating methods in migrations.py + +class MissingCurrentMigration(Exception): pass + + +MIGRATIONS = {} + + +class RegisterMigration(object): + """ + Tool for registering migrations + + Call like: + + @RegisterMigration(33) + def update_dwarves(database): + [...] + + This will register your migration with the default migration + registry. Alternately, to specify a very specific + migration_registry, you can pass in that as the second argument. + + Note, the number of your migration should NEVER be 0 or less than + 0. 0 is the default "no migrations" state! + """ + def __init__(self, migration_number, migration_registry=MIGRATIONS): + assert migration_number > 0, "Migration number must be > 0!" + assert not migration_registry.has_key(migration_number), \ + "Duplicate migration numbers detected! That's not allowed!" + + self.migration_number = migration_number + self.migration_registry = migration_registry + + def __call__(self, migration): + self.migration_registry[self.migration_number] = migration + return migration + + +class MigrationManager(object): + """ + Migration handling tool. + + Takes information about a database, lets you update the database + to the latest migrations, etc. + """ + def __init__(self, database, migration_registry=MIGRATIONS): + """ + Args: + - database: database we're going to migrate + - migration_registry: where we should find all migrations to + run + """ + self.database = database + self.migration_registry = migration_registry + self._sorted_migrations = None + + def _ensure_current_migration_record(self): + """ + If there isn't a database[u'app_metadata'] mediagoblin entry + with the 'current_migration', throw an error. + """ + if self.database_current_migration() is None: + raise MissingCurrentMigration( + "Tried to call function which requires " + "'current_migration' set in database") + + @property + def sorted_migrations(self): + """ + Sort migrations if necessary and store in self._sorted_migrations + """ + if not self._sorted_migrations: + self._sorted_migrations = sorted( + self.migration_registry.items(), + # sort on the key... the migration number + key=lambda migration_tuple: migration_tuple[0]) + + return self._sorted_migrations + + def latest_migration(self): + """ + Return a migration number for the latest migration, or 0 if + there are no migrations. + """ + if self.sorted_migrations: + return self.sorted_migrations[-1][0] + else: + # If no migrations have been set, we start at 0. + return 0 + + def set_current_migration(self, migration_number): + """ + Set the migration in the database to migration_number + """ + # Add the mediagoblin migration if necessary + self.database[u'app_metadata'].update( + {u'_id': u'mediagoblin'}, + {u'$set': {u'current_migration': migration_number}}, + upsert=True) + + def install_migration_version_if_missing(self): + """ + Sets the migration to the latest version if no migration + version at all is set. + """ + mgoblin_metadata = self.database[u'app_metadata'].find_one( + {u'_id': u'mediagoblin'}) + if not mgoblin_metadata: + latest_migration = self.latest_migration() + self.set_current_migration(latest_migration) + + def database_current_migration(self): + """ + Return the current migration in the database. + """ + mgoblin_metadata = self.database[u'app_metadata'].find_one( + {u'_id': u'mediagoblin'}) + if not mgoblin_metadata: + return None + else: + return mgoblin_metadata[u'current_migration'] + + def database_at_latest_migration(self): + """ + See if the database is at the latest migration. + Returns a boolean. + """ + current_migration = self.database_current_migration() + return current_migration == self.latest_migration() + + def migrations_to_run(self): + """ + Get a list of migrations to run still, if any. + + Note that calling this will set your migration version to the + latest version if it isn't installed to anything yet! + """ + self._ensure_current_migration_record() + + db_current_migration = self.database_current_migration() + + return [ + (migration_number, migration_func) + for migration_number, migration_func in self.sorted_migrations + if migration_number > db_current_migration] + + def migrate_new(self, pre_callback=None, post_callback=None): + """ + Run all migrations. + + Includes two optional args: + - pre_callback: if called, this is a callback on something to + run pre-migration. Takes (migration_number, migration_func) + as arguments + - pre_callback: if called, this is a callback on something to + run post-migration. Takes (migration_number, migration_func) + as arguments + """ + # If we aren't set to any version number, presume we're at the + # latest (which means we'll do nothing here...) + self.install_migration_version_if_missing() + + for migration_number, migration_func in self.migrations_to_run(): + if pre_callback: + pre_callback(migration_number, migration_func) + migration_func(self.database) + self.set_current_migration(migration_number) + if post_callback: + post_callback(migration_number, migration_func) |