diff options
Diffstat (limited to 'mediagoblin/db')
| -rw-r--r-- | mediagoblin/db/mongo/__init__.py | 15 | ||||
| -rw-r--r-- | mediagoblin/db/mongo/indexes.py (renamed from mediagoblin/db/indexes.py) | 0 | ||||
| -rw-r--r-- | mediagoblin/db/mongo/migrations.py (renamed from mediagoblin/db/migrations.py) | 2 | ||||
| -rw-r--r-- | mediagoblin/db/mongo/models.py (renamed from mediagoblin/db/models.py) | 24 | ||||
| -rw-r--r-- | mediagoblin/db/mongo/open.py | 78 | ||||
| -rw-r--r-- | mediagoblin/db/mongo/util.py | 292 | ||||
| -rw-r--r-- | mediagoblin/db/open.py | 41 | ||||
| -rw-r--r-- | mediagoblin/db/sql/base.py | 38 | ||||
| -rw-r--r-- | mediagoblin/db/sql/convert.py | 27 | ||||
| -rw-r--r-- | mediagoblin/db/sql/models.py | 24 | ||||
| -rw-r--r-- | mediagoblin/db/sql/open.py | 33 | ||||
| -rw-r--r-- | mediagoblin/db/util.py | 278 |
12 files changed, 509 insertions, 343 deletions
diff --git a/mediagoblin/db/mongo/__init__.py b/mediagoblin/db/mongo/__init__.py new file mode 100644 index 00000000..ba347c69 --- /dev/null +++ b/mediagoblin/db/mongo/__init__.py @@ -0,0 +1,15 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. diff --git a/mediagoblin/db/indexes.py b/mediagoblin/db/mongo/indexes.py index 1dd73f2b..1dd73f2b 100644 --- a/mediagoblin/db/indexes.py +++ b/mediagoblin/db/mongo/indexes.py diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/mongo/migrations.py index cfc01287..cf4e94ae 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/mongo/migrations.py @@ -14,7 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from mediagoblin.db.util import RegisterMigration +from mediagoblin.db.mongo.util import RegisterMigration from mediagoblin.tools.text import cleaned_markdown_conversion diff --git a/mediagoblin/db/models.py b/mediagoblin/db/mongo/models.py index 569c3600..5de59c12 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/mongo/models.py @@ -15,14 +15,13 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. import datetime -import uuid from mongokit import Document from mediagoblin.auth import lib as auth_lib from mediagoblin import mg_globals -from mediagoblin.db import migrations -from mediagoblin.db.util import ASCENDING, DESCENDING, ObjectId +from mediagoblin.db.mongo import migrations +from mediagoblin.db.mongo.util import ASCENDING, DESCENDING, ObjectId from mediagoblin.tools.pagination import Pagination from mediagoblin.tools import url, common @@ -88,7 +87,6 @@ class User(Document): 'created': datetime.datetime.utcnow, 'email_verified': False, 'status': u'needs_email_verification', - 'verification_key': lambda: unicode(uuid.uuid4()), 'is_admin': False} def check_login(self, password): @@ -263,7 +261,7 @@ class MediaEntry(Document): Use a slug if we have one, else use our '_id'. """ - uploader = self.get_uploader() + uploader = self.get_uploader if self.get('slug'): return urlgen( @@ -284,10 +282,8 @@ class MediaEntry(Document): 'uploader': self.uploader, 'state': 'processed'}).sort( '_id', ASCENDING).limit(1) - if cursor.count(): - return urlgen('mediagoblin.user_pages.media_home', - user=self.get_uploader().username, - media=unicode(cursor[0].slug)) + for media in cursor: + return media.url_for_self(urlgen) def url_to_next(self, urlgen): """ @@ -298,11 +294,10 @@ class MediaEntry(Document): 'state': 'processed'}).sort( '_id', DESCENDING).limit(1) - if cursor.count(): - return urlgen('mediagoblin.user_pages.media_home', - user=self.get_uploader().username, - media=unicode(cursor[0].slug)) + for media in cursor: + return media.url_for_self(urlgen) + @property def get_uploader(self): return self.db.User.find_one({'_id': self.uploader}) @@ -346,7 +341,8 @@ class MediaComment(Document): def media_entry(self): return self.db.MediaEntry.find_one({'_id': self['media_entry']}) - def author(self): + @property + def get_author(self): return self.db.User.find_one({'_id': self['author']}) diff --git a/mediagoblin/db/mongo/open.py b/mediagoblin/db/mongo/open.py new file mode 100644 index 00000000..48c909d9 --- /dev/null +++ b/mediagoblin/db/mongo/open.py @@ -0,0 +1,78 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import pymongo +import mongokit +from paste.deploy.converters import asint +from mediagoblin.db.mongo import models +from mediagoblin.db.mongo.util import MigrationManager + + +def connect_database_from_config(app_config, use_pymongo=False): + """ + Connect to the main database, take config from app_config + + Optionally use pymongo instead of mongokit for the connection. + """ + port = app_config.get('db_port') + if port: + port = asint(port) + + if use_pymongo: + connection = pymongo.Connection( + app_config.get('db_host'), port) + else: + connection = mongokit.Connection( + app_config.get('db_host'), port) + return connection + + +def setup_connection_and_db_from_config(app_config, use_pymongo=False): + """ + Setup connection and database from config. + + Optionally use pymongo instead of mongokit. + """ + connection = connect_database_from_config(app_config, use_pymongo) + database_path = app_config['db_name'] + db = connection[database_path] + + if not use_pymongo: + models.register_models(connection) + + return (connection, db) + + +def check_db_migrations_current(db): + # This MUST be imported so as to set up the appropriate migrations! + from mediagoblin.db.mongo import migrations + + # Init the migration number if necessary + migration_manager = MigrationManager(db) + migration_manager.install_migration_version_if_missing() + + # Tiny hack to warn user if our migration is out of date + if not migration_manager.database_at_latest_migration(): + db_migration_num = migration_manager.database_current_migration() + latest_migration_num = migration_manager.latest_migration() + if db_migration_num < latest_migration_num: + print ( + "*WARNING:* Your migrations are out of date, " + "maybe run ./bin/gmg migrate?") + elif db_migration_num > latest_migration_num: + print ( + "*WARNING:* Your migrations are out of date... " + "in fact they appear to be from the future?!") diff --git a/mediagoblin/db/mongo/util.py b/mediagoblin/db/mongo/util.py new file mode 100644 index 00000000..e2065693 --- /dev/null +++ b/mediagoblin/db/mongo/util.py @@ -0,0 +1,292 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +""" +Utilities for database operations. + +Some note on migration and indexing tools: + +We store information about what the state of the database is in the +'mediagoblin' document of the 'app_metadata' collection. Keys in that +document relevant to here: + + - 'migration_number': The integer representing the current state of + the migrations +""" + +import copy + +# Imports that other modules might use +from pymongo import ASCENDING, DESCENDING +from pymongo.errors import InvalidId +from mongokit import ObjectId + +from mediagoblin.db.mongo.indexes import ACTIVE_INDEXES, DEPRECATED_INDEXES + + +################ +# Indexing tools +################ + + +def add_new_indexes(database, active_indexes=ACTIVE_INDEXES): + """ + Add any new indexes to the database. + + Args: + - database: pymongo or mongokit database instance. + - active_indexes: indexes to possibly add in the pattern of: + {'collection_name': { + 'identifier': { + 'index': [index_foo_goes_here], + 'unique': True}} + where 'index' is the index to add and all other options are + arguments for collection.create_index. + + Returns: + A list of indexes added in form ('collection', 'index_name') + """ + indexes_added = [] + + for collection_name, indexes in active_indexes.iteritems(): + collection = database[collection_name] + collection_indexes = collection.index_information().keys() + + for index_name, index_data in indexes.iteritems(): + if not index_name in collection_indexes: + # Get a copy actually so we don't modify the actual + # structure + index_data = copy.copy(index_data) + index = index_data.pop('index') + collection.create_index( + index, name=index_name, **index_data) + + indexes_added.append((collection_name, index_name)) + + return indexes_added + + +def remove_deprecated_indexes(database, deprecated_indexes=DEPRECATED_INDEXES): + """ + Remove any deprecated indexes from the database. + + Args: + - database: pymongo or mongokit database instance. + - deprecated_indexes: the indexes to deprecate in the pattern of: + {'collection_name': { + 'identifier': { + 'index': [index_foo_goes_here], + 'unique': True}} + + (... although we really only need the 'identifier' here, as the + rest of the information isn't used in this case. But it's kept + around so we can remember what it was) + + Returns: + A list of indexes removed in form ('collection', 'index_name') + """ + indexes_removed = [] + + for collection_name, indexes in deprecated_indexes.iteritems(): + collection = database[collection_name] + collection_indexes = collection.index_information().keys() + + for index_name, index_data in indexes.iteritems(): + if index_name in collection_indexes: + collection.drop_index(index_name) + + indexes_removed.append((collection_name, index_name)) + + return indexes_removed + + +################# +# Migration tools +################# + +# The default migration registry... +# +# Don't set this yourself! RegisterMigration will automatically fill +# this with stuff via decorating methods in migrations.py + +class MissingCurrentMigration(Exception): + pass + + +MIGRATIONS = {} + + +class RegisterMigration(object): + """ + Tool for registering migrations + + Call like: + + @RegisterMigration(33) + def update_dwarves(database): + [...] + + This will register your migration with the default migration + registry. Alternately, to specify a very specific + migration_registry, you can pass in that as the second argument. + + Note, the number of your migration should NEVER be 0 or less than + 0. 0 is the default "no migrations" state! + """ + def __init__(self, migration_number, migration_registry=MIGRATIONS): + assert migration_number > 0, "Migration number must be > 0!" + assert migration_number not in migration_registry, \ + "Duplicate migration numbers detected! That's not allowed!" + + self.migration_number = migration_number + self.migration_registry = migration_registry + + def __call__(self, migration): + self.migration_registry[self.migration_number] = migration + return migration + + +class MigrationManager(object): + """ + Migration handling tool. + + Takes information about a database, lets you update the database + to the latest migrations, etc. + """ + def __init__(self, database, migration_registry=MIGRATIONS): + """ + Args: + - database: database we're going to migrate + - migration_registry: where we should find all migrations to + run + """ + self.database = database + self.migration_registry = migration_registry + self._sorted_migrations = None + + def _ensure_current_migration_record(self): + """ + If there isn't a database[u'app_metadata'] mediagoblin entry + with the 'current_migration', throw an error. + """ + if self.database_current_migration() is None: + raise MissingCurrentMigration( + "Tried to call function which requires " + "'current_migration' set in database") + + @property + def sorted_migrations(self): + """ + Sort migrations if necessary and store in self._sorted_migrations + """ + if not self._sorted_migrations: + self._sorted_migrations = sorted( + self.migration_registry.items(), + # sort on the key... the migration number + key=lambda migration_tuple: migration_tuple[0]) + + return self._sorted_migrations + + def latest_migration(self): + """ + Return a migration number for the latest migration, or 0 if + there are no migrations. + """ + if self.sorted_migrations: + return self.sorted_migrations[-1][0] + else: + # If no migrations have been set, we start at 0. + return 0 + + def set_current_migration(self, migration_number): + """ + Set the migration in the database to migration_number + """ + # Add the mediagoblin migration if necessary + self.database[u'app_metadata'].update( + {u'_id': u'mediagoblin'}, + {u'$set': {u'current_migration': migration_number}}, + upsert=True) + + def install_migration_version_if_missing(self): + """ + Sets the migration to the latest version if no migration + version at all is set. + """ + mgoblin_metadata = self.database[u'app_metadata'].find_one( + {u'_id': u'mediagoblin'}) + if not mgoblin_metadata: + latest_migration = self.latest_migration() + self.set_current_migration(latest_migration) + + def database_current_migration(self): + """ + Return the current migration in the database. + """ + mgoblin_metadata = self.database[u'app_metadata'].find_one( + {u'_id': u'mediagoblin'}) + if not mgoblin_metadata: + return None + else: + return mgoblin_metadata[u'current_migration'] + + def database_at_latest_migration(self): + """ + See if the database is at the latest migration. + Returns a boolean. + """ + current_migration = self.database_current_migration() + return current_migration == self.latest_migration() + + def migrations_to_run(self): + """ + Get a list of migrations to run still, if any. + + Note that calling this will set your migration version to the + latest version if it isn't installed to anything yet! + """ + self._ensure_current_migration_record() + + db_current_migration = self.database_current_migration() + + return [ + (migration_number, migration_func) + for migration_number, migration_func in self.sorted_migrations + if migration_number > db_current_migration] + + def migrate_new(self, pre_callback=None, post_callback=None): + """ + Run all migrations. + + Includes two optional args: + - pre_callback: if called, this is a callback on something to + run pre-migration. Takes (migration_number, migration_func) + as arguments + - pre_callback: if called, this is a callback on something to + run post-migration. Takes (migration_number, migration_func) + as arguments + """ + # If we aren't set to any version number, presume we're at the + # latest (which means we'll do nothing here...) + self.install_migration_version_if_missing() + + for migration_number, migration_func in self.migrations_to_run(): + if pre_callback: + pre_callback(migration_number, migration_func) + migration_func(self.database) + self.set_current_migration(migration_number) + if post_callback: + post_callback(migration_number, migration_func) diff --git a/mediagoblin/db/open.py b/mediagoblin/db/open.py index e677ba12..32827fcb 100644 --- a/mediagoblin/db/open.py +++ b/mediagoblin/db/open.py @@ -14,42 +14,5 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import pymongo -import mongokit -from paste.deploy.converters import asint -from mediagoblin.db import models - - -def connect_database_from_config(app_config, use_pymongo=False): - """ - Connect to the main database, take config from app_config - - Optionally use pymongo instead of mongokit for the connection. - """ - port = app_config.get('db_port') - if port: - port = asint(port) - - if use_pymongo: - connection = pymongo.Connection( - app_config.get('db_host'), port) - else: - connection = mongokit.Connection( - app_config.get('db_host'), port) - return connection - - -def setup_connection_and_db_from_config(app_config, use_pymongo=False): - """ - Setup connection and database from config. - - Optionally use pymongo instead of mongokit. - """ - connection = connect_database_from_config(app_config, use_pymongo) - database_path = app_config['db_name'] - db = connection[database_path] - - if not use_pymongo: - models.register_models(connection) - - return (connection, db) +from mediagoblin.db.mongo.open import \ + setup_connection_and_db_from_config, check_db_migrations_current diff --git a/mediagoblin/db/sql/base.py b/mediagoblin/db/sql/base.py new file mode 100644 index 00000000..40140327 --- /dev/null +++ b/mediagoblin/db/sql/base.py @@ -0,0 +1,38 @@ +from sqlalchemy.orm import scoped_session, sessionmaker, object_session + + +Session = scoped_session(sessionmaker()) + + +def _fix_query_dict(query_dict): + if '_id' in query_dict: + query_dict['id'] = query_dict.pop('_id') + + +class GMGTableBase(object): + query = Session.query_property() + + @classmethod + def find(cls, query_dict={}): + _fix_query_dict(query_dict) + return cls.query.filter_by(**query_dict) + + @classmethod + def find_one(cls, query_dict={}): + _fix_query_dict(query_dict) + return cls.query.filter_by(**query_dict).first() + + @classmethod + def one(cls, query_dict): + return cls.find(query_dict).one() + + def get(self, key): + return getattr(self, key) + + def save(self, validate = True): + assert validate + sess = object_session(self) + if sess is None: + sess = Session() + sess.add(self) + sess.commit() diff --git a/mediagoblin/db/sql/convert.py b/mediagoblin/db/sql/convert.py index 2ffa9fd7..6698b767 100644 --- a/mediagoblin/db/sql/convert.py +++ b/mediagoblin/db/sql/convert.py @@ -1,13 +1,13 @@ -from sqlalchemy import create_engine -from sqlalchemy.orm import sessionmaker - from mediagoblin.init import setup_global_and_app_config, setup_database -from mediagoblin.db.util import ObjectId +from mediagoblin.db.mongo.util import ObjectId from mediagoblin.db.sql.models import (Base, User, MediaEntry, MediaComment, Tag, MediaTag) - -Session = sessionmaker() +from mediagoblin.db.sql.open import setup_connection_and_db_from_config as \ + sql_connect +from mediagoblin.db.mongo.open import setup_connection_and_db_from_config as \ + mongo_connect +from mediagoblin.db.sql.base import Session obj_id_table = dict() @@ -61,7 +61,7 @@ def convert_media_entries(mk_db): copy_attrs(entry, new_entry, ('title', 'slug', 'created', 'description', 'description_html', - 'media_type', + 'media_type', 'state', 'fail_error', 'queued_task_id',)) copy_reference_attr(entry, new_entry, "uploader") @@ -124,19 +124,22 @@ def convert_media_comments(mk_db): def main(): - engine = create_engine('sqlite:///mediagoblin.db', echo=True) - Session.configure(bind=engine) + global_config, app_config = setup_global_and_app_config("mediagoblin.ini") - setup_global_and_app_config("mediagoblin.ini") + sql_conn, sql_db = sql_connect({'sql_engine': 'sqlite:///mediagoblin.db'}) - mk_conn, mk_db = setup_database() + mk_conn, mk_db = mongo_connect(app_config) - Base.metadata.create_all(engine) + Base.metadata.create_all(sql_db.engine) convert_users(mk_db) + Session.remove() convert_media_entries(mk_db) + Session.remove() convert_media_tags(mk_db) + Session.remove() convert_media_comments(mk_db) + Session.remove() if __name__ == '__main__': diff --git a/mediagoblin/db/sql/models.py b/mediagoblin/db/sql/models.py index 7723a753..31a6ed3b 100644 --- a/mediagoblin/db/sql/models.py +++ b/mediagoblin/db/sql/models.py @@ -4,9 +4,24 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy import ( Column, Integer, Unicode, UnicodeText, DateTime, Boolean, ForeignKey, UniqueConstraint) +from sqlalchemy.orm import relationship +from mediagoblin.db.sql.base import GMGTableBase -Base = declarative_base() + +Base = declarative_base(cls=GMGTableBase) + + +class SimpleFieldAlias(object): + """An alias for any field""" + def __init__(self, fieldname): + self.fieldname = fieldname + + def __get__(self, instance, cls): + return getattr(instance, self.fieldname) + + def __set__(self, instance, val): + setattr(instance, self.fieldname, val) class User(Base): @@ -30,6 +45,8 @@ class User(Base): ## TODO # plugin data would be in a separate model + _id = SimpleFieldAlias("id") + class MediaEntry(Base): __tablename__ = "media_entries" @@ -42,6 +59,7 @@ class MediaEntry(Base): description = Column(UnicodeText) # ?? description_html = Column(UnicodeText) # ?? media_type = Column(Unicode, nullable=False) + state = Column(Unicode, nullable=False) # or use sqlalchemy.types.Enum? fail_error = Column(Unicode) fail_metadata = Column(UnicodeText) @@ -54,6 +72,8 @@ class MediaEntry(Base): UniqueConstraint('uploader', 'slug'), {}) + get_uploader = relationship(User) + ## TODO # media_files # media_data @@ -95,6 +115,8 @@ class MediaComment(Base): content = Column(UnicodeText, nullable=False) content_html = Column(UnicodeText) + get_author = relationship(User) + def show_table_init(): from sqlalchemy import create_engine diff --git a/mediagoblin/db/sql/open.py b/mediagoblin/db/sql/open.py new file mode 100644 index 00000000..c682bd3b --- /dev/null +++ b/mediagoblin/db/sql/open.py @@ -0,0 +1,33 @@ +from sqlalchemy import create_engine + +from mediagoblin.db.sql.base import Session +from mediagoblin.db.sql.models import Base + + +class DatabaseMaster(object): + def __init__(self, engine): + self.engine = engine + + for k,v in Base._decl_class_registry.iteritems(): + setattr(self, k, v) + + def commit(self): + Session.commit() + + def save(self, obj): + Session.add(obj) + Session.flush() + + def reset_after_request(self): + Session.remove() + + +def setup_connection_and_db_from_config(app_config): + engine = create_engine(app_config['sql_engine'], echo=True) + Session.configure(bind=engine) + + return "dummy conn", DatabaseMaster(engine) + + +def check_db_migrations_current(db): + pass diff --git a/mediagoblin/db/util.py b/mediagoblin/db/util.py index 52e97f6d..1df9494c 100644 --- a/mediagoblin/db/util.py +++ b/mediagoblin/db/util.py @@ -14,279 +14,5 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Utilities for database operations. - -Some note on migration and indexing tools: - -We store information about what the state of the database is in the -'mediagoblin' document of the 'app_metadata' collection. Keys in that -document relevant to here: - - - 'migration_number': The integer representing the current state of - the migrations -""" - -import copy - -# Imports that other modules might use -from pymongo import ASCENDING, DESCENDING -from pymongo.errors import InvalidId -from mongokit import ObjectId - -from mediagoblin.db.indexes import ACTIVE_INDEXES, DEPRECATED_INDEXES - - -################ -# Indexing tools -################ - - -def add_new_indexes(database, active_indexes=ACTIVE_INDEXES): - """ - Add any new indexes to the database. - - Args: - - database: pymongo or mongokit database instance. - - active_indexes: indexes to possibly add in the pattern of: - {'collection_name': { - 'identifier': { - 'index': [index_foo_goes_here], - 'unique': True}} - where 'index' is the index to add and all other options are - arguments for collection.create_index. - - Returns: - A list of indexes added in form ('collection', 'index_name') - """ - indexes_added = [] - - for collection_name, indexes in active_indexes.iteritems(): - collection = database[collection_name] - collection_indexes = collection.index_information().keys() - - for index_name, index_data in indexes.iteritems(): - if not index_name in collection_indexes: - # Get a copy actually so we don't modify the actual - # structure - index_data = copy.copy(index_data) - index = index_data.pop('index') - collection.create_index( - index, name=index_name, **index_data) - - indexes_added.append((collection_name, index_name)) - - return indexes_added - - -def remove_deprecated_indexes(database, deprecated_indexes=DEPRECATED_INDEXES): - """ - Remove any deprecated indexes from the database. - - Args: - - database: pymongo or mongokit database instance. - - deprecated_indexes: the indexes to deprecate in the pattern of: - {'collection_name': { - 'identifier': { - 'index': [index_foo_goes_here], - 'unique': True}} - - (... although we really only need the 'identifier' here, as the - rest of the information isn't used in this case. But it's kept - around so we can remember what it was) - - Returns: - A list of indexes removed in form ('collection', 'index_name') - """ - indexes_removed = [] - - for collection_name, indexes in deprecated_indexes.iteritems(): - collection = database[collection_name] - collection_indexes = collection.index_information().keys() - - for index_name, index_data in indexes.iteritems(): - if index_name in collection_indexes: - collection.drop_index(index_name) - - indexes_removed.append((collection_name, index_name)) - - return indexes_removed - - -################# -# Migration tools -################# - -# The default migration registry... -# -# Don't set this yourself! RegisterMigration will automatically fill -# this with stuff via decorating methods in migrations.py - -class MissingCurrentMigration(Exception): - pass - - -MIGRATIONS = {} - - -class RegisterMigration(object): - """ - Tool for registering migrations - - Call like: - - @RegisterMigration(33) - def update_dwarves(database): - [...] - - This will register your migration with the default migration - registry. Alternately, to specify a very specific - migration_registry, you can pass in that as the second argument. - - Note, the number of your migration should NEVER be 0 or less than - 0. 0 is the default "no migrations" state! - """ - def __init__(self, migration_number, migration_registry=MIGRATIONS): - assert migration_number > 0, "Migration number must be > 0!" - assert migration_number not in migration_registry, \ - "Duplicate migration numbers detected! That's not allowed!" - - self.migration_number = migration_number - self.migration_registry = migration_registry - - def __call__(self, migration): - self.migration_registry[self.migration_number] = migration - return migration - - -class MigrationManager(object): - """ - Migration handling tool. - - Takes information about a database, lets you update the database - to the latest migrations, etc. - """ - def __init__(self, database, migration_registry=MIGRATIONS): - """ - Args: - - database: database we're going to migrate - - migration_registry: where we should find all migrations to - run - """ - self.database = database - self.migration_registry = migration_registry - self._sorted_migrations = None - - def _ensure_current_migration_record(self): - """ - If there isn't a database[u'app_metadata'] mediagoblin entry - with the 'current_migration', throw an error. - """ - if self.database_current_migration() is None: - raise MissingCurrentMigration( - "Tried to call function which requires " - "'current_migration' set in database") - - @property - def sorted_migrations(self): - """ - Sort migrations if necessary and store in self._sorted_migrations - """ - if not self._sorted_migrations: - self._sorted_migrations = sorted( - self.migration_registry.items(), - # sort on the key... the migration number - key=lambda migration_tuple: migration_tuple[0]) - - return self._sorted_migrations - - def latest_migration(self): - """ - Return a migration number for the latest migration, or 0 if - there are no migrations. - """ - if self.sorted_migrations: - return self.sorted_migrations[-1][0] - else: - # If no migrations have been set, we start at 0. - return 0 - - def set_current_migration(self, migration_number): - """ - Set the migration in the database to migration_number - """ - # Add the mediagoblin migration if necessary - self.database[u'app_metadata'].update( - {u'_id': u'mediagoblin'}, - {u'$set': {u'current_migration': migration_number}}, - upsert=True) - - def install_migration_version_if_missing(self): - """ - Sets the migration to the latest version if no migration - version at all is set. - """ - mgoblin_metadata = self.database[u'app_metadata'].find_one( - {u'_id': u'mediagoblin'}) - if not mgoblin_metadata: - latest_migration = self.latest_migration() - self.set_current_migration(latest_migration) - - def database_current_migration(self): - """ - Return the current migration in the database. - """ - mgoblin_metadata = self.database[u'app_metadata'].find_one( - {u'_id': u'mediagoblin'}) - if not mgoblin_metadata: - return None - else: - return mgoblin_metadata[u'current_migration'] - - def database_at_latest_migration(self): - """ - See if the database is at the latest migration. - Returns a boolean. - """ - current_migration = self.database_current_migration() - return current_migration == self.latest_migration() - - def migrations_to_run(self): - """ - Get a list of migrations to run still, if any. - - Note that calling this will set your migration version to the - latest version if it isn't installed to anything yet! - """ - self._ensure_current_migration_record() - - db_current_migration = self.database_current_migration() - - return [ - (migration_number, migration_func) - for migration_number, migration_func in self.sorted_migrations - if migration_number > db_current_migration] - - def migrate_new(self, pre_callback=None, post_callback=None): - """ - Run all migrations. - - Includes two optional args: - - pre_callback: if called, this is a callback on something to - run pre-migration. Takes (migration_number, migration_func) - as arguments - - pre_callback: if called, this is a callback on something to - run post-migration. Takes (migration_number, migration_func) - as arguments - """ - # If we aren't set to any version number, presume we're at the - # latest (which means we'll do nothing here...) - self.install_migration_version_if_missing() - - for migration_number, migration_func in self.migrations_to_run(): - if pre_callback: - pre_callback(migration_number, migration_func) - migration_func(self.database) - self.set_current_migration(migration_number) - if post_callback: - post_callback(migration_number, migration_func) +from mediagoblin.db.mongo.util import (ObjectId, InvalidId, + DESCENDING) |