diff options
| -rw-r--r-- | docs/hackinghowto.rst | 2 | ||||
| -rw-r--r-- | mediagoblin/app.py | 14 | ||||
| -rw-r--r-- | mediagoblin/db/migrations.py | 63 | ||||
| -rw-r--r-- | mediagoblin/db/models.py | 11 | ||||
| -rw-r--r-- | mediagoblin/db/open.py | 34 | ||||
| -rw-r--r-- | mediagoblin/db/util.py | 183 | ||||
| -rw-r--r-- | mediagoblin/edit/views.py | 17 | ||||
| -rw-r--r-- | mediagoblin/gmg_commands/migrate.py | 44 | ||||
| -rw-r--r-- | mediagoblin/templates/mediagoblin/utils/profile.html | 6 | ||||
| -rw-r--r-- | mediagoblin/tests/test_migrations.py | 402 | ||||
| -rw-r--r-- | mediagoblin/tests/tools.py | 32 |
11 files changed, 718 insertions, 90 deletions
diff --git a/docs/hackinghowto.rst b/docs/hackinghowto.rst index 46288882..4cdbf03c 100644 --- a/docs/hackinghowto.rst +++ b/docs/hackinghowto.rst @@ -85,7 +85,7 @@ After installing the requirements, follow these steps: 1. Clone the repository:: - git clone http://git.gitorious.org/mediagoblin/mediagoblin.git + git clone git://gitorious.org/mediagoblin/mediagoblin.git 2. Bootstrap and run buildout:: diff --git a/mediagoblin/app.py b/mediagoblin/app.py index 147db09c..1c38f778 100644 --- a/mediagoblin/app.py +++ b/mediagoblin/app.py @@ -22,11 +22,15 @@ from webob import Request, exc from mediagoblin import routing, util, storage from mediagoblin.db.open import setup_connection_and_db_from_config +from mediagoblin.db.util import MigrationManager from mediagoblin.mg_globals import setup_globals from mediagoblin.init.celery import setup_celery_from_config from mediagoblin.init import get_jinja_loader, get_staticdirector, \ setup_global_and_app_config, setup_workbench +# This MUST be imported so as to set up the appropriate migrations! +from mediagoblin.db import migrations + class MediaGoblinApp(object): """ @@ -59,6 +63,16 @@ class MediaGoblinApp(object): self.connection, self.db = setup_connection_and_db_from_config( app_config) + # Init the migration number if necessary + migration_manager = MigrationManager(self.db) + migration_manager.install_migration_version_if_missing() + + # Tiny hack to warn user if our migration is out of date + if not migration_manager.database_at_latest_migration(): + print ( + "*WARNING:* Your migrations are out of date, " + "maybe run ./bin/gmg migrate?") + # Get the template environment self.template_loader = get_jinja_loader( app_config.get('user_template_path')) diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py index 712f8ab4..f398f4b3 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/migrations.py @@ -14,56 +14,25 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +from mediagoblin.db.util import RegisterMigration from mediagoblin.util import cleaned_markdown_conversion -from mongokit import DocumentMigration +# Please see mediagoblin/tests/test_migrations.py for some examples of +# basic migrations. -class MediaEntryMigration(DocumentMigration): - def allmigration01_uploader_to_reference(self): - """ - Old MediaEntry['uploader'] accidentally embedded the User instead - of referencing it. Fix that! - """ - # uploader is an associative array - self.target = {'uploader': {'$type': 3}} - if not self.status: - for doc in self.collection.find(self.target): - self.update = { - '$set': { - 'uploader': doc['uploader']['_id']}} - self.collection.update( - self.target, self.update, multi=True, safe=True) - def allmigration02_add_description_html(self): - """ - Now that we can have rich descriptions via Markdown, we should - update all existing entries to record the rich description versions. - """ - self.target = {'description_html': {'$exists': False}, - 'description': {'$exists': True}} +@RegisterMigration(1) +def user_add_bio_html(database): + """ + Users now have richtext bios via Markdown, reflect appropriately. + """ + collection = database['users'] - if not self.status: - for doc in self.collection.find(self.target): - self.update = { - '$set': { - 'description_html': cleaned_markdown_conversion( - doc['description'])}} - -class UserMigration(DocumentMigration): - def allmigration01_add_bio_and_url_profile(self): - """ - User can elaborate profile with home page and biography - """ - self.target = {'url': {'$exists': False}, - 'bio': {'$exists': False}} - if not self.status: - for doc in self.collection.find(self.target): - self.update = { - '$set': {'url': '', - 'bio': ''}} - self.collection.update( - self.target, self.update, multi=True, safe=True) - - -MIGRATE_CLASSES = ['MediaEntry', 'User'] + target = collection.find( + {'bio_html': {'$exists': False}}) + + for document in target: + document['bio_html'] = cleaned_markdown_conversion( + document['bio']) + collection.save(document) diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 279cb9f2..ba8162ed 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -16,12 +16,11 @@ import datetime, uuid -from mongokit import Document, Set +from mongokit import Document from mediagoblin import util from mediagoblin.auth import lib as auth_lib from mediagoblin import mg_globals -from mediagoblin.db import migrations from mediagoblin.db.util import ASCENDING, DESCENDING, ObjectId ################### @@ -47,7 +46,8 @@ class User(Document): 'verification_key': unicode, 'is_admin': bool, 'url' : unicode, - 'bio' : unicode + 'bio' : unicode, # May contain markdown + 'bio_html': unicode, # May contain plaintext, or HTML } required_fields = ['username', 'created', 'pw_hash', 'email'] @@ -59,8 +59,6 @@ class User(Document): 'verification_key': lambda: unicode(uuid.uuid4()), 'is_admin': False} - migration_handler = migrations.UserMigration - def check_login(self, password): """ See if a user can login with this password @@ -106,8 +104,6 @@ class MediaEntry(Document): 'created': datetime.datetime.utcnow, 'state': u'unprocessed'} - migration_handler = migrations.MediaEntryMigration - def get_comments(self): return self.db.MediaComment.find({ 'media_entry': self['_id']}).sort('created', DESCENDING) @@ -196,6 +192,7 @@ class MediaComment(Document): def author(self): return self.db.User.find_one({'_id': self['author']}) + REGISTER_MODELS = [ MediaEntry, User, diff --git a/mediagoblin/db/open.py b/mediagoblin/db/open.py index b6987677..e5fde6f9 100644 --- a/mediagoblin/db/open.py +++ b/mediagoblin/db/open.py @@ -14,24 +14,42 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +import pymongo import mongokit from paste.deploy.converters import asint from mediagoblin.db import models -def connect_database_from_config(app_config): - """Connect to the main database, take config from app_config""" +def connect_database_from_config(app_config, use_pymongo=False): + """ + Connect to the main database, take config from app_config + + Optionally use pymongo instead of mongokit for the connection. + """ port = app_config.get('db_port') if port: port = asint(port) - connection = mongokit.Connection( - app_config.get('db_host'), port) + + if use_pymongo: + connection = pymongo.Connection( + app_config.get('db_host'), port) + else: + connection = mongokit.Connection( + app_config.get('db_host'), port) return connection -def setup_connection_and_db_from_config(app_config): - connection = connect_database_from_config(app_config) + +def setup_connection_and_db_from_config(app_config, use_pymongo=False): + """ + Setup connection and database from config. + + Optionally use pymongo instead of mongokit. + """ + connection = connect_database_from_config(app_config, use_pymongo) database_path = app_config['db_name'] db = connection[database_path] - models.register_models(connection) - # Could configure indexes here on db + + if not use_pymongo: + models.register_models(connection) + return (connection, db) diff --git a/mediagoblin/db/util.py b/mediagoblin/db/util.py index 37e6586f..0f3220d2 100644 --- a/mediagoblin/db/util.py +++ b/mediagoblin/db/util.py @@ -37,6 +37,11 @@ from mongokit import ObjectId from mediagoblin.db.indexes import ACTIVE_INDEXES, DEPRECATED_INDEXES +################ +# Indexing tools +################ + + def add_new_indexes(database, active_indexes=ACTIVE_INDEXES): """ Add any new indexes to the database. @@ -106,3 +111,181 @@ def remove_deprecated_indexes(database, deprecated_indexes=DEPRECATED_INDEXES): indexes_removed.append((collection_name, index_name)) return indexes_removed + + +################# +# Migration tools +################# + +# The default migration registry... +# +# Don't set this yourself! RegisterMigration will automatically fill +# this with stuff via decorating methods in migrations.py + +class MissingCurrentMigration(Exception): pass + + +MIGRATIONS = {} + + +class RegisterMigration(object): + """ + Tool for registering migrations + + Call like: + + @RegisterMigration(33) + def update_dwarves(database): + [...] + + This will register your migration with the default migration + registry. Alternately, to specify a very specific + migration_registry, you can pass in that as the second argument. + + Note, the number of your migration should NEVER be 0 or less than + 0. 0 is the default "no migrations" state! + """ + def __init__(self, migration_number, migration_registry=MIGRATIONS): + assert migration_number > 0, "Migration number must be > 0!" + assert not migration_registry.has_key(migration_number), \ + "Duplicate migration numbers detected! That's not allowed!" + + self.migration_number = migration_number + self.migration_registry = migration_registry + + def __call__(self, migration): + self.migration_registry[self.migration_number] = migration + return migration + + +class MigrationManager(object): + """ + Migration handling tool. + + Takes information about a database, lets you update the database + to the latest migrations, etc. + """ + def __init__(self, database, migration_registry=MIGRATIONS): + """ + Args: + - database: database we're going to migrate + - migration_registry: where we should find all migrations to + run + """ + self.database = database + self.migration_registry = migration_registry + self._sorted_migrations = None + + def _ensure_current_migration_record(self): + """ + If there isn't a database[u'app_metadata'] mediagoblin entry + with the 'current_migration', throw an error. + """ + if self.database_current_migration() is None: + raise MissingCurrentMigration( + "Tried to call function which requires " + "'current_migration' set in database") + + @property + def sorted_migrations(self): + """ + Sort migrations if necessary and store in self._sorted_migrations + """ + if not self._sorted_migrations: + self._sorted_migrations = sorted( + self.migration_registry.items(), + # sort on the key... the migration number + key=lambda migration_tuple: migration_tuple[0]) + + return self._sorted_migrations + + def latest_migration(self): + """ + Return a migration number for the latest migration, or 0 if + there are no migrations. + """ + if self.sorted_migrations: + return self.sorted_migrations[-1][0] + else: + # If no migrations have been set, we start at 0. + return 0 + + def set_current_migration(self, migration_number): + """ + Set the migration in the database to migration_number + """ + # Add the mediagoblin migration if necessary + self.database[u'app_metadata'].update( + {u'_id': u'mediagoblin'}, + {u'$set': {u'current_migration': migration_number}}, + upsert=True) + + def install_migration_version_if_missing(self): + """ + Sets the migration to the latest version if no migration + version at all is set. + """ + mgoblin_metadata = self.database[u'app_metadata'].find_one( + {u'_id': u'mediagoblin'}) + if not mgoblin_metadata: + latest_migration = self.latest_migration() + self.set_current_migration(latest_migration) + + def database_current_migration(self): + """ + Return the current migration in the database. + """ + mgoblin_metadata = self.database[u'app_metadata'].find_one( + {u'_id': u'mediagoblin'}) + if not mgoblin_metadata: + return None + else: + return mgoblin_metadata[u'current_migration'] + + def database_at_latest_migration(self): + """ + See if the database is at the latest migration. + Returns a boolean. + """ + current_migration = self.database_current_migration() + return current_migration == self.latest_migration() + + def migrations_to_run(self): + """ + Get a list of migrations to run still, if any. + + Note that calling this will set your migration version to the + latest version if it isn't installed to anything yet! + """ + self._ensure_current_migration_record() + + db_current_migration = self.database_current_migration() + + return [ + (migration_number, migration_func) + for migration_number, migration_func in self.sorted_migrations + if migration_number > db_current_migration] + + def migrate_new(self, pre_callback=None, post_callback=None): + """ + Run all migrations. + + Includes two optional args: + - pre_callback: if called, this is a callback on something to + run pre-migration. Takes (migration_number, migration_func) + as arguments + - pre_callback: if called, this is a callback on something to + run post-migration. Takes (migration_number, migration_func) + as arguments + """ + # If we aren't set to any version number, presume we're at the + # latest (which means we'll do nothing here...) + self.install_migration_version_if_missing() + + for migration_number, migration_func in self.migrations_to_run(): + if pre_callback: + pre_callback(migration_number, migration_func) + migration_func(self.database) + self.set_current_migration(migration_number) + if post_callback: + post_callback(migration_number, migration_func) diff --git a/mediagoblin/edit/views.py b/mediagoblin/edit/views.py index 3bcf788b..f372fbb9 100644 --- a/mediagoblin/edit/views.py +++ b/mediagoblin/edit/views.py @@ -18,13 +18,12 @@ from webob import exc from mediagoblin import messages -from mediagoblin.util import render_to_response, redirect, clean_html +from mediagoblin.util import ( + render_to_response, redirect, cleaned_markdown_conversion) from mediagoblin.edit import forms from mediagoblin.edit.lib import may_edit_media from mediagoblin.decorators import require_active_login, get_user_media_entry -import markdown - @get_user_media_entry @require_active_login @@ -51,12 +50,9 @@ def edit_media(request, media): else: media['title'] = request.POST['title'] media['description'] = request.POST.get('description') - - md = markdown.Markdown( - safe_mode = 'escape') - media['description_html'] = clean_html( - md.convert( - media['description'])) + + media['description_html'] = cleaned_markdown_conversion( + media['description']) media['slug'] = request.POST['slug'] media.save() @@ -101,6 +97,9 @@ def edit_profile(request): if request.method == 'POST' and form.validate(): user['url'] = request.POST['url'] user['bio'] = request.POST['bio'] + + user['bio_html'] = cleaned_markdown_conversion(user['bio']) + user.save() messages.add_message(request, diff --git a/mediagoblin/gmg_commands/migrate.py b/mediagoblin/gmg_commands/migrate.py index ab1a267b..94adc9e0 100644 --- a/mediagoblin/gmg_commands/migrate.py +++ b/mediagoblin/gmg_commands/migrate.py @@ -14,10 +14,14 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +import sys -from mediagoblin.db import migrations from mediagoblin.db import util as db_util -from mediagoblin.gmg_commands import util as commands_util +from mediagoblin.db.open import setup_connection_and_db_from_config +from mediagoblin.init.config import read_mediagoblin_config + +# This MUST be imported so as to set up the appropriate migrations! +from mediagoblin.db import migrations def migrate_parser_setup(subparser): @@ -26,31 +30,41 @@ def migrate_parser_setup(subparser): help="Config file used to set up environment") +def _print_started_migration(migration_number, migration_func): + sys.stdout.write( + "Running migration %s, '%s'... " % ( + migration_number, migration_func.func_name)) + sys.stdout.flush() + + +def _print_finished_migration(migration_number, migration_func): + sys.stdout.write("done.\n") + sys.stdout.flush() + + def migrate(args): - mgoblin_app = commands_util.setup_app(args) + config, validation_result = read_mediagoblin_config(args.conf_file) + connection, db = setup_connection_and_db_from_config( + config['mediagoblin'], use_pymongo=True) + migration_manager = db_util.MigrationManager(db) # Clear old indexes print "== Clearing old indexes... ==" - removed_indexes = db_util.remove_deprecated_indexes(mgoblin_app.db) + removed_indexes = db_util.remove_deprecated_indexes(db) for collection, index_name in removed_indexes: print "Removed index '%s' in collection '%s'" % ( index_name, collection) # Migrate - print "== Applying migrations... ==" - for model_name in migrations.MIGRATE_CLASSES: - model = getattr(mgoblin_app.db, model_name) - - if not hasattr(model, 'migration_handler') or not model.collection: - continue - - migration = model.migration_handler(model) - migration.migrate_all(collection=model.collection) + print "\n== Applying migrations... ==" + migration_manager.migrate_new( + pre_callback=_print_started_migration, + post_callback=_print_finished_migration) # Add new indexes - print "== Adding new indexes... ==" - new_indexes = db_util.add_new_indexes(mgoblin_app.db) + print "\n== Adding new indexes... ==" + new_indexes = db_util.add_new_indexes(db) for collection, index_name in new_indexes: print "Added index '%s' to collection '%s'" % ( diff --git a/mediagoblin/templates/mediagoblin/utils/profile.html b/mediagoblin/templates/mediagoblin/utils/profile.html index f44defa5..63024b77 100644 --- a/mediagoblin/templates/mediagoblin/utils/profile.html +++ b/mediagoblin/templates/mediagoblin/utils/profile.html @@ -18,9 +18,9 @@ {% block profile_content -%} {% if user.bio %} - <p> - {{ user.bio }} - </p> + {% autoescape False %} + <p>{{ user.bio_html }}</p> + {% endautoescape %} {% endif %} {% if user.url %} <p> diff --git a/mediagoblin/tests/test_migrations.py b/mediagoblin/tests/test_migrations.py new file mode 100644 index 00000000..127b90e1 --- /dev/null +++ b/mediagoblin/tests/test_migrations.py @@ -0,0 +1,402 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +from nose.tools import assert_raises +from pymongo import Connection + +from mediagoblin.tests.tools import ( + install_fixtures_simple, assert_db_meets_expected) +from mediagoblin.db.util import ( + RegisterMigration, MigrationManager, ObjectId, + MissingCurrentMigration) + +# This one will get filled with local migrations +TEST_MIGRATION_REGISTRY = {} +# this one won't get filled +TEST_EMPTY_MIGRATION_REGISTRY = {} + +MIGRATION_DB_NAME = u'__mediagoblin_test_migrations__' + + +###################### +# Fake test migrations +###################### + +@RegisterMigration(1, TEST_MIGRATION_REGISTRY) +def creature_add_magical_powers(database): + """ + Add lists of magical powers. + + This defaults to [], an empty list. Since we haven't declared any + magical powers, all existing monsters should + """ + database['creatures'].update( + {'magical_powers': {'$exists': False}}, + {'$set': {'magical_powers': []}}, + multi=True) + + +@RegisterMigration(2, TEST_MIGRATION_REGISTRY) +def creature_rename_num_legs_to_num_limbs(database): + """ + It turns out we want to track how many limbs a creature has, not + just how many legs. We don't care about the ambiguous distinction + between arms/legs currently. + """ + # $rename not available till 1.7.2+, Debian Stable only includes + # 1.4.4... we should do renames manually for now :( + + collection = database['creatures'] + target = collection.find( + {'num_legs': {'$exists': True}}) + + for document in target: + # A lame manual renaming. + document['num_limbs'] = document.pop('num_legs') + collection.save(document) + + +@RegisterMigration(3, TEST_MIGRATION_REGISTRY) +def creature_remove_is_demon(database): + """ + It turns out we don't care much about whether creatures are demons + or not. + """ + database['creatures'].update( + {'is_demon': {'$exists': True}}, + {'$unset': {'is_demon': 1}}, + multi=True) + + +@RegisterMigration(4, TEST_MIGRATION_REGISTRY) +def level_exits_dict_to_list(database): + """ + For the sake of the indexes we want to write, and because we + intend to add more flexible fields, we want to move level exits + from like: + + {'big_door': 'castle_level_id', + 'trapdoor': 'dungeon_level_id'} + + to like: + + [{'name': 'big_door', + 'exits_to': 'castle_level_id'}, + {'name': 'trapdoor', + 'exits_to': 'dungeon_level_id'}] + """ + collection = database['levels'] + target = collection.find( + {'exits': {'$type': 3}}) + + for level in target: + new_exits = [] + for exit_name, exits_to in level['exits'].items(): + new_exits.append( + {'name': exit_name, + 'exits_to': exits_to}) + + level['exits'] = new_exits + collection.save(level) + + +CENTIPEDE_OBJECTID = ObjectId() +WOLF_OBJECTID = ObjectId() +WIZARDSNAKE_OBJECTID = ObjectId() + +UNMIGRATED_DBDATA = { + 'creatures': [ + {'_id': CENTIPEDE_OBJECTID, + 'name': 'centipede', + 'num_legs': 100, + 'is_demon': False}, + {'_id': WOLF_OBJECTID, + 'name': 'wolf', + 'num_legs': 4, + 'is_demon': False}, + # don't ask me what a wizardsnake is. + {'_id': WIZARDSNAKE_OBJECTID, + 'name': 'wizardsnake', + 'num_legs': 0, + 'is_demon': True}], + 'levels': [ + {'_id': 'necroplex', + 'name': 'The Necroplex', + 'description': 'A complex full of pure deathzone.', + 'exits': { + 'deathwell': 'evilstorm', + 'portal': 'central_park'}}, + {'_id': 'evilstorm', + 'name': 'Evil Storm', + 'description': 'A storm full of pure evil.', + 'exits': {}}, # you can't escape the evilstorm + {'_id': 'central_park', + 'name': 'Central Park, NY, NY', + 'description': "New York's friendly Central Park.", + 'exits': { + 'portal': 'necroplex'}}]} + + +EXPECTED_POST_MIGRATION_UNMIGRATED_DBDATA = { + 'creatures': [ + {'_id': CENTIPEDE_OBJECTID, + 'name': 'centipede', + 'num_limbs': 100, + 'magical_powers': []}, + {'_id': WOLF_OBJECTID, + 'name': 'wolf', + 'num_limbs': 4, + # kept around namely to check that it *isn't* removed! + 'magical_powers': []}, + {'_id': WIZARDSNAKE_OBJECTID, + 'name': 'wizardsnake', + 'num_limbs': 0, + 'magical_powers': []}], + 'levels': [ + {'_id': 'necroplex', + 'name': 'The Necroplex', + 'description': 'A complex full of pure deathzone.', + 'exits': [ + {'name': 'deathwell', + 'exits_to': 'evilstorm'}, + {'name': 'portal', + 'exits_to': 'central_park'}]}, + {'_id': 'evilstorm', + 'name': 'Evil Storm', + 'description': 'A storm full of pure evil.', + 'exits': []}, # you can't escape the evilstorm + {'_id': 'central_park', + 'name': 'Central Park, NY, NY', + 'description': "New York's friendly Central Park.", + 'exits': [ + {'name': 'portal', + 'exits_to': 'necroplex'}]}]} + +# We want to make sure that if we're at migration 3, migration 3 +# doesn't get re-run. + +SEMI_MIGRATED_DBDATA = { + 'creatures': [ + {'_id': CENTIPEDE_OBJECTID, + 'name': 'centipede', + 'num_limbs': 100, + 'magical_powers': []}, + {'_id': WOLF_OBJECTID, + 'name': 'wolf', + 'num_limbs': 4, + # kept around namely to check that it *isn't* removed! + 'is_demon': False, + 'magical_powers': [ + 'ice_breath', 'death_stare']}, + {'_id': WIZARDSNAKE_OBJECTID, + 'name': 'wizardsnake', + 'num_limbs': 0, + 'magical_powers': [ + 'death_rattle', 'sneaky_stare', + 'slithery_smoke', 'treacherous_tremors'], + 'is_demon': True}], + 'levels': [ + {'_id': 'necroplex', + 'name': 'The Necroplex', + 'description': 'A complex full of pure deathzone.', + 'exits': { + 'deathwell': 'evilstorm', + 'portal': 'central_park'}}, + {'_id': 'evilstorm', + 'name': 'Evil Storm', + 'description': 'A storm full of pure evil.', + 'exits': {}}, # you can't escape the evilstorm + {'_id': 'central_park', + 'name': 'Central Park, NY, NY', + 'description': "New York's friendly Central Park.", + 'exits': { + 'portal': 'necroplex'}}]} + + +EXPECTED_POST_MIGRATION_SEMI_MIGRATED_DBDATA = { + 'creatures': [ + {'_id': CENTIPEDE_OBJECTID, + 'name': 'centipede', + 'num_limbs': 100, + 'magical_powers': []}, + {'_id': WOLF_OBJECTID, + 'name': 'wolf', + 'num_limbs': 4, + # kept around namely to check that it *isn't* removed! + 'is_demon': False, + 'magical_powers': [ + 'ice_breath', 'death_stare']}, + {'_id': WIZARDSNAKE_OBJECTID, + 'name': 'wizardsnake', + 'num_limbs': 0, + 'magical_powers': [ + 'death_rattle', 'sneaky_stare', + 'slithery_smoke', 'treacherous_tremors'], + 'is_demon': True}], + 'levels': [ + {'_id': 'necroplex', + 'name': 'The Necroplex', + 'description': 'A complex full of pure deathzone.', + 'exits': [ + {'name': 'deathwell', + 'exits_to': 'evilstorm'}, + {'name': 'portal', + 'exits_to': 'central_park'}]}, + {'_id': 'evilstorm', + 'name': 'Evil Storm', + 'description': 'A storm full of pure evil.', + 'exits': []}, # you can't escape the evilstorm + {'_id': 'central_park', + 'name': 'Central Park, NY, NY', + 'description': "New York's friendly Central Park.", + 'exits': [ + {'name': 'portal', + 'exits_to': 'necroplex'}]}]} + + +class TestMigrations(object): + def setUp(self): + # Set up the connection, drop an existing possible database + self.connection = Connection() + self.connection.drop_database(MIGRATION_DB_NAME) + self.db = Connection()[MIGRATION_DB_NAME] + self.migration_manager = MigrationManager( + self.db, TEST_MIGRATION_REGISTRY) + self.empty_migration_manager = MigrationManager( + self.db, TEST_EMPTY_MIGRATION_REGISTRY) + self.run_migrations = [] + + def tearDown(self): + self.connection.drop_database(MIGRATION_DB_NAME) + + def _record_migration(self, migration_number, migration_func): + self.run_migrations.append((migration_number, migration_func)) + + def test_migrations_registered_and_sorted(self): + """ + Make sure that migrations get registered and are sorted right + in the migration manager + """ + assert TEST_MIGRATION_REGISTRY == { + 1: creature_add_magical_powers, + 2: creature_rename_num_legs_to_num_limbs, + 3: creature_remove_is_demon, + 4: level_exits_dict_to_list} + assert self.migration_manager.sorted_migrations == [ + (1, creature_add_magical_powers), + (2, creature_rename_num_legs_to_num_limbs), + (3, creature_remove_is_demon), + (4, level_exits_dict_to_list)] + assert self.empty_migration_manager.sorted_migrations == [] + + def test_run_full_migrations(self): + """ + Make sure that running the full migration suite from 0 updates + everything + """ + self.migration_manager.set_current_migration(0) + assert self.migration_manager.database_current_migration() == 0 + install_fixtures_simple(self.db, UNMIGRATED_DBDATA) + self.migration_manager.migrate_new(post_callback=self._record_migration) + + assert self.run_migrations == [ + (1, creature_add_magical_powers), + (2, creature_rename_num_legs_to_num_limbs), + (3, creature_remove_is_demon), + (4, level_exits_dict_to_list)] + + assert_db_meets_expected( + self.db, EXPECTED_POST_MIGRATION_UNMIGRATED_DBDATA) + + # Make sure the migration is recorded correctly + assert self.migration_manager.database_current_migration() == 4 + + # run twice! It should do nothing the second time. + # ------------------------------------------------ + self.run_migrations = [] + self.migration_manager.migrate_new(post_callback=self._record_migration) + assert self.run_migrations == [] + assert_db_meets_expected( + self.db, EXPECTED_POST_MIGRATION_UNMIGRATED_DBDATA) + assert self.migration_manager.database_current_migration() == 4 + + + def test_run_partial_migrations(self): + """ + Make sure that running full migration suite from 3 only runs + last migration + """ + self.migration_manager.set_current_migration(3) + assert self.migration_manager.database_current_migration() == 3 + install_fixtures_simple(self.db, SEMI_MIGRATED_DBDATA) + self.migration_manager.migrate_new(post_callback=self._record_migration) + + assert self.run_migrations == [ + (4, level_exits_dict_to_list)] + + assert_db_meets_expected( + self.db, EXPECTED_POST_MIGRATION_SEMI_MIGRATED_DBDATA) + + # Make sure the migration is recorded correctly + assert self.migration_manager.database_current_migration() == 4 + + def test_migrations_recorded_as_latest(self): + """ + Make sure that if we don't have a migration_status + pre-recorded it's marked as the latest + """ + self.migration_manager.install_migration_version_if_missing() + assert self.migration_manager.database_current_migration() == 4 + + def test_no_migrations_recorded_as_zero(self): + """ + Make sure that if we don't have a migration_status + but there *are* no migrations that it's marked as 0 + """ + self.empty_migration_manager.install_migration_version_if_missing() + assert self.empty_migration_manager.database_current_migration() == 0 + + def test_migrations_to_run(self): + """ + Make sure we get the right list of migrations to run + """ + self.migration_manager.set_current_migration(0) + + assert self.migration_manager.migrations_to_run() == [ + (1, creature_add_magical_powers), + (2, creature_rename_num_legs_to_num_limbs), + (3, creature_remove_is_demon), + (4, level_exits_dict_to_list)] + + self.migration_manager.set_current_migration(3) + + assert self.migration_manager.migrations_to_run() == [ + (4, level_exits_dict_to_list)] + + self.migration_manager.set_current_migration(4) + + assert self.migration_manager.migrations_to_run() == [] + + + def test_no_migrations_raises_exception(self): + """ + If we don't have the current migration set in the database, + this should error out. + """ + assert_raises( + MissingCurrentMigration, + self.migration_manager.migrations_to_run) diff --git a/mediagoblin/tests/tools.py b/mediagoblin/tests/tools.py index e56af4de..4b61f259 100644 --- a/mediagoblin/tests/tools.py +++ b/mediagoblin/tests/tools.py @@ -118,3 +118,35 @@ def setup_fresh_app(func): return func(test_app, *args, **kwargs) return _make_safe(wrapper, func) + + +def install_fixtures_simple(db, fixtures): + """ + Very simply install fixtures in the database + """ + for collection_name, collection_fixtures in fixtures.iteritems(): + collection = db[collection_name] + for fixture in collection_fixtures: + collection.insert(fixture) + + +def assert_db_meets_expected(db, expected): + """ + Assert a database contains the things we expect it to. + + Objects are found via '_id', so you should make sure your document + has an _id. + + Args: + - db: pymongo or mongokit database connection + - expected: the data we expect. Formatted like: + {'collection_name': [ + {'_id': 'foo', + 'some_field': 'some_value'},]} + """ + for collection_name, collection_data in expected.iteritems(): + collection = db[collection_name] + for expected_document in collection_data: + document = collection.find_one({'_id': expected_document['_id']}) + assert document is not None # make sure it exists + assert document == expected_document # make sure it matches |