diff options
| -rw-r--r-- | mediagoblin/_version.py | 11 | ||||
| -rw-r--r-- | mediagoblin/db/mongo/__init__.py | 15 | ||||
| -rw-r--r-- | mediagoblin/db/mongo/indexes.py (renamed from mediagoblin/db/indexes.py) | 0 | ||||
| -rw-r--r-- | mediagoblin/db/mongo/migrations.py (renamed from mediagoblin/db/migrations.py) | 2 | ||||
| -rw-r--r-- | mediagoblin/db/mongo/models.py (renamed from mediagoblin/db/models.py) | 4 | ||||
| -rw-r--r-- | mediagoblin/db/mongo/open.py | 55 | ||||
| -rw-r--r-- | mediagoblin/db/mongo/util.py | 292 | ||||
| -rw-r--r-- | mediagoblin/db/open.py | 40 | ||||
| -rw-r--r-- | mediagoblin/db/sql/__init__.py | 15 | ||||
| -rw-r--r-- | mediagoblin/db/sql/base.py | 16 | ||||
| -rw-r--r-- | mediagoblin/db/sql/convert.py | 148 | ||||
| -rw-r--r-- | mediagoblin/db/sql/models.py | 109 | ||||
| -rw-r--r-- | mediagoblin/db/sql/open.py | 29 | ||||
| -rw-r--r-- | mediagoblin/db/util.py | 278 | ||||
| -rw-r--r-- | mediagoblin/gmg_commands/import_export.py | 10 | ||||
| -rw-r--r-- | mediagoblin/init/__init__.py | 2 | ||||
| -rw-r--r-- | mediagoblin/templates/mediagoblin/user_pages/media.html | 2 | ||||
| -rw-r--r-- | mediagoblin/tests/test_migrations.py | 4 |
18 files changed, 704 insertions, 328 deletions
diff --git a/mediagoblin/_version.py b/mediagoblin/_version.py index 5e3f4e5a..5e69f21a 100644 --- a/mediagoblin/_version.py +++ b/mediagoblin/_version.py @@ -14,4 +14,13 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -__version__ = "0.3.0-dev" +# valid version formats: +# * x.y - final release +# * x.ya1 - alpha 1 +# * x.yb1 - beta 1 +# * x.yrc1 - release candidate 1 +# * x.y.dev - dev + +# see http://www.python.org/dev/peps/pep-0386/ + +__version__ = "0.3.0.dev" diff --git a/mediagoblin/db/mongo/__init__.py b/mediagoblin/db/mongo/__init__.py new file mode 100644 index 00000000..ba347c69 --- /dev/null +++ b/mediagoblin/db/mongo/__init__.py @@ -0,0 +1,15 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. diff --git a/mediagoblin/db/indexes.py b/mediagoblin/db/mongo/indexes.py index 1dd73f2b..1dd73f2b 100644 --- a/mediagoblin/db/indexes.py +++ b/mediagoblin/db/mongo/indexes.py diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/mongo/migrations.py index cfc01287..cf4e94ae 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/mongo/migrations.py @@ -14,7 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from mediagoblin.db.util import RegisterMigration +from mediagoblin.db.mongo.util import RegisterMigration from mediagoblin.tools.text import cleaned_markdown_conversion diff --git a/mediagoblin/db/models.py b/mediagoblin/db/mongo/models.py index 569c3600..e2ac1b5a 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/mongo/models.py @@ -21,8 +21,8 @@ from mongokit import Document from mediagoblin.auth import lib as auth_lib from mediagoblin import mg_globals -from mediagoblin.db import migrations -from mediagoblin.db.util import ASCENDING, DESCENDING, ObjectId +from mediagoblin.db.mongo import migrations +from mediagoblin.db.mongo.util import ASCENDING, DESCENDING, ObjectId from mediagoblin.tools.pagination import Pagination from mediagoblin.tools import url, common diff --git a/mediagoblin/db/mongo/open.py b/mediagoblin/db/mongo/open.py new file mode 100644 index 00000000..63889292 --- /dev/null +++ b/mediagoblin/db/mongo/open.py @@ -0,0 +1,55 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import pymongo +import mongokit +from paste.deploy.converters import asint +from mediagoblin.db.mongo import models + + +def connect_database_from_config(app_config, use_pymongo=False): + """ + Connect to the main database, take config from app_config + + Optionally use pymongo instead of mongokit for the connection. + """ + port = app_config.get('db_port') + if port: + port = asint(port) + + if use_pymongo: + connection = pymongo.Connection( + app_config.get('db_host'), port) + else: + connection = mongokit.Connection( + app_config.get('db_host'), port) + return connection + + +def setup_connection_and_db_from_config(app_config, use_pymongo=False): + """ + Setup connection and database from config. + + Optionally use pymongo instead of mongokit. + """ + connection = connect_database_from_config(app_config, use_pymongo) + database_path = app_config['db_name'] + db = connection[database_path] + + if not use_pymongo: + models.register_models(connection) + + return (connection, db) diff --git a/mediagoblin/db/mongo/util.py b/mediagoblin/db/mongo/util.py new file mode 100644 index 00000000..e2065693 --- /dev/null +++ b/mediagoblin/db/mongo/util.py @@ -0,0 +1,292 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +""" +Utilities for database operations. + +Some note on migration and indexing tools: + +We store information about what the state of the database is in the +'mediagoblin' document of the 'app_metadata' collection. Keys in that +document relevant to here: + + - 'migration_number': The integer representing the current state of + the migrations +""" + +import copy + +# Imports that other modules might use +from pymongo import ASCENDING, DESCENDING +from pymongo.errors import InvalidId +from mongokit import ObjectId + +from mediagoblin.db.mongo.indexes import ACTIVE_INDEXES, DEPRECATED_INDEXES + + +################ +# Indexing tools +################ + + +def add_new_indexes(database, active_indexes=ACTIVE_INDEXES): + """ + Add any new indexes to the database. + + Args: + - database: pymongo or mongokit database instance. + - active_indexes: indexes to possibly add in the pattern of: + {'collection_name': { + 'identifier': { + 'index': [index_foo_goes_here], + 'unique': True}} + where 'index' is the index to add and all other options are + arguments for collection.create_index. + + Returns: + A list of indexes added in form ('collection', 'index_name') + """ + indexes_added = [] + + for collection_name, indexes in active_indexes.iteritems(): + collection = database[collection_name] + collection_indexes = collection.index_information().keys() + + for index_name, index_data in indexes.iteritems(): + if not index_name in collection_indexes: + # Get a copy actually so we don't modify the actual + # structure + index_data = copy.copy(index_data) + index = index_data.pop('index') + collection.create_index( + index, name=index_name, **index_data) + + indexes_added.append((collection_name, index_name)) + + return indexes_added + + +def remove_deprecated_indexes(database, deprecated_indexes=DEPRECATED_INDEXES): + """ + Remove any deprecated indexes from the database. + + Args: + - database: pymongo or mongokit database instance. + - deprecated_indexes: the indexes to deprecate in the pattern of: + {'collection_name': { + 'identifier': { + 'index': [index_foo_goes_here], + 'unique': True}} + + (... although we really only need the 'identifier' here, as the + rest of the information isn't used in this case. But it's kept + around so we can remember what it was) + + Returns: + A list of indexes removed in form ('collection', 'index_name') + """ + indexes_removed = [] + + for collection_name, indexes in deprecated_indexes.iteritems(): + collection = database[collection_name] + collection_indexes = collection.index_information().keys() + + for index_name, index_data in indexes.iteritems(): + if index_name in collection_indexes: + collection.drop_index(index_name) + + indexes_removed.append((collection_name, index_name)) + + return indexes_removed + + +################# +# Migration tools +################# + +# The default migration registry... +# +# Don't set this yourself! RegisterMigration will automatically fill +# this with stuff via decorating methods in migrations.py + +class MissingCurrentMigration(Exception): + pass + + +MIGRATIONS = {} + + +class RegisterMigration(object): + """ + Tool for registering migrations + + Call like: + + @RegisterMigration(33) + def update_dwarves(database): + [...] + + This will register your migration with the default migration + registry. Alternately, to specify a very specific + migration_registry, you can pass in that as the second argument. + + Note, the number of your migration should NEVER be 0 or less than + 0. 0 is the default "no migrations" state! + """ + def __init__(self, migration_number, migration_registry=MIGRATIONS): + assert migration_number > 0, "Migration number must be > 0!" + assert migration_number not in migration_registry, \ + "Duplicate migration numbers detected! That's not allowed!" + + self.migration_number = migration_number + self.migration_registry = migration_registry + + def __call__(self, migration): + self.migration_registry[self.migration_number] = migration + return migration + + +class MigrationManager(object): + """ + Migration handling tool. + + Takes information about a database, lets you update the database + to the latest migrations, etc. + """ + def __init__(self, database, migration_registry=MIGRATIONS): + """ + Args: + - database: database we're going to migrate + - migration_registry: where we should find all migrations to + run + """ + self.database = database + self.migration_registry = migration_registry + self._sorted_migrations = None + + def _ensure_current_migration_record(self): + """ + If there isn't a database[u'app_metadata'] mediagoblin entry + with the 'current_migration', throw an error. + """ + if self.database_current_migration() is None: + raise MissingCurrentMigration( + "Tried to call function which requires " + "'current_migration' set in database") + + @property + def sorted_migrations(self): + """ + Sort migrations if necessary and store in self._sorted_migrations + """ + if not self._sorted_migrations: + self._sorted_migrations = sorted( + self.migration_registry.items(), + # sort on the key... the migration number + key=lambda migration_tuple: migration_tuple[0]) + + return self._sorted_migrations + + def latest_migration(self): + """ + Return a migration number for the latest migration, or 0 if + there are no migrations. + """ + if self.sorted_migrations: + return self.sorted_migrations[-1][0] + else: + # If no migrations have been set, we start at 0. + return 0 + + def set_current_migration(self, migration_number): + """ + Set the migration in the database to migration_number + """ + # Add the mediagoblin migration if necessary + self.database[u'app_metadata'].update( + {u'_id': u'mediagoblin'}, + {u'$set': {u'current_migration': migration_number}}, + upsert=True) + + def install_migration_version_if_missing(self): + """ + Sets the migration to the latest version if no migration + version at all is set. + """ + mgoblin_metadata = self.database[u'app_metadata'].find_one( + {u'_id': u'mediagoblin'}) + if not mgoblin_metadata: + latest_migration = self.latest_migration() + self.set_current_migration(latest_migration) + + def database_current_migration(self): + """ + Return the current migration in the database. + """ + mgoblin_metadata = self.database[u'app_metadata'].find_one( + {u'_id': u'mediagoblin'}) + if not mgoblin_metadata: + return None + else: + return mgoblin_metadata[u'current_migration'] + + def database_at_latest_migration(self): + """ + See if the database is at the latest migration. + Returns a boolean. + """ + current_migration = self.database_current_migration() + return current_migration == self.latest_migration() + + def migrations_to_run(self): + """ + Get a list of migrations to run still, if any. + + Note that calling this will set your migration version to the + latest version if it isn't installed to anything yet! + """ + self._ensure_current_migration_record() + + db_current_migration = self.database_current_migration() + + return [ + (migration_number, migration_func) + for migration_number, migration_func in self.sorted_migrations + if migration_number > db_current_migration] + + def migrate_new(self, pre_callback=None, post_callback=None): + """ + Run all migrations. + + Includes two optional args: + - pre_callback: if called, this is a callback on something to + run pre-migration. Takes (migration_number, migration_func) + as arguments + - pre_callback: if called, this is a callback on something to + run post-migration. Takes (migration_number, migration_func) + as arguments + """ + # If we aren't set to any version number, presume we're at the + # latest (which means we'll do nothing here...) + self.install_migration_version_if_missing() + + for migration_number, migration_func in self.migrations_to_run(): + if pre_callback: + pre_callback(migration_number, migration_func) + migration_func(self.database) + self.set_current_migration(migration_number) + if post_callback: + post_callback(migration_number, migration_func) diff --git a/mediagoblin/db/open.py b/mediagoblin/db/open.py index e677ba12..a92a6ada 100644 --- a/mediagoblin/db/open.py +++ b/mediagoblin/db/open.py @@ -14,42 +14,4 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import pymongo -import mongokit -from paste.deploy.converters import asint -from mediagoblin.db import models - - -def connect_database_from_config(app_config, use_pymongo=False): - """ - Connect to the main database, take config from app_config - - Optionally use pymongo instead of mongokit for the connection. - """ - port = app_config.get('db_port') - if port: - port = asint(port) - - if use_pymongo: - connection = pymongo.Connection( - app_config.get('db_host'), port) - else: - connection = mongokit.Connection( - app_config.get('db_host'), port) - return connection - - -def setup_connection_and_db_from_config(app_config, use_pymongo=False): - """ - Setup connection and database from config. - - Optionally use pymongo instead of mongokit. - """ - connection = connect_database_from_config(app_config, use_pymongo) - database_path = app_config['db_name'] - db = connection[database_path] - - if not use_pymongo: - models.register_models(connection) - - return (connection, db) +from mediagoblin.db.mongo.open import setup_connection_and_db_from_config diff --git a/mediagoblin/db/sql/__init__.py b/mediagoblin/db/sql/__init__.py new file mode 100644 index 00000000..ba347c69 --- /dev/null +++ b/mediagoblin/db/sql/__init__.py @@ -0,0 +1,15 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. diff --git a/mediagoblin/db/sql/base.py b/mediagoblin/db/sql/base.py new file mode 100644 index 00000000..b8d5cc96 --- /dev/null +++ b/mediagoblin/db/sql/base.py @@ -0,0 +1,16 @@ +from sqlalchemy.orm import scoped_session, sessionmaker + + +Session = scoped_session(sessionmaker()) + + +class GMGTableBase(object): + query = Session.query_property() + + @classmethod + def find(cls, query_dict={}): + return cls.query.filter_by(**query_dict) + + @classmethod + def find_one(cls, query_dict={}): + return cls.query.filter_by(**query_dict).first() diff --git a/mediagoblin/db/sql/convert.py b/mediagoblin/db/sql/convert.py new file mode 100644 index 00000000..6de758ed --- /dev/null +++ b/mediagoblin/db/sql/convert.py @@ -0,0 +1,148 @@ +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from mediagoblin.init import setup_global_and_app_config, setup_database +from mediagoblin.db.util import ObjectId + +from mediagoblin.db.sql.models import (Base, User, MediaEntry, MediaComment, + Tag, MediaTag) + +# Session = sessionmaker() +from mediagoblin.db.sql.base import Session + + +obj_id_table = dict() + +def add_obj_ids(entry, new_entry): + global obj_id_table + print "%r -> %r" % (entry._id, new_entry.id) + obj_id_table[entry._id] = new_entry.id + + +def copy_attrs(entry, new_entry, attr_list): + for a in attr_list: + val = entry[a] + setattr(new_entry, a, val) + +def copy_reference_attr(entry, new_entry, ref_attr): + val = entry[ref_attr] + val = obj_id_table[val] + setattr(new_entry, ref_attr, val) + + +def convert_users(mk_db): + session = Session() + + for entry in mk_db.User.find(): + print entry.username + + new_entry = User() + copy_attrs(entry, new_entry, + ('username', 'email', 'created', 'pw_hash', 'email_verified', + 'status', 'verification_key', 'is_admin', 'url', + 'bio', 'bio_html', + 'fp_verification_key', 'fp_token_expire',)) + # new_entry.fp_verification_expire = entry.fp_token_expire + + session.add(new_entry) + session.flush() + add_obj_ids(entry, new_entry) + + session.commit() + session.close() + + +def convert_media_entries(mk_db): + session = Session() + + for entry in mk_db.MediaEntry.find(): + print repr(entry.title) + + new_entry = MediaEntry() + copy_attrs(entry, new_entry, + ('title', 'slug', 'created', + 'description', 'description_html', + 'media_type', + 'fail_error', + 'queued_task_id',)) + copy_reference_attr(entry, new_entry, "uploader") + + session.add(new_entry) + session.flush() + add_obj_ids(entry, new_entry) + + session.commit() + session.close() + + +def convert_media_tags(mk_db): + session = Session() + session.autoflush = False + + for media in mk_db.MediaEntry.find(): + print repr(media.title) + + for otag in media.tags: + print " ", repr((otag["slug"], otag["name"])) + + nslug = session.query(Tag).filter_by(slug=otag["slug"]).first() + print " ", repr(nslug) + if nslug is None: + nslug = Tag(slug=otag["slug"]) + session.add(nslug) + session.flush() + print " ", repr(nslug), nslug.id + + ntag = MediaTag() + ntag.tag = nslug.id + ntag.name = otag["name"] + ntag.media_entry = obj_id_table[media._id] + session.add(ntag) + + session.commit() + session.close() + + +def convert_media_comments(mk_db): + session = Session() + + for entry in mk_db.MediaComment.find(): + print repr(entry.content) + + new_entry = MediaComment() + copy_attrs(entry, new_entry, + ('created', + 'content', 'content_html',)) + copy_reference_attr(entry, new_entry, "media_entry") + copy_reference_attr(entry, new_entry, "author") + + session.add(new_entry) + session.flush() + add_obj_ids(entry, new_entry) + + session.commit() + session.close() + + +def main(): + engine = create_engine('sqlite:///mediagoblin.db', echo=True) + Session.configure(bind=engine) + + setup_global_and_app_config("mediagoblin.ini") + + mk_conn, mk_db = setup_database() + + Base.metadata.create_all(engine) + + convert_users(mk_db) + Session.remove() + convert_media_entries(mk_db) + Session.remove() + convert_media_tags(mk_db) + Session.remove() + convert_media_comments(mk_db) + Session.remove() + + +if __name__ == '__main__': + main() diff --git a/mediagoblin/db/sql/models.py b/mediagoblin/db/sql/models.py new file mode 100644 index 00000000..b87ff3aa --- /dev/null +++ b/mediagoblin/db/sql/models.py @@ -0,0 +1,109 @@ +import datetime + +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import ( + Column, Integer, Unicode, UnicodeText, DateTime, Boolean, ForeignKey, + UniqueConstraint) + +from mediagoblin.db.sql.base import GMGTableBase + + +Base = declarative_base(cls=GMGTableBase) + + +class User(Base): + __tablename__ = "users" + + id = Column(Integer, primary_key=True) + username = Column(Unicode, nullable=False, unique=True) + email = Column(Unicode, nullable=False) + created = Column(DateTime, nullable=False, default=datetime.datetime.now) + pw_hash = Column(Unicode, nullable=False) + email_verified = Column(Boolean) + status = Column(Unicode, default=u"needs_email_verification", nullable=False) + verification_key = Column(Unicode) + is_admin = Column(Boolean, default=False, nullable=False) + url = Column(Unicode) + bio = Column(UnicodeText) # ?? + bio_html = Column(UnicodeText) # ?? + fp_verification_key = Column(Unicode) + fp_token_expire = Column(DateTime) + + ## TODO + # plugin data would be in a separate model + + +class MediaEntry(Base): + __tablename__ = "media_entries" + + id = Column(Integer, primary_key=True) + uploader = Column(Integer, ForeignKey('users.id'), nullable=False) + title = Column(Unicode, nullable=False) + slug = Column(Unicode, nullable=False) + created = Column(DateTime, nullable=False, default=datetime.datetime.now) + description = Column(UnicodeText) # ?? + description_html = Column(UnicodeText) # ?? + media_type = Column(Unicode, nullable=False) + + fail_error = Column(Unicode) + fail_metadata = Column(UnicodeText) + + queued_media_file = Column(Unicode) + + queued_task_id = Column(Unicode) + + __table_args__ = ( + UniqueConstraint('uploader', 'slug'), + {}) + + ## TODO + # media_files + # media_data + # attachment_files + # fail_error + + +class Tag(Base): + __tablename__ = "tags" + + id = Column(Integer, primary_key=True) + slug = Column(Unicode, nullable=False, unique=True) + + +class MediaTag(Base): + __tablename__ = "media_tags" + + id = Column(Integer, primary_key=True) + tag = Column(Integer, ForeignKey('tags.id'), nullable=False) + name = Column(Unicode) + media_entry = Column( + Integer, ForeignKey('media_entries.id'), + nullable=False) + # created = Column(DateTime, nullable=False, default=datetime.datetime.now) + + __table_args__ = ( + UniqueConstraint('tag', 'media_entry'), + {}) + + +class MediaComment(Base): + __tablename__ = "media_comments" + + id = Column(Integer, primary_key=True) + media_entry = Column( + Integer, ForeignKey('media_entries.id'), nullable=False) + author = Column(Integer, ForeignKey('users.id'), nullable=False) + created = Column(DateTime, nullable=False, default=datetime.datetime.now) + content = Column(UnicodeText, nullable=False) + content_html = Column(UnicodeText) + + +def show_table_init(): + from sqlalchemy import create_engine + engine = create_engine('sqlite:///:memory:', echo=True) + + Base.metadata.create_all(engine) + + +if __name__ == '__main__': + show_table_init() diff --git a/mediagoblin/db/sql/open.py b/mediagoblin/db/sql/open.py new file mode 100644 index 00000000..57feaf50 --- /dev/null +++ b/mediagoblin/db/sql/open.py @@ -0,0 +1,29 @@ +from sqlalchemy import create_engine + +from mediagoblin.db.sql.base import Session +from mediagoblin.db.sql.models import Base + + +class DatabaseMaster(object): + def __init__(self, engine): + self.engine = engine + + for k,v in Base._decl_class_registry.iteritems(): + setattr(self, k, v) + + def commit(self): + Session.commit() + + def save(self, obj): + Session.add(obj) + Session.flush() + + def reset_after_request(self): + Session.remove() + + +def setup_connection_and_db_from_config(app_config): + engine = create_engine(app_config['sql_engine'], echo=True) + Session.configure(bind=engine) + + return "dummy conn", DatabaseMaster(engine) diff --git a/mediagoblin/db/util.py b/mediagoblin/db/util.py index 52e97f6d..3fd96a1d 100644 --- a/mediagoblin/db/util.py +++ b/mediagoblin/db/util.py @@ -14,279 +14,5 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Utilities for database operations. - -Some note on migration and indexing tools: - -We store information about what the state of the database is in the -'mediagoblin' document of the 'app_metadata' collection. Keys in that -document relevant to here: - - - 'migration_number': The integer representing the current state of - the migrations -""" - -import copy - -# Imports that other modules might use -from pymongo import ASCENDING, DESCENDING -from pymongo.errors import InvalidId -from mongokit import ObjectId - -from mediagoblin.db.indexes import ACTIVE_INDEXES, DEPRECATED_INDEXES - - -################ -# Indexing tools -################ - - -def add_new_indexes(database, active_indexes=ACTIVE_INDEXES): - """ - Add any new indexes to the database. - - Args: - - database: pymongo or mongokit database instance. - - active_indexes: indexes to possibly add in the pattern of: - {'collection_name': { - 'identifier': { - 'index': [index_foo_goes_here], - 'unique': True}} - where 'index' is the index to add and all other options are - arguments for collection.create_index. - - Returns: - A list of indexes added in form ('collection', 'index_name') - """ - indexes_added = [] - - for collection_name, indexes in active_indexes.iteritems(): - collection = database[collection_name] - collection_indexes = collection.index_information().keys() - - for index_name, index_data in indexes.iteritems(): - if not index_name in collection_indexes: - # Get a copy actually so we don't modify the actual - # structure - index_data = copy.copy(index_data) - index = index_data.pop('index') - collection.create_index( - index, name=index_name, **index_data) - - indexes_added.append((collection_name, index_name)) - - return indexes_added - - -def remove_deprecated_indexes(database, deprecated_indexes=DEPRECATED_INDEXES): - """ - Remove any deprecated indexes from the database. - - Args: - - database: pymongo or mongokit database instance. - - deprecated_indexes: the indexes to deprecate in the pattern of: - {'collection_name': { - 'identifier': { - 'index': [index_foo_goes_here], - 'unique': True}} - - (... although we really only need the 'identifier' here, as the - rest of the information isn't used in this case. But it's kept - around so we can remember what it was) - - Returns: - A list of indexes removed in form ('collection', 'index_name') - """ - indexes_removed = [] - - for collection_name, indexes in deprecated_indexes.iteritems(): - collection = database[collection_name] - collection_indexes = collection.index_information().keys() - - for index_name, index_data in indexes.iteritems(): - if index_name in collection_indexes: - collection.drop_index(index_name) - - indexes_removed.append((collection_name, index_name)) - - return indexes_removed - - -################# -# Migration tools -################# - -# The default migration registry... -# -# Don't set this yourself! RegisterMigration will automatically fill -# this with stuff via decorating methods in migrations.py - -class MissingCurrentMigration(Exception): - pass - - -MIGRATIONS = {} - - -class RegisterMigration(object): - """ - Tool for registering migrations - - Call like: - - @RegisterMigration(33) - def update_dwarves(database): - [...] - - This will register your migration with the default migration - registry. Alternately, to specify a very specific - migration_registry, you can pass in that as the second argument. - - Note, the number of your migration should NEVER be 0 or less than - 0. 0 is the default "no migrations" state! - """ - def __init__(self, migration_number, migration_registry=MIGRATIONS): - assert migration_number > 0, "Migration number must be > 0!" - assert migration_number not in migration_registry, \ - "Duplicate migration numbers detected! That's not allowed!" - - self.migration_number = migration_number - self.migration_registry = migration_registry - - def __call__(self, migration): - self.migration_registry[self.migration_number] = migration - return migration - - -class MigrationManager(object): - """ - Migration handling tool. - - Takes information about a database, lets you update the database - to the latest migrations, etc. - """ - def __init__(self, database, migration_registry=MIGRATIONS): - """ - Args: - - database: database we're going to migrate - - migration_registry: where we should find all migrations to - run - """ - self.database = database - self.migration_registry = migration_registry - self._sorted_migrations = None - - def _ensure_current_migration_record(self): - """ - If there isn't a database[u'app_metadata'] mediagoblin entry - with the 'current_migration', throw an error. - """ - if self.database_current_migration() is None: - raise MissingCurrentMigration( - "Tried to call function which requires " - "'current_migration' set in database") - - @property - def sorted_migrations(self): - """ - Sort migrations if necessary and store in self._sorted_migrations - """ - if not self._sorted_migrations: - self._sorted_migrations = sorted( - self.migration_registry.items(), - # sort on the key... the migration number - key=lambda migration_tuple: migration_tuple[0]) - - return self._sorted_migrations - - def latest_migration(self): - """ - Return a migration number for the latest migration, or 0 if - there are no migrations. - """ - if self.sorted_migrations: - return self.sorted_migrations[-1][0] - else: - # If no migrations have been set, we start at 0. - return 0 - - def set_current_migration(self, migration_number): - """ - Set the migration in the database to migration_number - """ - # Add the mediagoblin migration if necessary - self.database[u'app_metadata'].update( - {u'_id': u'mediagoblin'}, - {u'$set': {u'current_migration': migration_number}}, - upsert=True) - - def install_migration_version_if_missing(self): - """ - Sets the migration to the latest version if no migration - version at all is set. - """ - mgoblin_metadata = self.database[u'app_metadata'].find_one( - {u'_id': u'mediagoblin'}) - if not mgoblin_metadata: - latest_migration = self.latest_migration() - self.set_current_migration(latest_migration) - - def database_current_migration(self): - """ - Return the current migration in the database. - """ - mgoblin_metadata = self.database[u'app_metadata'].find_one( - {u'_id': u'mediagoblin'}) - if not mgoblin_metadata: - return None - else: - return mgoblin_metadata[u'current_migration'] - - def database_at_latest_migration(self): - """ - See if the database is at the latest migration. - Returns a boolean. - """ - current_migration = self.database_current_migration() - return current_migration == self.latest_migration() - - def migrations_to_run(self): - """ - Get a list of migrations to run still, if any. - - Note that calling this will set your migration version to the - latest version if it isn't installed to anything yet! - """ - self._ensure_current_migration_record() - - db_current_migration = self.database_current_migration() - - return [ - (migration_number, migration_func) - for migration_number, migration_func in self.sorted_migrations - if migration_number > db_current_migration] - - def migrate_new(self, pre_callback=None, post_callback=None): - """ - Run all migrations. - - Includes two optional args: - - pre_callback: if called, this is a callback on something to - run pre-migration. Takes (migration_number, migration_func) - as arguments - - pre_callback: if called, this is a callback on something to - run post-migration. Takes (migration_number, migration_func) - as arguments - """ - # If we aren't set to any version number, presume we're at the - # latest (which means we'll do nothing here...) - self.install_migration_version_if_missing() - - for migration_number, migration_func in self.migrations_to_run(): - if pre_callback: - pre_callback(migration_number, migration_func) - migration_func(self.database) - self.set_current_migration(migration_number) - if post_callback: - post_callback(migration_number, migration_func) +from mediagoblin.db.mongo.util import (MigrationManager, ObjectId, InvalidId, + DESCENDING) diff --git a/mediagoblin/gmg_commands/import_export.py b/mediagoblin/gmg_commands/import_export.py index 1308f09e..eda41f4c 100644 --- a/mediagoblin/gmg_commands/import_export.py +++ b/mediagoblin/gmg_commands/import_export.py @@ -64,7 +64,7 @@ def _import_media(db, args): queue_cache = BasicFileStorage( args._cache_path['queue']) - for entry in db.media_entries.find(): + for entry in db.MediaEntry.find(): for name, path in entry['media_files'].items(): _log.info('Importing: {0} - {1}'.format( entry.title, @@ -107,7 +107,7 @@ def env_import(args): global_config, app_config = setup_global_and_app_config(args.conf_file) connection, db = setup_connection_and_db_from_config( - app_config, use_pymongo=True) + app_config) tf = tarfile.open( args.tar_file, @@ -206,7 +206,7 @@ def _export_media(db, args): queue_cache = BasicFileStorage( args._cache_path['queue']) - for entry in db.media_entries.find(): + for entry in db.MediaEntry.find(): for name, path in entry['media_files'].items(): _log.info(u'Exporting {0} - {1}'.format( entry.title, @@ -215,7 +215,7 @@ def _export_media(db, args): mc_file = media_cache.get_file(path, mode='wb') mc_file.write( mg_globals.public_store.get_file(path, mode='rb').read()) - except e: + except Exception as e: _log.error('Failed: {0}'.format(e)) _log.info('...Media exported') @@ -246,7 +246,7 @@ def env_export(args): setup_storage() connection, db = setup_connection_and_db_from_config( - app_config, use_pymongo=True) + app_config) _export_database(db, args) diff --git a/mediagoblin/init/__init__.py b/mediagoblin/init/__init__.py index 08a0618d..5f7f83d4 100644 --- a/mediagoblin/init/__init__.py +++ b/mediagoblin/init/__init__.py @@ -57,7 +57,7 @@ def setup_database(): app_config = mg_globals.app_config # This MUST be imported so as to set up the appropriate migrations! - from mediagoblin.db import migrations + from mediagoblin.db.mongo import migrations # Set up the database connection, db = setup_connection_and_db_from_config(app_config) diff --git a/mediagoblin/templates/mediagoblin/user_pages/media.html b/mediagoblin/templates/mediagoblin/user_pages/media.html index 2c8c5033..b9e31667 100644 --- a/mediagoblin/templates/mediagoblin/user_pages/media.html +++ b/mediagoblin/templates/mediagoblin/user_pages/media.html @@ -81,7 +81,7 @@ <a class="button_action" href="{{ delete_url }}">{% trans %}Delete{% endtrans %}</a> {% endif %} </p> - <h3>{% trans %}23 comments{% endtrans %} + <h3>{% trans comment_count=comments.count() -%}{{ comment_count }} comments{%- endtrans %} <div class="right_align"> <a {% if not request.user %} diff --git a/mediagoblin/tests/test_migrations.py b/mediagoblin/tests/test_migrations.py index e7cef0a1..8e573f5a 100644 --- a/mediagoblin/tests/test_migrations.py +++ b/mediagoblin/tests/test_migrations.py @@ -20,10 +20,10 @@ from pymongo import Connection from mediagoblin.tests.tools import ( install_fixtures_simple, assert_db_meets_expected) -from mediagoblin.db.util import ( +from mediagoblin.db.mongo.util import ( RegisterMigration, MigrationManager, ObjectId, MissingCurrentMigration) -from mediagoblin.db.migrations import add_table_field +from mediagoblin.db.mongo.migrations import add_table_field # This one will get filled with local migrations TEST_MIGRATION_REGISTRY = {} |