diff options
Diffstat (limited to 'mediagoblin/db')
| -rw-r--r-- | mediagoblin/db/__init__.py | 2 | ||||
| -rw-r--r-- | mediagoblin/db/mixin.py | 92 | ||||
| -rw-r--r-- | mediagoblin/db/mongo/__init__.py | 15 | ||||
| -rw-r--r-- | mediagoblin/db/mongo/indexes.py (renamed from mediagoblin/db/indexes.py) | 5 | ||||
| -rw-r--r-- | mediagoblin/db/mongo/migrations.py (renamed from mediagoblin/db/migrations.py) | 10 | ||||
| -rw-r--r-- | mediagoblin/db/mongo/models.py (renamed from mediagoblin/db/models.py) | 0 | ||||
| -rw-r--r-- | mediagoblin/db/mongo/open.py | 78 | ||||
| -rw-r--r-- | mediagoblin/db/mongo/util.py | 292 | ||||
| -rw-r--r-- | mediagoblin/db/open.py | 41 | ||||
| -rw-r--r-- | mediagoblin/db/sql/__init__.py | 15 | ||||
| -rw-r--r-- | mediagoblin/db/sql/base.py | 38 | ||||
| -rw-r--r-- | mediagoblin/db/sql/convert.py | 151 | ||||
| -rw-r--r-- | mediagoblin/db/sql/extratypes.py | 18 | ||||
| -rw-r--r-- | mediagoblin/db/sql/models.py | 153 | ||||
| -rw-r--r-- | mediagoblin/db/sql/open.py | 33 | ||||
| -rw-r--r-- | mediagoblin/db/util.py | 277 |
16 files changed, 902 insertions, 318 deletions
diff --git a/mediagoblin/db/__init__.py b/mediagoblin/db/__init__.py index c5124b1a..27e8a90f 100644 --- a/mediagoblin/db/__init__.py +++ b/mediagoblin/db/__init__.py @@ -23,7 +23,7 @@ Database Abstraction/Wrapper Layer pymongo. Read beow for why, but note that nobody is actually doing this and there's no proof that we'll ever support more than MongoDB... it would be a huge amount of work to do so. - + If you really want to prove that possible, jump on IRC and talk to us about making such a branch. In the meanwhile, it doesn't hurt to have things as they are... if it ever makes it hard for us to diff --git a/mediagoblin/db/mixin.py b/mediagoblin/db/mixin.py new file mode 100644 index 00000000..5145289e --- /dev/null +++ b/mediagoblin/db/mixin.py @@ -0,0 +1,92 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011,2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +""" +This module contains some Mixin classes for the db objects. + +A bunch of functions on the db objects are really more like +"utility functions": They could live outside the classes +and be called "by hand" passing the appropiate reference. +They usually only use the public API of the object and +rarely use database related stuff. + +These functions now live here and get "mixed in" into the +real objects. +""" + +from mediagoblin.auth import lib as auth_lib +from mediagoblin.tools import common + + +class UserMixin(object): + def check_login(self, password): + """ + See if a user can login with this password + """ + return auth_lib.bcrypt_check_password( + password, self.pw_hash) + + +class MediaEntryMixin(object): + def get_display_media(self, media_map, + fetch_order=common.DISPLAY_IMAGE_FETCHING_ORDER): + """ + Find the best media for display. + + Args: + - media_map: a dict like + {u'image_size': [u'dir1', u'dir2', u'image.jpg']} + - fetch_order: the order we should try fetching images in + + Returns: + (media_size, media_path) + """ + media_sizes = media_map.keys() + + for media_size in common.DISPLAY_IMAGE_FETCHING_ORDER: + if media_size in media_sizes: + return media_map[media_size] + + def main_mediafile(self): + pass + + def url_for_self(self, urlgen, **extra_args): + """ + Generate an appropriate url for ourselves + + Use a slug if we have one, else use our '_id'. + """ + uploader = self.get_uploader + + if self.get('slug'): + return urlgen( + 'mediagoblin.user_pages.media_home', + user=uploader.username, + media=self.slug, + **extra_args) + else: + return urlgen( + 'mediagoblin.user_pages.media_home', + user=uploader.username, + media=unicode(self._id), + **extra_args) + + def get_fail_exception(self): + """ + Get the exception that's appropriate for this error + """ + if self['fail_error']: + return common.import_component(self['fail_error']) diff --git a/mediagoblin/db/mongo/__init__.py b/mediagoblin/db/mongo/__init__.py new file mode 100644 index 00000000..ba347c69 --- /dev/null +++ b/mediagoblin/db/mongo/__init__.py @@ -0,0 +1,15 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. diff --git a/mediagoblin/db/indexes.py b/mediagoblin/db/mongo/indexes.py index 75394a31..1dd73f2b 100644 --- a/mediagoblin/db/indexes.py +++ b/mediagoblin/db/mongo/indexes.py @@ -93,8 +93,9 @@ MEDIAENTRY_INDEXES = { ('created', DESCENDING)]}, 'state_uploader_tags_created': { - # Indexing on processed?, media uploader, associated tags, and timestamp - # Used for showing media items matching a tag search, most recent first. + # Indexing on processed?, media uploader, associated tags, and + # timestamp Used for showing media items matching a tag + # search, most recent first. 'index': [('state', ASCENDING), ('uploader', ASCENDING), ('tags.slug', DESCENDING), diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/mongo/migrations.py index edaf5630..cf4e94ae 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/mongo/migrations.py @@ -14,7 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from mediagoblin.db.util import RegisterMigration +from mediagoblin.db.mongo.util import RegisterMigration from mediagoblin.tools.text import cleaned_markdown_conversion @@ -100,3 +100,11 @@ def user_add_forgot_password_token_and_expires(database): """ add_table_field(database, 'users', 'fp_verification_key', None) add_table_field(database, 'users', 'fp_token_expire', None) + + +@RegisterMigration(7) +def media_type_image_to_multimedia_type_image(database): + database['media_entries'].update( + {'media_type': 'image'}, + {'$set': {'media_type': 'mediagoblin.media_types.image'}}, + multi=True) diff --git a/mediagoblin/db/models.py b/mediagoblin/db/mongo/models.py index e085840e..e085840e 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/mongo/models.py diff --git a/mediagoblin/db/mongo/open.py b/mediagoblin/db/mongo/open.py new file mode 100644 index 00000000..48c909d9 --- /dev/null +++ b/mediagoblin/db/mongo/open.py @@ -0,0 +1,78 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import pymongo +import mongokit +from paste.deploy.converters import asint +from mediagoblin.db.mongo import models +from mediagoblin.db.mongo.util import MigrationManager + + +def connect_database_from_config(app_config, use_pymongo=False): + """ + Connect to the main database, take config from app_config + + Optionally use pymongo instead of mongokit for the connection. + """ + port = app_config.get('db_port') + if port: + port = asint(port) + + if use_pymongo: + connection = pymongo.Connection( + app_config.get('db_host'), port) + else: + connection = mongokit.Connection( + app_config.get('db_host'), port) + return connection + + +def setup_connection_and_db_from_config(app_config, use_pymongo=False): + """ + Setup connection and database from config. + + Optionally use pymongo instead of mongokit. + """ + connection = connect_database_from_config(app_config, use_pymongo) + database_path = app_config['db_name'] + db = connection[database_path] + + if not use_pymongo: + models.register_models(connection) + + return (connection, db) + + +def check_db_migrations_current(db): + # This MUST be imported so as to set up the appropriate migrations! + from mediagoblin.db.mongo import migrations + + # Init the migration number if necessary + migration_manager = MigrationManager(db) + migration_manager.install_migration_version_if_missing() + + # Tiny hack to warn user if our migration is out of date + if not migration_manager.database_at_latest_migration(): + db_migration_num = migration_manager.database_current_migration() + latest_migration_num = migration_manager.latest_migration() + if db_migration_num < latest_migration_num: + print ( + "*WARNING:* Your migrations are out of date, " + "maybe run ./bin/gmg migrate?") + elif db_migration_num > latest_migration_num: + print ( + "*WARNING:* Your migrations are out of date... " + "in fact they appear to be from the future?!") diff --git a/mediagoblin/db/mongo/util.py b/mediagoblin/db/mongo/util.py new file mode 100644 index 00000000..e2065693 --- /dev/null +++ b/mediagoblin/db/mongo/util.py @@ -0,0 +1,292 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +""" +Utilities for database operations. + +Some note on migration and indexing tools: + +We store information about what the state of the database is in the +'mediagoblin' document of the 'app_metadata' collection. Keys in that +document relevant to here: + + - 'migration_number': The integer representing the current state of + the migrations +""" + +import copy + +# Imports that other modules might use +from pymongo import ASCENDING, DESCENDING +from pymongo.errors import InvalidId +from mongokit import ObjectId + +from mediagoblin.db.mongo.indexes import ACTIVE_INDEXES, DEPRECATED_INDEXES + + +################ +# Indexing tools +################ + + +def add_new_indexes(database, active_indexes=ACTIVE_INDEXES): + """ + Add any new indexes to the database. + + Args: + - database: pymongo or mongokit database instance. + - active_indexes: indexes to possibly add in the pattern of: + {'collection_name': { + 'identifier': { + 'index': [index_foo_goes_here], + 'unique': True}} + where 'index' is the index to add and all other options are + arguments for collection.create_index. + + Returns: + A list of indexes added in form ('collection', 'index_name') + """ + indexes_added = [] + + for collection_name, indexes in active_indexes.iteritems(): + collection = database[collection_name] + collection_indexes = collection.index_information().keys() + + for index_name, index_data in indexes.iteritems(): + if not index_name in collection_indexes: + # Get a copy actually so we don't modify the actual + # structure + index_data = copy.copy(index_data) + index = index_data.pop('index') + collection.create_index( + index, name=index_name, **index_data) + + indexes_added.append((collection_name, index_name)) + + return indexes_added + + +def remove_deprecated_indexes(database, deprecated_indexes=DEPRECATED_INDEXES): + """ + Remove any deprecated indexes from the database. + + Args: + - database: pymongo or mongokit database instance. + - deprecated_indexes: the indexes to deprecate in the pattern of: + {'collection_name': { + 'identifier': { + 'index': [index_foo_goes_here], + 'unique': True}} + + (... although we really only need the 'identifier' here, as the + rest of the information isn't used in this case. But it's kept + around so we can remember what it was) + + Returns: + A list of indexes removed in form ('collection', 'index_name') + """ + indexes_removed = [] + + for collection_name, indexes in deprecated_indexes.iteritems(): + collection = database[collection_name] + collection_indexes = collection.index_information().keys() + + for index_name, index_data in indexes.iteritems(): + if index_name in collection_indexes: + collection.drop_index(index_name) + + indexes_removed.append((collection_name, index_name)) + + return indexes_removed + + +################# +# Migration tools +################# + +# The default migration registry... +# +# Don't set this yourself! RegisterMigration will automatically fill +# this with stuff via decorating methods in migrations.py + +class MissingCurrentMigration(Exception): + pass + + +MIGRATIONS = {} + + +class RegisterMigration(object): + """ + Tool for registering migrations + + Call like: + + @RegisterMigration(33) + def update_dwarves(database): + [...] + + This will register your migration with the default migration + registry. Alternately, to specify a very specific + migration_registry, you can pass in that as the second argument. + + Note, the number of your migration should NEVER be 0 or less than + 0. 0 is the default "no migrations" state! + """ + def __init__(self, migration_number, migration_registry=MIGRATIONS): + assert migration_number > 0, "Migration number must be > 0!" + assert migration_number not in migration_registry, \ + "Duplicate migration numbers detected! That's not allowed!" + + self.migration_number = migration_number + self.migration_registry = migration_registry + + def __call__(self, migration): + self.migration_registry[self.migration_number] = migration + return migration + + +class MigrationManager(object): + """ + Migration handling tool. + + Takes information about a database, lets you update the database + to the latest migrations, etc. + """ + def __init__(self, database, migration_registry=MIGRATIONS): + """ + Args: + - database: database we're going to migrate + - migration_registry: where we should find all migrations to + run + """ + self.database = database + self.migration_registry = migration_registry + self._sorted_migrations = None + + def _ensure_current_migration_record(self): + """ + If there isn't a database[u'app_metadata'] mediagoblin entry + with the 'current_migration', throw an error. + """ + if self.database_current_migration() is None: + raise MissingCurrentMigration( + "Tried to call function which requires " + "'current_migration' set in database") + + @property + def sorted_migrations(self): + """ + Sort migrations if necessary and store in self._sorted_migrations + """ + if not self._sorted_migrations: + self._sorted_migrations = sorted( + self.migration_registry.items(), + # sort on the key... the migration number + key=lambda migration_tuple: migration_tuple[0]) + + return self._sorted_migrations + + def latest_migration(self): + """ + Return a migration number for the latest migration, or 0 if + there are no migrations. + """ + if self.sorted_migrations: + return self.sorted_migrations[-1][0] + else: + # If no migrations have been set, we start at 0. + return 0 + + def set_current_migration(self, migration_number): + """ + Set the migration in the database to migration_number + """ + # Add the mediagoblin migration if necessary + self.database[u'app_metadata'].update( + {u'_id': u'mediagoblin'}, + {u'$set': {u'current_migration': migration_number}}, + upsert=True) + + def install_migration_version_if_missing(self): + """ + Sets the migration to the latest version if no migration + version at all is set. + """ + mgoblin_metadata = self.database[u'app_metadata'].find_one( + {u'_id': u'mediagoblin'}) + if not mgoblin_metadata: + latest_migration = self.latest_migration() + self.set_current_migration(latest_migration) + + def database_current_migration(self): + """ + Return the current migration in the database. + """ + mgoblin_metadata = self.database[u'app_metadata'].find_one( + {u'_id': u'mediagoblin'}) + if not mgoblin_metadata: + return None + else: + return mgoblin_metadata[u'current_migration'] + + def database_at_latest_migration(self): + """ + See if the database is at the latest migration. + Returns a boolean. + """ + current_migration = self.database_current_migration() + return current_migration == self.latest_migration() + + def migrations_to_run(self): + """ + Get a list of migrations to run still, if any. + + Note that calling this will set your migration version to the + latest version if it isn't installed to anything yet! + """ + self._ensure_current_migration_record() + + db_current_migration = self.database_current_migration() + + return [ + (migration_number, migration_func) + for migration_number, migration_func in self.sorted_migrations + if migration_number > db_current_migration] + + def migrate_new(self, pre_callback=None, post_callback=None): + """ + Run all migrations. + + Includes two optional args: + - pre_callback: if called, this is a callback on something to + run pre-migration. Takes (migration_number, migration_func) + as arguments + - pre_callback: if called, this is a callback on something to + run post-migration. Takes (migration_number, migration_func) + as arguments + """ + # If we aren't set to any version number, presume we're at the + # latest (which means we'll do nothing here...) + self.install_migration_version_if_missing() + + for migration_number, migration_func in self.migrations_to_run(): + if pre_callback: + pre_callback(migration_number, migration_func) + migration_func(self.database) + self.set_current_migration(migration_number) + if post_callback: + post_callback(migration_number, migration_func) diff --git a/mediagoblin/db/open.py b/mediagoblin/db/open.py index e73b6258..32827fcb 100644 --- a/mediagoblin/db/open.py +++ b/mediagoblin/db/open.py @@ -14,42 +14,5 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import pymongo -import mongokit -from paste.deploy.converters import asint -from mediagoblin.db import models - - -def connect_database_from_config(app_config, use_pymongo=False): - """ - Connect to the main database, take config from app_config - - Optionally use pymongo instead of mongokit for the connection. - """ - port = app_config.get('db_port') - if port: - port = asint(port) - - if use_pymongo: - connection = pymongo.Connection( - app_config.get('db_host'), port) - else: - connection = mongokit.Connection( - app_config.get('db_host'), port) - return connection - - -def setup_connection_and_db_from_config(app_config, use_pymongo=False): - """ - Setup connection and database from config. - - Optionally use pymongo instead of mongokit. - """ - connection = connect_database_from_config(app_config, use_pymongo) - database_path = app_config['db_name'] - db = connection[database_path] - - if not use_pymongo: - models.register_models(connection) - - return (connection, db) +from mediagoblin.db.mongo.open import \ + setup_connection_and_db_from_config, check_db_migrations_current diff --git a/mediagoblin/db/sql/__init__.py b/mediagoblin/db/sql/__init__.py new file mode 100644 index 00000000..ba347c69 --- /dev/null +++ b/mediagoblin/db/sql/__init__.py @@ -0,0 +1,15 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. diff --git a/mediagoblin/db/sql/base.py b/mediagoblin/db/sql/base.py new file mode 100644 index 00000000..40140327 --- /dev/null +++ b/mediagoblin/db/sql/base.py @@ -0,0 +1,38 @@ +from sqlalchemy.orm import scoped_session, sessionmaker, object_session + + +Session = scoped_session(sessionmaker()) + + +def _fix_query_dict(query_dict): + if '_id' in query_dict: + query_dict['id'] = query_dict.pop('_id') + + +class GMGTableBase(object): + query = Session.query_property() + + @classmethod + def find(cls, query_dict={}): + _fix_query_dict(query_dict) + return cls.query.filter_by(**query_dict) + + @classmethod + def find_one(cls, query_dict={}): + _fix_query_dict(query_dict) + return cls.query.filter_by(**query_dict).first() + + @classmethod + def one(cls, query_dict): + return cls.find(query_dict).one() + + def get(self, key): + return getattr(self, key) + + def save(self, validate = True): + assert validate + sess = object_session(self) + if sess is None: + sess = Session() + sess.add(self) + sess.commit() diff --git a/mediagoblin/db/sql/convert.py b/mediagoblin/db/sql/convert.py new file mode 100644 index 00000000..88614fd4 --- /dev/null +++ b/mediagoblin/db/sql/convert.py @@ -0,0 +1,151 @@ +from mediagoblin.init import setup_global_and_app_config, setup_database +from mediagoblin.db.mongo.util import ObjectId + +from mediagoblin.db.sql.models import (Base, User, MediaEntry, MediaComment, + Tag, MediaTag, MediaFile) +from mediagoblin.db.sql.open import setup_connection_and_db_from_config as \ + sql_connect +from mediagoblin.db.mongo.open import setup_connection_and_db_from_config as \ + mongo_connect +from mediagoblin.db.sql.base import Session + + +obj_id_table = dict() + +def add_obj_ids(entry, new_entry): + global obj_id_table + print "%r -> %r" % (entry._id, new_entry.id) + obj_id_table[entry._id] = new_entry.id + + +def copy_attrs(entry, new_entry, attr_list): + for a in attr_list: + val = entry[a] + setattr(new_entry, a, val) + +def copy_reference_attr(entry, new_entry, ref_attr): + val = entry[ref_attr] + val = obj_id_table[val] + setattr(new_entry, ref_attr, val) + + +def convert_users(mk_db): + session = Session() + + for entry in mk_db.User.find(): + print entry.username + + new_entry = User() + copy_attrs(entry, new_entry, + ('username', 'email', 'created', 'pw_hash', 'email_verified', + 'status', 'verification_key', 'is_admin', 'url', + 'bio', 'bio_html', + 'fp_verification_key', 'fp_token_expire',)) + # new_entry.fp_verification_expire = entry.fp_token_expire + + session.add(new_entry) + session.flush() + add_obj_ids(entry, new_entry) + + session.commit() + session.close() + + +def convert_media_entries(mk_db): + session = Session() + + for entry in mk_db.MediaEntry.find(): + print repr(entry.title) + + new_entry = MediaEntry() + copy_attrs(entry, new_entry, + ('title', 'slug', 'created', + 'description', 'description_html', + 'media_type', 'state', + 'fail_error', + 'queued_task_id',)) + copy_reference_attr(entry, new_entry, "uploader") + + session.add(new_entry) + session.flush() + add_obj_ids(entry, new_entry) + + for key, value in entry.media_files.iteritems(): + new_file = MediaFile(name=key, file_path=value) + new_file.media_entry = new_entry.id + Session.add(new_file) + + session.commit() + session.close() + + +def convert_media_tags(mk_db): + session = Session() + session.autoflush = False + + for media in mk_db.MediaEntry.find(): + print repr(media.title) + + for otag in media.tags: + print " ", repr((otag["slug"], otag["name"])) + + nslug = session.query(Tag).filter_by(slug=otag["slug"]).first() + print " ", repr(nslug) + if nslug is None: + nslug = Tag(slug=otag["slug"]) + session.add(nslug) + session.flush() + print " ", repr(nslug), nslug.id + + ntag = MediaTag() + ntag.tag = nslug.id + ntag.name = otag["name"] + ntag.media_entry = obj_id_table[media._id] + session.add(ntag) + + session.commit() + session.close() + + +def convert_media_comments(mk_db): + session = Session() + + for entry in mk_db.MediaComment.find(): + print repr(entry.content) + + new_entry = MediaComment() + copy_attrs(entry, new_entry, + ('created', + 'content', 'content_html',)) + copy_reference_attr(entry, new_entry, "media_entry") + copy_reference_attr(entry, new_entry, "author") + + session.add(new_entry) + session.flush() + add_obj_ids(entry, new_entry) + + session.commit() + session.close() + + +def main(): + global_config, app_config = setup_global_and_app_config("mediagoblin.ini") + + sql_conn, sql_db = sql_connect({'sql_engine': 'sqlite:///mediagoblin.db'}) + + mk_conn, mk_db = mongo_connect(app_config) + + Base.metadata.create_all(sql_db.engine) + + convert_users(mk_db) + Session.remove() + convert_media_entries(mk_db) + Session.remove() + convert_media_tags(mk_db) + Session.remove() + convert_media_comments(mk_db) + Session.remove() + + +if __name__ == '__main__': + main() diff --git a/mediagoblin/db/sql/extratypes.py b/mediagoblin/db/sql/extratypes.py new file mode 100644 index 00000000..88f556d9 --- /dev/null +++ b/mediagoblin/db/sql/extratypes.py @@ -0,0 +1,18 @@ +from sqlalchemy.types import TypeDecorator, Unicode + + +class PathTupleWithSlashes(TypeDecorator): + "Represents a Tuple of strings as a slash separated string." + + impl = Unicode + + def process_bind_param(self, value, dialect): + if value is not None: + assert len(value), "Does not support empty lists" + value = '/'.join(value) + return value + + def process_result_value(self, value, dialect): + if value is not None: + value = tuple(value.split('/')) + return value diff --git a/mediagoblin/db/sql/models.py b/mediagoblin/db/sql/models.py new file mode 100644 index 00000000..91092f33 --- /dev/null +++ b/mediagoblin/db/sql/models.py @@ -0,0 +1,153 @@ +import datetime + +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import ( + Column, Integer, Unicode, UnicodeText, DateTime, Boolean, ForeignKey, + UniqueConstraint) +from sqlalchemy.orm import relationship +from sqlalchemy.orm.collections import attribute_mapped_collection +from sqlalchemy.ext.associationproxy import association_proxy + +from mediagoblin.db.sql.extratypes import PathTupleWithSlashes +from mediagoblin.db.sql.base import GMGTableBase +from mediagoblin.db.mixin import UserMixin, MediaEntryMixin + + +Base = declarative_base(cls=GMGTableBase) + + +class SimpleFieldAlias(object): + """An alias for any field""" + def __init__(self, fieldname): + self.fieldname = fieldname + + def __get__(self, instance, cls): + return getattr(instance, self.fieldname) + + def __set__(self, instance, val): + setattr(instance, self.fieldname, val) + + +class User(Base, UserMixin): + __tablename__ = "users" + + id = Column(Integer, primary_key=True) + username = Column(Unicode, nullable=False, unique=True) + email = Column(Unicode, nullable=False) + created = Column(DateTime, nullable=False, default=datetime.datetime.now) + pw_hash = Column(Unicode, nullable=False) + email_verified = Column(Boolean) + status = Column(Unicode, default=u"needs_email_verification", nullable=False) + verification_key = Column(Unicode) + is_admin = Column(Boolean, default=False, nullable=False) + url = Column(Unicode) + bio = Column(UnicodeText) # ?? + bio_html = Column(UnicodeText) # ?? + fp_verification_key = Column(Unicode) + fp_token_expire = Column(DateTime) + + ## TODO + # plugin data would be in a separate model + + _id = SimpleFieldAlias("id") + + +class MediaEntry(Base, MediaEntryMixin): + __tablename__ = "media_entries" + + id = Column(Integer, primary_key=True) + uploader = Column(Integer, ForeignKey('users.id'), nullable=False) + title = Column(Unicode, nullable=False) + slug = Column(Unicode, nullable=False) + created = Column(DateTime, nullable=False, default=datetime.datetime.now) + description = Column(UnicodeText) # ?? + description_html = Column(UnicodeText) # ?? + media_type = Column(Unicode, nullable=False) + state = Column(Unicode, nullable=False) # or use sqlalchemy.types.Enum? + + fail_error = Column(Unicode) + fail_metadata = Column(UnicodeText) + + queued_media_file = Column(PathTupleWithSlashes) + + queued_task_id = Column(Unicode) + + __table_args__ = ( + UniqueConstraint('uploader', 'slug'), + {}) + + get_uploader = relationship(User) + + media_files_helper = relationship("MediaFile", + collection_class=attribute_mapped_collection("name"), + cascade="all, delete-orphan" + ) + media_files = association_proxy('media_files_helper', 'file_path', + creator=lambda k,v: MediaFile(name=k, file_path=v) + ) + + ## TODO + # media_data + # attachment_files + # fail_error + + +class MediaFile(Base): + __tablename__ = "mediafiles" + + media_entry = Column( + Integer, ForeignKey(MediaEntry.id), + nullable=False, primary_key=True) + name = Column(Unicode, primary_key=True) + file_path = Column(PathTupleWithSlashes) + + def __repr__(self): + return "<MediaFile %s: %r>" % (self.name, self.file_path) + + +class Tag(Base): + __tablename__ = "tags" + + id = Column(Integer, primary_key=True) + slug = Column(Unicode, nullable=False, unique=True) + + +class MediaTag(Base): + __tablename__ = "media_tags" + + id = Column(Integer, primary_key=True) + tag = Column(Integer, ForeignKey('tags.id'), nullable=False) + name = Column(Unicode) + media_entry = Column( + Integer, ForeignKey('media_entries.id'), + nullable=False) + # created = Column(DateTime, nullable=False, default=datetime.datetime.now) + + __table_args__ = ( + UniqueConstraint('tag', 'media_entry'), + {}) + + +class MediaComment(Base): + __tablename__ = "media_comments" + + id = Column(Integer, primary_key=True) + media_entry = Column( + Integer, ForeignKey('media_entries.id'), nullable=False) + author = Column(Integer, ForeignKey('users.id'), nullable=False) + created = Column(DateTime, nullable=False, default=datetime.datetime.now) + content = Column(UnicodeText, nullable=False) + content_html = Column(UnicodeText) + + get_author = relationship(User) + + +def show_table_init(): + from sqlalchemy import create_engine + engine = create_engine('sqlite:///:memory:', echo=True) + + Base.metadata.create_all(engine) + + +if __name__ == '__main__': + show_table_init() diff --git a/mediagoblin/db/sql/open.py b/mediagoblin/db/sql/open.py new file mode 100644 index 00000000..c682bd3b --- /dev/null +++ b/mediagoblin/db/sql/open.py @@ -0,0 +1,33 @@ +from sqlalchemy import create_engine + +from mediagoblin.db.sql.base import Session +from mediagoblin.db.sql.models import Base + + +class DatabaseMaster(object): + def __init__(self, engine): + self.engine = engine + + for k,v in Base._decl_class_registry.iteritems(): + setattr(self, k, v) + + def commit(self): + Session.commit() + + def save(self, obj): + Session.add(obj) + Session.flush() + + def reset_after_request(self): + Session.remove() + + +def setup_connection_and_db_from_config(app_config): + engine = create_engine(app_config['sql_engine'], echo=True) + Session.configure(bind=engine) + + return "dummy conn", DatabaseMaster(engine) + + +def check_db_migrations_current(db): + pass diff --git a/mediagoblin/db/util.py b/mediagoblin/db/util.py index 84a6cbce..1df9494c 100644 --- a/mediagoblin/db/util.py +++ b/mediagoblin/db/util.py @@ -14,278 +14,5 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -""" -Utilities for database operations. - -Some note on migration and indexing tools: - -We store information about what the state of the database is in the -'mediagoblin' document of the 'app_metadata' collection. Keys in that -document relevant to here: - - - 'migration_number': The integer representing the current state of - the migrations -""" - -import copy - -# Imports that other modules might use -from pymongo import ASCENDING, DESCENDING -from pymongo.errors import InvalidId -from mongokit import ObjectId - -from mediagoblin.db.indexes import ACTIVE_INDEXES, DEPRECATED_INDEXES - - -################ -# Indexing tools -################ - - -def add_new_indexes(database, active_indexes=ACTIVE_INDEXES): - """ - Add any new indexes to the database. - - Args: - - database: pymongo or mongokit database instance. - - active_indexes: indexes to possibly add in the pattern of: - {'collection_name': { - 'identifier': { - 'index': [index_foo_goes_here], - 'unique': True}} - where 'index' is the index to add and all other options are - arguments for collection.create_index. - - Returns: - A list of indexes added in form ('collection', 'index_name') - """ - indexes_added = [] - - for collection_name, indexes in active_indexes.iteritems(): - collection = database[collection_name] - collection_indexes = collection.index_information().keys() - - for index_name, index_data in indexes.iteritems(): - if not index_name in collection_indexes: - # Get a copy actually so we don't modify the actual - # structure - index_data = copy.copy(index_data) - index = index_data.pop('index') - collection.create_index( - index, name=index_name, **index_data) - - indexes_added.append((collection_name, index_name)) - - return indexes_added - - -def remove_deprecated_indexes(database, deprecated_indexes=DEPRECATED_INDEXES): - """ - Remove any deprecated indexes from the database. - - Args: - - database: pymongo or mongokit database instance. - - deprecated_indexes: the indexes to deprecate in the pattern of: - {'collection_name': { - 'identifier': { - 'index': [index_foo_goes_here], - 'unique': True}} - - (... although we really only need the 'identifier' here, as the - rest of the information isn't used in this case. But it's kept - around so we can remember what it was) - - Returns: - A list of indexes removed in form ('collection', 'index_name') - """ - indexes_removed = [] - - for collection_name, indexes in deprecated_indexes.iteritems(): - collection = database[collection_name] - collection_indexes = collection.index_information().keys() - - for index_name, index_data in indexes.iteritems(): - if index_name in collection_indexes: - collection.drop_index(index_name) - - indexes_removed.append((collection_name, index_name)) - - return indexes_removed - - -################# -# Migration tools -################# - -# The default migration registry... -# -# Don't set this yourself! RegisterMigration will automatically fill -# this with stuff via decorating methods in migrations.py - -class MissingCurrentMigration(Exception): pass - - -MIGRATIONS = {} - - -class RegisterMigration(object): - """ - Tool for registering migrations - - Call like: - - @RegisterMigration(33) - def update_dwarves(database): - [...] - - This will register your migration with the default migration - registry. Alternately, to specify a very specific - migration_registry, you can pass in that as the second argument. - - Note, the number of your migration should NEVER be 0 or less than - 0. 0 is the default "no migrations" state! - """ - def __init__(self, migration_number, migration_registry=MIGRATIONS): - assert migration_number > 0, "Migration number must be > 0!" - assert not migration_registry.has_key(migration_number), \ - "Duplicate migration numbers detected! That's not allowed!" - - self.migration_number = migration_number - self.migration_registry = migration_registry - - def __call__(self, migration): - self.migration_registry[self.migration_number] = migration - return migration - - -class MigrationManager(object): - """ - Migration handling tool. - - Takes information about a database, lets you update the database - to the latest migrations, etc. - """ - def __init__(self, database, migration_registry=MIGRATIONS): - """ - Args: - - database: database we're going to migrate - - migration_registry: where we should find all migrations to - run - """ - self.database = database - self.migration_registry = migration_registry - self._sorted_migrations = None - - def _ensure_current_migration_record(self): - """ - If there isn't a database[u'app_metadata'] mediagoblin entry - with the 'current_migration', throw an error. - """ - if self.database_current_migration() is None: - raise MissingCurrentMigration( - "Tried to call function which requires " - "'current_migration' set in database") - - @property - def sorted_migrations(self): - """ - Sort migrations if necessary and store in self._sorted_migrations - """ - if not self._sorted_migrations: - self._sorted_migrations = sorted( - self.migration_registry.items(), - # sort on the key... the migration number - key=lambda migration_tuple: migration_tuple[0]) - - return self._sorted_migrations - - def latest_migration(self): - """ - Return a migration number for the latest migration, or 0 if - there are no migrations. - """ - if self.sorted_migrations: - return self.sorted_migrations[-1][0] - else: - # If no migrations have been set, we start at 0. - return 0 - - def set_current_migration(self, migration_number): - """ - Set the migration in the database to migration_number - """ - # Add the mediagoblin migration if necessary - self.database[u'app_metadata'].update( - {u'_id': u'mediagoblin'}, - {u'$set': {u'current_migration': migration_number}}, - upsert=True) - - def install_migration_version_if_missing(self): - """ - Sets the migration to the latest version if no migration - version at all is set. - """ - mgoblin_metadata = self.database[u'app_metadata'].find_one( - {u'_id': u'mediagoblin'}) - if not mgoblin_metadata: - latest_migration = self.latest_migration() - self.set_current_migration(latest_migration) - - def database_current_migration(self): - """ - Return the current migration in the database. - """ - mgoblin_metadata = self.database[u'app_metadata'].find_one( - {u'_id': u'mediagoblin'}) - if not mgoblin_metadata: - return None - else: - return mgoblin_metadata[u'current_migration'] - - def database_at_latest_migration(self): - """ - See if the database is at the latest migration. - Returns a boolean. - """ - current_migration = self.database_current_migration() - return current_migration == self.latest_migration() - - def migrations_to_run(self): - """ - Get a list of migrations to run still, if any. - - Note that calling this will set your migration version to the - latest version if it isn't installed to anything yet! - """ - self._ensure_current_migration_record() - - db_current_migration = self.database_current_migration() - - return [ - (migration_number, migration_func) - for migration_number, migration_func in self.sorted_migrations - if migration_number > db_current_migration] - - def migrate_new(self, pre_callback=None, post_callback=None): - """ - Run all migrations. - - Includes two optional args: - - pre_callback: if called, this is a callback on something to - run pre-migration. Takes (migration_number, migration_func) - as arguments - - pre_callback: if called, this is a callback on something to - run post-migration. Takes (migration_number, migration_func) - as arguments - """ - # If we aren't set to any version number, presume we're at the - # latest (which means we'll do nothing here...) - self.install_migration_version_if_missing() - - for migration_number, migration_func in self.migrations_to_run(): - if pre_callback: - pre_callback(migration_number, migration_func) - migration_func(self.database) - self.set_current_migration(migration_number) - if post_callback: - post_callback(migration_number, migration_func) +from mediagoblin.db.mongo.util import (ObjectId, InvalidId, + DESCENDING) |