diff options
Diffstat (limited to 'mediagoblin/db/sql')
| -rw-r--r-- | mediagoblin/db/sql/__init__.py | 15 | ||||
| -rw-r--r-- | mediagoblin/db/sql/base.py | 107 | ||||
| -rw-r--r-- | mediagoblin/db/sql/convert.py | 282 | ||||
| -rw-r--r-- | mediagoblin/db/sql/extratypes.py | 63 | ||||
| -rw-r--r-- | mediagoblin/db/sql/fake.py | 45 | ||||
| -rw-r--r-- | mediagoblin/db/sql/migrations.py | 130 | ||||
| -rw-r--r-- | mediagoblin/db/sql/models.py | 469 | ||||
| -rw-r--r-- | mediagoblin/db/sql/models_v0.py | 320 | ||||
| -rw-r--r-- | mediagoblin/db/sql/open.py | 78 | ||||
| -rw-r--r-- | mediagoblin/db/sql/util.py | 327 |
10 files changed, 0 insertions, 1836 deletions
diff --git a/mediagoblin/db/sql/__init__.py b/mediagoblin/db/sql/__init__.py deleted file mode 100644 index 621845ba..00000000 --- a/mediagoblin/db/sql/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. diff --git a/mediagoblin/db/sql/base.py b/mediagoblin/db/sql/base.py deleted file mode 100644 index e10e7739..00000000 --- a/mediagoblin/db/sql/base.py +++ /dev/null @@ -1,107 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import scoped_session, sessionmaker, object_session -from sqlalchemy.orm.query import Query -from sqlalchemy.sql.expression import desc -from mediagoblin.db.sql.fake import DESCENDING - - -def _get_query_model(query): - cols = query.column_descriptions - assert len(cols) == 1, "These functions work only on simple queries" - return cols[0]["type"] - - -class GMGQuery(Query): - def sort(self, key, direction): - key_col = getattr(_get_query_model(self), key) - if direction is DESCENDING: - key_col = desc(key_col) - return self.order_by(key_col) - - def skip(self, amount): - return self.offset(amount) - - -Session = scoped_session(sessionmaker(query_cls=GMGQuery)) - - -def _fix_query_dict(query_dict): - if '_id' in query_dict: - query_dict['id'] = query_dict.pop('_id') - - -class GMGTableBase(object): - query = Session.query_property() - - @classmethod - def find(cls, query_dict={}): - _fix_query_dict(query_dict) - return cls.query.filter_by(**query_dict) - - @classmethod - def find_one(cls, query_dict={}): - _fix_query_dict(query_dict) - return cls.query.filter_by(**query_dict).first() - - @classmethod - def one(cls, query_dict): - return cls.find(query_dict).one() - - def get(self, key): - return getattr(self, key) - - def setdefault(self, key, defaultvalue): - # The key *has* to exist on sql. - return getattr(self, key) - - def save(self, validate=True): - assert validate - sess = object_session(self) - if sess is None: - sess = Session() - sess.add(self) - sess.commit() - - def delete(self, commit=True): - """Delete the object and commit the change immediately by default""" - sess = object_session(self) - assert sess is not None, "Not going to delete detached %r" % self - sess.delete(self) - if commit: - sess.commit() - - -Base = declarative_base(cls=GMGTableBase) - - -class DictReadAttrProxy(object): - """ - Maps read accesses to obj['key'] to obj.key - and hides all the rest of the obj - """ - def __init__(self, proxied_obj): - self.proxied_obj = proxied_obj - - def __getitem__(self, key): - try: - return getattr(self.proxied_obj, key) - except AttributeError: - raise KeyError("%r is not an attribute on %r" - % (key, self.proxied_obj)) diff --git a/mediagoblin/db/sql/convert.py b/mediagoblin/db/sql/convert.py deleted file mode 100644 index ac64cf8d..00000000 --- a/mediagoblin/db/sql/convert.py +++ /dev/null @@ -1,282 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -from copy import copy -from itertools import chain, imap - -from mediagoblin.init import setup_global_and_app_config - -from mediagoblin.db.sql.base import Session -from mediagoblin.db.sql.models_v0 import Base_v0 -from mediagoblin.db.sql.models_v0 import (User, MediaEntry, MediaComment, - Tag, MediaTag, MediaFile, MediaAttachmentFile, MigrationData, - ImageData, VideoData, AsciiData, AudioData) -from mediagoblin.db.sql.open import setup_connection_and_db_from_config as \ - sql_connect -from mediagoblin.db.mongo.open import setup_connection_and_db_from_config as \ - mongo_connect - - -obj_id_table = dict() - - -def add_obj_ids(entry, new_entry): - global obj_id_table - print "\t%r -> SQL id %r" % (entry._id, new_entry.id) - obj_id_table[entry._id] = new_entry.id - - -def copy_attrs(entry, new_entry, attr_list): - for a in attr_list: - val = entry[a] - setattr(new_entry, a, val) - - -def copy_reference_attr(entry, new_entry, ref_attr): - val = entry[ref_attr] - val = obj_id_table[val] - setattr(new_entry, ref_attr, val) - - -def convert_users(mk_db): - session = Session() - - for entry in mk_db.User.find().sort('created'): - print entry.username - - new_entry = User() - copy_attrs(entry, new_entry, - ('username', 'email', 'created', 'pw_hash', 'email_verified', - 'status', 'verification_key', 'is_admin', 'url', - 'bio', - 'fp_verification_key', 'fp_token_expire',)) - # new_entry.fp_verification_expire = entry.fp_token_expire - - session.add(new_entry) - session.flush() - add_obj_ids(entry, new_entry) - - session.commit() - session.close() - - -def convert_media_entries(mk_db): - session = Session() - - for entry in mk_db.MediaEntry.find().sort('created'): - print repr(entry.title) - - new_entry = MediaEntry() - copy_attrs(entry, new_entry, - ('title', 'slug', 'created', - 'description', - 'media_type', 'state', 'license', - 'fail_error', 'fail_metadata', - 'queued_task_id',)) - copy_reference_attr(entry, new_entry, "uploader") - - session.add(new_entry) - session.flush() - add_obj_ids(entry, new_entry) - - for key, value in entry.media_files.iteritems(): - new_file = MediaFile(name=key, file_path=value) - new_file.media_entry = new_entry.id - Session.add(new_file) - - for attachment in entry.attachment_files: - new_attach = MediaAttachmentFile( - name=attachment["name"], - filepath=attachment["filepath"], - created=attachment["created"] - ) - new_attach.media_entry = new_entry.id - Session.add(new_attach) - - session.commit() - session.close() - - -def convert_image(mk_db): - session = Session() - - for media in mk_db.MediaEntry.find( - {'media_type': 'mediagoblin.media_types.image'}).sort('created'): - media_data = copy(media.media_data) - - if len(media_data): - media_data_row = ImageData(**media_data) - media_data_row.media_entry = obj_id_table[media['_id']] - session.add(media_data_row) - - session.commit() - session.close() - - -def convert_video(mk_db): - session = Session() - - for media in mk_db.MediaEntry.find( - {'media_type': 'mediagoblin.media_types.video'}).sort('created'): - media_data_row = VideoData(**media.media_data) - media_data_row.media_entry = obj_id_table[media['_id']] - session.add(media_data_row) - - session.commit() - session.close() - - -def convert_media_tags(mk_db): - session = Session() - session.autoflush = False - - for media in mk_db.MediaEntry.find().sort('created'): - print repr(media.title) - - for otag in media.tags: - print " ", repr((otag["slug"], otag["name"])) - - nslug = session.query(Tag).filter_by(slug=otag["slug"]).first() - print " ", repr(nslug) - if nslug is None: - nslug = Tag(slug=otag["slug"]) - session.add(nslug) - session.flush() - print " ", repr(nslug), nslug.id - - ntag = MediaTag() - ntag.tag = nslug.id - ntag.name = otag["name"] - ntag.media_entry = obj_id_table[media._id] - session.add(ntag) - - session.commit() - session.close() - - -def convert_media_comments(mk_db): - session = Session() - - for entry in mk_db.MediaComment.find().sort('created'): - print repr(entry.content) - - new_entry = MediaComment() - copy_attrs(entry, new_entry, - ('created', - 'content',)) - - try: - copy_reference_attr(entry, new_entry, "media_entry") - copy_reference_attr(entry, new_entry, "author") - except KeyError as e: - print('KeyError in convert_media_comments(): {0}'.format(e)) - else: - session.add(new_entry) - session.flush() - add_obj_ids(entry, new_entry) - - session.commit() - session.close() - - -media_types_tables = ( - ("mediagoblin.media_types.image", (ImageData,)), - ("mediagoblin.media_types.video", (VideoData,)), - ("mediagoblin.media_types.ascii", (AsciiData,)), - ("mediagoblin.media_types.audio", (AudioData,)), - ) - - -def convert_add_migration_versions(dummy_sql_db): - session = Session() - - for name in chain(("__main__",), - imap(lambda e: e[0], media_types_tables)): - print "\tAdding %s" % (name,) - m = MigrationData(name=unicode(name), version=0) - session.add(m) - - session.commit() - session.close() - - -def cleanup_sql_tables(sql_db): - for mt, table_list in media_types_tables: - session = Session() - - count = session.query(MediaEntry.media_type). \ - filter_by(media_type=unicode(mt)).count() - print " %s: %d entries" % (mt, count) - - if count == 0: - print "\tAnalyzing tables" - for tab in table_list: - cnt2 = session.query(tab).count() - print "\t %s: %d entries" % (tab.__tablename__, cnt2) - assert cnt2 == 0 - - print "\tRemoving migration info" - mi = session.query(MigrationData).filter_by(name=unicode(mt)).one() - session.delete(mi) - session.commit() - session.close() - - print "\tDropping tables" - tables = [model.__table__ for model in table_list] - Base_v0.metadata.drop_all(sql_db.engine, tables=tables) - - session.close() - - -def print_header(title): - print "\n=== %s ===" % (title,) - - -convert_call_list = ( - ("Converting Users", convert_users), - ("Converting Media Entries", convert_media_entries), - ("Converting Media Data for Images", convert_image), - ("Cnnverting Media Data for Videos", convert_video), - ("Converting Tags for Media", convert_media_tags), - ("Converting Media Comments", convert_media_comments), - ) - -sql_call_list = ( - ("Filling Migration Tables", convert_add_migration_versions), - ("Analyzing/Cleaning SQL Data", cleanup_sql_tables), - ) - -def run_conversion(config_name): - global_config, app_config = setup_global_and_app_config(config_name) - - sql_conn, sql_db = sql_connect(app_config) - mk_conn, mk_db = mongo_connect(app_config) - - Base_v0.metadata.create_all(sql_db.engine) - - for title, func in convert_call_list: - print_header(title) - func(mk_db) - Session.remove() - - for title, func in sql_call_list: - print_header(title) - func(sql_db) - Session.remove() - - -if __name__ == '__main__': - run_conversion("mediagoblin.ini") diff --git a/mediagoblin/db/sql/extratypes.py b/mediagoblin/db/sql/extratypes.py deleted file mode 100644 index f2304af0..00000000 --- a/mediagoblin/db/sql/extratypes.py +++ /dev/null @@ -1,63 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -from sqlalchemy.types import TypeDecorator, Unicode, TEXT -import json - - -class PathTupleWithSlashes(TypeDecorator): - "Represents a Tuple of strings as a slash separated string." - - impl = Unicode - - def process_bind_param(self, value, dialect): - if value is not None: - if len(value) == 0: - value = None - else: - value = '/'.join(value) - return value - - def process_result_value(self, value, dialect): - if value is not None: - value = tuple(value.split('/')) - return value - - -# The following class and only this one class is in very -# large parts based on example code from sqlalchemy. -# -# The original copyright notice and license follows: -# Copyright (C) 2005-2011 the SQLAlchemy authors and contributors <see AUTHORS file> -# -# This module is part of SQLAlchemy and is released under -# the MIT License: http://www.opensource.org/licenses/mit-license.php -# -class JSONEncoded(TypeDecorator): - "Represents an immutable structure as a json-encoded string." - - impl = TEXT - - def process_bind_param(self, value, dialect): - if value is not None: - value = json.dumps(value) - return value - - def process_result_value(self, value, dialect): - if value is not None: - value = json.loads(value) - return value diff --git a/mediagoblin/db/sql/fake.py b/mediagoblin/db/sql/fake.py deleted file mode 100644 index 0fd0cc41..00000000 --- a/mediagoblin/db/sql/fake.py +++ /dev/null @@ -1,45 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -""" -This module contains some fake classes and functions to -calm the rest of the code base. Or provide super minimal -implementations. - -Currently: -- ObjectId "class": It's a function mostly doing - int(init_arg) to convert string primary keys into - integer primary keys. -- InvalidId exception -- DESCENDING "constant" -""" - - -DESCENDING = object() # a unique object for this "constant" - - -class InvalidId(Exception): - pass - - -def ObjectId(value=None): - if value is None: - return None - try: - return int(value) - except ValueError: - raise InvalidId("%r is an invalid id" % value) diff --git a/mediagoblin/db/sql/migrations.py b/mediagoblin/db/sql/migrations.py deleted file mode 100644 index bc68caa3..00000000 --- a/mediagoblin/db/sql/migrations.py +++ /dev/null @@ -1,130 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -import datetime - -from sqlalchemy import (MetaData, Table, Column, Boolean, SmallInteger, - Integer, Unicode, UnicodeText, DateTime, - ForeignKey, UniqueConstraint) -from sqlalchemy.ext.declarative import declarative_base - -from mediagoblin.db.sql.util import RegisterMigration -from mediagoblin.db.sql.models import MediaEntry, Collection, User - -MIGRATIONS = {} - - -@RegisterMigration(1, MIGRATIONS) -def ogg_to_webm_audio(db_conn): - metadata = MetaData(bind=db_conn.bind) - - file_keynames = Table('core__file_keynames', metadata, autoload=True, - autoload_with=db_conn.bind) - - db_conn.execute( - file_keynames.update().where(file_keynames.c.name == 'ogg'). - values(name='webm_audio') - ) - db_conn.commit() - - -@RegisterMigration(2, MIGRATIONS) -def add_wants_notification_column(db_conn): - metadata = MetaData(bind=db_conn.bind) - - users = Table('core__users', metadata, autoload=True, - autoload_with=db_conn.bind) - - col = Column('wants_comment_notification', Boolean, - default=True, nullable=True) - col.create(users, populate_defaults=True) - db_conn.commit() - - -@RegisterMigration(3, MIGRATIONS) -def add_transcoding_progress(db_conn): - metadata = MetaData(bind=db_conn.bind) - - media_entry = Table('core__media_entries', metadata, autoload=True, - autoload_with=db_conn.bind) - - col = Column('transcoding_progress', SmallInteger) - col.create(media_entry) - db_conn.commit() - - -class Collection_v0(declarative_base()): - __tablename__ = "core__collections" - - id = Column(Integer, primary_key=True) - title = Column(Unicode, nullable=False) - slug = Column(Unicode) - created = Column(DateTime, nullable=False, default=datetime.datetime.now, - index=True) - description = Column(UnicodeText) - creator = Column(Integer, ForeignKey(User.id), nullable=False) - items = Column(Integer, default=0) - -class CollectionItem_v0(declarative_base()): - __tablename__ = "core__collection_items" - - id = Column(Integer, primary_key=True) - media_entry = Column( - Integer, ForeignKey(MediaEntry.id), nullable=False, index=True) - collection = Column(Integer, ForeignKey(Collection.id), nullable=False) - note = Column(UnicodeText, nullable=True) - added = Column(DateTime, nullable=False, default=datetime.datetime.now) - position = Column(Integer) - - ## This should be activated, normally. - ## But this would change the way the next migration used to work. - ## So it's commented for now. - # __table_args__ = ( - # UniqueConstraint('collection', 'media_entry'), - # {}) - -@RegisterMigration(4, MIGRATIONS) -def add_collection_tables(db_conn): - Collection_v0.__table__.create(db_conn.bind) - CollectionItem_v0.__table__.create(db_conn.bind) - - db_conn.commit() - - -@RegisterMigration(5, MIGRATIONS) -def add_mediaentry_collected(db_conn): - metadata = MetaData(bind=db_conn.bind) - - media_entry = Table('core__media_entries', metadata, autoload=True, - autoload_with=db_conn.bind) - - col = Column('collected', Integer, default=0) - col.create(media_entry) - db_conn.commit() - - -class ProcessingMetaData_v0(declarative_base()): - __tablename__ = 'core__processing_metadata' - - id = Column(Integer, primary_key=True) - media_entry_id = Column(Integer, ForeignKey(MediaEntry.id), nullable=False, - index=True) - callback_url = Column(Unicode) - -@RegisterMigration(6, MIGRATIONS) -def create_processing_metadata_table(db): - ProcessingMetaData_v0.__table__.create(db.bind) - db.commit() diff --git a/mediagoblin/db/sql/models.py b/mediagoblin/db/sql/models.py deleted file mode 100644 index b48c1fbe..00000000 --- a/mediagoblin/db/sql/models.py +++ /dev/null @@ -1,469 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -""" -TODO: indexes on foreignkeys, where useful. -""" - - -import datetime -import sys - -from sqlalchemy import Column, Integer, Unicode, UnicodeText, DateTime, \ - Boolean, ForeignKey, UniqueConstraint, PrimaryKeyConstraint, \ - SmallInteger -from sqlalchemy.orm import relationship, backref -from sqlalchemy.orm.collections import attribute_mapped_collection -from sqlalchemy.sql.expression import desc -from sqlalchemy.ext.associationproxy import association_proxy -from sqlalchemy.util import memoized_property - -from mediagoblin.db.sql.extratypes import PathTupleWithSlashes, JSONEncoded -from mediagoblin.db.sql.base import Base, DictReadAttrProxy -from mediagoblin.db.mixin import UserMixin, MediaEntryMixin, MediaCommentMixin, CollectionMixin, CollectionItemMixin -from mediagoblin.db.sql.base import Session - -# It's actually kind of annoying how sqlalchemy-migrate does this, if -# I understand it right, but whatever. Anyway, don't remove this :P -# -# We could do migration calls more manually instead of relying on -# this import-based meddling... -from migrate import changeset - - -class SimpleFieldAlias(object): - """An alias for any field""" - def __init__(self, fieldname): - self.fieldname = fieldname - - def __get__(self, instance, cls): - return getattr(instance, self.fieldname) - - def __set__(self, instance, val): - setattr(instance, self.fieldname, val) - - -class User(Base, UserMixin): - """ - TODO: We should consider moving some rarely used fields - into some sort of "shadow" table. - """ - __tablename__ = "core__users" - - id = Column(Integer, primary_key=True) - username = Column(Unicode, nullable=False, unique=True) - email = Column(Unicode, nullable=False) - created = Column(DateTime, nullable=False, default=datetime.datetime.now) - pw_hash = Column(Unicode, nullable=False) - email_verified = Column(Boolean, default=False) - status = Column(Unicode, default=u"needs_email_verification", nullable=False) - # Intented to be nullable=False, but migrations would not work for it - # set to nullable=True implicitly. - wants_comment_notification = Column(Boolean, default=True) - verification_key = Column(Unicode) - is_admin = Column(Boolean, default=False, nullable=False) - url = Column(Unicode) - bio = Column(UnicodeText) # ?? - fp_verification_key = Column(Unicode) - fp_token_expire = Column(DateTime) - - ## TODO - # plugin data would be in a separate model - - _id = SimpleFieldAlias("id") - - def __repr__(self): - return '<{0} #{1} {2} {3} "{4}">'.format( - self.__class__.__name__, - self.id, - 'verified' if self.email_verified else 'non-verified', - 'admin' if self.is_admin else 'user', - self.username) - - -class MediaEntry(Base, MediaEntryMixin): - """ - TODO: Consider fetching the media_files using join - """ - __tablename__ = "core__media_entries" - - id = Column(Integer, primary_key=True) - uploader = Column(Integer, ForeignKey(User.id), nullable=False, index=True) - title = Column(Unicode, nullable=False) - slug = Column(Unicode) - created = Column(DateTime, nullable=False, default=datetime.datetime.now, - index=True) - description = Column(UnicodeText) # ?? - media_type = Column(Unicode, nullable=False) - state = Column(Unicode, default=u'unprocessed', nullable=False) - # or use sqlalchemy.types.Enum? - license = Column(Unicode) - collected = Column(Integer, default=0) - - fail_error = Column(Unicode) - fail_metadata = Column(JSONEncoded) - - transcoding_progress = Column(SmallInteger) - - queued_media_file = Column(PathTupleWithSlashes) - - queued_task_id = Column(Unicode) - - __table_args__ = ( - UniqueConstraint('uploader', 'slug'), - {}) - - get_uploader = relationship(User) - - media_files_helper = relationship("MediaFile", - collection_class=attribute_mapped_collection("name"), - cascade="all, delete-orphan" - ) - media_files = association_proxy('media_files_helper', 'file_path', - creator=lambda k, v: MediaFile(name=k, file_path=v) - ) - - attachment_files_helper = relationship("MediaAttachmentFile", - cascade="all, delete-orphan", - order_by="MediaAttachmentFile.created" - ) - attachment_files = association_proxy("attachment_files_helper", "dict_view", - creator=lambda v: MediaAttachmentFile( - name=v["name"], filepath=v["filepath"]) - ) - - tags_helper = relationship("MediaTag", - cascade="all, delete-orphan" - ) - tags = association_proxy("tags_helper", "dict_view", - creator=lambda v: MediaTag(name=v["name"], slug=v["slug"]) - ) - - collections_helper = relationship("CollectionItem", - cascade="all, delete-orphan" - ) - collections = association_proxy("collections_helper", "in_collection") - - ## TODO - # media_data - # fail_error - - _id = SimpleFieldAlias("id") - - def get_comments(self, ascending=False): - order_col = MediaComment.created - if not ascending: - order_col = desc(order_col) - return MediaComment.query.filter_by( - media_entry=self.id).order_by(order_col) - - def url_to_prev(self, urlgen): - """get the next 'newer' entry by this user""" - media = MediaEntry.query.filter( - (MediaEntry.uploader == self.uploader) - & (MediaEntry.state == u'processed') - & (MediaEntry.id > self.id)).order_by(MediaEntry.id).first() - - if media is not None: - return media.url_for_self(urlgen) - - def url_to_next(self, urlgen): - """get the next 'older' entry by this user""" - media = MediaEntry.query.filter( - (MediaEntry.uploader == self.uploader) - & (MediaEntry.state == u'processed') - & (MediaEntry.id < self.id)).order_by(desc(MediaEntry.id)).first() - - if media is not None: - return media.url_for_self(urlgen) - - #@memoized_property - @property - def media_data(self): - session = Session() - - return session.query(self.media_data_table).filter_by( - media_entry=self.id).first() - - def media_data_init(self, **kwargs): - """ - Initialize or update the contents of a media entry's media_data row - """ - session = Session() - - media_data = session.query(self.media_data_table).filter_by( - media_entry=self.id).first() - - # No media data, so actually add a new one - if media_data is None: - media_data = self.media_data_table( - media_entry=self.id, - **kwargs) - session.add(media_data) - # Update old media data - else: - for field, value in kwargs.iteritems(): - setattr(media_data, field, value) - - @memoized_property - def media_data_table(self): - # TODO: memoize this - models_module = self.media_type + '.models' - __import__(models_module) - return sys.modules[models_module].DATA_MODEL - - def __repr__(self): - safe_title = self.title.encode('ascii', 'replace') - - return '<{classname} {id}: {title}>'.format( - classname=self.__class__.__name__, - id=self.id, - title=safe_title) - - -class FileKeynames(Base): - """ - keywords for various places. - currently the MediaFile keys - """ - __tablename__ = "core__file_keynames" - id = Column(Integer, primary_key=True) - name = Column(Unicode, unique=True) - - def __repr__(self): - return "<FileKeyname %r: %r>" % (self.id, self.name) - - @classmethod - def find_or_new(cls, name): - t = cls.query.filter_by(name=name).first() - if t is not None: - return t - return cls(name=name) - - -class MediaFile(Base): - """ - TODO: Highly consider moving "name" into a new table. - TODO: Consider preloading said table in software - """ - __tablename__ = "core__mediafiles" - - media_entry = Column( - Integer, ForeignKey(MediaEntry.id), - nullable=False) - name_id = Column(SmallInteger, ForeignKey(FileKeynames.id), nullable=False) - file_path = Column(PathTupleWithSlashes) - - __table_args__ = ( - PrimaryKeyConstraint('media_entry', 'name_id'), - {}) - - def __repr__(self): - return "<MediaFile %s: %r>" % (self.name, self.file_path) - - name_helper = relationship(FileKeynames, lazy="joined", innerjoin=True) - name = association_proxy('name_helper', 'name', - creator=FileKeynames.find_or_new - ) - - -class MediaAttachmentFile(Base): - __tablename__ = "core__attachment_files" - - id = Column(Integer, primary_key=True) - media_entry = Column( - Integer, ForeignKey(MediaEntry.id), - nullable=False) - name = Column(Unicode, nullable=False) - filepath = Column(PathTupleWithSlashes) - created = Column(DateTime, nullable=False, default=datetime.datetime.now) - - @property - def dict_view(self): - """A dict like view on this object""" - return DictReadAttrProxy(self) - - -class Tag(Base): - __tablename__ = "core__tags" - - id = Column(Integer, primary_key=True) - slug = Column(Unicode, nullable=False, unique=True) - - def __repr__(self): - return "<Tag %r: %r>" % (self.id, self.slug) - - @classmethod - def find_or_new(cls, slug): - t = cls.query.filter_by(slug=slug).first() - if t is not None: - return t - return cls(slug=slug) - - -class MediaTag(Base): - __tablename__ = "core__media_tags" - - id = Column(Integer, primary_key=True) - media_entry = Column( - Integer, ForeignKey(MediaEntry.id), - nullable=False, index=True) - tag = Column(Integer, ForeignKey(Tag.id), nullable=False, index=True) - name = Column(Unicode) - # created = Column(DateTime, nullable=False, default=datetime.datetime.now) - - __table_args__ = ( - UniqueConstraint('tag', 'media_entry'), - {}) - - tag_helper = relationship(Tag) - slug = association_proxy('tag_helper', 'slug', - creator=Tag.find_or_new - ) - - def __init__(self, name=None, slug=None): - Base.__init__(self) - if name is not None: - self.name = name - if slug is not None: - self.tag_helper = Tag.find_or_new(slug) - - @property - def dict_view(self): - """A dict like view on this object""" - return DictReadAttrProxy(self) - - -class MediaComment(Base, MediaCommentMixin): - __tablename__ = "core__media_comments" - - id = Column(Integer, primary_key=True) - media_entry = Column( - Integer, ForeignKey(MediaEntry.id), nullable=False, index=True) - author = Column(Integer, ForeignKey(User.id), nullable=False) - created = Column(DateTime, nullable=False, default=datetime.datetime.now) - content = Column(UnicodeText, nullable=False) - - get_author = relationship(User) - - _id = SimpleFieldAlias("id") - - -class Collection(Base, CollectionMixin): - __tablename__ = "core__collections" - - id = Column(Integer, primary_key=True) - title = Column(Unicode, nullable=False) - slug = Column(Unicode) - created = Column(DateTime, nullable=False, default=datetime.datetime.now, - index=True) - description = Column(UnicodeText) - creator = Column(Integer, ForeignKey(User.id), nullable=False) - items = Column(Integer, default=0) - - get_creator = relationship(User) - - def get_collection_items(self, ascending=False): - order_col = CollectionItem.position - if not ascending: - order_col = desc(order_col) - return CollectionItem.query.filter_by( - collection=self.id).order_by(order_col) - - _id = SimpleFieldAlias("id") - - -class CollectionItem(Base, CollectionItemMixin): - __tablename__ = "core__collection_items" - - id = Column(Integer, primary_key=True) - media_entry = Column( - Integer, ForeignKey(MediaEntry.id), nullable=False, index=True) - collection = Column(Integer, ForeignKey(Collection.id), nullable=False) - note = Column(UnicodeText, nullable=True) - added = Column(DateTime, nullable=False, default=datetime.datetime.now) - position = Column(Integer) - in_collection = relationship("Collection") - - get_media_entry = relationship(MediaEntry) - - _id = SimpleFieldAlias("id") - - __table_args__ = ( - UniqueConstraint('collection', 'media_entry'), - {}) - - @property - def dict_view(self): - """A dict like view on this object""" - return DictReadAttrProxy(self) - - -class ProcessingMetaData(Base): - __tablename__ = 'core__processing_metadata' - - id = Column(Integer, primary_key=True) - media_entry_id = Column(Integer, ForeignKey(MediaEntry.id), nullable=False, - index=True) - media_entry = relationship(MediaEntry, - backref=backref('processing_metadata', - cascade='all, delete-orphan')) - callback_url = Column(Unicode) - - @property - def dict_view(self): - """A dict like view on this object""" - return DictReadAttrProxy(self) - - -MODELS = [ - User, MediaEntry, Tag, MediaTag, MediaComment, Collection, CollectionItem, MediaFile, FileKeynames, - MediaAttachmentFile, ProcessingMetaData] - - -###################################################### -# Special, migrations-tracking table -# -# Not listed in MODELS because this is special and not -# really migrated, but used for migrations (for now) -###################################################### - -class MigrationData(Base): - __tablename__ = "core__migrations" - - name = Column(Unicode, primary_key=True) - version = Column(Integer, nullable=False, default=0) - -###################################################### - - -def show_table_init(engine_uri): - if engine_uri is None: - engine_uri = 'sqlite:///:memory:' - from sqlalchemy import create_engine - engine = create_engine(engine_uri, echo=True) - - Base.metadata.create_all(engine) - - -if __name__ == '__main__': - from sys import argv - print repr(argv) - if len(argv) == 2: - uri = argv[1] - else: - uri = None - show_table_init(uri) diff --git a/mediagoblin/db/sql/models_v0.py b/mediagoblin/db/sql/models_v0.py deleted file mode 100644 index 06f87d28..00000000 --- a/mediagoblin/db/sql/models_v0.py +++ /dev/null @@ -1,320 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -""" -TODO: indexes on foreignkeys, where useful. -""" - - -import datetime -import sys - -from sqlalchemy import ( - Column, Integer, Unicode, UnicodeText, DateTime, Boolean, ForeignKey, - UniqueConstraint, PrimaryKeyConstraint, SmallInteger, Float) -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import relationship, backref -from sqlalchemy.orm.collections import attribute_mapped_collection -from sqlalchemy.ext.associationproxy import association_proxy -from sqlalchemy.util import memoized_property - -from mediagoblin.db.sql.extratypes import PathTupleWithSlashes, JSONEncoded -from mediagoblin.db.sql.base import GMGTableBase -from mediagoblin.db.sql.base import Session - - -Base_v0 = declarative_base(cls=GMGTableBase) - - -class User(Base_v0): - """ - TODO: We should consider moving some rarely used fields - into some sort of "shadow" table. - """ - __tablename__ = "core__users" - - id = Column(Integer, primary_key=True) - username = Column(Unicode, nullable=False, unique=True) - email = Column(Unicode, nullable=False) - created = Column(DateTime, nullable=False, default=datetime.datetime.now) - pw_hash = Column(Unicode, nullable=False) - email_verified = Column(Boolean, default=False) - status = Column(Unicode, default=u"needs_email_verification", nullable=False) - verification_key = Column(Unicode) - is_admin = Column(Boolean, default=False, nullable=False) - url = Column(Unicode) - bio = Column(UnicodeText) # ?? - fp_verification_key = Column(Unicode) - fp_token_expire = Column(DateTime) - - ## TODO - # plugin data would be in a separate model - - -class MediaEntry(Base_v0): - """ - TODO: Consider fetching the media_files using join - """ - __tablename__ = "core__media_entries" - - id = Column(Integer, primary_key=True) - uploader = Column(Integer, ForeignKey(User.id), nullable=False, index=True) - title = Column(Unicode, nullable=False) - slug = Column(Unicode) - created = Column(DateTime, nullable=False, default=datetime.datetime.now, - index=True) - description = Column(UnicodeText) # ?? - media_type = Column(Unicode, nullable=False) - state = Column(Unicode, default=u'unprocessed', nullable=False) - # or use sqlalchemy.types.Enum? - license = Column(Unicode) - - fail_error = Column(Unicode) - fail_metadata = Column(JSONEncoded) - - queued_media_file = Column(PathTupleWithSlashes) - - queued_task_id = Column(Unicode) - - __table_args__ = ( - UniqueConstraint('uploader', 'slug'), - {}) - - get_uploader = relationship(User) - - media_files_helper = relationship("MediaFile", - collection_class=attribute_mapped_collection("name"), - cascade="all, delete-orphan" - ) - - attachment_files_helper = relationship("MediaAttachmentFile", - cascade="all, delete-orphan", - order_by="MediaAttachmentFile.created" - ) - - tags_helper = relationship("MediaTag", - cascade="all, delete-orphan" - ) - - def media_data_init(self, **kwargs): - """ - Initialize or update the contents of a media entry's media_data row - """ - session = Session() - - media_data = session.query(self.media_data_table).filter_by( - media_entry=self.id).first() - - # No media data, so actually add a new one - if media_data is None: - media_data = self.media_data_table( - media_entry=self.id, - **kwargs) - session.add(media_data) - # Update old media data - else: - for field, value in kwargs.iteritems(): - setattr(media_data, field, value) - - @memoized_property - def media_data_table(self): - # TODO: memoize this - models_module = self.media_type + '.models' - __import__(models_module) - return sys.modules[models_module].DATA_MODEL - - -class FileKeynames(Base_v0): - """ - keywords for various places. - currently the MediaFile keys - """ - __tablename__ = "core__file_keynames" - id = Column(Integer, primary_key=True) - name = Column(Unicode, unique=True) - - def __repr__(self): - return "<FileKeyname %r: %r>" % (self.id, self.name) - - @classmethod - def find_or_new(cls, name): - t = cls.query.filter_by(name=name).first() - if t is not None: - return t - return cls(name=name) - - -class MediaFile(Base_v0): - """ - TODO: Highly consider moving "name" into a new table. - TODO: Consider preloading said table in software - """ - __tablename__ = "core__mediafiles" - - media_entry = Column( - Integer, ForeignKey(MediaEntry.id), - nullable=False) - name_id = Column(SmallInteger, ForeignKey(FileKeynames.id), nullable=False) - file_path = Column(PathTupleWithSlashes) - - __table_args__ = ( - PrimaryKeyConstraint('media_entry', 'name_id'), - {}) - - def __repr__(self): - return "<MediaFile %s: %r>" % (self.name, self.file_path) - - name_helper = relationship(FileKeynames, lazy="joined", innerjoin=True) - name = association_proxy('name_helper', 'name', - creator=FileKeynames.find_or_new - ) - - -class MediaAttachmentFile(Base_v0): - __tablename__ = "core__attachment_files" - - id = Column(Integer, primary_key=True) - media_entry = Column( - Integer, ForeignKey(MediaEntry.id), - nullable=False) - name = Column(Unicode, nullable=False) - filepath = Column(PathTupleWithSlashes) - created = Column(DateTime, nullable=False, default=datetime.datetime.now) - - -class Tag(Base_v0): - __tablename__ = "core__tags" - - id = Column(Integer, primary_key=True) - slug = Column(Unicode, nullable=False, unique=True) - - def __repr__(self): - return "<Tag %r: %r>" % (self.id, self.slug) - - @classmethod - def find_or_new(cls, slug): - t = cls.query.filter_by(slug=slug).first() - if t is not None: - return t - return cls(slug=slug) - - -class MediaTag(Base_v0): - __tablename__ = "core__media_tags" - - id = Column(Integer, primary_key=True) - media_entry = Column( - Integer, ForeignKey(MediaEntry.id), - nullable=False, index=True) - tag = Column(Integer, ForeignKey(Tag.id), nullable=False, index=True) - name = Column(Unicode) - # created = Column(DateTime, nullable=False, default=datetime.datetime.now) - - __table_args__ = ( - UniqueConstraint('tag', 'media_entry'), - {}) - - tag_helper = relationship(Tag) - slug = association_proxy('tag_helper', 'slug', - creator=Tag.find_or_new - ) - - def __init__(self, name=None, slug=None): - Base_v0.__init__(self) - if name is not None: - self.name = name - if slug is not None: - self.tag_helper = Tag.find_or_new(slug) - - -class MediaComment(Base_v0): - __tablename__ = "core__media_comments" - - id = Column(Integer, primary_key=True) - media_entry = Column( - Integer, ForeignKey(MediaEntry.id), nullable=False, index=True) - author = Column(Integer, ForeignKey(User.id), nullable=False) - created = Column(DateTime, nullable=False, default=datetime.datetime.now) - content = Column(UnicodeText, nullable=False) - - get_author = relationship(User) - - -class ImageData(Base_v0): - __tablename__ = "image__mediadata" - - # The primary key *and* reference to the main media_entry - media_entry = Column(Integer, ForeignKey('core__media_entries.id'), - primary_key=True) - get_media_entry = relationship("MediaEntry", - backref=backref("image__media_data", cascade="all, delete-orphan")) - - width = Column(Integer) - height = Column(Integer) - exif_all = Column(JSONEncoded) - gps_longitude = Column(Float) - gps_latitude = Column(Float) - gps_altitude = Column(Float) - gps_direction = Column(Float) - - -class VideoData(Base_v0): - __tablename__ = "video__mediadata" - - # The primary key *and* reference to the main media_entry - media_entry = Column(Integer, ForeignKey('core__media_entries.id'), - primary_key=True) - get_media_entry = relationship("MediaEntry", - backref=backref("video__media_data", cascade="all, delete-orphan")) - - width = Column(SmallInteger) - height = Column(SmallInteger) - - -class AsciiData(Base_v0): - __tablename__ = "ascii__mediadata" - - # The primary key *and* reference to the main media_entry - media_entry = Column(Integer, ForeignKey('core__media_entries.id'), - primary_key=True) - get_media_entry = relationship("MediaEntry", - backref=backref("ascii__media_data", cascade="all, delete-orphan")) - - -class AudioData(Base_v0): - __tablename__ = "audio__mediadata" - - # The primary key *and* reference to the main media_entry - media_entry = Column(Integer, ForeignKey('core__media_entries.id'), - primary_key=True) - get_media_entry = relationship("MediaEntry", - backref=backref("audio__media_data", cascade="all, delete-orphan")) - - -###################################################### -# Special, migrations-tracking table -# -# Not listed in MODELS because this is special and not -# really migrated, but used for migrations (for now) -###################################################### - -class MigrationData(Base_v0): - __tablename__ = "core__migrations" - - name = Column(Unicode, primary_key=True) - version = Column(Integer, nullable=False, default=0) - -###################################################### diff --git a/mediagoblin/db/sql/open.py b/mediagoblin/db/sql/open.py deleted file mode 100644 index 9db21c56..00000000 --- a/mediagoblin/db/sql/open.py +++ /dev/null @@ -1,78 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -from sqlalchemy import create_engine -import logging - -from mediagoblin.db.sql.base import Base, Session -from mediagoblin import mg_globals - -_log = logging.getLogger(__name__) - - -class DatabaseMaster(object): - def __init__(self, engine): - self.engine = engine - - for k, v in Base._decl_class_registry.iteritems(): - setattr(self, k, v) - - def commit(self): - Session.commit() - - def save(self, obj): - Session.add(obj) - Session.flush() - - def check_session_clean(self): - for dummy in Session(): - _log.warn("STRANGE: There are elements in the sql session. " - "Please report this and help us track this down.") - break - - def reset_after_request(self): - Session.rollback() - Session.remove() - - -def load_models(app_config): - import mediagoblin.db.sql.models - - for media_type in app_config['media_types']: - _log.debug("Loading %s.models", media_type) - __import__(media_type + ".models") - - for plugin in mg_globals.global_config.get('plugins', {}).keys(): - _log.debug("Loading %s.models", plugin) - try: - __import__(plugin + ".models") - except ImportError as exc: - _log.debug("Could not load {0}.models: {1}".format( - plugin, - exc)) - - -def setup_connection_and_db_from_config(app_config): - engine = create_engine(app_config['sql_engine']) - # logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) - Session.configure(bind=engine) - - return "dummy conn", DatabaseMaster(engine) - - -def check_db_migrations_current(db): - pass diff --git a/mediagoblin/db/sql/util.py b/mediagoblin/db/sql/util.py deleted file mode 100644 index c6d8562e..00000000 --- a/mediagoblin/db/sql/util.py +++ /dev/null @@ -1,327 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -import sys -from mediagoblin.db.sql.base import Session -from mediagoblin.db.sql.models import MediaEntry, Tag, MediaTag, Collection - -from mediagoblin.tools.common import simple_printer - - -class MigrationManager(object): - """ - Migration handling tool. - - Takes information about a database, lets you update the database - to the latest migrations, etc. - """ - - def __init__(self, name, models, migration_registry, session, - printer=simple_printer): - """ - Args: - - name: identifier of this section of the database - - session: session we're going to migrate - - migration_registry: where we should find all migrations to - run - """ - self.name = unicode(name) - self.models = models - self.session = session - self.migration_registry = migration_registry - self._sorted_migrations = None - self.printer = printer - - # For convenience - from mediagoblin.db.sql.models import MigrationData - - self.migration_model = MigrationData - self.migration_table = MigrationData.__table__ - - @property - def sorted_migrations(self): - """ - Sort migrations if necessary and store in self._sorted_migrations - """ - if not self._sorted_migrations: - self._sorted_migrations = sorted( - self.migration_registry.items(), - # sort on the key... the migration number - key=lambda migration_tuple: migration_tuple[0]) - - return self._sorted_migrations - - @property - def migration_data(self): - """ - Get the migration row associated with this object, if any. - """ - return self.session.query( - self.migration_model).filter_by(name=self.name).first() - - @property - def latest_migration(self): - """ - Return a migration number for the latest migration, or 0 if - there are no migrations. - """ - if self.sorted_migrations: - return self.sorted_migrations[-1][0] - else: - # If no migrations have been set, we start at 0. - return 0 - - @property - def database_current_migration(self): - """ - Return the current migration in the database. - """ - # If the table doesn't even exist, return None. - if not self.migration_table.exists(self.session.bind): - return None - - # Also return None if self.migration_data is None. - if self.migration_data is None: - return None - - return self.migration_data.version - - def set_current_migration(self, migration_number=None): - """ - Set the migration in the database to migration_number - (or, the latest available) - """ - self.migration_data.version = migration_number or self.latest_migration - self.session.commit() - - def migrations_to_run(self): - """ - Get a list of migrations to run still, if any. - - Note that this will fail if there's no migration record for - this class! - """ - assert self.database_current_migration is not None - - db_current_migration = self.database_current_migration - - return [ - (migration_number, migration_func) - for migration_number, migration_func in self.sorted_migrations - if migration_number > db_current_migration] - - - def init_tables(self): - """ - Create all tables relative to this package - """ - # sanity check before we proceed, none of these should be created - for model in self.models: - # Maybe in the future just print out a "Yikes!" or something? - assert not model.__table__.exists(self.session.bind) - - self.migration_model.metadata.create_all( - self.session.bind, - tables=[model.__table__ for model in self.models]) - - def create_new_migration_record(self): - """ - Create a new migration record for this migration set - """ - migration_record = self.migration_model( - name=self.name, - version=self.latest_migration) - self.session.add(migration_record) - self.session.commit() - - def dry_run(self): - """ - Print out a dry run of what we would have upgraded. - """ - if self.database_current_migration is None: - self.printer( - u'~> Woulda initialized: %s\n' % self.name_for_printing()) - return u'inited' - - migrations_to_run = self.migrations_to_run() - if migrations_to_run: - self.printer( - u'~> Woulda updated %s:\n' % self.name_for_printing()) - - for migration_number, migration_func in migrations_to_run(): - self.printer( - u' + Would update %s, "%s"\n' % ( - migration_number, migration_func.func_name)) - - return u'migrated' - - def name_for_printing(self): - if self.name == u'__main__': - return u"main mediagoblin tables" - else: - # TODO: Use the friendlier media manager "human readable" name - return u'media type "%s"' % self.name - - def init_or_migrate(self): - """ - Initialize the database or migrate if appropriate. - - Returns information about whether or not we initialized - ('inited'), migrated ('migrated'), or did nothing (None) - """ - assure_migrations_table_setup(self.session) - - # Find out what migration number, if any, this database data is at, - # and what the latest is. - migration_number = self.database_current_migration - - # Is this our first time? Is there even a table entry for - # this identifier? - # If so: - # - create all tables - # - create record in migrations registry - # - print / inform the user - # - return 'inited' - if migration_number is None: - self.printer(u"-> Initializing %s... " % self.name_for_printing()) - - self.init_tables() - # auto-set at latest migration number - self.create_new_migration_record() - - self.printer(u"done.\n") - self.set_current_migration() - return u'inited' - - # Run migrations, if appropriate. - migrations_to_run = self.migrations_to_run() - if migrations_to_run: - self.printer( - u'-> Updating %s:\n' % self.name_for_printing()) - for migration_number, migration_func in migrations_to_run: - self.printer( - u' + Running migration %s, "%s"... ' % ( - migration_number, migration_func.func_name)) - migration_func(self.session) - self.printer('done.\n') - - self.set_current_migration() - return u'migrated' - - # Otherwise return None. Well it would do this anyway, but - # for clarity... ;) - return None - - -class RegisterMigration(object): - """ - Tool for registering migrations - - Call like: - - @RegisterMigration(33) - def update_dwarves(database): - [...] - - This will register your migration with the default migration - registry. Alternately, to specify a very specific - migration_registry, you can pass in that as the second argument. - - Note, the number of your migration should NEVER be 0 or less than - 0. 0 is the default "no migrations" state! - """ - def __init__(self, migration_number, migration_registry): - assert migration_number > 0, "Migration number must be > 0!" - assert migration_number not in migration_registry, \ - "Duplicate migration numbers detected! That's not allowed!" - - self.migration_number = migration_number - self.migration_registry = migration_registry - - def __call__(self, migration): - self.migration_registry[self.migration_number] = migration - return migration - - -def assure_migrations_table_setup(db): - """ - Make sure the migrations table is set up in the database. - """ - from mediagoblin.db.sql.models import MigrationData - - if not MigrationData.__table__.exists(db.bind): - MigrationData.metadata.create_all( - db.bind, tables=[MigrationData.__table__]) - - -########################## -# Random utility functions -########################## - - -def atomic_update(table, query_dict, update_values): - table.find(query_dict).update(update_values, - synchronize_session=False) - Session.commit() - - -def check_media_slug_used(dummy_db, uploader_id, slug, ignore_m_id): - filt = (MediaEntry.uploader == uploader_id) \ - & (MediaEntry.slug == slug) - if ignore_m_id is not None: - filt = filt & (MediaEntry.id != ignore_m_id) - does_exist = Session.query(MediaEntry.id).filter(filt).first() is not None - return does_exist - - -def media_entries_for_tag_slug(dummy_db, tag_slug): - return MediaEntry.query \ - .join(MediaEntry.tags_helper) \ - .join(MediaTag.tag_helper) \ - .filter( - (MediaEntry.state == u'processed') - & (Tag.slug == tag_slug)) - - -def clean_orphan_tags(commit=True): - """Search for unused MediaTags and delete them""" - q1 = Session.query(Tag).outerjoin(MediaTag).filter(MediaTag.id==None) - for t in q1: - Session.delete(t) - # The "let the db do all the work" version: - # q1 = Session.query(Tag.id).outerjoin(MediaTag).filter(MediaTag.id==None) - # q2 = Session.query(Tag).filter(Tag.id.in_(q1)) - # q2.delete(synchronize_session = False) - if commit: - Session.commit() - - -def check_collection_slug_used(dummy_db, creator_id, slug, ignore_c_id): - filt = (Collection.creator == creator_id) \ - & (Collection.slug == slug) - if ignore_c_id is not None: - filt = filt & (Collection.id != ignore_c_id) - does_exist = Session.query(Collection.id).filter(filt).first() is not None - return does_exist - - -if __name__ == '__main__': - from mediagoblin.db.sql.open import setup_connection_and_db_from_config - - conn,db = setup_connection_and_db_from_config({'sql_engine':'sqlite:///mediagoblin.db'}) - - clean_orphan_tags() |