diff options
Diffstat (limited to 'mediagoblin/db/sql')
-rw-r--r-- | mediagoblin/db/sql/__init__.py | 15 | ||||
-rw-r--r-- | mediagoblin/db/sql/base.py | 38 | ||||
-rw-r--r-- | mediagoblin/db/sql/convert.py | 151 | ||||
-rw-r--r-- | mediagoblin/db/sql/extratypes.py | 18 | ||||
-rw-r--r-- | mediagoblin/db/sql/models.py | 153 | ||||
-rw-r--r-- | mediagoblin/db/sql/open.py | 33 |
6 files changed, 408 insertions, 0 deletions
diff --git a/mediagoblin/db/sql/__init__.py b/mediagoblin/db/sql/__init__.py new file mode 100644 index 00000000..ba347c69 --- /dev/null +++ b/mediagoblin/db/sql/__init__.py @@ -0,0 +1,15 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. diff --git a/mediagoblin/db/sql/base.py b/mediagoblin/db/sql/base.py new file mode 100644 index 00000000..40140327 --- /dev/null +++ b/mediagoblin/db/sql/base.py @@ -0,0 +1,38 @@ +from sqlalchemy.orm import scoped_session, sessionmaker, object_session + + +Session = scoped_session(sessionmaker()) + + +def _fix_query_dict(query_dict): + if '_id' in query_dict: + query_dict['id'] = query_dict.pop('_id') + + +class GMGTableBase(object): + query = Session.query_property() + + @classmethod + def find(cls, query_dict={}): + _fix_query_dict(query_dict) + return cls.query.filter_by(**query_dict) + + @classmethod + def find_one(cls, query_dict={}): + _fix_query_dict(query_dict) + return cls.query.filter_by(**query_dict).first() + + @classmethod + def one(cls, query_dict): + return cls.find(query_dict).one() + + def get(self, key): + return getattr(self, key) + + def save(self, validate = True): + assert validate + sess = object_session(self) + if sess is None: + sess = Session() + sess.add(self) + sess.commit() diff --git a/mediagoblin/db/sql/convert.py b/mediagoblin/db/sql/convert.py new file mode 100644 index 00000000..88614fd4 --- /dev/null +++ b/mediagoblin/db/sql/convert.py @@ -0,0 +1,151 @@ +from mediagoblin.init import setup_global_and_app_config, setup_database +from mediagoblin.db.mongo.util import ObjectId + +from mediagoblin.db.sql.models import (Base, User, MediaEntry, MediaComment, + Tag, MediaTag, MediaFile) +from mediagoblin.db.sql.open import setup_connection_and_db_from_config as \ + sql_connect +from mediagoblin.db.mongo.open import setup_connection_and_db_from_config as \ + mongo_connect +from mediagoblin.db.sql.base import Session + + +obj_id_table = dict() + +def add_obj_ids(entry, new_entry): + global obj_id_table + print "%r -> %r" % (entry._id, new_entry.id) + obj_id_table[entry._id] = new_entry.id + + +def copy_attrs(entry, new_entry, attr_list): + for a in attr_list: + val = entry[a] + setattr(new_entry, a, val) + +def copy_reference_attr(entry, new_entry, ref_attr): + val = entry[ref_attr] + val = obj_id_table[val] + setattr(new_entry, ref_attr, val) + + +def convert_users(mk_db): + session = Session() + + for entry in mk_db.User.find(): + print entry.username + + new_entry = User() + copy_attrs(entry, new_entry, + ('username', 'email', 'created', 'pw_hash', 'email_verified', + 'status', 'verification_key', 'is_admin', 'url', + 'bio', 'bio_html', + 'fp_verification_key', 'fp_token_expire',)) + # new_entry.fp_verification_expire = entry.fp_token_expire + + session.add(new_entry) + session.flush() + add_obj_ids(entry, new_entry) + + session.commit() + session.close() + + +def convert_media_entries(mk_db): + session = Session() + + for entry in mk_db.MediaEntry.find(): + print repr(entry.title) + + new_entry = MediaEntry() + copy_attrs(entry, new_entry, + ('title', 'slug', 'created', + 'description', 'description_html', + 'media_type', 'state', + 'fail_error', + 'queued_task_id',)) + copy_reference_attr(entry, new_entry, "uploader") + + session.add(new_entry) + session.flush() + add_obj_ids(entry, new_entry) + + for key, value in entry.media_files.iteritems(): + new_file = MediaFile(name=key, file_path=value) + new_file.media_entry = new_entry.id + Session.add(new_file) + + session.commit() + session.close() + + +def convert_media_tags(mk_db): + session = Session() + session.autoflush = False + + for media in mk_db.MediaEntry.find(): + print repr(media.title) + + for otag in media.tags: + print " ", repr((otag["slug"], otag["name"])) + + nslug = session.query(Tag).filter_by(slug=otag["slug"]).first() + print " ", repr(nslug) + if nslug is None: + nslug = Tag(slug=otag["slug"]) + session.add(nslug) + session.flush() + print " ", repr(nslug), nslug.id + + ntag = MediaTag() + ntag.tag = nslug.id + ntag.name = otag["name"] + ntag.media_entry = obj_id_table[media._id] + session.add(ntag) + + session.commit() + session.close() + + +def convert_media_comments(mk_db): + session = Session() + + for entry in mk_db.MediaComment.find(): + print repr(entry.content) + + new_entry = MediaComment() + copy_attrs(entry, new_entry, + ('created', + 'content', 'content_html',)) + copy_reference_attr(entry, new_entry, "media_entry") + copy_reference_attr(entry, new_entry, "author") + + session.add(new_entry) + session.flush() + add_obj_ids(entry, new_entry) + + session.commit() + session.close() + + +def main(): + global_config, app_config = setup_global_and_app_config("mediagoblin.ini") + + sql_conn, sql_db = sql_connect({'sql_engine': 'sqlite:///mediagoblin.db'}) + + mk_conn, mk_db = mongo_connect(app_config) + + Base.metadata.create_all(sql_db.engine) + + convert_users(mk_db) + Session.remove() + convert_media_entries(mk_db) + Session.remove() + convert_media_tags(mk_db) + Session.remove() + convert_media_comments(mk_db) + Session.remove() + + +if __name__ == '__main__': + main() diff --git a/mediagoblin/db/sql/extratypes.py b/mediagoblin/db/sql/extratypes.py new file mode 100644 index 00000000..88f556d9 --- /dev/null +++ b/mediagoblin/db/sql/extratypes.py @@ -0,0 +1,18 @@ +from sqlalchemy.types import TypeDecorator, Unicode + + +class PathTupleWithSlashes(TypeDecorator): + "Represents a Tuple of strings as a slash separated string." + + impl = Unicode + + def process_bind_param(self, value, dialect): + if value is not None: + assert len(value), "Does not support empty lists" + value = '/'.join(value) + return value + + def process_result_value(self, value, dialect): + if value is not None: + value = tuple(value.split('/')) + return value diff --git a/mediagoblin/db/sql/models.py b/mediagoblin/db/sql/models.py new file mode 100644 index 00000000..91092f33 --- /dev/null +++ b/mediagoblin/db/sql/models.py @@ -0,0 +1,153 @@ +import datetime + +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import ( + Column, Integer, Unicode, UnicodeText, DateTime, Boolean, ForeignKey, + UniqueConstraint) +from sqlalchemy.orm import relationship +from sqlalchemy.orm.collections import attribute_mapped_collection +from sqlalchemy.ext.associationproxy import association_proxy + +from mediagoblin.db.sql.extratypes import PathTupleWithSlashes +from mediagoblin.db.sql.base import GMGTableBase +from mediagoblin.db.mixin import UserMixin, MediaEntryMixin + + +Base = declarative_base(cls=GMGTableBase) + + +class SimpleFieldAlias(object): + """An alias for any field""" + def __init__(self, fieldname): + self.fieldname = fieldname + + def __get__(self, instance, cls): + return getattr(instance, self.fieldname) + + def __set__(self, instance, val): + setattr(instance, self.fieldname, val) + + +class User(Base, UserMixin): + __tablename__ = "users" + + id = Column(Integer, primary_key=True) + username = Column(Unicode, nullable=False, unique=True) + email = Column(Unicode, nullable=False) + created = Column(DateTime, nullable=False, default=datetime.datetime.now) + pw_hash = Column(Unicode, nullable=False) + email_verified = Column(Boolean) + status = Column(Unicode, default=u"needs_email_verification", nullable=False) + verification_key = Column(Unicode) + is_admin = Column(Boolean, default=False, nullable=False) + url = Column(Unicode) + bio = Column(UnicodeText) # ?? + bio_html = Column(UnicodeText) # ?? + fp_verification_key = Column(Unicode) + fp_token_expire = Column(DateTime) + + ## TODO + # plugin data would be in a separate model + + _id = SimpleFieldAlias("id") + + +class MediaEntry(Base, MediaEntryMixin): + __tablename__ = "media_entries" + + id = Column(Integer, primary_key=True) + uploader = Column(Integer, ForeignKey('users.id'), nullable=False) + title = Column(Unicode, nullable=False) + slug = Column(Unicode, nullable=False) + created = Column(DateTime, nullable=False, default=datetime.datetime.now) + description = Column(UnicodeText) # ?? + description_html = Column(UnicodeText) # ?? + media_type = Column(Unicode, nullable=False) + state = Column(Unicode, nullable=False) # or use sqlalchemy.types.Enum? + + fail_error = Column(Unicode) + fail_metadata = Column(UnicodeText) + + queued_media_file = Column(PathTupleWithSlashes) + + queued_task_id = Column(Unicode) + + __table_args__ = ( + UniqueConstraint('uploader', 'slug'), + {}) + + get_uploader = relationship(User) + + media_files_helper = relationship("MediaFile", + collection_class=attribute_mapped_collection("name"), + cascade="all, delete-orphan" + ) + media_files = association_proxy('media_files_helper', 'file_path', + creator=lambda k,v: MediaFile(name=k, file_path=v) + ) + + ## TODO + # media_data + # attachment_files + # fail_error + + +class MediaFile(Base): + __tablename__ = "mediafiles" + + media_entry = Column( + Integer, ForeignKey(MediaEntry.id), + nullable=False, primary_key=True) + name = Column(Unicode, primary_key=True) + file_path = Column(PathTupleWithSlashes) + + def __repr__(self): + return "<MediaFile %s: %r>" % (self.name, self.file_path) + + +class Tag(Base): + __tablename__ = "tags" + + id = Column(Integer, primary_key=True) + slug = Column(Unicode, nullable=False, unique=True) + + +class MediaTag(Base): + __tablename__ = "media_tags" + + id = Column(Integer, primary_key=True) + tag = Column(Integer, ForeignKey('tags.id'), nullable=False) + name = Column(Unicode) + media_entry = Column( + Integer, ForeignKey('media_entries.id'), + nullable=False) + # created = Column(DateTime, nullable=False, default=datetime.datetime.now) + + __table_args__ = ( + UniqueConstraint('tag', 'media_entry'), + {}) + + +class MediaComment(Base): + __tablename__ = "media_comments" + + id = Column(Integer, primary_key=True) + media_entry = Column( + Integer, ForeignKey('media_entries.id'), nullable=False) + author = Column(Integer, ForeignKey('users.id'), nullable=False) + created = Column(DateTime, nullable=False, default=datetime.datetime.now) + content = Column(UnicodeText, nullable=False) + content_html = Column(UnicodeText) + + get_author = relationship(User) + + +def show_table_init(): + from sqlalchemy import create_engine + engine = create_engine('sqlite:///:memory:', echo=True) + + Base.metadata.create_all(engine) + + +if __name__ == '__main__': + show_table_init() diff --git a/mediagoblin/db/sql/open.py b/mediagoblin/db/sql/open.py new file mode 100644 index 00000000..c682bd3b --- /dev/null +++ b/mediagoblin/db/sql/open.py @@ -0,0 +1,33 @@ +from sqlalchemy import create_engine + +from mediagoblin.db.sql.base import Session +from mediagoblin.db.sql.models import Base + + +class DatabaseMaster(object): + def __init__(self, engine): + self.engine = engine + + for k,v in Base._decl_class_registry.iteritems(): + setattr(self, k, v) + + def commit(self): + Session.commit() + + def save(self, obj): + Session.add(obj) + Session.flush() + + def reset_after_request(self): + Session.remove() + + +def setup_connection_and_db_from_config(app_config): + engine = create_engine(app_config['sql_engine'], echo=True) + Session.configure(bind=engine) + + return "dummy conn", DatabaseMaster(engine) + + +def check_db_migrations_current(db): + pass |