aboutsummaryrefslogtreecommitdiffstats
path: root/mediagoblin/db/sql
diff options
context:
space:
mode:
Diffstat (limited to 'mediagoblin/db/sql')
-rw-r--r--mediagoblin/db/sql/__init__.py15
-rw-r--r--mediagoblin/db/sql/base.py38
-rw-r--r--mediagoblin/db/sql/convert.py151
-rw-r--r--mediagoblin/db/sql/extratypes.py18
-rw-r--r--mediagoblin/db/sql/models.py153
-rw-r--r--mediagoblin/db/sql/open.py33
6 files changed, 408 insertions, 0 deletions
diff --git a/mediagoblin/db/sql/__init__.py b/mediagoblin/db/sql/__init__.py
new file mode 100644
index 00000000..ba347c69
--- /dev/null
+++ b/mediagoblin/db/sql/__init__.py
@@ -0,0 +1,15 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
diff --git a/mediagoblin/db/sql/base.py b/mediagoblin/db/sql/base.py
new file mode 100644
index 00000000..40140327
--- /dev/null
+++ b/mediagoblin/db/sql/base.py
@@ -0,0 +1,38 @@
+from sqlalchemy.orm import scoped_session, sessionmaker, object_session
+
+
+Session = scoped_session(sessionmaker())
+
+
+def _fix_query_dict(query_dict):
+ if '_id' in query_dict:
+ query_dict['id'] = query_dict.pop('_id')
+
+
+class GMGTableBase(object):
+ query = Session.query_property()
+
+ @classmethod
+ def find(cls, query_dict={}):
+ _fix_query_dict(query_dict)
+ return cls.query.filter_by(**query_dict)
+
+ @classmethod
+ def find_one(cls, query_dict={}):
+ _fix_query_dict(query_dict)
+ return cls.query.filter_by(**query_dict).first()
+
+ @classmethod
+ def one(cls, query_dict):
+ return cls.find(query_dict).one()
+
+ def get(self, key):
+ return getattr(self, key)
+
+ def save(self, validate = True):
+ assert validate
+ sess = object_session(self)
+ if sess is None:
+ sess = Session()
+ sess.add(self)
+ sess.commit()
diff --git a/mediagoblin/db/sql/convert.py b/mediagoblin/db/sql/convert.py
new file mode 100644
index 00000000..88614fd4
--- /dev/null
+++ b/mediagoblin/db/sql/convert.py
@@ -0,0 +1,151 @@
+from mediagoblin.init import setup_global_and_app_config, setup_database
+from mediagoblin.db.mongo.util import ObjectId
+
+from mediagoblin.db.sql.models import (Base, User, MediaEntry, MediaComment,
+ Tag, MediaTag, MediaFile)
+from mediagoblin.db.sql.open import setup_connection_and_db_from_config as \
+ sql_connect
+from mediagoblin.db.mongo.open import setup_connection_and_db_from_config as \
+ mongo_connect
+from mediagoblin.db.sql.base import Session
+
+
+obj_id_table = dict()
+
+def add_obj_ids(entry, new_entry):
+ global obj_id_table
+ print "%r -> %r" % (entry._id, new_entry.id)
+ obj_id_table[entry._id] = new_entry.id
+
+
+def copy_attrs(entry, new_entry, attr_list):
+ for a in attr_list:
+ val = entry[a]
+ setattr(new_entry, a, val)
+
+def copy_reference_attr(entry, new_entry, ref_attr):
+ val = entry[ref_attr]
+ val = obj_id_table[val]
+ setattr(new_entry, ref_attr, val)
+
+
+def convert_users(mk_db):
+ session = Session()
+
+ for entry in mk_db.User.find():
+ print entry.username
+
+ new_entry = User()
+ copy_attrs(entry, new_entry,
+ ('username', 'email', 'created', 'pw_hash', 'email_verified',
+ 'status', 'verification_key', 'is_admin', 'url',
+ 'bio', 'bio_html',
+ 'fp_verification_key', 'fp_token_expire',))
+ # new_entry.fp_verification_expire = entry.fp_token_expire
+
+ session.add(new_entry)
+ session.flush()
+ add_obj_ids(entry, new_entry)
+
+ session.commit()
+ session.close()
+
+
+def convert_media_entries(mk_db):
+ session = Session()
+
+ for entry in mk_db.MediaEntry.find():
+ print repr(entry.title)
+
+ new_entry = MediaEntry()
+ copy_attrs(entry, new_entry,
+ ('title', 'slug', 'created',
+ 'description', 'description_html',
+ 'media_type', 'state',
+ 'fail_error',
+ 'queued_task_id',))
+ copy_reference_attr(entry, new_entry, "uploader")
+
+ session.add(new_entry)
+ session.flush()
+ add_obj_ids(entry, new_entry)
+
+ for key, value in entry.media_files.iteritems():
+ new_file = MediaFile(name=key, file_path=value)
+ new_file.media_entry = new_entry.id
+ Session.add(new_file)
+
+ session.commit()
+ session.close()
+
+
+def convert_media_tags(mk_db):
+ session = Session()
+ session.autoflush = False
+
+ for media in mk_db.MediaEntry.find():
+ print repr(media.title)
+
+ for otag in media.tags:
+ print " ", repr((otag["slug"], otag["name"]))
+
+ nslug = session.query(Tag).filter_by(slug=otag["slug"]).first()
+ print " ", repr(nslug)
+ if nslug is None:
+ nslug = Tag(slug=otag["slug"])
+ session.add(nslug)
+ session.flush()
+ print " ", repr(nslug), nslug.id
+
+ ntag = MediaTag()
+ ntag.tag = nslug.id
+ ntag.name = otag["name"]
+ ntag.media_entry = obj_id_table[media._id]
+ session.add(ntag)
+
+ session.commit()
+ session.close()
+
+
+def convert_media_comments(mk_db):
+ session = Session()
+
+ for entry in mk_db.MediaComment.find():
+ print repr(entry.content)
+
+ new_entry = MediaComment()
+ copy_attrs(entry, new_entry,
+ ('created',
+ 'content', 'content_html',))
+ copy_reference_attr(entry, new_entry, "media_entry")
+ copy_reference_attr(entry, new_entry, "author")
+
+ session.add(new_entry)
+ session.flush()
+ add_obj_ids(entry, new_entry)
+
+ session.commit()
+ session.close()
+
+
+def main():
+ global_config, app_config = setup_global_and_app_config("mediagoblin.ini")
+
+ sql_conn, sql_db = sql_connect({'sql_engine': 'sqlite:///mediagoblin.db'})
+
+ mk_conn, mk_db = mongo_connect(app_config)
+
+ Base.metadata.create_all(sql_db.engine)
+
+ convert_users(mk_db)
+ Session.remove()
+ convert_media_entries(mk_db)
+ Session.remove()
+ convert_media_tags(mk_db)
+ Session.remove()
+ convert_media_comments(mk_db)
+ Session.remove()
+
+
+if __name__ == '__main__':
+ main()
diff --git a/mediagoblin/db/sql/extratypes.py b/mediagoblin/db/sql/extratypes.py
new file mode 100644
index 00000000..88f556d9
--- /dev/null
+++ b/mediagoblin/db/sql/extratypes.py
@@ -0,0 +1,18 @@
+from sqlalchemy.types import TypeDecorator, Unicode
+
+
+class PathTupleWithSlashes(TypeDecorator):
+ "Represents a Tuple of strings as a slash separated string."
+
+ impl = Unicode
+
+ def process_bind_param(self, value, dialect):
+ if value is not None:
+ assert len(value), "Does not support empty lists"
+ value = '/'.join(value)
+ return value
+
+ def process_result_value(self, value, dialect):
+ if value is not None:
+ value = tuple(value.split('/'))
+ return value
diff --git a/mediagoblin/db/sql/models.py b/mediagoblin/db/sql/models.py
new file mode 100644
index 00000000..91092f33
--- /dev/null
+++ b/mediagoblin/db/sql/models.py
@@ -0,0 +1,153 @@
+import datetime
+
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy import (
+ Column, Integer, Unicode, UnicodeText, DateTime, Boolean, ForeignKey,
+ UniqueConstraint)
+from sqlalchemy.orm import relationship
+from sqlalchemy.orm.collections import attribute_mapped_collection
+from sqlalchemy.ext.associationproxy import association_proxy
+
+from mediagoblin.db.sql.extratypes import PathTupleWithSlashes
+from mediagoblin.db.sql.base import GMGTableBase
+from mediagoblin.db.mixin import UserMixin, MediaEntryMixin
+
+
+Base = declarative_base(cls=GMGTableBase)
+
+
+class SimpleFieldAlias(object):
+ """An alias for any field"""
+ def __init__(self, fieldname):
+ self.fieldname = fieldname
+
+ def __get__(self, instance, cls):
+ return getattr(instance, self.fieldname)
+
+ def __set__(self, instance, val):
+ setattr(instance, self.fieldname, val)
+
+
+class User(Base, UserMixin):
+ __tablename__ = "users"
+
+ id = Column(Integer, primary_key=True)
+ username = Column(Unicode, nullable=False, unique=True)
+ email = Column(Unicode, nullable=False)
+ created = Column(DateTime, nullable=False, default=datetime.datetime.now)
+ pw_hash = Column(Unicode, nullable=False)
+ email_verified = Column(Boolean)
+ status = Column(Unicode, default=u"needs_email_verification", nullable=False)
+ verification_key = Column(Unicode)
+ is_admin = Column(Boolean, default=False, nullable=False)
+ url = Column(Unicode)
+ bio = Column(UnicodeText) # ??
+ bio_html = Column(UnicodeText) # ??
+ fp_verification_key = Column(Unicode)
+ fp_token_expire = Column(DateTime)
+
+ ## TODO
+ # plugin data would be in a separate model
+
+ _id = SimpleFieldAlias("id")
+
+
+class MediaEntry(Base, MediaEntryMixin):
+ __tablename__ = "media_entries"
+
+ id = Column(Integer, primary_key=True)
+ uploader = Column(Integer, ForeignKey('users.id'), nullable=False)
+ title = Column(Unicode, nullable=False)
+ slug = Column(Unicode, nullable=False)
+ created = Column(DateTime, nullable=False, default=datetime.datetime.now)
+ description = Column(UnicodeText) # ??
+ description_html = Column(UnicodeText) # ??
+ media_type = Column(Unicode, nullable=False)
+ state = Column(Unicode, nullable=False) # or use sqlalchemy.types.Enum?
+
+ fail_error = Column(Unicode)
+ fail_metadata = Column(UnicodeText)
+
+ queued_media_file = Column(PathTupleWithSlashes)
+
+ queued_task_id = Column(Unicode)
+
+ __table_args__ = (
+ UniqueConstraint('uploader', 'slug'),
+ {})
+
+ get_uploader = relationship(User)
+
+ media_files_helper = relationship("MediaFile",
+ collection_class=attribute_mapped_collection("name"),
+ cascade="all, delete-orphan"
+ )
+ media_files = association_proxy('media_files_helper', 'file_path',
+ creator=lambda k,v: MediaFile(name=k, file_path=v)
+ )
+
+ ## TODO
+ # media_data
+ # attachment_files
+ # fail_error
+
+
+class MediaFile(Base):
+ __tablename__ = "mediafiles"
+
+ media_entry = Column(
+ Integer, ForeignKey(MediaEntry.id),
+ nullable=False, primary_key=True)
+ name = Column(Unicode, primary_key=True)
+ file_path = Column(PathTupleWithSlashes)
+
+ def __repr__(self):
+ return "<MediaFile %s: %r>" % (self.name, self.file_path)
+
+
+class Tag(Base):
+ __tablename__ = "tags"
+
+ id = Column(Integer, primary_key=True)
+ slug = Column(Unicode, nullable=False, unique=True)
+
+
+class MediaTag(Base):
+ __tablename__ = "media_tags"
+
+ id = Column(Integer, primary_key=True)
+ tag = Column(Integer, ForeignKey('tags.id'), nullable=False)
+ name = Column(Unicode)
+ media_entry = Column(
+ Integer, ForeignKey('media_entries.id'),
+ nullable=False)
+ # created = Column(DateTime, nullable=False, default=datetime.datetime.now)
+
+ __table_args__ = (
+ UniqueConstraint('tag', 'media_entry'),
+ {})
+
+
+class MediaComment(Base):
+ __tablename__ = "media_comments"
+
+ id = Column(Integer, primary_key=True)
+ media_entry = Column(
+ Integer, ForeignKey('media_entries.id'), nullable=False)
+ author = Column(Integer, ForeignKey('users.id'), nullable=False)
+ created = Column(DateTime, nullable=False, default=datetime.datetime.now)
+ content = Column(UnicodeText, nullable=False)
+ content_html = Column(UnicodeText)
+
+ get_author = relationship(User)
+
+
+def show_table_init():
+ from sqlalchemy import create_engine
+ engine = create_engine('sqlite:///:memory:', echo=True)
+
+ Base.metadata.create_all(engine)
+
+
+if __name__ == '__main__':
+ show_table_init()
diff --git a/mediagoblin/db/sql/open.py b/mediagoblin/db/sql/open.py
new file mode 100644
index 00000000..c682bd3b
--- /dev/null
+++ b/mediagoblin/db/sql/open.py
@@ -0,0 +1,33 @@
+from sqlalchemy import create_engine
+
+from mediagoblin.db.sql.base import Session
+from mediagoblin.db.sql.models import Base
+
+
+class DatabaseMaster(object):
+ def __init__(self, engine):
+ self.engine = engine
+
+ for k,v in Base._decl_class_registry.iteritems():
+ setattr(self, k, v)
+
+ def commit(self):
+ Session.commit()
+
+ def save(self, obj):
+ Session.add(obj)
+ Session.flush()
+
+ def reset_after_request(self):
+ Session.remove()
+
+
+def setup_connection_and_db_from_config(app_config):
+ engine = create_engine(app_config['sql_engine'], echo=True)
+ Session.configure(bind=engine)
+
+ return "dummy conn", DatabaseMaster(engine)
+
+
+def check_db_migrations_current(db):
+ pass