aboutsummaryrefslogtreecommitdiffstats
path: root/mediagoblin/db/sql
diff options
context:
space:
mode:
Diffstat (limited to 'mediagoblin/db/sql')
-rw-r--r--mediagoblin/db/sql/convert.py57
-rw-r--r--mediagoblin/db/sql/models.py64
-rw-r--r--mediagoblin/db/sql/open.py16
-rw-r--r--mediagoblin/db/sql/util.py9
4 files changed, 123 insertions, 23 deletions
diff --git a/mediagoblin/db/sql/convert.py b/mediagoblin/db/sql/convert.py
index 250c559b..ebf3037c 100644
--- a/mediagoblin/db/sql/convert.py
+++ b/mediagoblin/db/sql/convert.py
@@ -14,18 +14,20 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
+from copy import copy
from mediagoblin.init import setup_global_and_app_config, setup_database
from mediagoblin.db.mongo.util import ObjectId
-from mediagoblin.db.sql.models import (Base, User, MediaEntry, MediaComment,
- Tag, MediaTag, MediaFile, MediaAttachmentFile)
+from mediagoblin.db.sql.base import Base, Session
+from mediagoblin.db.sql.models import (User, MediaEntry, MediaComment,
+ Tag, MediaTag, MediaFile, MediaAttachmentFile, MigrationData)
+from mediagoblin.media_types.image.models import ImageData
from mediagoblin.media_types.video.models import VideoData
from mediagoblin.db.sql.open import setup_connection_and_db_from_config as \
sql_connect
from mediagoblin.db.mongo.open import setup_connection_and_db_from_config as \
mongo_connect
-from mediagoblin.db.sql.base import Session
obj_id_table = dict()
@@ -106,6 +108,35 @@ def convert_media_entries(mk_db):
session.close()
+def convert_image(mk_db):
+ session = Session()
+
+ for media in mk_db.MediaEntry.find(
+ {'media_type': 'mediagoblin.media_types.image'}).sort('created'):
+ media_data = copy(media.media_data)
+
+ if len(media_data):
+ media_data_row = ImageData(**media_data)
+ media_data_row.media_entry = obj_id_table[media['_id']]
+ session.add(media_data_row)
+
+ session.commit()
+ session.close()
+
+
+def convert_video(mk_db):
+ session = Session()
+
+ for media in mk_db.MediaEntry.find(
+ {'media_type': 'mediagoblin.media_types.video'}).sort('created'):
+ media_data_row = VideoData(**media.media_data)
+ media_data_row.media_entry = obj_id_table[media['_id']]
+ session.add(media_data_row)
+
+ session.commit()
+ session.close()
+
+
def convert_media_tags(mk_db):
session = Session()
session.autoflush = False
@@ -155,6 +186,20 @@ def convert_media_comments(mk_db):
session.close()
+def convert_add_migration_versions():
+ session = Session()
+
+ for name in ("__main__",
+ "mediagoblin.media_types.image",
+ "mediagoblin.media_types.video",
+ ):
+ m = MigrationData(name=name, version=0)
+ session.add(m)
+
+ session.commit()
+ session.close()
+
+
def run_conversion(config_name):
global_config, app_config = setup_global_and_app_config(config_name)
@@ -167,10 +212,16 @@ def run_conversion(config_name):
Session.remove()
convert_media_entries(mk_db)
Session.remove()
+ convert_image(mk_db)
+ Session.remove()
+ convert_video(mk_db)
+ Session.remove()
convert_media_tags(mk_db)
Session.remove()
convert_media_comments(mk_db)
Session.remove()
+ convert_add_migration_versions()
+ Session.remove()
if __name__ == '__main__':
diff --git a/mediagoblin/db/sql/models.py b/mediagoblin/db/sql/models.py
index 089bbb40..e87aaddb 100644
--- a/mediagoblin/db/sql/models.py
+++ b/mediagoblin/db/sql/models.py
@@ -20,6 +20,7 @@ TODO: indexes on foreignkeys, where useful.
import datetime
+import sys
from sqlalchemy import (
Column, Integer, Unicode, UnicodeText, DateTime, Boolean, ForeignKey,
@@ -28,10 +29,12 @@ from sqlalchemy.orm import relationship
from sqlalchemy.orm.collections import attribute_mapped_collection
from sqlalchemy.sql.expression import desc
from sqlalchemy.ext.associationproxy import association_proxy
+from sqlalchemy.util import memoized_property
from mediagoblin.db.sql.extratypes import PathTupleWithSlashes, JSONEncoded
from mediagoblin.db.sql.base import Base, DictReadAttrProxy
from mediagoblin.db.mixin import UserMixin, MediaEntryMixin, MediaCommentMixin
+from mediagoblin.db.sql.base import Session
# It's actually kind of annoying how sqlalchemy-migrate does this, if
# I understand it right, but whatever. Anyway, don't remove this :P
@@ -58,7 +61,7 @@ class User(Base, UserMixin):
TODO: We should consider moving some rarely used fields
into some sort of "shadow" table.
"""
- __tablename__ = "users"
+ __tablename__ = "core__users"
id = Column(Integer, primary_key=True)
username = Column(Unicode, nullable=False, unique=True)
@@ -84,13 +87,14 @@ class MediaEntry(Base, MediaEntryMixin):
"""
TODO: Consider fetching the media_files using join
"""
- __tablename__ = "media_entries"
+ __tablename__ = "core__media_entries"
id = Column(Integer, primary_key=True)
- uploader = Column(Integer, ForeignKey('users.id'), nullable=False)
+ uploader = Column(Integer, ForeignKey(User.id), nullable=False, index=True)
title = Column(Unicode, nullable=False)
slug = Column(Unicode)
- created = Column(DateTime, nullable=False, default=datetime.datetime.now)
+ created = Column(DateTime, nullable=False, default=datetime.datetime.now,
+ index=True)
description = Column(UnicodeText) # ??
media_type = Column(Unicode, nullable=False)
state = Column(Unicode, default=u'unprocessed', nullable=False)
@@ -167,14 +171,40 @@ class MediaEntry(Base, MediaEntryMixin):
if media is not None:
return media.url_for_self(urlgen)
+ #@memoized_property
@property
def media_data(self):
- # TODO: Replace with proper code to read the correct table
- return {}
+ session = Session()
+
+ return session.query(self.media_data_table).filter_by(
+ media_entry=self.id).first()
def media_data_init(self, **kwargs):
- # TODO: Implement this
- pass
+ """
+ Initialize or update the contents of a media entry's media_data row
+ """
+ session = Session()
+
+ media_data = session.query(self.media_data_table).filter_by(
+ media_entry=self.id).first()
+
+ # No media data, so actually add a new one
+ if media_data is None:
+ media_data = self.media_data_table(
+ media_entry=self.id,
+ **kwargs)
+ session.add(media_data)
+ # Update old media data
+ else:
+ for field, value in kwargs.iteritems():
+ setattr(media_data, field, value)
+
+ @memoized_property
+ def media_data_table(self):
+ # TODO: memoize this
+ models_module = self.media_type + '.models'
+ __import__(models_module)
+ return sys.modules[models_module].DATA_MODEL
class FileKeynames(Base):
@@ -202,7 +232,7 @@ class MediaFile(Base):
TODO: Highly consider moving "name" into a new table.
TODO: Consider preloading said table in software
"""
- __tablename__ = "mediafiles"
+ __tablename__ = "core__mediafiles"
media_entry = Column(
Integer, ForeignKey(MediaEntry.id),
@@ -241,7 +271,7 @@ class MediaAttachmentFile(Base):
class Tag(Base):
- __tablename__ = "tags"
+ __tablename__ = "core__tags"
id = Column(Integer, primary_key=True)
slug = Column(Unicode, nullable=False, unique=True)
@@ -258,13 +288,13 @@ class Tag(Base):
class MediaTag(Base):
- __tablename__ = "media_tags"
+ __tablename__ = "core__media_tags"
id = Column(Integer, primary_key=True)
media_entry = Column(
Integer, ForeignKey(MediaEntry.id),
- nullable=False)
- tag = Column(Integer, ForeignKey('tags.id'), nullable=False)
+ nullable=False, index=True)
+ tag = Column(Integer, ForeignKey(Tag.id), nullable=False, index=True)
name = Column(Unicode)
# created = Column(DateTime, nullable=False, default=datetime.datetime.now)
@@ -291,12 +321,12 @@ class MediaTag(Base):
class MediaComment(Base, MediaCommentMixin):
- __tablename__ = "media_comments"
+ __tablename__ = "core__media_comments"
id = Column(Integer, primary_key=True)
media_entry = Column(
- Integer, ForeignKey('media_entries.id'), nullable=False)
- author = Column(Integer, ForeignKey('users.id'), nullable=False)
+ Integer, ForeignKey(MediaEntry.id), nullable=False, index=True)
+ author = Column(Integer, ForeignKey(User.id), nullable=False)
created = Column(DateTime, nullable=False, default=datetime.datetime.now)
content = Column(UnicodeText, nullable=False)
@@ -318,7 +348,7 @@ MODELS = [
######################################################
class MigrationData(Base):
- __tablename__ = "migrations"
+ __tablename__ = "core__migrations"
name = Column(Unicode, primary_key=True)
version = Column(Integer, nullable=False, default=0)
diff --git a/mediagoblin/db/sql/open.py b/mediagoblin/db/sql/open.py
index b1f389e8..edbf0785 100644
--- a/mediagoblin/db/sql/open.py
+++ b/mediagoblin/db/sql/open.py
@@ -18,8 +18,9 @@
from sqlalchemy import create_engine
import logging
-from mediagoblin.db.sql.base import Session
-from mediagoblin.db.sql.models import Base
+from mediagoblin.db.sql.base import Base, Session
+
+_log = logging.getLogger(__name__)
class DatabaseMaster(object):
@@ -41,9 +42,18 @@ class DatabaseMaster(object):
Session.remove()
+def load_models(app_config):
+ import mediagoblin.db.sql.models
+
+ if True:
+ for media_type in app_config['media_types']:
+ _log.debug("Loading %s.models", media_type)
+ __import__(media_type + ".models")
+
+
def setup_connection_and_db_from_config(app_config):
engine = create_engine(app_config['sql_engine'])
- logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
+ # logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
Session.configure(bind=engine)
return "dummy conn", DatabaseMaster(engine)
diff --git a/mediagoblin/db/sql/util.py b/mediagoblin/db/sql/util.py
index efb0fafe..60024b28 100644
--- a/mediagoblin/db/sql/util.py
+++ b/mediagoblin/db/sql/util.py
@@ -294,6 +294,15 @@ def check_media_slug_used(dummy_db, uploader_id, slug, ignore_m_id):
return does_exist
+def media_entries_for_tag_slug(dummy_db, tag_slug):
+ return MediaEntry.query \
+ .join(MediaEntry.tags_helper) \
+ .join(MediaTag.tag_helper) \
+ .filter(
+ (MediaEntry.state == u'processed')
+ & (Tag.slug == tag_slug))
+
+
def clean_orphan_tags():
q1 = Session.query(Tag).outerjoin(MediaTag).filter(MediaTag.id==None)
for t in q1:
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-31 09:06:01 +0000