From 2f5ce68c6c3d8710bbb504754987b35302d0fdd2 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sat, 17 Mar 2012 16:48:28 -0500 Subject: Make mediagoblin SQL models all use a consistent table naming scheme --- mediagoblin/db/sql/models.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'mediagoblin/db/sql') diff --git a/mediagoblin/db/sql/models.py b/mediagoblin/db/sql/models.py index a2feeebb..949933bc 100644 --- a/mediagoblin/db/sql/models.py +++ b/mediagoblin/db/sql/models.py @@ -61,7 +61,7 @@ class User(Base, UserMixin): TODO: We should consider moving some rarely used fields into some sort of "shadow" table. """ - __tablename__ = "users" + __tablename__ = "core__users" id = Column(Integer, primary_key=True) username = Column(Unicode, nullable=False, unique=True) @@ -87,10 +87,10 @@ class MediaEntry(Base, MediaEntryMixin): """ TODO: Consider fetching the media_files using join """ - __tablename__ = "media_entries" + __tablename__ = "core__media_entries" id = Column(Integer, primary_key=True) - uploader = Column(Integer, ForeignKey('users.id'), nullable=False) + uploader = Column(Integer, ForeignKey('core__users.id'), nullable=False) title = Column(Unicode, nullable=False) slug = Column(Unicode) created = Column(DateTime, nullable=False, default=datetime.datetime.now) @@ -230,7 +230,7 @@ class MediaFile(Base): TODO: Highly consider moving "name" into a new table. TODO: Consider preloading said table in software """ - __tablename__ = "mediafiles" + __tablename__ = "core__mediafiles" media_entry = Column( Integer, ForeignKey(MediaEntry.id), @@ -269,7 +269,7 @@ class MediaAttachmentFile(Base): class Tag(Base): - __tablename__ = "tags" + __tablename__ = "core__tags" id = Column(Integer, primary_key=True) slug = Column(Unicode, nullable=False, unique=True) @@ -286,13 +286,13 @@ class Tag(Base): class MediaTag(Base): - __tablename__ = "media_tags" + __tablename__ = "core__media_tags" id = Column(Integer, primary_key=True) media_entry = Column( Integer, ForeignKey(MediaEntry.id), nullable=False) - tag = Column(Integer, ForeignKey('tags.id'), nullable=False) + tag = Column(Integer, ForeignKey('core__tags.id'), nullable=False) name = Column(Unicode) # created = Column(DateTime, nullable=False, default=datetime.datetime.now) @@ -319,12 +319,12 @@ class MediaTag(Base): class MediaComment(Base, MediaCommentMixin): - __tablename__ = "media_comments" + __tablename__ = "core__media_comments" id = Column(Integer, primary_key=True) media_entry = Column( - Integer, ForeignKey('media_entries.id'), nullable=False) - author = Column(Integer, ForeignKey('users.id'), nullable=False) + Integer, ForeignKey('core__media_entries.id'), nullable=False) + author = Column(Integer, ForeignKey('core__users.id'), nullable=False) created = Column(DateTime, nullable=False, default=datetime.datetime.now) content = Column(UnicodeText, nullable=False) @@ -346,7 +346,7 @@ MODELS = [ ###################################################### class MigrationData(Base): - __tablename__ = "migrations" + __tablename__ = "core__migrations" name = Column(Unicode, primary_key=True) version = Column(Integer, nullable=False, default=0) -- cgit v1.2.3 From 7bf819a99579a3aec2c539305c08759e2c0b39e3 Mon Sep 17 00:00:00 2001 From: Elrond Date: Sat, 17 Mar 2012 22:30:48 +0100 Subject: Let convert_mongo_to_sql add Migration Records The mongosql tool is really dumping directly into the sql database and is trying not to use too much logic that might change later. So this means, it needs to create the migration records on its own! So add a bunch of records with version=0. --- mediagoblin/db/sql/convert.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) (limited to 'mediagoblin/db/sql') diff --git a/mediagoblin/db/sql/convert.py b/mediagoblin/db/sql/convert.py index dca93f3f..d1492977 100644 --- a/mediagoblin/db/sql/convert.py +++ b/mediagoblin/db/sql/convert.py @@ -20,7 +20,7 @@ from mediagoblin.init import setup_global_and_app_config, setup_database from mediagoblin.db.mongo.util import ObjectId from mediagoblin.db.sql.models import (Base, User, MediaEntry, MediaComment, - Tag, MediaTag, MediaFile, MediaAttachmentFile) + Tag, MediaTag, MediaFile, MediaAttachmentFile, MigrationData) from mediagoblin.media_types.image.models import ImageData from mediagoblin.media_types.video.models import VideoData from mediagoblin.db.sql.open import setup_connection_and_db_from_config as \ @@ -189,6 +189,20 @@ def convert_media_comments(mk_db): session.close() +def convert_add_migration_versions(): + session = Session() + + for name in ("__main__", + "mediagoblin.media_types.image", + "mediagoblin.media_types.video", + ): + m = MigrationData(name=name, version=0) + session.add(m) + + session.commit() + session.close() + + def run_conversion(config_name): global_config, app_config = setup_global_and_app_config(config_name) @@ -209,6 +223,8 @@ def run_conversion(config_name): Session.remove() convert_media_comments(mk_db) Session.remove() + convert_add_migration_versions() + Session.remove() if __name__ == '__main__': -- cgit v1.2.3 From df7c405845e1fe6436083e7781abb3d1b178624f Mon Sep 17 00:00:00 2001 From: Elrond Date: Tue, 20 Mar 2012 12:43:09 +0100 Subject: Disable query logging for now, so we get nicer logs As the queries are quite verbose, disable them for now. Reenabling them should be done in the central logging config, which is another story for celery and bin/gmg. --- mediagoblin/db/sql/open.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mediagoblin/db/sql') diff --git a/mediagoblin/db/sql/open.py b/mediagoblin/db/sql/open.py index b1f389e8..9819b468 100644 --- a/mediagoblin/db/sql/open.py +++ b/mediagoblin/db/sql/open.py @@ -43,7 +43,7 @@ class DatabaseMaster(object): def setup_connection_and_db_from_config(app_config): engine = create_engine(app_config['sql_engine']) - logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + # logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) Session.configure(bind=engine) return "dummy conn", DatabaseMaster(engine) -- cgit v1.2.3 From 39fd817ab8c4745187e5e1992d2f2866de38e2c2 Mon Sep 17 00:00:00 2001 From: Elrond Date: Tue, 20 Mar 2012 12:32:11 +0100 Subject: Import "Base" from base instead of models. Import the "Base" class for models from db.sql.base instead of db.sql.models. --- mediagoblin/db/sql/convert.py | 4 ++-- mediagoblin/db/sql/open.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) (limited to 'mediagoblin/db/sql') diff --git a/mediagoblin/db/sql/convert.py b/mediagoblin/db/sql/convert.py index d1492977..232e424f 100644 --- a/mediagoblin/db/sql/convert.py +++ b/mediagoblin/db/sql/convert.py @@ -19,7 +19,8 @@ from copy import copy from mediagoblin.init import setup_global_and_app_config, setup_database from mediagoblin.db.mongo.util import ObjectId -from mediagoblin.db.sql.models import (Base, User, MediaEntry, MediaComment, +from mediagoblin.db.sql.base import Base, Session +from mediagoblin.db.sql.models import (User, MediaEntry, MediaComment, Tag, MediaTag, MediaFile, MediaAttachmentFile, MigrationData) from mediagoblin.media_types.image.models import ImageData from mediagoblin.media_types.video.models import VideoData @@ -27,7 +28,6 @@ from mediagoblin.db.sql.open import setup_connection_and_db_from_config as \ sql_connect from mediagoblin.db.mongo.open import setup_connection_and_db_from_config as \ mongo_connect -from mediagoblin.db.sql.base import Session obj_id_table = dict() diff --git a/mediagoblin/db/sql/open.py b/mediagoblin/db/sql/open.py index 9819b468..7362b1c5 100644 --- a/mediagoblin/db/sql/open.py +++ b/mediagoblin/db/sql/open.py @@ -18,8 +18,7 @@ from sqlalchemy import create_engine import logging -from mediagoblin.db.sql.base import Session -from mediagoblin.db.sql.models import Base +from mediagoblin.db.sql.base import Base, Session class DatabaseMaster(object): -- cgit v1.2.3 From b829595334e10454a6528bdac944d781f6102145 Mon Sep 17 00:00:00 2001 From: Elrond Date: Tue, 20 Mar 2012 12:39:15 +0100 Subject: Create load_models(). So all models are ready when connecting to the db and so our "db" object has all models listed on it, create a function to load all models from the media_types, etc. Call it in setup_database() Problem: This gives celery warnings, because celery is imported before being setup properly. No idea how to fix this now. So media-type loading is excluded from load_models for now. --- mediagoblin/db/sql/open.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'mediagoblin/db/sql') diff --git a/mediagoblin/db/sql/open.py b/mediagoblin/db/sql/open.py index 7362b1c5..2520bb49 100644 --- a/mediagoblin/db/sql/open.py +++ b/mediagoblin/db/sql/open.py @@ -20,6 +20,8 @@ import logging from mediagoblin.db.sql.base import Base, Session +_log = logging.getLogger(__name__) + class DatabaseMaster(object): def __init__(self, engine): @@ -40,6 +42,16 @@ class DatabaseMaster(object): Session.remove() +def load_models(app_config): + import mediagoblin.db.sql.models + + # TODO/Fix: This breaks celery + if False: + for media_type in app_config['media_types']: + _log.debug("Loading %s.models", media_type) + __import__(media_type + ".models") + + def setup_connection_and_db_from_config(app_config): engine = create_engine(app_config['sql_engine']) # logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) -- cgit v1.2.3 From e136ad3ed613b1c2b42093be687e3aea9889e5b3 Mon Sep 17 00:00:00 2001 From: Elrond Date: Wed, 21 Mar 2012 21:46:07 +0100 Subject: Finally load all models. Load all models for the media_types. This was stopped by a celery problem. But that is now fixed. --- mediagoblin/db/sql/open.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'mediagoblin/db/sql') diff --git a/mediagoblin/db/sql/open.py b/mediagoblin/db/sql/open.py index 2520bb49..edbf0785 100644 --- a/mediagoblin/db/sql/open.py +++ b/mediagoblin/db/sql/open.py @@ -45,8 +45,7 @@ class DatabaseMaster(object): def load_models(app_config): import mediagoblin.db.sql.models - # TODO/Fix: This breaks celery - if False: + if True: for media_type in app_config['media_types']: _log.debug("Loading %s.models", media_type) __import__(media_type + ".models") -- cgit v1.2.3 From ecd538bb6596f064ed791d865f1efc7a24a637c4 Mon Sep 17 00:00:00 2001 From: Elrond Date: Wed, 21 Mar 2012 22:46:38 +0100 Subject: Add index=True for some columns. These are the columns that seem to make the most sense to have an index on them. --- mediagoblin/db/sql/models.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'mediagoblin/db/sql') diff --git a/mediagoblin/db/sql/models.py b/mediagoblin/db/sql/models.py index 949933bc..8d198fd6 100644 --- a/mediagoblin/db/sql/models.py +++ b/mediagoblin/db/sql/models.py @@ -90,10 +90,11 @@ class MediaEntry(Base, MediaEntryMixin): __tablename__ = "core__media_entries" id = Column(Integer, primary_key=True) - uploader = Column(Integer, ForeignKey('core__users.id'), nullable=False) + uploader = Column(Integer, ForeignKey(User.id), nullable=False, index=True) title = Column(Unicode, nullable=False) slug = Column(Unicode) - created = Column(DateTime, nullable=False, default=datetime.datetime.now) + created = Column(DateTime, nullable=False, default=datetime.datetime.now, + index=True) description = Column(UnicodeText) # ?? media_type = Column(Unicode, nullable=False) state = Column(Unicode, default=u'unprocessed', nullable=False) @@ -291,8 +292,8 @@ class MediaTag(Base): id = Column(Integer, primary_key=True) media_entry = Column( Integer, ForeignKey(MediaEntry.id), - nullable=False) - tag = Column(Integer, ForeignKey('core__tags.id'), nullable=False) + nullable=False, index=True) + tag = Column(Integer, ForeignKey(Tag.id), nullable=False, index=True) name = Column(Unicode) # created = Column(DateTime, nullable=False, default=datetime.datetime.now) @@ -323,8 +324,8 @@ class MediaComment(Base, MediaCommentMixin): id = Column(Integer, primary_key=True) media_entry = Column( - Integer, ForeignKey('core__media_entries.id'), nullable=False) - author = Column(Integer, ForeignKey('core__users.id'), nullable=False) + Integer, ForeignKey(MediaEntry.id), nullable=False, index=True) + author = Column(Integer, ForeignKey(User.id), nullable=False) created = Column(DateTime, nullable=False, default=datetime.datetime.now) content = Column(UnicodeText, nullable=False) -- cgit v1.2.3 From 99c2f9f0c4e141adf1771f91fa1105e52108634b Mon Sep 17 00:00:00 2001 From: Elrond Date: Thu, 22 Mar 2012 21:19:47 +0100 Subject: New media_data row needs to know its MediaEntry's id. When creating a new media_data row, the new row needs to know the MediaEntry it is associated with. I have no idea, why this worked before at all. Maybe some implicit tricks by sqlalchemy? --- mediagoblin/db/sql/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'mediagoblin/db/sql') diff --git a/mediagoblin/db/sql/models.py b/mediagoblin/db/sql/models.py index 8d198fd6..e87aaddb 100644 --- a/mediagoblin/db/sql/models.py +++ b/mediagoblin/db/sql/models.py @@ -189,8 +189,9 @@ class MediaEntry(Base, MediaEntryMixin): media_entry=self.id).first() # No media data, so actually add a new one - if not media_data: + if media_data is None: media_data = self.media_data_table( + media_entry=self.id, **kwargs) session.add(media_data) # Update old media data -- cgit v1.2.3 From e63656fcf74ac0b6f7de3922e60481f7dfc21c7c Mon Sep 17 00:00:00 2001 From: Elrond Date: Sat, 24 Mar 2012 10:55:34 +0100 Subject: Cleanup mongo->sql converter. 1. No need to drop media_data['exif'], we only have and want media_data['exif_all']. 2. Use media['_id'] instead of media._id (better not use dot-notation on mongo objects in such a low level tool). --- mediagoblin/db/sql/convert.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'mediagoblin/db/sql') diff --git a/mediagoblin/db/sql/convert.py b/mediagoblin/db/sql/convert.py index 232e424f..ebf3037c 100644 --- a/mediagoblin/db/sql/convert.py +++ b/mediagoblin/db/sql/convert.py @@ -115,12 +115,9 @@ def convert_image(mk_db): {'media_type': 'mediagoblin.media_types.image'}).sort('created'): media_data = copy(media.media_data) - # TODO: Fix after exif is migrated - media_data.pop('exif', None) - if len(media_data): media_data_row = ImageData(**media_data) - media_data_row.media_entry = obj_id_table[media._id] + media_data_row.media_entry = obj_id_table[media['_id']] session.add(media_data_row) session.commit() @@ -133,7 +130,7 @@ def convert_video(mk_db): for media in mk_db.MediaEntry.find( {'media_type': 'mediagoblin.media_types.video'}).sort('created'): media_data_row = VideoData(**media.media_data) - media_data_row.media_entry = obj_id_table[media._id] + media_data_row.media_entry = obj_id_table[media['_id']] session.add(media_data_row) session.commit() -- cgit v1.2.3