diff options
Diffstat (limited to 'mediagoblin/db/sql/convert.py')
| -rw-r--r-- | mediagoblin/db/sql/convert.py | 131 |
1 files changed, 91 insertions, 40 deletions
diff --git a/mediagoblin/db/sql/convert.py b/mediagoblin/db/sql/convert.py index d1492977..ac64cf8d 100644 --- a/mediagoblin/db/sql/convert.py +++ b/mediagoblin/db/sql/convert.py @@ -15,26 +15,27 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. from copy import copy +from itertools import chain, imap -from mediagoblin.init import setup_global_and_app_config, setup_database -from mediagoblin.db.mongo.util import ObjectId +from mediagoblin.init import setup_global_and_app_config -from mediagoblin.db.sql.models import (Base, User, MediaEntry, MediaComment, - Tag, MediaTag, MediaFile, MediaAttachmentFile, MigrationData) -from mediagoblin.media_types.image.models import ImageData -from mediagoblin.media_types.video.models import VideoData +from mediagoblin.db.sql.base import Session +from mediagoblin.db.sql.models_v0 import Base_v0 +from mediagoblin.db.sql.models_v0 import (User, MediaEntry, MediaComment, + Tag, MediaTag, MediaFile, MediaAttachmentFile, MigrationData, + ImageData, VideoData, AsciiData, AudioData) from mediagoblin.db.sql.open import setup_connection_and_db_from_config as \ sql_connect from mediagoblin.db.mongo.open import setup_connection_and_db_from_config as \ mongo_connect -from mediagoblin.db.sql.base import Session obj_id_table = dict() + def add_obj_ids(entry, new_entry): global obj_id_table - print "%r -> %r" % (entry._id, new_entry.id) + print "\t%r -> SQL id %r" % (entry._id, new_entry.id) obj_id_table[entry._id] = new_entry.id @@ -43,6 +44,7 @@ def copy_attrs(entry, new_entry, attr_list): val = entry[a] setattr(new_entry, a, val) + def copy_reference_attr(entry, new_entry, ref_attr): val = entry[ref_attr] val = obj_id_table[val] @@ -115,12 +117,9 @@ def convert_image(mk_db): {'media_type': 'mediagoblin.media_types.image'}).sort('created'): media_data = copy(media.media_data) - # TODO: Fix after exif is migrated - media_data.pop('exif', None) - if len(media_data): media_data_row = ImageData(**media_data) - media_data_row.media_entry = obj_id_table[media._id] + media_data_row.media_entry = obj_id_table[media['_id']] session.add(media_data_row) session.commit() @@ -133,7 +132,7 @@ def convert_video(mk_db): for media in mk_db.MediaEntry.find( {'media_type': 'mediagoblin.media_types.video'}).sort('created'): media_data_row = VideoData(**media.media_data) - media_data_row.media_entry = obj_id_table[media._id] + media_data_row.media_entry = obj_id_table[media['_id']] session.add(media_data_row) session.commit() @@ -178,53 +177,105 @@ def convert_media_comments(mk_db): copy_attrs(entry, new_entry, ('created', 'content',)) - copy_reference_attr(entry, new_entry, "media_entry") - copy_reference_attr(entry, new_entry, "author") - session.add(new_entry) - session.flush() - add_obj_ids(entry, new_entry) + try: + copy_reference_attr(entry, new_entry, "media_entry") + copy_reference_attr(entry, new_entry, "author") + except KeyError as e: + print('KeyError in convert_media_comments(): {0}'.format(e)) + else: + session.add(new_entry) + session.flush() + add_obj_ids(entry, new_entry) session.commit() session.close() -def convert_add_migration_versions(): +media_types_tables = ( + ("mediagoblin.media_types.image", (ImageData,)), + ("mediagoblin.media_types.video", (VideoData,)), + ("mediagoblin.media_types.ascii", (AsciiData,)), + ("mediagoblin.media_types.audio", (AudioData,)), + ) + + +def convert_add_migration_versions(dummy_sql_db): session = Session() - for name in ("__main__", - "mediagoblin.media_types.image", - "mediagoblin.media_types.video", - ): - m = MigrationData(name=name, version=0) + for name in chain(("__main__",), + imap(lambda e: e[0], media_types_tables)): + print "\tAdding %s" % (name,) + m = MigrationData(name=unicode(name), version=0) session.add(m) session.commit() session.close() +def cleanup_sql_tables(sql_db): + for mt, table_list in media_types_tables: + session = Session() + + count = session.query(MediaEntry.media_type). \ + filter_by(media_type=unicode(mt)).count() + print " %s: %d entries" % (mt, count) + + if count == 0: + print "\tAnalyzing tables" + for tab in table_list: + cnt2 = session.query(tab).count() + print "\t %s: %d entries" % (tab.__tablename__, cnt2) + assert cnt2 == 0 + + print "\tRemoving migration info" + mi = session.query(MigrationData).filter_by(name=unicode(mt)).one() + session.delete(mi) + session.commit() + session.close() + + print "\tDropping tables" + tables = [model.__table__ for model in table_list] + Base_v0.metadata.drop_all(sql_db.engine, tables=tables) + + session.close() + + +def print_header(title): + print "\n=== %s ===" % (title,) + + +convert_call_list = ( + ("Converting Users", convert_users), + ("Converting Media Entries", convert_media_entries), + ("Converting Media Data for Images", convert_image), + ("Cnnverting Media Data for Videos", convert_video), + ("Converting Tags for Media", convert_media_tags), + ("Converting Media Comments", convert_media_comments), + ) + +sql_call_list = ( + ("Filling Migration Tables", convert_add_migration_versions), + ("Analyzing/Cleaning SQL Data", cleanup_sql_tables), + ) + def run_conversion(config_name): global_config, app_config = setup_global_and_app_config(config_name) sql_conn, sql_db = sql_connect(app_config) mk_conn, mk_db = mongo_connect(app_config) - Base.metadata.create_all(sql_db.engine) - - convert_users(mk_db) - Session.remove() - convert_media_entries(mk_db) - Session.remove() - convert_image(mk_db) - Session.remove() - convert_video(mk_db) - Session.remove() - convert_media_tags(mk_db) - Session.remove() - convert_media_comments(mk_db) - Session.remove() - convert_add_migration_versions() - Session.remove() + Base_v0.metadata.create_all(sql_db.engine) + + for title, func in convert_call_list: + print_header(title) + func(mk_db) + Session.remove() + + for title, func in sql_call_list: + print_header(title) + func(sql_db) + Session.remove() if __name__ == '__main__': |