diff options
Diffstat (limited to 'mediagoblin/db')
| -rw-r--r-- | mediagoblin/db/mixin.py | 11 | ||||
| -rw-r--r-- | mediagoblin/db/mongo/migrations.py | 45 | ||||
| -rw-r--r-- | mediagoblin/db/mongo/open.py | 4 | ||||
| -rw-r--r-- | mediagoblin/db/mongo/util.py | 6 | ||||
| -rw-r--r-- | mediagoblin/db/open.py | 6 | ||||
| -rw-r--r-- | mediagoblin/db/sql/convert.py | 57 | ||||
| -rw-r--r-- | mediagoblin/db/sql/models.py | 64 | ||||
| -rw-r--r-- | mediagoblin/db/sql/open.py | 16 | ||||
| -rw-r--r-- | mediagoblin/db/sql/util.py | 9 | ||||
| -rw-r--r-- | mediagoblin/db/util.py | 6 |
10 files changed, 197 insertions, 27 deletions
diff --git a/mediagoblin/db/mixin.py b/mediagoblin/db/mixin.py index 4f9e1b11..a5aded02 100644 --- a/mediagoblin/db/mixin.py +++ b/mediagoblin/db/mixin.py @@ -123,6 +123,17 @@ class MediaEntryMixin(object): """Return license dict for requested license""" return licenses.SUPPORTED_LICENSES[self.license or ""] + def exif_display_iter(self): + from mediagoblin.tools.exif import USEFUL_TAGS + + if not self.media_data: + return + exif_all = self.media_data.get("exif_all") + + for key in USEFUL_TAGS: + if key in exif_all: + yield key, exif_all[key] + class MediaCommentMixin(object): @property diff --git a/mediagoblin/db/mongo/migrations.py b/mediagoblin/db/mongo/migrations.py index c5766b0d..732f5846 100644 --- a/mediagoblin/db/mongo/migrations.py +++ b/mediagoblin/db/mongo/migrations.py @@ -119,6 +119,7 @@ def media_type_image_to_multimedia_type_image(database): {'$set': {'media_type': 'mediagoblin.media_types.image'}}, multi=True) + @RegisterMigration(8) def mediaentry_add_license(database): """ @@ -140,6 +141,7 @@ def remove_calculated_html(database): drop_table_field(database, 'media_entries', 'description_html') drop_table_field(database, 'media_comments', 'content_html') + @RegisterMigration(10) def convert_video_media_data(database): """ @@ -153,3 +155,46 @@ def convert_video_media_data(database): assert len(document['media_data']) == 1 document['media_data'] = document['media_data']['video'] collection.save(document) + + +@RegisterMigration(11) +def convert_gps_media_data(database): + """ + Move media_data["gps"]["*"] to media_data["gps_*"]. + In preparation for media_data.gps_* + """ + collection = database['media_entries'] + target = collection.find( + {'media_data.gps': {'$exists': True}}) + + for document in target: + for key, value in document['media_data']['gps'].iteritems(): + document['media_data']['gps_' + key] = value + del document['media_data']['gps'] + collection.save(document) + + +@RegisterMigration(12) +def convert_exif_media_data(database): + """ + Move media_data["exif"]["clean"] to media_data["exif_all"]. + Drop media_data["exif"]["useful"] + In preparation for media_data.exif_all + """ + collection = database['media_entries'] + target = collection.find( + {'media_data.exif.clean': {'$exists': True}}) + + for document in target: + media_data = document['media_data'] + + exif_all = media_data['exif'].pop('clean') + if len(exif_all): + media_data['exif_all'] = exif_all + + del media_data['exif']['useful'] + + assert len(media_data['exif']) == 0 + del media_data['exif'] + + collection.save(document) diff --git a/mediagoblin/db/mongo/open.py b/mediagoblin/db/mongo/open.py index bedc497b..c4f37b42 100644 --- a/mediagoblin/db/mongo/open.py +++ b/mediagoblin/db/mongo/open.py @@ -21,6 +21,10 @@ from mediagoblin.db.mongo import models from mediagoblin.db.mongo.util import MigrationManager +def load_models(app_config): + pass + + def connect_database_from_config(app_config, use_pymongo=False): """ Connect to the main database, take config from app_config diff --git a/mediagoblin/db/mongo/util.py b/mediagoblin/db/mongo/util.py index 2d47f648..f61ae6be 100644 --- a/mediagoblin/db/mongo/util.py +++ b/mediagoblin/db/mongo/util.py @@ -310,3 +310,9 @@ def check_media_slug_used(db, uploader_id, slug, ignore_m_id): existing_user_slug_entries = db.MediaEntry.find( query_dict).count() return existing_user_slug_entries + + +def media_entries_for_tag_slug(db, tag_slug): + return db.MediaEntry.find( + {u'state': u'processed', + u'tags.slug': tag_slug}) diff --git a/mediagoblin/db/open.py b/mediagoblin/db/open.py index 0163469f..f4c38511 100644 --- a/mediagoblin/db/open.py +++ b/mediagoblin/db/open.py @@ -21,7 +21,9 @@ except ImportError: if use_sql: from mediagoblin.db.sql.open import \ - setup_connection_and_db_from_config, check_db_migrations_current + setup_connection_and_db_from_config, check_db_migrations_current, \ + load_models else: from mediagoblin.db.mongo.open import \ - setup_connection_and_db_from_config, check_db_migrations_current + setup_connection_and_db_from_config, check_db_migrations_current, \ + load_models diff --git a/mediagoblin/db/sql/convert.py b/mediagoblin/db/sql/convert.py index 250c559b..ebf3037c 100644 --- a/mediagoblin/db/sql/convert.py +++ b/mediagoblin/db/sql/convert.py @@ -14,18 +14,20 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +from copy import copy from mediagoblin.init import setup_global_and_app_config, setup_database from mediagoblin.db.mongo.util import ObjectId -from mediagoblin.db.sql.models import (Base, User, MediaEntry, MediaComment, - Tag, MediaTag, MediaFile, MediaAttachmentFile) +from mediagoblin.db.sql.base import Base, Session +from mediagoblin.db.sql.models import (User, MediaEntry, MediaComment, + Tag, MediaTag, MediaFile, MediaAttachmentFile, MigrationData) +from mediagoblin.media_types.image.models import ImageData from mediagoblin.media_types.video.models import VideoData from mediagoblin.db.sql.open import setup_connection_and_db_from_config as \ sql_connect from mediagoblin.db.mongo.open import setup_connection_and_db_from_config as \ mongo_connect -from mediagoblin.db.sql.base import Session obj_id_table = dict() @@ -106,6 +108,35 @@ def convert_media_entries(mk_db): session.close() +def convert_image(mk_db): + session = Session() + + for media in mk_db.MediaEntry.find( + {'media_type': 'mediagoblin.media_types.image'}).sort('created'): + media_data = copy(media.media_data) + + if len(media_data): + media_data_row = ImageData(**media_data) + media_data_row.media_entry = obj_id_table[media['_id']] + session.add(media_data_row) + + session.commit() + session.close() + + +def convert_video(mk_db): + session = Session() + + for media in mk_db.MediaEntry.find( + {'media_type': 'mediagoblin.media_types.video'}).sort('created'): + media_data_row = VideoData(**media.media_data) + media_data_row.media_entry = obj_id_table[media['_id']] + session.add(media_data_row) + + session.commit() + session.close() + + def convert_media_tags(mk_db): session = Session() session.autoflush = False @@ -155,6 +186,20 @@ def convert_media_comments(mk_db): session.close() +def convert_add_migration_versions(): + session = Session() + + for name in ("__main__", + "mediagoblin.media_types.image", + "mediagoblin.media_types.video", + ): + m = MigrationData(name=name, version=0) + session.add(m) + + session.commit() + session.close() + + def run_conversion(config_name): global_config, app_config = setup_global_and_app_config(config_name) @@ -167,10 +212,16 @@ def run_conversion(config_name): Session.remove() convert_media_entries(mk_db) Session.remove() + convert_image(mk_db) + Session.remove() + convert_video(mk_db) + Session.remove() convert_media_tags(mk_db) Session.remove() convert_media_comments(mk_db) Session.remove() + convert_add_migration_versions() + Session.remove() if __name__ == '__main__': diff --git a/mediagoblin/db/sql/models.py b/mediagoblin/db/sql/models.py index 089bbb40..e87aaddb 100644 --- a/mediagoblin/db/sql/models.py +++ b/mediagoblin/db/sql/models.py @@ -20,6 +20,7 @@ TODO: indexes on foreignkeys, where useful. import datetime +import sys from sqlalchemy import ( Column, Integer, Unicode, UnicodeText, DateTime, Boolean, ForeignKey, @@ -28,10 +29,12 @@ from sqlalchemy.orm import relationship from sqlalchemy.orm.collections import attribute_mapped_collection from sqlalchemy.sql.expression import desc from sqlalchemy.ext.associationproxy import association_proxy +from sqlalchemy.util import memoized_property from mediagoblin.db.sql.extratypes import PathTupleWithSlashes, JSONEncoded from mediagoblin.db.sql.base import Base, DictReadAttrProxy from mediagoblin.db.mixin import UserMixin, MediaEntryMixin, MediaCommentMixin +from mediagoblin.db.sql.base import Session # It's actually kind of annoying how sqlalchemy-migrate does this, if # I understand it right, but whatever. Anyway, don't remove this :P @@ -58,7 +61,7 @@ class User(Base, UserMixin): TODO: We should consider moving some rarely used fields into some sort of "shadow" table. """ - __tablename__ = "users" + __tablename__ = "core__users" id = Column(Integer, primary_key=True) username = Column(Unicode, nullable=False, unique=True) @@ -84,13 +87,14 @@ class MediaEntry(Base, MediaEntryMixin): """ TODO: Consider fetching the media_files using join """ - __tablename__ = "media_entries" + __tablename__ = "core__media_entries" id = Column(Integer, primary_key=True) - uploader = Column(Integer, ForeignKey('users.id'), nullable=False) + uploader = Column(Integer, ForeignKey(User.id), nullable=False, index=True) title = Column(Unicode, nullable=False) slug = Column(Unicode) - created = Column(DateTime, nullable=False, default=datetime.datetime.now) + created = Column(DateTime, nullable=False, default=datetime.datetime.now, + index=True) description = Column(UnicodeText) # ?? media_type = Column(Unicode, nullable=False) state = Column(Unicode, default=u'unprocessed', nullable=False) @@ -167,14 +171,40 @@ class MediaEntry(Base, MediaEntryMixin): if media is not None: return media.url_for_self(urlgen) + #@memoized_property @property def media_data(self): - # TODO: Replace with proper code to read the correct table - return {} + session = Session() + + return session.query(self.media_data_table).filter_by( + media_entry=self.id).first() def media_data_init(self, **kwargs): - # TODO: Implement this - pass + """ + Initialize or update the contents of a media entry's media_data row + """ + session = Session() + + media_data = session.query(self.media_data_table).filter_by( + media_entry=self.id).first() + + # No media data, so actually add a new one + if media_data is None: + media_data = self.media_data_table( + media_entry=self.id, + **kwargs) + session.add(media_data) + # Update old media data + else: + for field, value in kwargs.iteritems(): + setattr(media_data, field, value) + + @memoized_property + def media_data_table(self): + # TODO: memoize this + models_module = self.media_type + '.models' + __import__(models_module) + return sys.modules[models_module].DATA_MODEL class FileKeynames(Base): @@ -202,7 +232,7 @@ class MediaFile(Base): TODO: Highly consider moving "name" into a new table. TODO: Consider preloading said table in software """ - __tablename__ = "mediafiles" + __tablename__ = "core__mediafiles" media_entry = Column( Integer, ForeignKey(MediaEntry.id), @@ -241,7 +271,7 @@ class MediaAttachmentFile(Base): class Tag(Base): - __tablename__ = "tags" + __tablename__ = "core__tags" id = Column(Integer, primary_key=True) slug = Column(Unicode, nullable=False, unique=True) @@ -258,13 +288,13 @@ class Tag(Base): class MediaTag(Base): - __tablename__ = "media_tags" + __tablename__ = "core__media_tags" id = Column(Integer, primary_key=True) media_entry = Column( Integer, ForeignKey(MediaEntry.id), - nullable=False) - tag = Column(Integer, ForeignKey('tags.id'), nullable=False) + nullable=False, index=True) + tag = Column(Integer, ForeignKey(Tag.id), nullable=False, index=True) name = Column(Unicode) # created = Column(DateTime, nullable=False, default=datetime.datetime.now) @@ -291,12 +321,12 @@ class MediaTag(Base): class MediaComment(Base, MediaCommentMixin): - __tablename__ = "media_comments" + __tablename__ = "core__media_comments" id = Column(Integer, primary_key=True) media_entry = Column( - Integer, ForeignKey('media_entries.id'), nullable=False) - author = Column(Integer, ForeignKey('users.id'), nullable=False) + Integer, ForeignKey(MediaEntry.id), nullable=False, index=True) + author = Column(Integer, ForeignKey(User.id), nullable=False) created = Column(DateTime, nullable=False, default=datetime.datetime.now) content = Column(UnicodeText, nullable=False) @@ -318,7 +348,7 @@ MODELS = [ ###################################################### class MigrationData(Base): - __tablename__ = "migrations" + __tablename__ = "core__migrations" name = Column(Unicode, primary_key=True) version = Column(Integer, nullable=False, default=0) diff --git a/mediagoblin/db/sql/open.py b/mediagoblin/db/sql/open.py index b1f389e8..edbf0785 100644 --- a/mediagoblin/db/sql/open.py +++ b/mediagoblin/db/sql/open.py @@ -18,8 +18,9 @@ from sqlalchemy import create_engine import logging -from mediagoblin.db.sql.base import Session -from mediagoblin.db.sql.models import Base +from mediagoblin.db.sql.base import Base, Session + +_log = logging.getLogger(__name__) class DatabaseMaster(object): @@ -41,9 +42,18 @@ class DatabaseMaster(object): Session.remove() +def load_models(app_config): + import mediagoblin.db.sql.models + + if True: + for media_type in app_config['media_types']: + _log.debug("Loading %s.models", media_type) + __import__(media_type + ".models") + + def setup_connection_and_db_from_config(app_config): engine = create_engine(app_config['sql_engine']) - logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + # logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) Session.configure(bind=engine) return "dummy conn", DatabaseMaster(engine) diff --git a/mediagoblin/db/sql/util.py b/mediagoblin/db/sql/util.py index efb0fafe..60024b28 100644 --- a/mediagoblin/db/sql/util.py +++ b/mediagoblin/db/sql/util.py @@ -294,6 +294,15 @@ def check_media_slug_used(dummy_db, uploader_id, slug, ignore_m_id): return does_exist +def media_entries_for_tag_slug(dummy_db, tag_slug): + return MediaEntry.query \ + .join(MediaEntry.tags_helper) \ + .join(MediaTag.tag_helper) \ + .filter( + (MediaEntry.state == u'processed') + & (Tag.slug == tag_slug)) + + def clean_orphan_tags(): q1 = Session.query(Tag).outerjoin(MediaTag).filter(MediaTag.id==None) for t in q1: diff --git a/mediagoblin/db/util.py b/mediagoblin/db/util.py index ff7aaf79..540a9244 100644 --- a/mediagoblin/db/util.py +++ b/mediagoblin/db/util.py @@ -21,7 +21,9 @@ except ImportError: if use_sql: from mediagoblin.db.sql.fake import ObjectId, InvalidId, DESCENDING - from mediagoblin.db.sql.util import atomic_update, check_media_slug_used + from mediagoblin.db.sql.util import atomic_update, check_media_slug_used, \ + media_entries_for_tag_slug else: from mediagoblin.db.mongo.util import \ - ObjectId, InvalidId, DESCENDING, atomic_update, check_media_slug_used + ObjectId, InvalidId, DESCENDING, atomic_update, \ + check_media_slug_used, media_entries_for_tag_slug |