diff options
Diffstat (limited to 'mediagoblin/db/mongo')
| -rw-r--r-- | mediagoblin/db/mongo/migrations.py | 83 | ||||
| -rw-r--r-- | mediagoblin/db/mongo/models.py | 43 | ||||
| -rw-r--r-- | mediagoblin/db/mongo/open.py | 4 | ||||
| -rw-r--r-- | mediagoblin/db/mongo/util.py | 26 |
4 files changed, 134 insertions, 22 deletions
diff --git a/mediagoblin/db/mongo/migrations.py b/mediagoblin/db/mongo/migrations.py index 261e21a5..732f5846 100644 --- a/mediagoblin/db/mongo/migrations.py +++ b/mediagoblin/db/mongo/migrations.py @@ -29,6 +29,16 @@ def add_table_field(db, table_name, field_name, default_value): multi=True) +def drop_table_field(db, table_name, field_name): + """ + Drop an old field from a table/collection + """ + db[table_name].update( + {field_name: {'$exists': True}}, + {'$unset': {field_name: 1}}, + multi=True) + + # Please see mediagoblin/tests/test_migrations.py for some examples of # basic migrations. @@ -109,9 +119,82 @@ def media_type_image_to_multimedia_type_image(database): {'$set': {'media_type': 'mediagoblin.media_types.image'}}, multi=True) + @RegisterMigration(8) def mediaentry_add_license(database): """ Add the 'license' field for entries that don't have it. """ add_table_field(database, 'media_entries', 'license', None) + + +@RegisterMigration(9) +def remove_calculated_html(database): + """ + Drop pre-rendered html again and calculate things + on the fly (and cache): + - User.bio_html + - MediaEntry.description_html + - MediaComment.content_html + """ + drop_table_field(database, 'users', 'bio_html') + drop_table_field(database, 'media_entries', 'description_html') + drop_table_field(database, 'media_comments', 'content_html') + + +@RegisterMigration(10) +def convert_video_media_data(database): + """ + Move media_data["video"] directly into media_data + """ + collection = database['media_entries'] + target = collection.find( + {'media_data.video': {'$exists': True}}) + + for document in target: + assert len(document['media_data']) == 1 + document['media_data'] = document['media_data']['video'] + collection.save(document) + + +@RegisterMigration(11) +def convert_gps_media_data(database): + """ + Move media_data["gps"]["*"] to media_data["gps_*"]. + In preparation for media_data.gps_* + """ + collection = database['media_entries'] + target = collection.find( + {'media_data.gps': {'$exists': True}}) + + for document in target: + for key, value in document['media_data']['gps'].iteritems(): + document['media_data']['gps_' + key] = value + del document['media_data']['gps'] + collection.save(document) + + +@RegisterMigration(12) +def convert_exif_media_data(database): + """ + Move media_data["exif"]["clean"] to media_data["exif_all"]. + Drop media_data["exif"]["useful"] + In preparation for media_data.exif_all + """ + collection = database['media_entries'] + target = collection.find( + {'media_data.exif.clean': {'$exists': True}}) + + for document in target: + media_data = document['media_data'] + + exif_all = media_data['exif'].pop('clean') + if len(exif_all): + media_data['exif_all'] = exif_all + + del media_data['exif']['useful'] + + assert len(media_data['exif']) == 0 + del media_data['exif'] + + collection.save(document) diff --git a/mediagoblin/db/mongo/models.py b/mediagoblin/db/mongo/models.py index 541086bc..2e35a2b8 100644 --- a/mediagoblin/db/mongo/models.py +++ b/mediagoblin/db/mongo/models.py @@ -18,12 +18,21 @@ import datetime from mongokit import Document -from mediagoblin import mg_globals from mediagoblin.db.mongo import migrations from mediagoblin.db.mongo.util import ASCENDING, DESCENDING, ObjectId from mediagoblin.tools.pagination import Pagination -from mediagoblin.tools import url -from mediagoblin.db.mixin import UserMixin, MediaEntryMixin +from mediagoblin.db.mixin import UserMixin, MediaEntryMixin, MediaCommentMixin + + +class MongoPK(object): + """An alias for the _id primary key""" + def __get__(self, instance, cls): + return instance['_id'] + def __set__(self, instance, val): + instance['_id'] = val + def __delete__(self, instance): + del instance['_id'] + ################### # Custom validators @@ -59,7 +68,6 @@ class User(Document, UserMixin): - is_admin: Whether or not this user is an administrator or not. - url: this user's personal webpage/website, if appropriate. - bio: biography of this user (plaintext, in markdown) - - bio_html: biography of the user converted to proper HTML. """ __collection__ = 'users' use_dot_notation = True @@ -76,7 +84,6 @@ class User(Document, UserMixin): 'is_admin': bool, 'url': unicode, 'bio': unicode, # May contain markdown - 'bio_html': unicode, # May contain plaintext, or HTML 'fp_verification_key': unicode, # forgotten password verification key 'fp_token_expire': datetime.datetime, } @@ -89,6 +96,8 @@ class User(Document, UserMixin): 'status': u'needs_email_verification', 'is_admin': False} + id = MongoPK() + class MediaEntry(Document, MediaEntryMixin): """ @@ -112,9 +121,6 @@ class MediaEntry(Document, MediaEntryMixin): up with MarkDown for slight fanciness (links, boldness, italics, paragraphs...) - - description_html: Rendered version of the description, run through - Markdown and cleaned with our cleaning tool. - - media_type: What type of media is this? Currently we only support 'image' ;) @@ -179,7 +185,6 @@ class MediaEntry(Document, MediaEntryMixin): 'slug': unicode, 'created': datetime.datetime, 'description': unicode, # May contain markdown/up - 'description_html': unicode, # May contain plaintext, or HTML 'media_type': unicode, 'media_data': dict, # extra data relevant to this media_type 'plugin_data': dict, # plugins can dump stuff here. @@ -211,6 +216,11 @@ class MediaEntry(Document, MediaEntryMixin): 'created': datetime.datetime.utcnow, 'state': u'unprocessed'} + id = MongoPK() + + def media_data_init(self, **kwargs): + self.media_data.update(kwargs) + def get_comments(self, ascending=False): if ascending: order = ASCENDING @@ -220,15 +230,6 @@ class MediaEntry(Document, MediaEntryMixin): return self.db.MediaComment.find({ 'media_entry': self._id}).sort('created', order) - def generate_slug(self): - self.slug = url.slugify(self.title) - - duplicate = mg_globals.database.media_entries.find_one( - {'slug': self.slug}) - - if duplicate: - self.slug = "%s-%s" % (self._id, self.slug) - def url_to_prev(self, urlgen): """ Provide a url to the previous entry from this user, if there is one @@ -257,7 +258,7 @@ class MediaEntry(Document, MediaEntryMixin): return self.db.User.find_one({'_id': self.uploader}) -class MediaComment(Document): +class MediaComment(Document, MediaCommentMixin): """ A comment on a MediaEntry. @@ -266,8 +267,6 @@ class MediaComment(Document): - author: user who posted this comment - created: when the comment was created - content: plaintext (but markdown'able) version of the comment's content. - - content_html: the actual html-rendered version of the comment displayed. - Run through Markdown and the HTML cleaner. """ __collection__ = 'media_comments' @@ -278,7 +277,7 @@ class MediaComment(Document): 'author': ObjectId, 'created': datetime.datetime, 'content': unicode, - 'content_html': unicode} + } required_fields = [ 'media_entry', 'author', 'created', 'content'] diff --git a/mediagoblin/db/mongo/open.py b/mediagoblin/db/mongo/open.py index bedc497b..c4f37b42 100644 --- a/mediagoblin/db/mongo/open.py +++ b/mediagoblin/db/mongo/open.py @@ -21,6 +21,10 @@ from mediagoblin.db.mongo import models from mediagoblin.db.mongo.util import MigrationManager +def load_models(app_config): + pass + + def connect_database_from_config(app_config, use_pymongo=False): """ Connect to the main database, take config from app_config diff --git a/mediagoblin/db/mongo/util.py b/mediagoblin/db/mongo/util.py index 4daf616a..f61ae6be 100644 --- a/mediagoblin/db/mongo/util.py +++ b/mediagoblin/db/mongo/util.py @@ -290,3 +290,29 @@ class MigrationManager(object): self.set_current_migration(migration_number) if post_callback: post_callback(migration_number, migration_func) + + +########################## +# Random utility functions +########################## + + +def atomic_update(table, query_dict, update_values): + table.collection.update( + query_dict, + {"$set": update_values}) + + +def check_media_slug_used(db, uploader_id, slug, ignore_m_id): + query_dict = {'uploader': uploader_id, 'slug': slug} + if ignore_m_id is not None: + query_dict['_id'] = {'$ne': ignore_m_id} + existing_user_slug_entries = db.MediaEntry.find( + query_dict).count() + return existing_user_slug_entries + + +def media_entries_for_tag_slug(db, tag_slug): + return db.MediaEntry.find( + {u'state': u'processed', + u'tags.slug': tag_slug}) |