diff options
Diffstat (limited to 'mediagoblin/db')
22 files changed, 704 insertions, 1854 deletions
diff --git a/mediagoblin/db/__init__.py b/mediagoblin/db/__init__.py index d149f62a..27ca4b06 100644 --- a/mediagoblin/db/__init__.py +++ b/mediagoblin/db/__init__.py @@ -18,18 +18,6 @@ Database Abstraction/Wrapper Layer ================================== - **NOTE from Chris Webber:** I asked Elrond to explain why he put - ASCENDING and DESCENDING in db/util.py when we could just import from - pymongo. Read beow for why, but note that nobody is actually doing - this and there's no proof that we'll ever support more than - MongoDB... it would be a huge amount of work to do so. - - If you really want to prove that possible, jump on IRC and talk to - us about making such a branch. In the meanwhile, it doesn't hurt to - have things as they are... if it ever makes it hard for us to - actually do things, we might revisit or remove this. But for more - information, read below. - This submodule is for most of the db specific stuff. There are two main ideas here: diff --git a/mediagoblin/db/sql/base.py b/mediagoblin/db/base.py index 838080b0..699a503a 100644 --- a/mediagoblin/db/sql/base.py +++ b/mediagoblin/db/base.py @@ -17,47 +17,19 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import scoped_session, sessionmaker, object_session -from sqlalchemy.orm.query import Query -from sqlalchemy.sql.expression import desc -from mediagoblin.db.sql.fake import DESCENDING - -def _get_query_model(query): - cols = query.column_descriptions - assert len(cols) == 1, "These functions work only on simple queries" - return cols[0]["type"] - - -class GMGQuery(Query): - def sort(self, key, direction): - key_col = getattr(_get_query_model(self), key) - if direction is DESCENDING: - key_col = desc(key_col) - return self.order_by(key_col) - - def skip(self, amount): - return self.offset(amount) - - -Session = scoped_session(sessionmaker(query_cls=GMGQuery)) - - -def _fix_query_dict(query_dict): - if '_id' in query_dict: - query_dict['id'] = query_dict.pop('_id') +Session = scoped_session(sessionmaker()) class GMGTableBase(object): query = Session.query_property() @classmethod - def find(cls, query_dict={}): - _fix_query_dict(query_dict) + def find(cls, query_dict): return cls.query.filter_by(**query_dict) @classmethod - def find_one(cls, query_dict={}): - _fix_query_dict(query_dict) + def find_one(cls, query_dict): return cls.query.filter_by(**query_dict).first() @classmethod @@ -71,19 +43,20 @@ class GMGTableBase(object): # The key *has* to exist on sql. return getattr(self, key) - def save(self, validate=True): - assert validate + def save(self): sess = object_session(self) if sess is None: sess = Session() sess.add(self) sess.commit() - def delete(self): + def delete(self, commit=True): + """Delete the object and commit the change immediately by default""" sess = object_session(self) assert sess is not None, "Not going to delete detached %r" % self sess.delete(self) - sess.commit() + if commit: + sess.commit() Base = declarative_base(cls=GMGTableBase) diff --git a/mediagoblin/db/sql/extratypes.py b/mediagoblin/db/extratypes.py index f2304af0..f2304af0 100644 --- a/mediagoblin/db/sql/extratypes.py +++ b/mediagoblin/db/extratypes.py diff --git a/mediagoblin/db/sql/util.py b/mediagoblin/db/migration_tools.py index 74b5d73e..c0c7e998 100644 --- a/mediagoblin/db/sql/util.py +++ b/mediagoblin/db/migration_tools.py @@ -14,12 +14,11 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. - -import sys -from mediagoblin.db.sql.base import Session -from mediagoblin.db.sql.models import MediaEntry, Tag, MediaTag, Collection - from mediagoblin.tools.common import simple_printer +from sqlalchemy import Table + +class TableAlreadyExists(Exception): + pass class MigrationManager(object): @@ -39,7 +38,7 @@ class MigrationManager(object): - migration_registry: where we should find all migrations to run """ - self.name = name + self.name = unicode(name) self.models = models self.session = session self.migration_registry = migration_registry @@ -47,7 +46,7 @@ class MigrationManager(object): self.printer = printer # For convenience - from mediagoblin.db.sql.models import MigrationData + from mediagoblin.db.models import MigrationData self.migration_model = MigrationData self.migration_table = MigrationData.__table__ @@ -132,7 +131,10 @@ class MigrationManager(object): # sanity check before we proceed, none of these should be created for model in self.models: # Maybe in the future just print out a "Yikes!" or something? - assert not model.__table__.exists(self.session.bind) + if model.__table__.exists(self.session.bind): + raise TableAlreadyExists( + u"Intended to create table '%s' but it already exists" % + model.__table__.name) self.migration_model.metadata.create_all( self.session.bind, @@ -217,9 +219,9 @@ class MigrationManager(object): u' + Running migration %s, "%s"... ' % ( migration_number, migration_func.func_name)) migration_func(self.session) + self.set_current_migration(migration_number) self.printer('done.\n') - self.set_current_migration() return u'migrated' # Otherwise return None. Well it would do this anyway, but @@ -261,67 +263,14 @@ def assure_migrations_table_setup(db): """ Make sure the migrations table is set up in the database. """ - from mediagoblin.db.sql.models import MigrationData + from mediagoblin.db.models import MigrationData if not MigrationData.__table__.exists(db.bind): MigrationData.metadata.create_all( db.bind, tables=[MigrationData.__table__]) -########################## -# Random utility functions -########################## - - -def atomic_update(table, query_dict, update_values): - table.find(query_dict).update(update_values, - synchronize_session=False) - Session.commit() - - -def check_media_slug_used(dummy_db, uploader_id, slug, ignore_m_id): - filt = (MediaEntry.uploader == uploader_id) \ - & (MediaEntry.slug == slug) - if ignore_m_id is not None: - filt = filt & (MediaEntry.id != ignore_m_id) - does_exist = Session.query(MediaEntry.id).filter(filt).first() is not None - return does_exist - - -def media_entries_for_tag_slug(dummy_db, tag_slug): - return MediaEntry.query \ - .join(MediaEntry.tags_helper) \ - .join(MediaTag.tag_helper) \ - .filter( - (MediaEntry.state == u'processed') - & (Tag.slug == tag_slug)) - - -def clean_orphan_tags(): - q1 = Session.query(Tag).outerjoin(MediaTag).filter(MediaTag.id==None) - for t in q1: - Session.delete(t) - - # The "let the db do all the work" version: - # q1 = Session.query(Tag.id).outerjoin(MediaTag).filter(MediaTag.id==None) - # q2 = Session.query(Tag).filter(Tag.id.in_(q1)) - # q2.delete(synchronize_session = False) - - Session.commit() - - -def check_collection_slug_used(dummy_db, creator_id, slug, ignore_c_id): - filt = (Collection.creator == creator_id) \ - & (Collection.slug == slug) - if ignore_c_id is not None: - filt = filt & (Collection.id != ignore_c_id) - does_exist = Session.query(Collection.id).filter(filt).first() is not None - return does_exist - - -if __name__ == '__main__': - from mediagoblin.db.sql.open import setup_connection_and_db_from_config - - conn,db = setup_connection_and_db_from_config({'sql_engine':'sqlite:///mediagoblin.db'}) - - clean_orphan_tags() +def inspect_table(metadata, table_name): + """Simple helper to get a ref to an already existing table""" + return Table(table_name, metadata, autoload=True, + autoload_with=metadata.bind) diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py new file mode 100644 index 00000000..2c553396 --- /dev/null +++ b/mediagoblin/db/migrations.py @@ -0,0 +1,289 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import datetime +import uuid + +from sqlalchemy import (MetaData, Table, Column, Boolean, SmallInteger, + Integer, Unicode, UnicodeText, DateTime, + ForeignKey) +from sqlalchemy.exc import ProgrammingError +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.sql import and_ +from migrate.changeset.constraint import UniqueConstraint + +from mediagoblin.db.migration_tools import RegisterMigration, inspect_table +from mediagoblin.db.models import MediaEntry, Collection, User + +MIGRATIONS = {} + + +@RegisterMigration(1, MIGRATIONS) +def ogg_to_webm_audio(db_conn): + metadata = MetaData(bind=db_conn.bind) + + file_keynames = Table('core__file_keynames', metadata, autoload=True, + autoload_with=db_conn.bind) + + db_conn.execute( + file_keynames.update().where(file_keynames.c.name == 'ogg'). + values(name='webm_audio') + ) + db_conn.commit() + + +@RegisterMigration(2, MIGRATIONS) +def add_wants_notification_column(db_conn): + metadata = MetaData(bind=db_conn.bind) + + users = Table('core__users', metadata, autoload=True, + autoload_with=db_conn.bind) + + col = Column('wants_comment_notification', Boolean, + default=True, nullable=True) + col.create(users, populate_defaults=True) + db_conn.commit() + + +@RegisterMigration(3, MIGRATIONS) +def add_transcoding_progress(db_conn): + metadata = MetaData(bind=db_conn.bind) + + media_entry = inspect_table(metadata, 'core__media_entries') + + col = Column('transcoding_progress', SmallInteger) + col.create(media_entry) + db_conn.commit() + + +class Collection_v0(declarative_base()): + __tablename__ = "core__collections" + + id = Column(Integer, primary_key=True) + title = Column(Unicode, nullable=False) + slug = Column(Unicode) + created = Column(DateTime, nullable=False, default=datetime.datetime.now, + index=True) + description = Column(UnicodeText) + creator = Column(Integer, ForeignKey(User.id), nullable=False) + items = Column(Integer, default=0) + +class CollectionItem_v0(declarative_base()): + __tablename__ = "core__collection_items" + + id = Column(Integer, primary_key=True) + media_entry = Column( + Integer, ForeignKey(MediaEntry.id), nullable=False, index=True) + collection = Column(Integer, ForeignKey(Collection.id), nullable=False) + note = Column(UnicodeText, nullable=True) + added = Column(DateTime, nullable=False, default=datetime.datetime.now) + position = Column(Integer) + + ## This should be activated, normally. + ## But this would change the way the next migration used to work. + ## So it's commented for now. + __table_args__ = ( + UniqueConstraint('collection', 'media_entry'), + {}) + +collectionitem_unique_constraint_done = False + +@RegisterMigration(4, MIGRATIONS) +def add_collection_tables(db_conn): + Collection_v0.__table__.create(db_conn.bind) + CollectionItem_v0.__table__.create(db_conn.bind) + + global collectionitem_unique_constraint_done + collectionitem_unique_constraint_done = True + + db_conn.commit() + + +@RegisterMigration(5, MIGRATIONS) +def add_mediaentry_collected(db_conn): + metadata = MetaData(bind=db_conn.bind) + + media_entry = inspect_table(metadata, 'core__media_entries') + + col = Column('collected', Integer, default=0) + col.create(media_entry) + db_conn.commit() + + +class ProcessingMetaData_v0(declarative_base()): + __tablename__ = 'core__processing_metadata' + + id = Column(Integer, primary_key=True) + media_entry_id = Column(Integer, ForeignKey(MediaEntry.id), nullable=False, + index=True) + callback_url = Column(Unicode) + +@RegisterMigration(6, MIGRATIONS) +def create_processing_metadata_table(db): + ProcessingMetaData_v0.__table__.create(db.bind) + db.commit() + + +# Okay, problem being: +# Migration #4 forgot to add the uniqueconstraint for the +# new tables. While creating the tables from scratch had +# the constraint enabled. +# +# So we have four situations that should end up at the same +# db layout: +# +# 1. Fresh install. +# Well, easy. Just uses the tables in models.py +# 2. Fresh install using a git version just before this migration +# The tables are all there, the unique constraint is also there. +# This migration should do nothing. +# But as we can't detect the uniqueconstraint easily, +# this migration just adds the constraint again. +# And possibly fails very loud. But ignores the failure. +# 3. old install, not using git, just releases. +# This one will get the new tables in #4 (now with constraint!) +# And this migration is just skipped silently. +# 4. old install, always on latest git. +# This one has the tables, but lacks the constraint. +# So this migration adds the constraint. +@RegisterMigration(7, MIGRATIONS) +def fix_CollectionItem_v0_constraint(db_conn): + """Add the forgotten Constraint on CollectionItem""" + + global collectionitem_unique_constraint_done + if collectionitem_unique_constraint_done: + # Reset it. Maybe the whole thing gets run again + # For a different db? + collectionitem_unique_constraint_done = False + return + + metadata = MetaData(bind=db_conn.bind) + + CollectionItem_table = inspect_table(metadata, 'core__collection_items') + + constraint = UniqueConstraint('collection', 'media_entry', + name='core__collection_items_collection_media_entry_key', + table=CollectionItem_table) + + try: + constraint.create() + except ProgrammingError: + # User probably has an install that was run since the + # collection tables were added, so we don't need to run this migration. + pass + + db_conn.commit() + + +@RegisterMigration(8, MIGRATIONS) +def add_license_preference(db): + metadata = MetaData(bind=db.bind) + + user_table = inspect_table(metadata, 'core__users') + + col = Column('license_preference', Unicode) + col.create(user_table) + db.commit() + + +@RegisterMigration(9, MIGRATIONS) +def mediaentry_new_slug_era(db): + """ + Update for the new era for media type slugs. + + Entries without slugs now display differently in the url like: + /u/cwebber/m/id=251/ + + ... because of this, we should back-convert: + - entries without slugs should be converted to use the id, if possible, to + make old urls still work + - slugs with = (or also : which is now also not allowed) to have those + stripped out (small possibility of breakage here sadly) + """ + + def slug_and_user_combo_exists(slug, uploader): + return db.execute( + media_table.select( + and_(media_table.c.uploader==uploader, + media_table.c.slug==slug))).first() is not None + + def append_garbage_till_unique(row, new_slug): + """ + Attach junk to this row until it's unique, then save it + """ + if slug_and_user_combo_exists(new_slug, row.uploader): + # okay, still no success; + # let's whack junk on there till it's unique. + new_slug += '-' + uuid.uuid4().hex[:4] + # keep going if necessary! + while slug_and_user_combo_exists(new_slug, row.uploader): + new_slug += uuid.uuid4().hex[:4] + + db.execute( + media_table.update(). \ + where(media_table.c.id==row.id). \ + values(slug=new_slug)) + + metadata = MetaData(bind=db.bind) + + media_table = inspect_table(metadata, 'core__media_entries') + + for row in db.execute(media_table.select()): + # no slug, try setting to an id + if not row.slug: + append_garbage_till_unique(row, unicode(row.id)) + # has "=" or ":" in it... we're getting rid of those + elif u"=" in row.slug or u":" in row.slug: + append_garbage_till_unique( + row, row.slug.replace(u"=", u"-").replace(u":", u"-")) + + db.commit() + + +@RegisterMigration(10, MIGRATIONS) +def unique_collections_slug(db): + """Add unique constraint to collection slug""" + metadata = MetaData(bind=db.bind) + collection_table = inspect_table(metadata, "core__collections") + existing_slugs = {} + slugs_to_change = [] + + for row in db.execute(collection_table.select()): + # if duplicate slug, generate a unique slug + if row.creator in existing_slugs and row.slug in \ + existing_slugs[row.creator]: + slugs_to_change.append(row.id) + else: + if not row.creator in existing_slugs: + existing_slugs[row.creator] = [row.slug] + else: + existing_slugs[row.creator].append(row.slug) + + for row_id in slugs_to_change: + new_slug = unicode(uuid.uuid4()) + db.execute(collection_table.update(). + where(collection_table.c.id == row_id). + values(slug=new_slug)) + # sqlite does not like to change the schema when a transaction(update) is + # not yet completed + db.commit() + + constraint = UniqueConstraint('creator', 'slug', + name='core__collection_creator_slug_key', + table=collection_table) + constraint.create() + + db.commit() diff --git a/mediagoblin/db/mixin.py b/mediagoblin/db/mixin.py index f6a15a12..388bac89 100644 --- a/mediagoblin/db/mixin.py +++ b/mediagoblin/db/mixin.py @@ -27,8 +27,13 @@ These functions now live here and get "mixed in" into the real objects. """ +import uuid + +from werkzeug.utils import cached_property + from mediagoblin import mg_globals from mediagoblin.auth import lib as auth_lib +from mediagoblin.media_types import get_media_managers, FileTypeNotSupported from mediagoblin.tools import common, licenses from mediagoblin.tools.text import cleaned_markdown_conversion from mediagoblin.tools.url import slugify @@ -47,22 +52,83 @@ class UserMixin(object): return cleaned_markdown_conversion(self.bio) -class MediaEntryMixin(object): +class GenerateSlugMixin(object): + """ + Mixin to add a generate_slug method to objects. + + Depends on: + - self.slug + - self.title + - self.check_slug_used(new_slug) + """ def generate_slug(self): + """ + Generate a unique slug for this object. + + This one does not *force* slugs, but usually it will probably result + in a niceish one. + + The end *result* of the algorithm will result in these resolutions for + these situations: + - If we have a slug, make sure it's clean and sanitized, and if it's + unique, we'll use that. + - If we have a title, slugify it, and if it's unique, we'll use that. + - If we can't get any sort of thing that looks like it'll be a useful + slug out of a title or an existing slug, bail, and don't set the + slug at all. Don't try to create something just because. Make + sure we have a reasonable basis for a slug first. + - If we have a reasonable basis for a slug (either based on existing + slug or slugified title) but it's not unique, first try appending + the entry's id, if that exists + - If that doesn't result in something unique, tack on some randomly + generated bits until it's unique. That'll be a little bit of junk, + but at least it has the basis of a nice slug. + """ + #Is already a slug assigned? Check if it is valid + if self.slug: + self.slug = slugify(self.slug) + + # otherwise, try to use the title. + elif self.title: + # assign slug based on title + self.slug = slugify(self.title) + + # We don't want any empty string slugs + if self.slug == u"": + self.slug = None + + # Do we have anything at this point? + # If not, we're not going to get a slug + # so just return... we're not going to force one. + if not self.slug: + return # giving up! + + # Otherwise, let's see if this is unique. + if self.check_slug_used(self.slug): + # It looks like it's being used... lame. + + # Can we just append the object's id to the end? + if self.id: + slug_with_id = u"%s-%s" % (self.slug, self.id) + if not self.check_slug_used(slug_with_id): + self.slug = slug_with_id + return # success! + + # okay, still no success; + # let's whack junk on there till it's unique. + self.slug += '-' + uuid.uuid4().hex[:4] + # keep going if necessary! + while self.check_slug_used(self.slug): + self.slug += uuid.uuid4().hex[:4] + + +class MediaEntryMixin(GenerateSlugMixin): + def check_slug_used(self, slug): # import this here due to a cyclic import issue # (db.models -> db.mixin -> db.util -> db.models) from mediagoblin.db.util import check_media_slug_used - self.slug = slugify(self.title) - - duplicate = check_media_slug_used(mg_globals.database, - self.uploader, self.slug, self.id) - - if duplicate: - if self.id is not None: - self.slug = u"%s-%s" % (self.id, self.slug) - else: - self.slug = None + return check_media_slug_used(self.uploader, slug, self.id) @property def description_html(self): @@ -72,37 +138,44 @@ class MediaEntryMixin(object): """ return cleaned_markdown_conversion(self.description) - def get_display_media(self, media_map, - fetch_order=common.DISPLAY_IMAGE_FETCHING_ORDER): - """ - Find the best media for display. + def get_display_media(self): + """Find the best media for display. - Args: - - media_map: a dict like - {u'image_size': [u'dir1', u'dir2', u'image.jpg']} - - fetch_order: the order we should try fetching images in + We try checking self.media_manager.fetching_order if it exists to + pull down the order. Returns: - (media_size, media_path) + (media_size, media_path) + or, if not found, None. + """ - media_sizes = media_map.keys() + fetch_order = self.media_manager.media_fetch_order - for media_size in common.DISPLAY_IMAGE_FETCHING_ORDER: + # No fetching order found? well, give up! + if not fetch_order: + return None + + media_sizes = self.media_files.keys() + + for media_size in fetch_order: if media_size in media_sizes: - return media_map[media_size] + return media_size, self.media_files[media_size] def main_mediafile(self): pass @property def slug_or_id(self): - return (self.slug or self._id) + if self.slug: + return self.slug + else: + return u'id:%s' % self.id def url_for_self(self, urlgen, **extra_args): """ Generate an appropriate url for ourselves - Use a slug if we have one, else use our '_id'. + Use a slug if we have one, else use our 'id'. """ uploader = self.get_uploader @@ -112,6 +185,38 @@ class MediaEntryMixin(object): media=self.slug_or_id, **extra_args) + @property + def thumb_url(self): + """Return the thumbnail URL (for usage in templates) + Will return either the real thumbnail or a default fallback icon.""" + # TODO: implement generic fallback in case MEDIA_MANAGER does + # not specify one? + if u'thumb' in self.media_files: + thumb_url = mg_globals.app.public_store.file_url( + self.media_files[u'thumb']) + else: + # No thumbnail in media available. Get the media's + # MEDIA_MANAGER for the fallback icon and return static URL + # Raises FileTypeNotSupported in case no such manager is enabled + manager = self.media_manager + thumb_url = mg_globals.app.staticdirector(manager[u'default_thumb']) + return thumb_url + + @cached_property + def media_manager(self): + """Returns the MEDIA_MANAGER of the media's media_type + + Raises FileTypeNotSupported in case no such manager is enabled + """ + # TODO, we should be able to make this a simple lookup rather + # than iterating through all media managers. + for media_type, manager in get_media_managers(): + if media_type == self.media_type: + return manager(self) + # Not found? Then raise an error + raise FileTypeNotSupported( + "MediaManager not in enabled types. Check media_types in config?") + def get_fail_exception(self): """ Get the exception that's appropriate for this error @@ -121,7 +226,7 @@ class MediaEntryMixin(object): def get_license_data(self): """Return license dict for requested license""" - return licenses.SUPPORTED_LICENSES[self.license or ""] + return licenses.get_license_by_url(self.license or "") def exif_display_iter(self): from mediagoblin.tools.exif import USEFUL_TAGS @@ -145,22 +250,13 @@ class MediaCommentMixin(object): return cleaned_markdown_conversion(self.content) -class CollectionMixin(object): - def generate_slug(self): +class CollectionMixin(GenerateSlugMixin): + def check_slug_used(self, slug): # import this here due to a cyclic import issue # (db.models -> db.mixin -> db.util -> db.models) from mediagoblin.db.util import check_collection_slug_used - self.slug = slugify(self.title) - - duplicate = check_collection_slug_used(mg_globals.database, - self.creator, self.slug, self.id) - - if duplicate: - if self.id is not None: - self.slug = u"%s-%s" % (self.id, self.slug) - else: - self.slug = None + return check_collection_slug_used(self.creator, slug, self.id) @property def description_html(self): @@ -172,13 +268,13 @@ class CollectionMixin(object): @property def slug_or_id(self): - return (self.slug or self._id) + return (self.slug or self.id) def url_for_self(self, urlgen, **extra_args): """ Generate an appropriate url for ourselves - Use a slug if we have one, else use our '_id'. + Use a slug if we have one, else use our 'id'. """ creator = self.get_creator diff --git a/mediagoblin/db/sql/models.py b/mediagoblin/db/models.py index b48c1fbe..2b925983 100644 --- a/mediagoblin/db/sql/models.py +++ b/mediagoblin/db/models.py @@ -18,9 +18,8 @@ TODO: indexes on foreignkeys, where useful. """ - +import logging import datetime -import sys from sqlalchemy import Column, Integer, Unicode, UnicodeText, DateTime, \ Boolean, ForeignKey, UniqueConstraint, PrimaryKeyConstraint, \ @@ -31,10 +30,11 @@ from sqlalchemy.sql.expression import desc from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy.util import memoized_property -from mediagoblin.db.sql.extratypes import PathTupleWithSlashes, JSONEncoded -from mediagoblin.db.sql.base import Base, DictReadAttrProxy +from mediagoblin.db.extratypes import PathTupleWithSlashes, JSONEncoded +from mediagoblin.db.base import Base, DictReadAttrProxy from mediagoblin.db.mixin import UserMixin, MediaEntryMixin, MediaCommentMixin, CollectionMixin, CollectionItemMixin -from mediagoblin.db.sql.base import Session +from mediagoblin.tools.files import delete_media_files +from mediagoblin.tools.common import import_component # It's actually kind of annoying how sqlalchemy-migrate does this, if # I understand it right, but whatever. Anyway, don't remove this :P @@ -43,17 +43,7 @@ from mediagoblin.db.sql.base import Session # this import-based meddling... from migrate import changeset - -class SimpleFieldAlias(object): - """An alias for any field""" - def __init__(self, fieldname): - self.fieldname = fieldname - - def __get__(self, instance, cls): - return getattr(instance, self.fieldname) - - def __set__(self, instance, val): - setattr(instance, self.fieldname, val) +_log = logging.getLogger(__name__) class User(Base, UserMixin): @@ -65,6 +55,10 @@ class User(Base, UserMixin): id = Column(Integer, primary_key=True) username = Column(Unicode, nullable=False, unique=True) + # Note: no db uniqueness constraint on email because it's not + # reliable (many email systems case insensitive despite against + # the RFC) and because it would be a mess to implement at this + # point. email = Column(Unicode, nullable=False) created = Column(DateTime, nullable=False, default=datetime.datetime.now) pw_hash = Column(Unicode, nullable=False) @@ -73,6 +67,7 @@ class User(Base, UserMixin): # Intented to be nullable=False, but migrations would not work for it # set to nullable=True implicitly. wants_comment_notification = Column(Boolean, default=True) + license_preference = Column(Unicode) verification_key = Column(Unicode) is_admin = Column(Boolean, default=False, nullable=False) url = Column(Unicode) @@ -83,8 +78,6 @@ class User(Base, UserMixin): ## TODO # plugin data would be in a separate model - _id = SimpleFieldAlias("id") - def __repr__(self): return '<{0} #{1} {2} {3} "{4}">'.format( self.__class__.__name__, @@ -93,6 +86,25 @@ class User(Base, UserMixin): 'admin' if self.is_admin else 'user', self.username) + def delete(self, **kwargs): + """Deletes a User and all related entries/comments/files/...""" + # Collections get deleted by relationships. + + media_entries = MediaEntry.query.filter(MediaEntry.uploader == self.id) + for media in media_entries: + # TODO: Make sure that "MediaEntry.delete()" also deletes + # all related files/Comments + media.delete(del_orphan_tags=False, commit=False) + + # Delete now unused tags + # TODO: import here due to cyclic imports!!! This cries for refactoring + from mediagoblin.db.util import clean_orphan_tags + clean_orphan_tags(commit=False) + + # Delete user, pass through commit=False/True in kwargs + super(User, self).delete(**kwargs) + _log.info('Deleted user "{0}" account'.format(self.username)) + class MediaEntry(Base, MediaEntryMixin): """ @@ -146,7 +158,7 @@ class MediaEntry(Base, MediaEntryMixin): ) tags_helper = relationship("MediaTag", - cascade="all, delete-orphan" + cascade="all, delete-orphan" # should be automatically deleted ) tags = association_proxy("tags_helper", "dict_view", creator=lambda v: MediaTag(name=v["name"], slug=v["slug"]) @@ -158,17 +170,13 @@ class MediaEntry(Base, MediaEntryMixin): collections = association_proxy("collections_helper", "in_collection") ## TODO - # media_data # fail_error - _id = SimpleFieldAlias("id") - def get_comments(self, ascending=False): order_col = MediaComment.created if not ascending: order_col = desc(order_col) - return MediaComment.query.filter_by( - media_entry=self.id).order_by(order_col) + return self.all_comments.order_by(order_col) def url_to_prev(self, urlgen): """get the next 'newer' entry by this user""" @@ -190,40 +198,31 @@ class MediaEntry(Base, MediaEntryMixin): if media is not None: return media.url_for_self(urlgen) - #@memoized_property @property def media_data(self): - session = Session() - - return session.query(self.media_data_table).filter_by( - media_entry=self.id).first() + return getattr(self, self.media_data_ref) def media_data_init(self, **kwargs): """ Initialize or update the contents of a media entry's media_data row """ - session = Session() - - media_data = session.query(self.media_data_table).filter_by( - media_entry=self.id).first() + media_data = self.media_data - # No media data, so actually add a new one if media_data is None: - media_data = self.media_data_table( - media_entry=self.id, - **kwargs) - session.add(media_data) - # Update old media data + # Get the correct table: + table = import_component(self.media_type + '.models:DATA_MODEL') + # No media data, so actually add a new one + media_data = table(**kwargs) + # Get the relationship set up. + media_data.get_media_entry = self else: + # Update old media data for field, value in kwargs.iteritems(): setattr(media_data, field, value) @memoized_property - def media_data_table(self): - # TODO: memoize this - models_module = self.media_type + '.models' - __import__(models_module) - return sys.modules[models_module].DATA_MODEL + def media_data_ref(self): + return import_component(self.media_type + '.models:BACKREF_NAME') def __repr__(self): safe_title = self.title.encode('ascii', 'replace') @@ -233,6 +232,35 @@ class MediaEntry(Base, MediaEntryMixin): id=self.id, title=safe_title) + def delete(self, del_orphan_tags=True, **kwargs): + """Delete MediaEntry and all related files/attachments/comments + + This will *not* automatically delete unused collections, which + can remain empty... + + :param del_orphan_tags: True/false if we delete unused Tags too + :param commit: True/False if this should end the db transaction""" + # User's CollectionItems are automatically deleted via "cascade". + # Comments on this Media are deleted by cascade, hopefully. + + # Delete all related files/attachments + try: + delete_media_files(self) + except OSError, error: + # Returns list of files we failed to delete + _log.error('No such files from the user "{1}" to delete: ' + '{0}'.format(str(error), self.get_uploader)) + _log.info('Deleted Media entry id "{0}"'.format(self.id)) + # Related MediaTag's are automatically cleaned, but we might + # want to clean out unused Tag's too. + if del_orphan_tags: + # TODO: Import here due to cyclic imports!!! + # This cries for refactoring + from mediagoblin.db.util import clean_orphan_tags + clean_orphan_tags(commit=False) + # pass through commit=False/True in kwargs + super(MediaEntry, self).delete(**kwargs) + class FileKeynames(Base): """ @@ -357,34 +385,58 @@ class MediaComment(Base, MediaCommentMixin): created = Column(DateTime, nullable=False, default=datetime.datetime.now) content = Column(UnicodeText, nullable=False) - get_author = relationship(User) + # Cascade: Comments are owned by their creator. So do the full thing. + # lazy=dynamic: People might post a *lot* of comments, + # so make the "posted_comments" a query-like thing. + get_author = relationship(User, + backref=backref("posted_comments", + lazy="dynamic", + cascade="all, delete-orphan")) - _id = SimpleFieldAlias("id") + # Cascade: Comments are somewhat owned by their MediaEntry. + # So do the full thing. + # lazy=dynamic: MediaEntries might have many comments, + # so make the "all_comments" a query-like thing. + get_media_entry = relationship(MediaEntry, + backref=backref("all_comments", + lazy="dynamic", + cascade="all, delete-orphan")) class Collection(Base, CollectionMixin): + """An 'album' or 'set' of media by a user. + + On deletion, contained CollectionItems get automatically reaped via + SQL cascade""" __tablename__ = "core__collections" id = Column(Integer, primary_key=True) title = Column(Unicode, nullable=False) slug = Column(Unicode) created = Column(DateTime, nullable=False, default=datetime.datetime.now, - index=True) + index=True) description = Column(UnicodeText) creator = Column(Integer, ForeignKey(User.id), nullable=False) + # TODO: No of items in Collection. Badly named, can we migrate to num_items? items = Column(Integer, default=0) - get_creator = relationship(User) + # Cascade: Collections are owned by their creator. So do the full thing. + get_creator = relationship(User, + backref=backref("collections", + cascade="all, delete-orphan")) + + __table_args__ = ( + UniqueConstraint('creator', 'slug'), + {}) def get_collection_items(self, ascending=False): + #TODO, is this still needed with self.collection_items being available? order_col = CollectionItem.position if not ascending: order_col = desc(order_col) return CollectionItem.query.filter_by( collection=self.id).order_by(order_col) - _id = SimpleFieldAlias("id") - class CollectionItem(Base, CollectionItemMixin): __tablename__ = "core__collection_items" @@ -396,11 +448,14 @@ class CollectionItem(Base, CollectionItemMixin): note = Column(UnicodeText, nullable=True) added = Column(DateTime, nullable=False, default=datetime.datetime.now) position = Column(Integer) - in_collection = relationship("Collection") - get_media_entry = relationship(MediaEntry) + # Cascade: CollectionItems are owned by their Collection. So do the full thing. + in_collection = relationship(Collection, + backref=backref( + "collection_items", + cascade="all, delete-orphan")) - _id = SimpleFieldAlias("id") + get_media_entry = relationship(MediaEntry) __table_args__ = ( UniqueConstraint('collection', 'media_entry'), diff --git a/mediagoblin/db/sql/models_v0.py b/mediagoblin/db/models_v0.py index 06f87d28..ec51a1f5 100644 --- a/mediagoblin/db/sql/models_v0.py +++ b/mediagoblin/db/models_v0.py @@ -31,9 +31,8 @@ from sqlalchemy.orm.collections import attribute_mapped_collection from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy.util import memoized_property -from mediagoblin.db.sql.extratypes import PathTupleWithSlashes, JSONEncoded -from mediagoblin.db.sql.base import GMGTableBase -from mediagoblin.db.sql.base import Session +from mediagoblin.db.extratypes import PathTupleWithSlashes, JSONEncoded +from mediagoblin.db.base import GMGTableBase, Session Base_v0 = declarative_base(cls=GMGTableBase) diff --git a/mediagoblin/db/mongo/__init__.py b/mediagoblin/db/mongo/__init__.py deleted file mode 100644 index 621845ba..00000000 --- a/mediagoblin/db/mongo/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. diff --git a/mediagoblin/db/mongo/indexes.py b/mediagoblin/db/mongo/indexes.py deleted file mode 100644 index a63c24ae..00000000 --- a/mediagoblin/db/mongo/indexes.py +++ /dev/null @@ -1,146 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -""" -Indexes for the local database. - -To add new indexes ------------------- - -Indexes are recorded in the following format: - -ACTIVE_INDEXES = { - 'collection_name': { - 'identifier': { # key identifier used for possibly deprecating later - 'index': [index_foo_goes_here]}} - -... and anything else being parameters to the create_index function -(including unique=True, etc) - -Current indexes must be registered in ACTIVE_INDEXES... deprecated -indexes should be marked in DEPRECATED_INDEXES. - -Remember, ordering of compound indexes MATTERS. Read below for more. - -REQUIRED READING: - - http://kylebanker.com/blog/2010/09/21/the-joy-of-mongodb-indexes/ - - - http://www.mongodb.org/display/DOCS/Indexes - - http://www.mongodb.org/display/DOCS/Indexing+Advice+and+FAQ - - -To remove deprecated indexes ----------------------------- - -Removing deprecated indexes is the same, just move the index into the -deprecated indexes mapping. - -DEPRECATED_INDEXES = { - 'collection_name': { - 'deprecated_index_identifier1': { - 'index': [index_foo_goes_here]}} - -... etc. - -If an index has been deprecated that identifier should NEVER BE USED -AGAIN. Eg, if you previously had 'awesomepants_unique', you shouldn't -use 'awesomepants_unique' again, you should create a totally new name -or at worst use 'awesomepants_unique2'. -""" - -from pymongo import ASCENDING, DESCENDING - - -################ -# Active indexes -################ -ACTIVE_INDEXES = {} - -# MediaEntry indexes -# ------------------ - -MEDIAENTRY_INDEXES = { - 'uploader_slug_unique': { - # Matching an object to an uploader + slug. - # MediaEntries are unique on these two combined, eg: - # /u/${myuser}/m/${myslugname}/ - 'index': [('uploader', ASCENDING), - ('slug', ASCENDING)], - 'unique': True}, - - 'created': { - # A global index for all media entries created, in descending - # order. This is used for the site's frontpage. - 'index': [('created', DESCENDING)]}, - - 'uploader_created': { - # Indexing on uploaders and when media entries are created. - # Used for showing a user gallery, etc. - 'index': [('uploader', ASCENDING), - ('created', DESCENDING)]}, - - 'state_uploader_tags_created': { - # Indexing on processed?, media uploader, associated tags, and - # timestamp Used for showing media items matching a tag - # search, most recent first. - 'index': [('state', ASCENDING), - ('uploader', ASCENDING), - ('tags.slug', DESCENDING), - ('created', DESCENDING)]}, - - 'state_tags_created': { - # Indexing on processed?, media tags, and timestamp (across all users) - # This is used for a front page tag search. - 'index': [('state', ASCENDING), - ('tags.slug', DESCENDING), - ('created', DESCENDING)]}} - - -ACTIVE_INDEXES['media_entries'] = MEDIAENTRY_INDEXES - - -# User indexes -# ------------ - -USER_INDEXES = { - 'username_unique': { - # Index usernames, and make sure they're unique. - # ... I guess we might need to adjust this once we're federated :) - 'index': 'username', - 'unique': True}, - 'created': { - # All most recently created users - 'index': 'created'}} - - -ACTIVE_INDEXES['users'] = USER_INDEXES - - -# MediaComment indexes - -MEDIA_COMMENT_INDEXES = { - 'mediaentry_created': { - 'index': [('media_entry', ASCENDING), - ('created', DESCENDING)]}} - -ACTIVE_INDEXES['media_comments'] = MEDIA_COMMENT_INDEXES - - -#################### -# Deprecated indexes -#################### - -DEPRECATED_INDEXES = {} diff --git a/mediagoblin/db/mongo/migrations.py b/mediagoblin/db/mongo/migrations.py deleted file mode 100644 index 569dec88..00000000 --- a/mediagoblin/db/mongo/migrations.py +++ /dev/null @@ -1,208 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -from mediagoblin.db.mongo.util import RegisterMigration -from mediagoblin.tools.text import cleaned_markdown_conversion - - -def add_table_field(db, table_name, field_name, default_value): - """ - Add a new field to the table/collection named table_name. - The field will have the name field_name and the value default_value - """ - db[table_name].update( - {field_name: {'$exists': False}}, - {'$set': {field_name: default_value}}, - multi=True) - - -def drop_table_field(db, table_name, field_name): - """ - Drop an old field from a table/collection - """ - db[table_name].update( - {field_name: {'$exists': True}}, - {'$unset': {field_name: 1}}, - multi=True) - - -# Please see mediagoblin/tests/test_migrations.py for some examples of -# basic migrations. - - -@RegisterMigration(1) -def user_add_bio_html(database): - """ - Users now have richtext bios via Markdown, reflect appropriately. - """ - collection = database['users'] - - target = collection.find( - {'bio_html': {'$exists': False}}) - - for document in target: - document['bio_html'] = cleaned_markdown_conversion( - document['bio']) - collection.save(document) - - -@RegisterMigration(2) -def mediaentry_mediafiles_main_to_original(database): - """ - Rename "main" media file to "original". - """ - collection = database['media_entries'] - target = collection.find( - {'media_files.main': {'$exists': True}}) - - for document in target: - original = document['media_files'].pop('main') - document['media_files']['original'] = original - - collection.save(document) - - -@RegisterMigration(3) -def mediaentry_remove_thumbnail_file(database): - """ - Use media_files['thumb'] instead of media_entries['thumbnail_file'] - """ - database['media_entries'].update( - {'thumbnail_file': {'$exists': True}}, - {'$unset': {'thumbnail_file': 1}}, - multi=True) - - -@RegisterMigration(4) -def mediaentry_add_queued_task_id(database): - """ - Add the 'queued_task_id' field for entries that don't have it. - """ - add_table_field(database, 'media_entries', 'queued_task_id', None) - - -@RegisterMigration(5) -def mediaentry_add_fail_error_and_metadata(database): - """ - Add 'fail_error' and 'fail_metadata' fields to media entries - """ - add_table_field(database, 'media_entries', 'fail_error', None) - add_table_field(database, 'media_entries', 'fail_metadata', {}) - - -@RegisterMigration(6) -def user_add_forgot_password_token_and_expires(database): - """ - Add token and expiration fields to help recover forgotten passwords - """ - add_table_field(database, 'users', 'fp_verification_key', None) - add_table_field(database, 'users', 'fp_token_expire', None) - - -@RegisterMigration(7) -def media_type_image_to_multimedia_type_image(database): - database['media_entries'].update( - {'media_type': 'image'}, - {'$set': {'media_type': 'mediagoblin.media_types.image'}}, - multi=True) - - -@RegisterMigration(8) -def mediaentry_add_license(database): - """ - Add the 'license' field for entries that don't have it. - """ - add_table_field(database, 'media_entries', 'license', None) - - -@RegisterMigration(9) -def remove_calculated_html(database): - """ - Drop pre-rendered html again and calculate things - on the fly (and cache): - - User.bio_html - - MediaEntry.description_html - - MediaComment.content_html - """ - drop_table_field(database, 'users', 'bio_html') - drop_table_field(database, 'media_entries', 'description_html') - drop_table_field(database, 'media_comments', 'content_html') - - -@RegisterMigration(10) -def convert_video_media_data(database): - """ - Move media_data["video"] directly into media_data - """ - collection = database['media_entries'] - target = collection.find( - {'media_data.video': {'$exists': True}}) - - for document in target: - assert len(document['media_data']) == 1 - document['media_data'] = document['media_data']['video'] - collection.save(document) - - -@RegisterMigration(11) -def convert_gps_media_data(database): - """ - Move media_data["gps"]["*"] to media_data["gps_*"]. - In preparation for media_data.gps_* - """ - collection = database['media_entries'] - target = collection.find( - {'media_data.gps': {'$exists': True}}) - - for document in target: - for key, value in document['media_data']['gps'].iteritems(): - document['media_data']['gps_' + key] = value - del document['media_data']['gps'] - collection.save(document) - - -@RegisterMigration(12) -def convert_exif_media_data(database): - """ - Move media_data["exif"]["clean"] to media_data["exif_all"]. - Drop media_data["exif"]["useful"] - In preparation for media_data.exif_all - """ - collection = database['media_entries'] - target = collection.find( - {'media_data.exif.clean': {'$exists': True}}) - - for document in target: - media_data = document['media_data'] - - exif_all = media_data['exif'].pop('clean') - if len(exif_all): - media_data['exif_all'] = exif_all - - del media_data['exif']['useful'] - - assert len(media_data['exif']) == 0 - del media_data['exif'] - - collection.save(document) - - -@RegisterMigration(13) -def user_add_wants_comment_notification(database): - """ - Add wants_comment_notification to user model - """ - add_table_field(database, 'users', 'wants_comment_notification', True) diff --git a/mediagoblin/db/mongo/models.py b/mediagoblin/db/mongo/models.py deleted file mode 100644 index 3f1363d5..00000000 --- a/mediagoblin/db/mongo/models.py +++ /dev/null @@ -1,310 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -import datetime - -from mongokit import Document - -from mediagoblin.db.mongo import migrations -from mediagoblin.db.mongo.util import ASCENDING, DESCENDING, ObjectId -from mediagoblin.tools.pagination import Pagination -from mediagoblin.db.mixin import UserMixin, MediaEntryMixin, MediaCommentMixin - - -class MongoPK(object): - """An alias for the _id primary key""" - def __get__(self, instance, cls): - return instance['_id'] - def __set__(self, instance, val): - instance['_id'] = val - def __delete__(self, instance): - del instance['_id'] - - -################### -# Custom validators -################### - -######## -# Models -######## - - -class User(Document, UserMixin): - """ - A user of MediaGoblin. - - Structure: - - username: The username of this user, should be unique to this instance. - - email: Email address of this user - - created: When the user was created - - plugin_data: a mapping of extra plugin information for this User. - Nothing uses this yet as we don't have plugins, but someday we - might... :) - - pw_hash: Hashed version of user's password. - - email_verified: Whether or not the user has verified their email or not. - Most parts of the site are disabled for users who haven't yet. - - status: whether or not the user is active, etc. Currently only has two - values, 'needs_email_verification' or 'active'. (In the future, maybe - we'll change this to a boolean with a key of 'active' and have a - separate field for a reason the user's been disabled if that's - appropriate... email_verified is already separate, after all.) - - wants_comment_notification: The user has selected that they want to be - notified when comments are posted on their media. - - verification_key: If the user is awaiting email verification, the user - will have to provide this key (which will be encoded in the presented - URL) in order to confirm their email as active. - - is_admin: Whether or not this user is an administrator or not. - - url: this user's personal webpage/website, if appropriate. - - bio: biography of this user (plaintext, in markdown) - """ - __collection__ = 'users' - use_dot_notation = True - - structure = { - 'username': unicode, - 'email': unicode, - 'created': datetime.datetime, - 'plugin_data': dict, # plugins can dump stuff here. - 'pw_hash': unicode, - 'email_verified': bool, - 'status': unicode, - 'wants_comment_notification': bool, - 'verification_key': unicode, - 'is_admin': bool, - 'url': unicode, - 'bio': unicode, # May contain markdown - 'fp_verification_key': unicode, # forgotten password verification key - 'fp_token_expire': datetime.datetime, - } - - required_fields = ['username', 'created', 'pw_hash', 'email'] - - default_values = { - 'created': datetime.datetime.utcnow, - 'email_verified': False, - 'wants_comment_notification': True, - 'status': u'needs_email_verification', - 'is_admin': False} - - id = MongoPK() - - -class MediaEntry(Document, MediaEntryMixin): - """ - Record of a piece of media. - - Structure: - - uploader: A reference to a User who uploaded this. - - - title: Title of this work - - - slug: A normalized "slug" which can be used as part of a URL to retrieve - this work, such as 'my-works-name-in-slug-form' may be viewable by - 'http://mg.example.org/u/username/m/my-works-name-in-slug-form/' - Note that since URLs are constructed this way, slugs must be unique - per-uploader. (An index is provided to enforce that but code should be - written on the python side to ensure this as well.) - - - created: Date and time of when this piece of work was uploaded. - - - description: Uploader-set description of this work. This can be marked - up with MarkDown for slight fanciness (links, boldness, italics, - paragraphs...) - - - media_type: What type of media is this? Currently we only support - 'image' ;) - - - media_data: Extra information that's media-format-dependent. - For example, images might contain some EXIF data that's not appropriate - to other formats. You might store it like: - - mediaentry.media_data['exif'] = { - 'manufacturer': 'CASIO', - 'model': 'QV-4000', - 'exposure_time': .659} - - Alternately for video you might store: - - # play length in seconds - mediaentry.media_data['play_length'] = 340 - - ... so what's appropriate here really depends on the media type. - - - plugin_data: a mapping of extra plugin information for this User. - Nothing uses this yet as we don't have plugins, but someday we - might... :) - - - tags: A list of tags. Each tag is stored as a dictionary that has a key - for the actual name and the normalized name-as-slug, so ultimately this - looks like: - [{'name': 'Gully Gardens', - 'slug': 'gully-gardens'}, - {'name': 'Castle Adventure Time?!", - 'slug': 'castle-adventure-time'}] - - - state: What's the state of this file? Active, inactive, disabled, etc... - But really for now there are only two states: - "unprocessed": uploaded but needs to go through processing for display - "processed": processed and able to be displayed - - - license: URI for media's license. - - - queued_media_file: storage interface style filepath describing a file - queued for processing. This is stored in the mg_globals.queue_store - storage system. - - - queued_task_id: celery task id. Use this to fetch the task state. - - - media_files: Files relevant to this that have actually been processed - and are available for various types of display. Stored like: - {'thumb': ['dir1', 'dir2', 'pic.png'} - - - attachment_files: A list of "attachment" files, ones that aren't - critical to this piece of media but may be usefully relevant to people - viewing the work. (currently unused.) - - - fail_error: path to the exception raised - - fail_metadata: - """ - __collection__ = 'media_entries' - use_dot_notation = True - - structure = { - 'uploader': ObjectId, - 'title': unicode, - 'slug': unicode, - 'created': datetime.datetime, - 'description': unicode, # May contain markdown/up - 'media_type': unicode, - 'media_data': dict, # extra data relevant to this media_type - 'plugin_data': dict, # plugins can dump stuff here. - 'tags': [dict], - 'state': unicode, - 'license': unicode, - - # For now let's assume there can only be one main file queued - # at a time - 'queued_media_file': [unicode], - 'queued_task_id': unicode, - - # A dictionary of logical names to filepaths - 'media_files': dict, - - # The following should be lists of lists, in appropriate file - # record form - 'attachment_files': list, - - # If things go badly in processing things, we'll store that - # data here - 'fail_error': unicode, - 'fail_metadata': dict} - - required_fields = [ - 'uploader', 'created', 'media_type', 'slug'] - - default_values = { - 'created': datetime.datetime.utcnow, - 'state': u'unprocessed'} - - id = MongoPK() - - def media_data_init(self, **kwargs): - self.media_data.update(kwargs) - - def get_comments(self, ascending=False): - if ascending: - order = ASCENDING - else: - order = DESCENDING - - return self.db.MediaComment.find({ - 'media_entry': self._id}).sort('created', order) - - def url_to_prev(self, urlgen): - """ - Provide a url to the previous entry from this user, if there is one - """ - cursor = self.db.MediaEntry.find({'_id': {"$gt": self._id}, - 'uploader': self.uploader, - 'state': 'processed'}).sort( - '_id', ASCENDING).limit(1) - for media in cursor: - return media.url_for_self(urlgen) - - def url_to_next(self, urlgen): - """ - Provide a url to the next entry from this user, if there is one - """ - cursor = self.db.MediaEntry.find({'_id': {"$lt": self._id}, - 'uploader': self.uploader, - 'state': 'processed'}).sort( - '_id', DESCENDING).limit(1) - - for media in cursor: - return media.url_for_self(urlgen) - - @property - def get_uploader(self): - return self.db.User.find_one({'_id': self.uploader}) - - -class MediaComment(Document, MediaCommentMixin): - """ - A comment on a MediaEntry. - - Structure: - - media_entry: The media entry this comment is attached to - - author: user who posted this comment - - created: when the comment was created - - content: plaintext (but markdown'able) version of the comment's content. - """ - - __collection__ = 'media_comments' - use_dot_notation = True - - structure = { - 'media_entry': ObjectId, - 'author': ObjectId, - 'created': datetime.datetime, - 'content': unicode, - } - - required_fields = [ - 'media_entry', 'author', 'created', 'content'] - - default_values = { - 'created': datetime.datetime.utcnow} - - def media_entry(self): - return self.db.MediaEntry.find_one({'_id': self['media_entry']}) - - @property - def get_author(self): - return self.db.User.find_one({'_id': self['author']}) - - -REGISTER_MODELS = [ - MediaEntry, - User, - MediaComment] - - -def register_models(connection): - """ - Register all models in REGISTER_MODELS with this connection. - """ - connection.register(REGISTER_MODELS) diff --git a/mediagoblin/db/mongo/open.py b/mediagoblin/db/mongo/open.py deleted file mode 100644 index c4f37b42..00000000 --- a/mediagoblin/db/mongo/open.py +++ /dev/null @@ -1,82 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -import pymongo -import mongokit -from paste.deploy.converters import asint -from mediagoblin.db.mongo import models -from mediagoblin.db.mongo.util import MigrationManager - - -def load_models(app_config): - pass - - -def connect_database_from_config(app_config, use_pymongo=False): - """ - Connect to the main database, take config from app_config - - Optionally use pymongo instead of mongokit for the connection. - """ - port = app_config.get('db_port') - if port: - port = asint(port) - - if use_pymongo: - connection = pymongo.Connection( - app_config.get('db_host'), port) - else: - connection = mongokit.Connection( - app_config.get('db_host'), port) - return connection - - -def setup_connection_and_db_from_config(app_config, use_pymongo=False): - """ - Setup connection and database from config. - - Optionally use pymongo instead of mongokit. - """ - connection = connect_database_from_config(app_config, use_pymongo) - database_path = app_config['db_name'] - db = connection[database_path] - - if not use_pymongo: - models.register_models(connection) - - return (connection, db) - - -def check_db_migrations_current(db): - # This MUST be imported so as to set up the appropriate migrations! - from mediagoblin.db.mongo import migrations - - # Init the migration number if necessary - migration_manager = MigrationManager(db) - migration_manager.install_migration_version_if_missing() - - # Tiny hack to warn user if our migration is out of date - if not migration_manager.database_at_latest_migration(): - db_migration_num = migration_manager.database_current_migration() - latest_migration_num = migration_manager.latest_migration() - if db_migration_num < latest_migration_num: - print ( - "*WARNING:* Your migrations are out of date, " - "maybe run ./bin/gmg migrate?") - elif db_migration_num > latest_migration_num: - print ( - "*WARNING:* Your migrations are out of date... " - "in fact they appear to be from the future?!") diff --git a/mediagoblin/db/mongo/util.py b/mediagoblin/db/mongo/util.py deleted file mode 100644 index f61ae6be..00000000 --- a/mediagoblin/db/mongo/util.py +++ /dev/null @@ -1,318 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -""" -Utilities for database operations. - -Some note on migration and indexing tools: - -We store information about what the state of the database is in the -'mediagoblin' document of the 'app_metadata' collection. Keys in that -document relevant to here: - - - 'migration_number': The integer representing the current state of - the migrations -""" - -import copy - -# Imports that other modules might use -from pymongo import ASCENDING, DESCENDING -from pymongo.errors import InvalidId -from mongokit import ObjectId - -from mediagoblin.db.mongo.indexes import ACTIVE_INDEXES, DEPRECATED_INDEXES - - -################ -# Indexing tools -################ - - -def add_new_indexes(database, active_indexes=ACTIVE_INDEXES): - """ - Add any new indexes to the database. - - Args: - - database: pymongo or mongokit database instance. - - active_indexes: indexes to possibly add in the pattern of: - {'collection_name': { - 'identifier': { - 'index': [index_foo_goes_here], - 'unique': True}} - where 'index' is the index to add and all other options are - arguments for collection.create_index. - - Returns: - A list of indexes added in form ('collection', 'index_name') - """ - indexes_added = [] - - for collection_name, indexes in active_indexes.iteritems(): - collection = database[collection_name] - collection_indexes = collection.index_information().keys() - - for index_name, index_data in indexes.iteritems(): - if not index_name in collection_indexes: - # Get a copy actually so we don't modify the actual - # structure - index_data = copy.copy(index_data) - index = index_data.pop('index') - collection.create_index( - index, name=index_name, **index_data) - - indexes_added.append((collection_name, index_name)) - - return indexes_added - - -def remove_deprecated_indexes(database, deprecated_indexes=DEPRECATED_INDEXES): - """ - Remove any deprecated indexes from the database. - - Args: - - database: pymongo or mongokit database instance. - - deprecated_indexes: the indexes to deprecate in the pattern of: - {'collection_name': { - 'identifier': { - 'index': [index_foo_goes_here], - 'unique': True}} - - (... although we really only need the 'identifier' here, as the - rest of the information isn't used in this case. But it's kept - around so we can remember what it was) - - Returns: - A list of indexes removed in form ('collection', 'index_name') - """ - indexes_removed = [] - - for collection_name, indexes in deprecated_indexes.iteritems(): - collection = database[collection_name] - collection_indexes = collection.index_information().keys() - - for index_name, index_data in indexes.iteritems(): - if index_name in collection_indexes: - collection.drop_index(index_name) - - indexes_removed.append((collection_name, index_name)) - - return indexes_removed - - -################# -# Migration tools -################# - -# The default migration registry... -# -# Don't set this yourself! RegisterMigration will automatically fill -# this with stuff via decorating methods in migrations.py - -class MissingCurrentMigration(Exception): - pass - - -MIGRATIONS = {} - - -class RegisterMigration(object): - """ - Tool for registering migrations - - Call like: - - @RegisterMigration(33) - def update_dwarves(database): - [...] - - This will register your migration with the default migration - registry. Alternately, to specify a very specific - migration_registry, you can pass in that as the second argument. - - Note, the number of your migration should NEVER be 0 or less than - 0. 0 is the default "no migrations" state! - """ - def __init__(self, migration_number, migration_registry=MIGRATIONS): - assert migration_number > 0, "Migration number must be > 0!" - assert migration_number not in migration_registry, \ - "Duplicate migration numbers detected! That's not allowed!" - - self.migration_number = migration_number - self.migration_registry = migration_registry - - def __call__(self, migration): - self.migration_registry[self.migration_number] = migration - return migration - - -class MigrationManager(object): - """ - Migration handling tool. - - Takes information about a database, lets you update the database - to the latest migrations, etc. - """ - def __init__(self, database, migration_registry=MIGRATIONS): - """ - Args: - - database: database we're going to migrate - - migration_registry: where we should find all migrations to - run - """ - self.database = database - self.migration_registry = migration_registry - self._sorted_migrations = None - - def _ensure_current_migration_record(self): - """ - If there isn't a database[u'app_metadata'] mediagoblin entry - with the 'current_migration', throw an error. - """ - if self.database_current_migration() is None: - raise MissingCurrentMigration( - "Tried to call function which requires " - "'current_migration' set in database") - - @property - def sorted_migrations(self): - """ - Sort migrations if necessary and store in self._sorted_migrations - """ - if not self._sorted_migrations: - self._sorted_migrations = sorted( - self.migration_registry.items(), - # sort on the key... the migration number - key=lambda migration_tuple: migration_tuple[0]) - - return self._sorted_migrations - - def latest_migration(self): - """ - Return a migration number for the latest migration, or 0 if - there are no migrations. - """ - if self.sorted_migrations: - return self.sorted_migrations[-1][0] - else: - # If no migrations have been set, we start at 0. - return 0 - - def set_current_migration(self, migration_number): - """ - Set the migration in the database to migration_number - """ - # Add the mediagoblin migration if necessary - self.database[u'app_metadata'].update( - {u'_id': u'mediagoblin'}, - {u'$set': {u'current_migration': migration_number}}, - upsert=True) - - def install_migration_version_if_missing(self): - """ - Sets the migration to the latest version if no migration - version at all is set. - """ - mgoblin_metadata = self.database[u'app_metadata'].find_one( - {u'_id': u'mediagoblin'}) - if not mgoblin_metadata: - latest_migration = self.latest_migration() - self.set_current_migration(latest_migration) - - def database_current_migration(self): - """ - Return the current migration in the database. - """ - mgoblin_metadata = self.database[u'app_metadata'].find_one( - {u'_id': u'mediagoblin'}) - if not mgoblin_metadata: - return None - else: - return mgoblin_metadata[u'current_migration'] - - def database_at_latest_migration(self): - """ - See if the database is at the latest migration. - Returns a boolean. - """ - current_migration = self.database_current_migration() - return current_migration == self.latest_migration() - - def migrations_to_run(self): - """ - Get a list of migrations to run still, if any. - - Note that calling this will set your migration version to the - latest version if it isn't installed to anything yet! - """ - self._ensure_current_migration_record() - - db_current_migration = self.database_current_migration() - - return [ - (migration_number, migration_func) - for migration_number, migration_func in self.sorted_migrations - if migration_number > db_current_migration] - - def migrate_new(self, pre_callback=None, post_callback=None): - """ - Run all migrations. - - Includes two optional args: - - pre_callback: if called, this is a callback on something to - run pre-migration. Takes (migration_number, migration_func) - as arguments - - pre_callback: if called, this is a callback on something to - run post-migration. Takes (migration_number, migration_func) - as arguments - """ - # If we aren't set to any version number, presume we're at the - # latest (which means we'll do nothing here...) - self.install_migration_version_if_missing() - - for migration_number, migration_func in self.migrations_to_run(): - if pre_callback: - pre_callback(migration_number, migration_func) - migration_func(self.database) - self.set_current_migration(migration_number) - if post_callback: - post_callback(migration_number, migration_func) - - -########################## -# Random utility functions -########################## - - -def atomic_update(table, query_dict, update_values): - table.collection.update( - query_dict, - {"$set": update_values}) - - -def check_media_slug_used(db, uploader_id, slug, ignore_m_id): - query_dict = {'uploader': uploader_id, 'slug': slug} - if ignore_m_id is not None: - query_dict['_id'] = {'$ne': ignore_m_id} - existing_user_slug_entries = db.MediaEntry.find( - query_dict).count() - return existing_user_slug_entries - - -def media_entries_for_tag_slug(db, tag_slug): - return db.MediaEntry.find( - {u'state': u'processed', - u'tags.slug': tag_slug}) diff --git a/mediagoblin/db/open.py b/mediagoblin/db/open.py index f4c38511..0b1679fb 100644 --- a/mediagoblin/db/open.py +++ b/mediagoblin/db/open.py @@ -14,16 +14,88 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -try: - from mediagoblin.db.sql_switch import use_sql -except ImportError: - use_sql = False - -if use_sql: - from mediagoblin.db.sql.open import \ - setup_connection_and_db_from_config, check_db_migrations_current, \ - load_models -else: - from mediagoblin.db.mongo.open import \ - setup_connection_and_db_from_config, check_db_migrations_current, \ - load_models + +from sqlalchemy import create_engine, event +import logging + +from mediagoblin.db.base import Base, Session +from mediagoblin import mg_globals + +_log = logging.getLogger(__name__) + + +class DatabaseMaster(object): + def __init__(self, engine): + self.engine = engine + + for k, v in Base._decl_class_registry.iteritems(): + setattr(self, k, v) + + def commit(self): + Session.commit() + + def save(self, obj): + Session.add(obj) + Session.flush() + + def check_session_clean(self): + for dummy in Session(): + _log.warn("STRANGE: There are elements in the sql session. " + "Please report this and help us track this down.") + break + + def reset_after_request(self): + Session.rollback() + Session.remove() + + +def load_models(app_config): + import mediagoblin.db.models + + for media_type in app_config['media_types']: + _log.debug("Loading %s.models", media_type) + __import__(media_type + ".models") + + for plugin in mg_globals.global_config.get('plugins', {}).keys(): + _log.debug("Loading %s.models", plugin) + try: + __import__(plugin + ".models") + except ImportError as exc: + _log.debug("Could not load {0}.models: {1}".format( + plugin, + exc)) + + +def _sqlite_fk_pragma_on_connect(dbapi_con, con_record): + """Enable foreign key checking on each new sqlite connection""" + dbapi_con.execute('pragma foreign_keys=on') + + +def _sqlite_disable_fk_pragma_on_connect(dbapi_con, con_record): + """ + Disable foreign key checking on each new sqlite connection + (Good for migrations!) + """ + dbapi_con.execute('pragma foreign_keys=off') + + +def setup_connection_and_db_from_config(app_config, migrations=False): + engine = create_engine(app_config['sql_engine']) + + # Enable foreign key checking for sqlite + if app_config['sql_engine'].startswith('sqlite://'): + if migrations: + event.listen(engine, 'connect', + _sqlite_disable_fk_pragma_on_connect) + else: + event.listen(engine, 'connect', _sqlite_fk_pragma_on_connect) + + # logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) + + Session.configure(bind=engine) + + return DatabaseMaster(engine) + + +def check_db_migrations_current(db): + pass diff --git a/mediagoblin/db/sql/__init__.py b/mediagoblin/db/sql/__init__.py deleted file mode 100644 index 621845ba..00000000 --- a/mediagoblin/db/sql/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. diff --git a/mediagoblin/db/sql/convert.py b/mediagoblin/db/sql/convert.py deleted file mode 100644 index ac64cf8d..00000000 --- a/mediagoblin/db/sql/convert.py +++ /dev/null @@ -1,282 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -from copy import copy -from itertools import chain, imap - -from mediagoblin.init import setup_global_and_app_config - -from mediagoblin.db.sql.base import Session -from mediagoblin.db.sql.models_v0 import Base_v0 -from mediagoblin.db.sql.models_v0 import (User, MediaEntry, MediaComment, - Tag, MediaTag, MediaFile, MediaAttachmentFile, MigrationData, - ImageData, VideoData, AsciiData, AudioData) -from mediagoblin.db.sql.open import setup_connection_and_db_from_config as \ - sql_connect -from mediagoblin.db.mongo.open import setup_connection_and_db_from_config as \ - mongo_connect - - -obj_id_table = dict() - - -def add_obj_ids(entry, new_entry): - global obj_id_table - print "\t%r -> SQL id %r" % (entry._id, new_entry.id) - obj_id_table[entry._id] = new_entry.id - - -def copy_attrs(entry, new_entry, attr_list): - for a in attr_list: - val = entry[a] - setattr(new_entry, a, val) - - -def copy_reference_attr(entry, new_entry, ref_attr): - val = entry[ref_attr] - val = obj_id_table[val] - setattr(new_entry, ref_attr, val) - - -def convert_users(mk_db): - session = Session() - - for entry in mk_db.User.find().sort('created'): - print entry.username - - new_entry = User() - copy_attrs(entry, new_entry, - ('username', 'email', 'created', 'pw_hash', 'email_verified', - 'status', 'verification_key', 'is_admin', 'url', - 'bio', - 'fp_verification_key', 'fp_token_expire',)) - # new_entry.fp_verification_expire = entry.fp_token_expire - - session.add(new_entry) - session.flush() - add_obj_ids(entry, new_entry) - - session.commit() - session.close() - - -def convert_media_entries(mk_db): - session = Session() - - for entry in mk_db.MediaEntry.find().sort('created'): - print repr(entry.title) - - new_entry = MediaEntry() - copy_attrs(entry, new_entry, - ('title', 'slug', 'created', - 'description', - 'media_type', 'state', 'license', - 'fail_error', 'fail_metadata', - 'queued_task_id',)) - copy_reference_attr(entry, new_entry, "uploader") - - session.add(new_entry) - session.flush() - add_obj_ids(entry, new_entry) - - for key, value in entry.media_files.iteritems(): - new_file = MediaFile(name=key, file_path=value) - new_file.media_entry = new_entry.id - Session.add(new_file) - - for attachment in entry.attachment_files: - new_attach = MediaAttachmentFile( - name=attachment["name"], - filepath=attachment["filepath"], - created=attachment["created"] - ) - new_attach.media_entry = new_entry.id - Session.add(new_attach) - - session.commit() - session.close() - - -def convert_image(mk_db): - session = Session() - - for media in mk_db.MediaEntry.find( - {'media_type': 'mediagoblin.media_types.image'}).sort('created'): - media_data = copy(media.media_data) - - if len(media_data): - media_data_row = ImageData(**media_data) - media_data_row.media_entry = obj_id_table[media['_id']] - session.add(media_data_row) - - session.commit() - session.close() - - -def convert_video(mk_db): - session = Session() - - for media in mk_db.MediaEntry.find( - {'media_type': 'mediagoblin.media_types.video'}).sort('created'): - media_data_row = VideoData(**media.media_data) - media_data_row.media_entry = obj_id_table[media['_id']] - session.add(media_data_row) - - session.commit() - session.close() - - -def convert_media_tags(mk_db): - session = Session() - session.autoflush = False - - for media in mk_db.MediaEntry.find().sort('created'): - print repr(media.title) - - for otag in media.tags: - print " ", repr((otag["slug"], otag["name"])) - - nslug = session.query(Tag).filter_by(slug=otag["slug"]).first() - print " ", repr(nslug) - if nslug is None: - nslug = Tag(slug=otag["slug"]) - session.add(nslug) - session.flush() - print " ", repr(nslug), nslug.id - - ntag = MediaTag() - ntag.tag = nslug.id - ntag.name = otag["name"] - ntag.media_entry = obj_id_table[media._id] - session.add(ntag) - - session.commit() - session.close() - - -def convert_media_comments(mk_db): - session = Session() - - for entry in mk_db.MediaComment.find().sort('created'): - print repr(entry.content) - - new_entry = MediaComment() - copy_attrs(entry, new_entry, - ('created', - 'content',)) - - try: - copy_reference_attr(entry, new_entry, "media_entry") - copy_reference_attr(entry, new_entry, "author") - except KeyError as e: - print('KeyError in convert_media_comments(): {0}'.format(e)) - else: - session.add(new_entry) - session.flush() - add_obj_ids(entry, new_entry) - - session.commit() - session.close() - - -media_types_tables = ( - ("mediagoblin.media_types.image", (ImageData,)), - ("mediagoblin.media_types.video", (VideoData,)), - ("mediagoblin.media_types.ascii", (AsciiData,)), - ("mediagoblin.media_types.audio", (AudioData,)), - ) - - -def convert_add_migration_versions(dummy_sql_db): - session = Session() - - for name in chain(("__main__",), - imap(lambda e: e[0], media_types_tables)): - print "\tAdding %s" % (name,) - m = MigrationData(name=unicode(name), version=0) - session.add(m) - - session.commit() - session.close() - - -def cleanup_sql_tables(sql_db): - for mt, table_list in media_types_tables: - session = Session() - - count = session.query(MediaEntry.media_type). \ - filter_by(media_type=unicode(mt)).count() - print " %s: %d entries" % (mt, count) - - if count == 0: - print "\tAnalyzing tables" - for tab in table_list: - cnt2 = session.query(tab).count() - print "\t %s: %d entries" % (tab.__tablename__, cnt2) - assert cnt2 == 0 - - print "\tRemoving migration info" - mi = session.query(MigrationData).filter_by(name=unicode(mt)).one() - session.delete(mi) - session.commit() - session.close() - - print "\tDropping tables" - tables = [model.__table__ for model in table_list] - Base_v0.metadata.drop_all(sql_db.engine, tables=tables) - - session.close() - - -def print_header(title): - print "\n=== %s ===" % (title,) - - -convert_call_list = ( - ("Converting Users", convert_users), - ("Converting Media Entries", convert_media_entries), - ("Converting Media Data for Images", convert_image), - ("Cnnverting Media Data for Videos", convert_video), - ("Converting Tags for Media", convert_media_tags), - ("Converting Media Comments", convert_media_comments), - ) - -sql_call_list = ( - ("Filling Migration Tables", convert_add_migration_versions), - ("Analyzing/Cleaning SQL Data", cleanup_sql_tables), - ) - -def run_conversion(config_name): - global_config, app_config = setup_global_and_app_config(config_name) - - sql_conn, sql_db = sql_connect(app_config) - mk_conn, mk_db = mongo_connect(app_config) - - Base_v0.metadata.create_all(sql_db.engine) - - for title, func in convert_call_list: - print_header(title) - func(mk_db) - Session.remove() - - for title, func in sql_call_list: - print_header(title) - func(sql_db) - Session.remove() - - -if __name__ == '__main__': - run_conversion("mediagoblin.ini") diff --git a/mediagoblin/db/sql/fake.py b/mediagoblin/db/sql/fake.py deleted file mode 100644 index 0fd0cc41..00000000 --- a/mediagoblin/db/sql/fake.py +++ /dev/null @@ -1,45 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -""" -This module contains some fake classes and functions to -calm the rest of the code base. Or provide super minimal -implementations. - -Currently: -- ObjectId "class": It's a function mostly doing - int(init_arg) to convert string primary keys into - integer primary keys. -- InvalidId exception -- DESCENDING "constant" -""" - - -DESCENDING = object() # a unique object for this "constant" - - -class InvalidId(Exception): - pass - - -def ObjectId(value=None): - if value is None: - return None - try: - return int(value) - except ValueError: - raise InvalidId("%r is an invalid id" % value) diff --git a/mediagoblin/db/sql/migrations.py b/mediagoblin/db/sql/migrations.py deleted file mode 100644 index 1d822cd9..00000000 --- a/mediagoblin/db/sql/migrations.py +++ /dev/null @@ -1,118 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -import datetime - -from sqlalchemy import (MetaData, Table, Column, Boolean, SmallInteger, - Integer, Unicode, UnicodeText, DateTime, ForeignKey) - -from mediagoblin.db.sql.util import RegisterMigration -from mediagoblin.db.sql.models import MediaEntry, Collection, User, \ - ProcessingMetaData - -MIGRATIONS = {} - - -@RegisterMigration(1, MIGRATIONS) -def ogg_to_webm_audio(db_conn): - metadata = MetaData(bind=db_conn.bind) - - file_keynames = Table('core__file_keynames', metadata, autoload=True, - autoload_with=db_conn.bind) - - db_conn.execute( - file_keynames.update().where(file_keynames.c.name == 'ogg'). - values(name='webm_audio') - ) - db_conn.commit() - - -@RegisterMigration(2, MIGRATIONS) -def add_wants_notification_column(db_conn): - metadata = MetaData(bind=db_conn.bind) - - users = Table('core__users', metadata, autoload=True, - autoload_with=db_conn.bind) - - col = Column('wants_comment_notification', Boolean, - default=True, nullable=True) - col.create(users, populate_defaults=True) - db_conn.commit() - - -@RegisterMigration(3, MIGRATIONS) -def add_transcoding_progress(db_conn): - metadata = MetaData(bind=db_conn.bind) - - media_entry = Table('core__media_entries', metadata, autoload=True, - autoload_with=db_conn.bind) - - col = Column('transcoding_progress', SmallInteger) - col.create(media_entry) - db_conn.commit() - - -@RegisterMigration(4, MIGRATIONS) -def add_collection_tables(db_conn): - metadata = MetaData(bind=db_conn.bind) - - collection = Table('core__collections', metadata, - Column('id', Integer, primary_key=True), - Column('title', Unicode, nullable=False), - Column('slug', Unicode), - Column('created', DateTime, nullable=False, default=datetime.datetime.now, index=True), - Column('description', UnicodeText), - Column('creator', Integer, ForeignKey(User.id), nullable=False), - Column('items', Integer, default=0)) - - collection_item = Table('core__collection_items', metadata, - Column('id', Integer, primary_key=True), - Column('media_entry', Integer, ForeignKey(MediaEntry.id), nullable=False, index=True), - Column('collection', Integer, ForeignKey(Collection.id), nullable=False), - Column('note', UnicodeText, nullable=True), - Column('added', DateTime, nullable=False, default=datetime.datetime.now), - Column('position', Integer)) - - collection.create() - collection_item.create() - - db_conn.commit() - - -@RegisterMigration(5, MIGRATIONS) -def add_mediaentry_collected(db_conn): - metadata = MetaData(bind=db_conn.bind) - - media_entry = Table('core__media_entries', metadata, autoload=True, - autoload_with=db_conn.bind) - - col = Column('collected', Integer, default=0) - col.create(media_entry) - db_conn.commit() - - -@RegisterMigration(6, MIGRATIONS) -def create_processing_metadata_table(db): - metadata = MetaData(bind=db.bind) - - metadata_table = Table('core__processing_metadata', metadata, - Column('id', Integer, primary_key=True), - Column('media_entry_id', Integer, ForeignKey(MediaEntry.id), - nullable=False, index=True), - Column('callback_url', Unicode)) - - metadata_table.create() - db.commit() diff --git a/mediagoblin/db/sql/open.py b/mediagoblin/db/sql/open.py deleted file mode 100644 index 9db21c56..00000000 --- a/mediagoblin/db/sql/open.py +++ /dev/null @@ -1,78 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -from sqlalchemy import create_engine -import logging - -from mediagoblin.db.sql.base import Base, Session -from mediagoblin import mg_globals - -_log = logging.getLogger(__name__) - - -class DatabaseMaster(object): - def __init__(self, engine): - self.engine = engine - - for k, v in Base._decl_class_registry.iteritems(): - setattr(self, k, v) - - def commit(self): - Session.commit() - - def save(self, obj): - Session.add(obj) - Session.flush() - - def check_session_clean(self): - for dummy in Session(): - _log.warn("STRANGE: There are elements in the sql session. " - "Please report this and help us track this down.") - break - - def reset_after_request(self): - Session.rollback() - Session.remove() - - -def load_models(app_config): - import mediagoblin.db.sql.models - - for media_type in app_config['media_types']: - _log.debug("Loading %s.models", media_type) - __import__(media_type + ".models") - - for plugin in mg_globals.global_config.get('plugins', {}).keys(): - _log.debug("Loading %s.models", plugin) - try: - __import__(plugin + ".models") - except ImportError as exc: - _log.debug("Could not load {0}.models: {1}".format( - plugin, - exc)) - - -def setup_connection_and_db_from_config(app_config): - engine = create_engine(app_config['sql_engine']) - # logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) - Session.configure(bind=engine) - - return "dummy conn", DatabaseMaster(engine) - - -def check_db_migrations_current(db): - pass diff --git a/mediagoblin/db/sql_switch.py b/mediagoblin/db/sql_switch.py deleted file mode 100644 index 571adbdb..00000000 --- a/mediagoblin/db/sql_switch.py +++ /dev/null @@ -1 +0,0 @@ -use_sql = True diff --git a/mediagoblin/db/util.py b/mediagoblin/db/util.py index a8c8c92b..6ffec44d 100644 --- a/mediagoblin/db/util.py +++ b/mediagoblin/db/util.py @@ -14,16 +14,63 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -try: - from mediagoblin.db.sql_switch import use_sql -except ImportError: - use_sql = False - -if use_sql: - from mediagoblin.db.sql.fake import ObjectId, InvalidId, DESCENDING - from mediagoblin.db.sql.util import atomic_update, check_media_slug_used, \ - media_entries_for_tag_slug, check_collection_slug_used -else: - from mediagoblin.db.mongo.util import \ - ObjectId, InvalidId, DESCENDING, atomic_update, \ - check_media_slug_used, media_entries_for_tag_slug +from mediagoblin.db.base import Session +from mediagoblin.db.models import MediaEntry, Tag, MediaTag, Collection + + +########################## +# Random utility functions +########################## + + +def atomic_update(table, query_dict, update_values): + table.find(query_dict).update(update_values, + synchronize_session=False) + Session.commit() + + +def check_media_slug_used(uploader_id, slug, ignore_m_id): + query = MediaEntry.query.filter_by(uploader=uploader_id, slug=slug) + if ignore_m_id is not None: + query = query.filter(MediaEntry.id != ignore_m_id) + does_exist = query.first() is not None + return does_exist + + +def media_entries_for_tag_slug(dummy_db, tag_slug): + return MediaEntry.query \ + .join(MediaEntry.tags_helper) \ + .join(MediaTag.tag_helper) \ + .filter( + (MediaEntry.state == u'processed') + & (Tag.slug == tag_slug)) + + +def clean_orphan_tags(commit=True): + """Search for unused MediaTags and delete them""" + q1 = Session.query(Tag).outerjoin(MediaTag).filter(MediaTag.id==None) + for t in q1: + Session.delete(t) + # The "let the db do all the work" version: + # q1 = Session.query(Tag.id).outerjoin(MediaTag).filter(MediaTag.id==None) + # q2 = Session.query(Tag).filter(Tag.id.in_(q1)) + # q2.delete(synchronize_session = False) + if commit: + Session.commit() + + +def check_collection_slug_used(creator_id, slug, ignore_c_id): + filt = (Collection.creator == creator_id) \ + & (Collection.slug == slug) + if ignore_c_id is not None: + filt = filt & (Collection.id != ignore_c_id) + does_exist = Session.query(Collection.id).filter(filt).first() is not None + return does_exist + + +if __name__ == '__main__': + from mediagoblin.db.open import setup_connection_and_db_from_config + + db = setup_connection_and_db_from_config({'sql_engine':'sqlite:///mediagoblin.db'}) + + clean_orphan_tags() |