Collection changes and migration for federation

- Adds a "type" column to the Collection object and allows the
CollectionItem model to contain any object.
- Changes "items" to "num_items" as per TODO
- Renames "uploader", "creator" and "user" to a common "actor" in most places

5 files changed, 235 insertions, 50 deletions

diff --git a/mediagoblin/db/__init__.py b/mediagoblin/db/__init__.py
index 719b56e7..621845ba 100644
--- a/mediagoblin/db/__init__.py
+++ b/mediagoblin/db/__init__.py
@@ -13,4 +13,3 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py
index ce7174da..4e9d3a2a 100644
--- a/mediagoblin/db/migrations.py
+++ b/mediagoblin/db/migrations.py
@@ -1676,3 +1676,147 @@ def create_oauth1_dummies(db):
 
     # Commit the changes
     db.commit()
+
+@RegisterMigration(37, MIGRATIONS)
+def federation_collection_schema(db):
+    """ Converts the Collection and CollectionItem """
+    metadata = MetaData(bind=db.bind)
+    collection_table = inspect_table(metadata, "core__collections")
+    collection_items_table = inspect_table(metadata, "core__collection_items")
+    media_entry_table = inspect_table(metadata, "core__media_entries")
+    gmr_table = inspect_table(metadata, "core__generic_model_reference")
+
+    ##
+    # Collection Table
+    ##
+
+    # Add the fields onto the Collection model, we need to set these as
+    # not null to avoid DB integreity errors. We will add the not null
+    # constraint later.
+    updated_column = Column(
+        "updated",
+        DateTime,
+        default=datetime.datetime.utcnow
+    )
+    updated_column.create(collection_table)
+
+    type_column = Column(
+        "type",
+        Unicode,
+    )
+    type_column.create(collection_table)
+
+    db.commit()
+
+    # Iterate over the items and set the updated and type fields
+    for collection in db.execute(collection_table.select()):
+        db.execute(collection_table.update().where(
+            collection_table.c.id==collection.id
+        ).values(
+            updated=collection.created,
+            type="core-user-defined"
+        ))
+
+    db.commit()
+
+    # Add the not null constraint onto the fields
+    updated_column = collection_table.columns["updated"]
+    updated_column.alter(nullable=False)
+
+    type_column = collection_table.columns["type"]
+    type_column.alter(nullable=False)
+
+    db.commit()
+
+    # Rename the "items" to "num_items" as per the TODO
+    num_items_field = collection_table.columns["items"]
+    num_items_field.alter(name="num_items")
+    db.commit()
+
+    ##
+    # CollectionItem
+    ##
+    # Adding the object ID column, this again will have not null added later.
+    object_id = Column(
+        "object_id",
+        Integer,
+        ForeignKey(GenericModelReference_V0.id),
+    )
+    object_id.create(
+        collection_items_table,
+    )
+
+    db.commit()
+
+    # Iterate through and convert the Media reference to object_id
+    for item in db.execute(collection_items_table.select()):
+        # Check if there is a GMR for the MediaEntry
+        object_gmr = db.execute(gmr_table.select(
+            and_(
+                gmr_table.c.obj_pk == item.media_entry,
+                gmr_table.c.model_type == "core__media_entries"
+            )
+        )).first()
+
+        if object_gmr:
+            object_gmr = object_gmr[0]
+        else:
+            # Create a GenericModelReference
+            object_gmr = db.execute(gmr_table.insert().values(
+                obj_pk=item.media_entry,
+                model_type="core__media_entries"
+            )).inserted_primary_key[0]
+
+        # Now set the object_id column to the ID of the GMR
+        db.execute(collection_items_table.update().where(
+            collection_items_table.c.id==item.id
+        ).values(
+            object_id=object_gmr
+        ))
+
+    db.commit()
+
+    # Add not null constraint
+    object_id = collection_items_table.columns["object_id"]
+    object_id.alter(nullable=False)
+
+    db.commit()
+
+    # Now remove the old media_entry column
+    media_entry_column = collection_items_table.columns["media_entry"]
+    media_entry_column.drop()
+
+    db.commit()
+
+@RegisterMigration(38, MIGRATIONS)
+def federation_actor(db):
+    """ Renames refereces to the user to actor """
+    metadata = MetaData(bind=db.bind)
+
+    # RequestToken: user -> actor
+    request_token_table = inspect_table(metadata, "core__request_tokens")
+    rt_user_column = request_token_table.columns["user"]
+    rt_user_column.alter(name="actor")
+
+    # AccessToken: user -> actor
+    access_token_table = inspect_table(metadata, "core__access_tokens")
+    at_user_column = access_token_table.columns["user"]
+    at_user_column.alter(name="actor")
+
+    # MediaEntry: uploader -> actor
+    media_entry_table = inspect_table(metadata, "core__media_entries")
+    me_user_column = media_entry_table.columns["uploader"]
+    me_user_column.alter(name="actor")
+
+    # MediaComment: author -> actor
+    media_comment_table = inspect_table(metadata, "core__media_comments")
+    mc_user_column = media_comment_table.columns["author"]
+    mc_user_column.alter(name="actor")
+
+    # Collection: creator -> actor
+    collection_table = inspect_table(metadata, "core__collections")
+    mc_user_column = collection_table.columns["creator"]
+    mc_user_column.alter(name="actor")
+
+    # commit changes to db.
+    db.commit()
diff --git a/mediagoblin/db/mixin.py b/mediagoblin/db/mixin.py
index b954ab90..7960061e 100644
--- a/mediagoblin/db/mixin.py
+++ b/mediagoblin/db/mixin.py
@@ -134,7 +134,7 @@ class MediaEntryMixin(GenerateSlugMixin):
         # (db.models -> db.mixin -> db.util -> db.models)
         from mediagoblin.db.util import check_media_slug_used
 
-        return check_media_slug_used(self.uploader, slug, self.id)
+        return check_media_slug_used(self.actor, slug, self.id)
 
     @property
     def object_type(self):
@@ -188,7 +188,7 @@ class MediaEntryMixin(GenerateSlugMixin):
 
         Use a slug if we have one, else use our 'id'.
         """
-        uploader = self.get_uploader
+        uploader = self.get_actor
 
         return urlgen(
             'mediagoblin.user_pages.media_home',
@@ -338,17 +338,17 @@ class MediaCommentMixin(object):
         return cleaned_markdown_conversion(self.content)
 
     def __unicode__(self):
-        return u'<{klass} #{id} {author} "{comment}">'.format(
+        return u'<{klass} #{id} {actor} "{comment}">'.format(
             klass=self.__class__.__name__,
             id=self.id,
-            author=self.get_author,
+            actor=self.get_actor,
             comment=self.content)
 
     def __repr__(self):
-        return '<{klass} #{id} {author} "{comment}">'.format(
+        return '<{klass} #{id} {actor} "{comment}">'.format(
             klass=self.__class__.__name__,
             id=self.id,
-            author=self.get_author,
+            actor=self.get_actor,
             comment=self.content)
 
 
@@ -360,7 +360,7 @@ class CollectionMixin(GenerateSlugMixin):
         # (db.models -> db.mixin -> db.util -> db.models)
         from mediagoblin.db.util import check_collection_slug_used
 
-        return check_collection_slug_used(self.creator, slug, self.id)
+        return check_collection_slug_used(self.actor, slug, self.id)
 
     @property
     def description_html(self):
@@ -380,7 +380,7 @@ class CollectionMixin(GenerateSlugMixin):
 
         Use a slug if we have one, else use our 'id'.
         """
-        creator = self.get_creator
+        creator = self.get_actor
 
         return urlgen(
             'mediagoblin.user_pages.user_collection',
diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py
index 8c8e42e5..9f4a144c 100644
--- a/mediagoblin/db/models.py
+++ b/mediagoblin/db/models.py
@@ -29,6 +29,7 @@ from sqlalchemy import Column, Integer, Unicode, UnicodeText, DateTime, \
 from sqlalchemy.orm import relationship, backref, with_polymorphic, validates, \
         class_mapper
 from sqlalchemy.orm.collections import attribute_mapped_collection
+from sqlalchemy.sql import and_
 from sqlalchemy.sql.expression import desc
 from sqlalchemy.ext.associationproxy import association_proxy
 from sqlalchemy.util import memoized_property
@@ -257,7 +258,7 @@ class User(Base, UserMixin):
         """Deletes a User and all related entries/comments/files/..."""
         # Collections get deleted by relationships.
 
-        media_entries = MediaEntry.query.filter(MediaEntry.uploader == self.id)
+        media_entries = MediaEntry.query.filter(MediaEntry.actor == self.id)
         for media in media_entries:
             # TODO: Make sure that "MediaEntry.delete()" also deletes
             # all related files/Comments
@@ -455,7 +456,7 @@ class RequestToken(Base):
     token = Column(Unicode, primary_key=True)
     secret = Column(Unicode, nullable=False)
     client = Column(Unicode, ForeignKey(Client.id))
-    user = Column(Integer, ForeignKey(User.id), nullable=True)
+    actor = Column(Integer, ForeignKey(User.id), nullable=True)
     used = Column(Boolean, default=False)
     authenticated = Column(Boolean, default=False)
     verifier = Column(Unicode, nullable=True)
@@ -473,7 +474,7 @@ class AccessToken(Base):
 
     token = Column(Unicode, nullable=False, primary_key=True)
     secret = Column(Unicode, nullable=False)
-    user = Column(Integer, ForeignKey(User.id))
+    actor = Column(Integer, ForeignKey(User.id))
     request_token = Column(Unicode, ForeignKey(RequestToken.token))
     created = Column(DateTime, nullable=False, default=datetime.datetime.utcnow)
     updated = Column(DateTime, nullable=False, default=datetime.datetime.utcnow)
@@ -500,7 +501,7 @@ class MediaEntry(Base, MediaEntryMixin):
     public_id = Column(Unicode, unique=True, nullable=True)
     remote = Column(Boolean, default=False)
 
-    uploader = Column(Integer, ForeignKey(User.id), nullable=False, index=True)
+    actor = Column(Integer, ForeignKey(User.id), nullable=False, index=True)
     title = Column(Unicode, nullable=False)
     slug = Column(Unicode)
     description = Column(UnicodeText) # ??
@@ -526,10 +527,10 @@ class MediaEntry(Base, MediaEntryMixin):
     queued_task_id = Column(Unicode)
 
     __table_args__ = (
-        UniqueConstraint('uploader', 'slug'),
+        UniqueConstraint('actor', 'slug'),
         {})
 
-    get_uploader = relationship(User)
+    get_actor = relationship(User)
 
     media_files_helper = relationship("MediaFile",
         collection_class=attribute_mapped_collection("name"),
@@ -555,16 +556,24 @@ class MediaEntry(Base, MediaEntryMixin):
         creator=lambda v: MediaTag(name=v["name"], slug=v["slug"])
         )
 
-    collections_helper = relationship("CollectionItem",
-        cascade="all, delete-orphan"
-        )
-    collections = association_proxy("collections_helper", "in_collection")
     media_metadata = Column(MutationDict.as_mutable(JSONEncoded),
         default=MutationDict())
 
     ## TODO
     # fail_error
 
+    @property
+    def collections(self):
+        """ Get any collections that this MediaEntry is in """
+        return list(Collection.query.join(Collection.collection_items).join(
+            CollectionItem.object_helper
+        ).filter(
+            and_(
+                GenericModelReference.model_type == self.__tablename__,
+                GenericModelReference.obj_pk == self.id
+            )
+        ))
+
     def get_comments(self, ascending=False):
         order_col = MediaComment.created
         if not ascending:
@@ -574,7 +583,7 @@ class MediaEntry(Base, MediaEntryMixin):
     def url_to_prev(self, urlgen):
         """get the next 'newer' entry by this user"""
         media = MediaEntry.query.filter(
-            (MediaEntry.uploader == self.uploader)
+            (MediaEntry.actor == self.actor)
             & (MediaEntry.state == u'processed')
             & (MediaEntry.id > self.id)).order_by(MediaEntry.id).first()
 
@@ -584,7 +593,7 @@ class MediaEntry(Base, MediaEntryMixin):
     def url_to_next(self, urlgen):
         """get the next 'older' entry by this user"""
         media = MediaEntry.query.filter(
-            (MediaEntry.uploader == self.uploader)
+            (MediaEntry.actor == self.actor)
             & (MediaEntry.state == u'processed')
             & (MediaEntry.id < self.id)).order_by(desc(MediaEntry.id)).first()
 
@@ -675,7 +684,7 @@ class MediaEntry(Base, MediaEntryMixin):
         except OSError as error:
             # Returns list of files we failed to delete
             _log.error('No such files from the user "{1}" to delete: '
-                       '{0}'.format(str(error), self.get_uploader))
+                       '{0}'.format(str(error), self.get_actor))
         _log.info('Deleted Media entry id "{0}"'.format(self.id))
         # Related MediaTag's are automatically cleaned, but we might
         # want to clean out unused Tag's too.
@@ -689,7 +698,7 @@ class MediaEntry(Base, MediaEntryMixin):
 
     def serialize(self, request, show_comments=True):
         """ Unserialize MediaEntry to object """
-        author = self.get_uploader
+        author = self.get_actor
         published = UTC.localize(self.created)
         updated = UTC.localize(self.updated)
         public_id = self.get_public_id(request)
@@ -898,7 +907,7 @@ class MediaComment(Base, MediaCommentMixin):
     id = Column(Integer, primary_key=True)
     media_entry = Column(
         Integer, ForeignKey(MediaEntry.id), nullable=False, index=True)
-    author = Column(Integer, ForeignKey(User.id), nullable=False)
+    actor = Column(Integer, ForeignKey(User.id), nullable=False)
     created = Column(DateTime, nullable=False, default=datetime.datetime.utcnow)
     content = Column(UnicodeText, nullable=False)
     location = Column(Integer, ForeignKey("core__locations.id"))
@@ -907,7 +916,7 @@ class MediaComment(Base, MediaCommentMixin):
     # Cascade: Comments are owned by their creator. So do the full thing.
     # lazy=dynamic: People might post a *lot* of comments,
     #     so make the "posted_comments" a query-like thing.
-    get_author = relationship(User,
+    get_actor = relationship(User,
                               backref=backref("posted_comments",
                                               lazy="dynamic",
                                               cascade="all, delete-orphan"))
@@ -934,7 +943,7 @@ class MediaComment(Base, MediaCommentMixin):
             qualified=True
         )
         media = MediaEntry.query.filter_by(id=self.media_entry).first()
-        author = self.get_author
+        author = self.get_actor
         published = UTC.localize(self.created)
         context = {
             "id": href,
@@ -981,7 +990,15 @@ class MediaComment(Base, MediaCommentMixin):
 
 
 class Collection(Base, CollectionMixin):
-    """An 'album' or 'set' of media by a user.
+    """A representation of a collection of objects.
+
+    This holds a group/collection of objects that could be a user defined album
+    or their inbox, outbox, followers, etc. These are always ordered and accessable
+    via the API and web.
+
+    The collection has a number of types which determine what kind of collection
+    it is, for example the users inbox will be of `Collection.INBOX_TYPE` that will
+    be stored on the `Collection.type` field. It's important to set the correct type.
 
     On deletion, contained CollectionItems get automatically reaped via
     SQL cascade"""
@@ -992,23 +1009,37 @@ class Collection(Base, CollectionMixin):
     slug = Column(Unicode)
     created = Column(DateTime, nullable=False, default=datetime.datetime.utcnow,
                      index=True)
+    updated = Column(DateTime, nullable=False, default=datetime.datetime.utcnow)
     description = Column(UnicodeText)
-    creator = Column(Integer, ForeignKey(User.id), nullable=False)
+    actor = Column(Integer, ForeignKey(User.id), nullable=False)
+    num_items = Column(Integer, default=0)
+
+    # There are lots of different special types of collections in the pump.io API
+    # for example: followers, following, inbox, outbox, etc. See type constants
+    # below the fields on this model.
+    type = Column(Unicode, nullable=False)
+
+    # Location
     location = Column(Integer, ForeignKey("core__locations.id"))
     get_location = relationship("Location", lazy="joined")
 
-    # TODO: No of items in Collection. Badly named, can we migrate to num_items?
-    items = Column(Integer, default=0)
-
     # Cascade: Collections are owned by their creator. So do the full thing.
-    get_creator = relationship(User,
+    get_actor = relationship(User,
                                backref=backref("collections",
                                                cascade="all, delete-orphan"))
-
     __table_args__ = (
-        UniqueConstraint('creator', 'slug'),
+        UniqueConstraint('actor', 'slug'),
         {})
 
+    # These are the types, It's strongly suggested if new ones are invented they
+    # are prefixed to ensure they're unique from other types. Any types used in
+    # the main mediagoblin should be prefixed "core-"
+    INBOX_TYPE = "core-inbox"
+    OUTBOX_TYPE = "core-outbox"
+    FOLLOWER_TYPE = "core-followers"
+    FOLLOWING_TYPE = "core-following"
+    USER_DEFINED_TYPE = "core-user-defined"
+
     def get_collection_items(self, ascending=False):
         #TODO, is this still needed with self.collection_items being available?
         order_col = CollectionItem.position
@@ -1019,18 +1050,15 @@ class Collection(Base, CollectionMixin):
 
     def __repr__(self):
         safe_title = self.title.encode('ascii', 'replace')
-        return '<{classname} #{id}: {title} by {creator}>'.format(
+        return '<{classname} #{id}: {title} by {actor}>'.format(
             id=self.id,
             classname=self.__class__.__name__,
-            creator=self.creator,
+            actor=self.actor,
             title=safe_title)
 
     def serialize(self, request):
         # Get all serialized output in a list
-        items = []
-        for item in self.get_collection_items():
-            items.append(item.serialize(request))
-
+        items = [i.serialize(request) for i in self.get_collection_items()]
         return {
             "totalItems": self.items,
             "url": self.url_for_self(request.urlgen, qualified=True),
@@ -1042,23 +1070,36 @@ class CollectionItem(Base, CollectionItemMixin):
     __tablename__ = "core__collection_items"
 
     id = Column(Integer, primary_key=True)
-    media_entry = Column(
-        Integer, ForeignKey(MediaEntry.id), nullable=False, index=True)
+
     collection = Column(Integer, ForeignKey(Collection.id), nullable=False)
     note = Column(UnicodeText, nullable=True)
     added = Column(DateTime, nullable=False, default=datetime.datetime.utcnow)
     position = Column(Integer)
-
     # Cascade: CollectionItems are owned by their Collection. So do the full thing.
     in_collection = relationship(Collection,
                                  backref=backref(
                                      "collection_items",
                                      cascade="all, delete-orphan"))
 
-    get_media_entry = relationship(MediaEntry)
+    # Link to the object (could be anything.
+    object_id = Column(
+        Integer,
+        ForeignKey(GenericModelReference.id),
+        nullable=False,
+        index=True
+    )
+    object_helper = relationship(
+        GenericModelReference,
+        foreign_keys=[object_id]
+    )
+    get_object = association_proxy(
+        "object_helper",
+        "get_object",
+        creator=GenericModelReference.find_or_new
+    )
 
     __table_args__ = (
-        UniqueConstraint('collection', 'media_entry'),
+        UniqueConstraint('collection', 'object_id'),
         {})
 
     @property
@@ -1067,14 +1108,15 @@ class CollectionItem(Base, CollectionItemMixin):
         return DictReadAttrProxy(self)
 
     def __repr__(self):
-        return '<{classname} #{id}: Entry {entry} in {collection}>'.format(
+        return '<{classname} #{id}: Object {obj} in {collection}>'.format(
             id=self.id,
             classname=self.__class__.__name__,
             collection=self.collection,
-            entry=self.media_entry)
+            obj=self.get_object()
+        )
 
     def serialize(self, request):
-        return self.get_media_entry.serialize(request)
+        return self.get_object().serialize(request)
 
 
 class ProcessingMetaData(Base):
diff --git a/mediagoblin/db/util.py b/mediagoblin/db/util.py
index 7c026691..57e6b942 100644
--- a/mediagoblin/db/util.py
+++ b/mediagoblin/db/util.py
@@ -37,7 +37,7 @@ def atomic_update(table, query_dict, update_values):
 
 
 def check_media_slug_used(uploader_id, slug, ignore_m_id):
-    query = MediaEntry.query.filter_by(uploader=uploader_id, slug=slug)
+    query = MediaEntry.query.filter_by(actor=uploader_id, slug=slug)
     if ignore_m_id is not None:
         query = query.filter(MediaEntry.id != ignore_m_id)
     does_exist = query.first() is not None
@@ -67,7 +67,7 @@ def clean_orphan_tags(commit=True):
 
 
 def check_collection_slug_used(creator_id, slug, ignore_c_id):
-    filt = (Collection.creator == creator_id) \
+    filt = (Collection.actor == creator_id) \
         & (Collection.slug == slug)
     if ignore_c_id is not None:
         filt = filt & (Collection.id != ignore_c_id)