Merge branch 'master' into jwandborg-f482_media_attachments

Conflicts:
	mediagoblin/config_spec.ini
	mediagoblin/edit/forms.py
	mediagoblin/edit/views.py
	mediagoblin/submit/views.py
	mediagoblin/templates/mediagoblin/user_pages/media.html

1 files changed, 226 insertions, 27 deletions

diff --git a/mediagoblin/storage.py b/mediagoblin/storage.py
index 5d6faa4c..d484be1f 100644
--- a/mediagoblin/storage.py
+++ b/mediagoblin/storage.py
@@ -15,10 +15,10 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
-import re
 import shutil
 import urlparse
 import uuid
+import cloudfiles
 
 from werkzeug.utils import secure_filename
 
@@ -28,11 +28,21 @@ from mediagoblin import util
 # Errors
 ########
 
-class Error(Exception): pass
-class InvalidFilepath(Error): pass
-class NoWebServing(Error): pass
 
-class NotImplementedError(Error): pass
+class Error(Exception):
+    pass
+
+
+class InvalidFilepath(Error):
+    pass
+
+
+class NoWebServing(Error):
+    pass
+
+
+class NotImplementedError(Error):
+    pass
 
 
 ###############################################
@@ -117,7 +127,7 @@ class StorageInterface(object):
         Eg, if the filename doesn't exist:
         >>> storage_handler.get_unique_filename(['dir1', 'dir2', 'fname.jpg'])
         [u'dir1', u'dir2', u'fname.jpg']
-        
+
         But if a file does exist, let's get one back with at uuid tacked on:
         >>> storage_handler.get_unique_filename(['dir1', 'dir2', 'fname.jpg'])
         [u'dir1', u'dir2', u'd02c3571-dd62-4479-9d62-9e3012dada29-fname.jpg']
@@ -184,7 +194,7 @@ class BasicFileStorage(StorageInterface):
         """
         return os.path.join(
             self.base_dir, *clean_listy_filepath(filepath))
-        
+
     def file_exists(self, filepath):
         return os.path.exists(self._resolve_filepath(filepath))
 
@@ -216,6 +226,204 @@ class BasicFileStorage(StorageInterface):
         return self._resolve_filepath(filepath)
 
 
+class CloudFilesStorage(StorageInterface):
+    def __init__(self, **kwargs):
+        self.param_container = kwargs.get('cloudfiles_container')
+        self.param_user = kwargs.get('cloudfiles_user')
+        self.param_api_key = kwargs.get('cloudfiles_api_key')
+        self.param_host = kwargs.get('cloudfiles_host')
+        self.param_use_servicenet = kwargs.get('cloudfiles_use_servicenet')
+
+        if not self.param_host:
+            print('No CloudFiles host URL specified, '
+                  'defaulting to Rackspace US')
+
+        self.connection = cloudfiles.get_connection(
+            username=self.param_user,
+            api_key=self.param_api_key,
+            servicenet=True if self.param_use_servicenet == 'true' or \
+                self.param_use_servicenet == True else False)
+
+        if not self.param_container == \
+                self.connection.get_container(self.param_container):
+            self.container = self.connection.create_container(
+                self.param_container)
+            self.container.make_public(
+                ttl=60 * 60 * 2)
+        else:
+            self.container = self.connection.get_container(
+                self.param_container)
+
+    def _resolve_filepath(self, filepath):
+        return '/'.join(
+            clean_listy_filepath(filepath))
+
+    def file_exists(self, filepath):
+        try:
+            object = self.container.get_object(
+                self._resolve_filepath(filepath))
+            return True
+        except cloudfiles.errors.NoSuchObject:
+            return False
+
+    def get_file(self, filepath, mode='r'):
+        try:
+            obj = self.container.get_object(
+                self._resolve_filepath(filepath))
+        except cloudfiles.errors.NoSuchObject:
+            obj = self.container.create_object(
+                self._resolve_filepath(filepath))
+
+        return obj
+
+    def delete_file(self, filepath):
+        # TODO: Also delete unused directories if empty (safely, with
+        # checks to avoid race conditions).
+        self.container.delete_object(filepath)
+
+    def file_url(self, filepath):
+        return self.get_file(filepath).public_uri()
+
+
+class MountStorage(StorageInterface):
+    """
+    Experimental "Mount" virtual Storage Interface
+    
+    This isn't an interface to some real storage, instead it's a
+    redirecting interface, that redirects requests to other
+    "StorageInterface"s.
+
+    For example, say you have the paths:
+
+     1. ['user_data', 'cwebber', 'avatar.jpg']
+     2. ['user_data', 'elrond', 'avatar.jpg']
+     3. ['media_entries', '34352f304c3f4d0ad8ad0f043522b6f2', 'thumb.jpg']
+
+    You could mount media_entries under CloudFileStorage and user_data
+    under BasicFileStorage.  Then 1 would be passed to
+    BasicFileStorage under the path ['cwebber', 'avatar.jpg'] and 3
+    would be passed to CloudFileStorage under
+    ['34352f304c3f4d0ad8ad0f043522b6f2', 'thumb.jpg'].
+
+    In other words, this is kind of like mounting /home/ and /etc/
+    under different filesystems on your operating system... but with
+    mediagoblin filestorages :)
+    
+    To set this up, you currently need to call the mount() method with
+    the target path and a backend, that shall be available under that
+    target path.  You have to mount things in a sensible order,
+    especially you can't mount ["a", "b"] before ["a"].
+    """
+    def __init__(self, **kwargs):
+        self.mounttab = {}
+
+    def mount(self, dirpath, backend):
+        """
+        Mount a new backend under dirpath
+        """
+        new_ent = clean_listy_filepath(dirpath)
+
+        print "Mounting:", repr(new_ent)
+        already, rem_1, table, rem_2 = self._resolve_to_backend(new_ent, True)
+        print "===", repr(already), repr(rem_1), repr(rem_2), len(table)
+
+        assert (len(rem_2) > 0) or (None not in table), \
+            "That path is already mounted"
+        assert (len(rem_2) > 0) or (len(table)==0), \
+            "A longer path is already mounted here"
+
+        for part in rem_2:
+            table[part] = {}
+            table = table[part]
+        table[None] = backend
+
+    def _resolve_to_backend(self, filepath, extra_info = False):
+        """
+        extra_info = True is for internal use!
+
+        Normally, returns the backend and the filepath inside that backend.
+
+        With extra_info = True it returns the last directory node and the
+        remaining filepath from there in addition.
+        """
+        table = self.mounttab
+        filepath = filepath[:]
+        res_fp = None
+        while True:
+            new_be = table.get(None)
+            if (new_be is not None) or res_fp is None:
+                res_be = new_be
+                res_fp = filepath[:]
+                res_extra = (table, filepath[:])
+                # print "... New res: %r, %r, %r" % (res_be, res_fp, res_extra)
+            if len(filepath) == 0:
+                break
+            query = filepath.pop(0)
+            entry = table.get(query)
+            if entry is not None:
+                table = entry
+                res_extra = (table, filepath[:])
+            else:
+                break
+        if extra_info:
+            return (res_be, res_fp) + res_extra
+        else:
+            return (res_be, res_fp)
+
+    def resolve_to_backend(self, filepath):
+        backend, filepath = self._resolve_to_backend(filepath)
+        if backend is None:
+            raise Error("Path not mounted")
+        return backend, filepath
+
+    def __repr__(self, table = None, indent = []):
+        res = []
+        if table is None:
+            res.append("MountStorage<")
+            table = self.mounttab
+        v = table.get(None)
+        if v:
+            res.append("  " * len(indent) + repr(indent) + ": " + repr(v))
+        for k, v in table.iteritems():
+            if k == None:
+                continue
+            res.append("  " * len(indent) + repr(k) + ":")
+            res += self.__repr__(v, indent + [k])
+        if table is self.mounttab:
+            res.append(">")
+            return "\n".join(res)
+        else:
+            return res
+
+    def file_exists(self, filepath):
+        backend, filepath = self.resolve_to_backend(filepath)
+        return backend.file_exists(filepath)
+
+    def get_file(self, filepath, mode='r'):
+        backend, filepath = self.resolve_to_backend(filepath)
+        return backend.get_file(filepath, mode)
+
+    def delete_file(self, filepath):
+        backend, filepath = self.resolve_to_backend(filepath)
+        return backend.delete_file(filepath)
+
+    def file_url(self, filepath):
+        backend, filepath = self.resolve_to_backend(filepath)
+        return backend.file_url(filepath)
+
+    def get_local_path(self, filepath):
+        backend, filepath = self.resolve_to_backend(filepath)
+        return backend.get_local_path(filepath)
+
+    def copy_locally(self, filepath, dest_path):
+        """
+        Need to override copy_locally, because the local_storage
+        attribute is not correct.
+        """
+        backend, filepath = self.resolve_to_backend(filepath)
+        backend.copy_locally(filepath, dest_path)
+
+
 ###########
 # Utilities
 ###########
@@ -247,43 +455,36 @@ def clean_listy_filepath(listy_filepath):
     return cleaned_filepath
 
 
-def storage_system_from_config(paste_config, storage_prefix):
+def storage_system_from_config(config_section):
     """
-    Utility for setting up a storage system from the paste app config.
+    Utility for setting up a storage system from a config section.
 
-    Note that a special argument may be passed in to the paste_config
-    which is "${storage_prefix}_storage_class" which will provide an
+    Note that a special argument may be passed in to
+    the config_section which is "storage_class" which will provide an
     import path to a storage system.  This defaults to
     "mediagoblin.storage:BasicFileStorage" if otherwise undefined.
 
     Arguments:
-     - paste_config: dictionary of config parameters
-     - storage_prefix: the storage system we're setting up / will be
-       getting keys/arguments from.  For example 'publicstore' will
-       grab all arguments that are like 'publicstore_FOO'.
+     - config_section: dictionary of config parameters
 
     Returns:
       An instantiated storage system.
 
     Example:
       storage_system_from_config(
-        {'publicstore_base_url': '/media/',
-         'publicstore_base_dir': '/var/whatever/media/'},
-        'publicstore')
+        {'base_url': '/media/',
+         'base_dir': '/var/whatever/media/'})
 
        Will return:
          BasicFileStorage(
            base_url='/media/',
            base_dir='/var/whatever/media')
     """
-    prefix_re = re.compile('^%s_(.+)$' % re.escape(storage_prefix))
+    # This construct is needed, because dict(config) does
+    # not replace the variables in the config items.
+    config_params = dict(config_section.iteritems())
 
-    config_params = dict(
-        [(prefix_re.match(key).groups()[0], value)
-         for key, value in paste_config.iteritems()
-         if prefix_re.match(key)])
-
-    if config_params.has_key('storage_class'):
+    if 'storage_class' in config_params:
         storage_class = config_params['storage_class']
         config_params.pop('storage_class')
     else:
@@ -291,5 +492,3 @@ def storage_system_from_config(paste_config, storage_prefix):
 
     storage_class = util.import_component(storage_class)
     return storage_class(**config_params)
-
-