diff options
Diffstat (limited to 'mediagoblin/storage.py')
| -rw-r--r-- | mediagoblin/storage.py | 167 |
1 files changed, 149 insertions, 18 deletions
diff --git a/mediagoblin/storage.py b/mediagoblin/storage.py index e449eda3..d484be1f 100644 --- a/mediagoblin/storage.py +++ b/mediagoblin/storage.py @@ -15,7 +15,6 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. import os -import re import shutil import urlparse import uuid @@ -286,6 +285,145 @@ class CloudFilesStorage(StorageInterface): return self.get_file(filepath).public_uri() +class MountStorage(StorageInterface): + """ + Experimental "Mount" virtual Storage Interface + + This isn't an interface to some real storage, instead it's a + redirecting interface, that redirects requests to other + "StorageInterface"s. + + For example, say you have the paths: + + 1. ['user_data', 'cwebber', 'avatar.jpg'] + 2. ['user_data', 'elrond', 'avatar.jpg'] + 3. ['media_entries', '34352f304c3f4d0ad8ad0f043522b6f2', 'thumb.jpg'] + + You could mount media_entries under CloudFileStorage and user_data + under BasicFileStorage. Then 1 would be passed to + BasicFileStorage under the path ['cwebber', 'avatar.jpg'] and 3 + would be passed to CloudFileStorage under + ['34352f304c3f4d0ad8ad0f043522b6f2', 'thumb.jpg']. + + In other words, this is kind of like mounting /home/ and /etc/ + under different filesystems on your operating system... but with + mediagoblin filestorages :) + + To set this up, you currently need to call the mount() method with + the target path and a backend, that shall be available under that + target path. You have to mount things in a sensible order, + especially you can't mount ["a", "b"] before ["a"]. + """ + def __init__(self, **kwargs): + self.mounttab = {} + + def mount(self, dirpath, backend): + """ + Mount a new backend under dirpath + """ + new_ent = clean_listy_filepath(dirpath) + + print "Mounting:", repr(new_ent) + already, rem_1, table, rem_2 = self._resolve_to_backend(new_ent, True) + print "===", repr(already), repr(rem_1), repr(rem_2), len(table) + + assert (len(rem_2) > 0) or (None not in table), \ + "That path is already mounted" + assert (len(rem_2) > 0) or (len(table)==0), \ + "A longer path is already mounted here" + + for part in rem_2: + table[part] = {} + table = table[part] + table[None] = backend + + def _resolve_to_backend(self, filepath, extra_info = False): + """ + extra_info = True is for internal use! + + Normally, returns the backend and the filepath inside that backend. + + With extra_info = True it returns the last directory node and the + remaining filepath from there in addition. + """ + table = self.mounttab + filepath = filepath[:] + res_fp = None + while True: + new_be = table.get(None) + if (new_be is not None) or res_fp is None: + res_be = new_be + res_fp = filepath[:] + res_extra = (table, filepath[:]) + # print "... New res: %r, %r, %r" % (res_be, res_fp, res_extra) + if len(filepath) == 0: + break + query = filepath.pop(0) + entry = table.get(query) + if entry is not None: + table = entry + res_extra = (table, filepath[:]) + else: + break + if extra_info: + return (res_be, res_fp) + res_extra + else: + return (res_be, res_fp) + + def resolve_to_backend(self, filepath): + backend, filepath = self._resolve_to_backend(filepath) + if backend is None: + raise Error("Path not mounted") + return backend, filepath + + def __repr__(self, table = None, indent = []): + res = [] + if table is None: + res.append("MountStorage<") + table = self.mounttab + v = table.get(None) + if v: + res.append(" " * len(indent) + repr(indent) + ": " + repr(v)) + for k, v in table.iteritems(): + if k == None: + continue + res.append(" " * len(indent) + repr(k) + ":") + res += self.__repr__(v, indent + [k]) + if table is self.mounttab: + res.append(">") + return "\n".join(res) + else: + return res + + def file_exists(self, filepath): + backend, filepath = self.resolve_to_backend(filepath) + return backend.file_exists(filepath) + + def get_file(self, filepath, mode='r'): + backend, filepath = self.resolve_to_backend(filepath) + return backend.get_file(filepath, mode) + + def delete_file(self, filepath): + backend, filepath = self.resolve_to_backend(filepath) + return backend.delete_file(filepath) + + def file_url(self, filepath): + backend, filepath = self.resolve_to_backend(filepath) + return backend.file_url(filepath) + + def get_local_path(self, filepath): + backend, filepath = self.resolve_to_backend(filepath) + return backend.get_local_path(filepath) + + def copy_locally(self, filepath, dest_path): + """ + Need to override copy_locally, because the local_storage + attribute is not correct. + """ + backend, filepath = self.resolve_to_backend(filepath) + backend.copy_locally(filepath, dest_path) + + ########### # Utilities ########### @@ -317,41 +455,34 @@ def clean_listy_filepath(listy_filepath): return cleaned_filepath -def storage_system_from_config(paste_config, storage_prefix): +def storage_system_from_config(config_section): """ - Utility for setting up a storage system from the paste app config. + Utility for setting up a storage system from a config section. - Note that a special argument may be passed in to the paste_config - which is "${storage_prefix}_storage_class" which will provide an + Note that a special argument may be passed in to + the config_section which is "storage_class" which will provide an import path to a storage system. This defaults to "mediagoblin.storage:BasicFileStorage" if otherwise undefined. Arguments: - - paste_config: dictionary of config parameters - - storage_prefix: the storage system we're setting up / will be - getting keys/arguments from. For example 'publicstore' will - grab all arguments that are like 'publicstore_FOO'. + - config_section: dictionary of config parameters Returns: An instantiated storage system. Example: storage_system_from_config( - {'publicstore_base_url': '/media/', - 'publicstore_base_dir': '/var/whatever/media/'}, - 'publicstore') + {'base_url': '/media/', + 'base_dir': '/var/whatever/media/'}) Will return: BasicFileStorage( base_url='/media/', base_dir='/var/whatever/media') """ - prefix_re = re.compile('^%s_(.+)$' % re.escape(storage_prefix)) - - config_params = dict( - [(prefix_re.match(key).groups()[0], value) - for key, value in paste_config.iteritems() - if prefix_re.match(key)]) + # This construct is needed, because dict(config) does + # not replace the variables in the config items. + config_params = dict(config_section.iteritems()) if 'storage_class' in config_params: storage_class = config_params['storage_class'] |