diff options
| -rw-r--r-- | mediagoblin/process_media/__init__.py | 13 | ||||
| -rw-r--r-- | mediagoblin/storage.py | 86 | ||||
| -rw-r--r-- | setup.py | 1 |
3 files changed, 88 insertions, 12 deletions
diff --git a/mediagoblin/process_media/__init__.py b/mediagoblin/process_media/__init__.py index 125b24e0..8e12ca4d 100644 --- a/mediagoblin/process_media/__init__.py +++ b/mediagoblin/process_media/__init__.py @@ -19,6 +19,7 @@ from mediagoblin.db.util import ObjectId from celery.task import task from mediagoblin import mg_globals as mgg +from contextlib import contextmanager THUMB_SIZE = 180, 180 @@ -31,6 +32,12 @@ def create_pub_filepath(entry, filename): unicode(entry['_id']), filename]) +@contextmanager +def closing(callback): + try: + yield callback + finally: + pass @task def process_media_initial(media_id): @@ -53,7 +60,7 @@ def process_media_initial(media_id): thumb_filepath = create_pub_filepath(entry, 'thumbnail.jpg') thumb_file = mgg.public_store.get_file(thumb_filepath, 'w') - with thumb_file: + with closing(thumb_file): thumb.save(thumb_file, "JPEG", quality=90) """ @@ -73,7 +80,7 @@ def process_media_initial(media_id): medium_filepath = create_pub_filepath(entry, 'medium.jpg') medium_file = mgg.public_store.get_file(medium_filepath, 'w') - with medium_file: + with closing(medium_file): medium.save(medium_file, "JPEG", quality=90) medium_processed = True @@ -84,7 +91,7 @@ def process_media_initial(media_id): with queued_file: original_filepath = create_pub_filepath(entry, queued_filepath[-1]) - with mgg.public_store.get_file(original_filepath, 'wb') as original_file: + with closing(mgg.public_store.get_file(original_filepath, 'wb')) as original_file: original_file.write(queued_file.read()) mgg.queue_store.delete_file(queued_filepath) diff --git a/mediagoblin/storage.py b/mediagoblin/storage.py index 5d6faa4c..e449eda3 100644 --- a/mediagoblin/storage.py +++ b/mediagoblin/storage.py @@ -19,6 +19,7 @@ import re import shutil import urlparse import uuid +import cloudfiles from werkzeug.utils import secure_filename @@ -28,11 +29,21 @@ from mediagoblin import util # Errors ######## -class Error(Exception): pass -class InvalidFilepath(Error): pass -class NoWebServing(Error): pass -class NotImplementedError(Error): pass +class Error(Exception): + pass + + +class InvalidFilepath(Error): + pass + + +class NoWebServing(Error): + pass + + +class NotImplementedError(Error): + pass ############################################### @@ -117,7 +128,7 @@ class StorageInterface(object): Eg, if the filename doesn't exist: >>> storage_handler.get_unique_filename(['dir1', 'dir2', 'fname.jpg']) [u'dir1', u'dir2', u'fname.jpg'] - + But if a file does exist, let's get one back with at uuid tacked on: >>> storage_handler.get_unique_filename(['dir1', 'dir2', 'fname.jpg']) [u'dir1', u'dir2', u'd02c3571-dd62-4479-9d62-9e3012dada29-fname.jpg'] @@ -184,7 +195,7 @@ class BasicFileStorage(StorageInterface): """ return os.path.join( self.base_dir, *clean_listy_filepath(filepath)) - + def file_exists(self, filepath): return os.path.exists(self._resolve_filepath(filepath)) @@ -216,6 +227,65 @@ class BasicFileStorage(StorageInterface): return self._resolve_filepath(filepath) +class CloudFilesStorage(StorageInterface): + def __init__(self, **kwargs): + self.param_container = kwargs.get('cloudfiles_container') + self.param_user = kwargs.get('cloudfiles_user') + self.param_api_key = kwargs.get('cloudfiles_api_key') + self.param_host = kwargs.get('cloudfiles_host') + self.param_use_servicenet = kwargs.get('cloudfiles_use_servicenet') + + if not self.param_host: + print('No CloudFiles host URL specified, ' + 'defaulting to Rackspace US') + + self.connection = cloudfiles.get_connection( + username=self.param_user, + api_key=self.param_api_key, + servicenet=True if self.param_use_servicenet == 'true' or \ + self.param_use_servicenet == True else False) + + if not self.param_container == \ + self.connection.get_container(self.param_container): + self.container = self.connection.create_container( + self.param_container) + self.container.make_public( + ttl=60 * 60 * 2) + else: + self.container = self.connection.get_container( + self.param_container) + + def _resolve_filepath(self, filepath): + return '/'.join( + clean_listy_filepath(filepath)) + + def file_exists(self, filepath): + try: + object = self.container.get_object( + self._resolve_filepath(filepath)) + return True + except cloudfiles.errors.NoSuchObject: + return False + + def get_file(self, filepath, mode='r'): + try: + obj = self.container.get_object( + self._resolve_filepath(filepath)) + except cloudfiles.errors.NoSuchObject: + obj = self.container.create_object( + self._resolve_filepath(filepath)) + + return obj + + def delete_file(self, filepath): + # TODO: Also delete unused directories if empty (safely, with + # checks to avoid race conditions). + self.container.delete_object(filepath) + + def file_url(self, filepath): + return self.get_file(filepath).public_uri() + + ########### # Utilities ########### @@ -283,7 +353,7 @@ def storage_system_from_config(paste_config, storage_prefix): for key, value in paste_config.iteritems() if prefix_re.match(key)]) - if config_params.has_key('storage_class'): + if 'storage_class' in config_params: storage_class = config_params['storage_class'] config_params.pop('storage_class') else: @@ -291,5 +361,3 @@ def storage_system_from_config(paste_config, storage_prefix): storage_class = util.import_component(storage_class) return storage_class(**config_params) - - @@ -44,6 +44,7 @@ setup( 'webtest', 'ConfigObj', 'Markdown', + 'python-cloudfiles', ## For now we're expecting that users will install this from ## their package managers. # 'lxml', |