aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJoar Wandborg <git@wandborg.com>2011-08-04 01:32:34 +0200
committerJoar Wandborg <git@wandborg.com>2011-08-04 01:32:34 +0200
commit851c51a354d1355683093cdb04d48fcaafc20c28 (patch)
tree5dab0d6a8898d43af8601317a6eb46bb2baa348e
parent4d74812dfc5a671aa50f54951ffe9e0ee520f8f7 (diff)
downloadmediagoblin-851c51a354d1355683093cdb04d48fcaafc20c28.tar.lz
mediagoblin-851c51a354d1355683093cdb04d48fcaafc20c28.tar.xz
mediagoblin-851c51a354d1355683093cdb04d48fcaafc20c28.zip
Feature 477 - Support Cloud Files public storage
* Added configuration options to mediagoblin.ini * process_media supports the python-cloudfiles almost-file-like objects by wrapping them in a contextlib.contextmanager-decorated func. * storage now has the CloudFilesStorage * New dependency added to setup.py; `python-cloudfiles`
-rw-r--r--mediagoblin.ini28
-rw-r--r--mediagoblin/process_media/__init__.py13
-rw-r--r--mediagoblin/storage.py56
-rw-r--r--setup.py1
4 files changed, 94 insertions, 4 deletions
diff --git a/mediagoblin.ini b/mediagoblin.ini
index e889646a..100e9376 100644
--- a/mediagoblin.ini
+++ b/mediagoblin.ini
@@ -1,7 +1,33 @@
[mediagoblin]
-queuestore_base_dir = %(here)s/user_dev/media/queue
+##
+# BEGIN CloudFiles public storage
+##
+# Uncomment the following line and fill in your details to enable Cloud Files
+# (or OpenStack Object Storage [Swift])
+# -
+# publicstore_storage_class = mediagoblin.storage:CloudFilesStorage
+publicstore_cloudfiles_user = user
+publicstore_cloudfiles_api_key = 1a2b3c4d5e6f7g8h9i
+publicstore_cloudfiles_container = mediagoblin
+
+# Only applicable if you run MediaGoblin on a Rackspace Cloud Server
+# it routes traffic through the internal Rackspace network, this
+# means that the bandwith betis free.
+publicstore_cloudfiles_use_servicenet = false
+##
+# END CloudFiles
+##
+
+##
+# BEGIN filesystem public storage
+##
publicstore_base_dir = %(here)s/user_dev/media/public
publicstore_base_url = /mgoblin_media/
+##
+# END
+##
+
+queuestore_base_dir = %(here)s/user_dev/media/queue
direct_remote_path = /mgoblin_static/
email_sender_address = "notice@mediagoblin.example.org"
diff --git a/mediagoblin/process_media/__init__.py b/mediagoblin/process_media/__init__.py
index 125b24e0..8e12ca4d 100644
--- a/mediagoblin/process_media/__init__.py
+++ b/mediagoblin/process_media/__init__.py
@@ -19,6 +19,7 @@ from mediagoblin.db.util import ObjectId
from celery.task import task
from mediagoblin import mg_globals as mgg
+from contextlib import contextmanager
THUMB_SIZE = 180, 180
@@ -31,6 +32,12 @@ def create_pub_filepath(entry, filename):
unicode(entry['_id']),
filename])
+@contextmanager
+def closing(callback):
+ try:
+ yield callback
+ finally:
+ pass
@task
def process_media_initial(media_id):
@@ -53,7 +60,7 @@ def process_media_initial(media_id):
thumb_filepath = create_pub_filepath(entry, 'thumbnail.jpg')
thumb_file = mgg.public_store.get_file(thumb_filepath, 'w')
- with thumb_file:
+ with closing(thumb_file):
thumb.save(thumb_file, "JPEG", quality=90)
"""
@@ -73,7 +80,7 @@ def process_media_initial(media_id):
medium_filepath = create_pub_filepath(entry, 'medium.jpg')
medium_file = mgg.public_store.get_file(medium_filepath, 'w')
- with medium_file:
+ with closing(medium_file):
medium.save(medium_file, "JPEG", quality=90)
medium_processed = True
@@ -84,7 +91,7 @@ def process_media_initial(media_id):
with queued_file:
original_filepath = create_pub_filepath(entry, queued_filepath[-1])
- with mgg.public_store.get_file(original_filepath, 'wb') as original_file:
+ with closing(mgg.public_store.get_file(original_filepath, 'wb')) as original_file:
original_file.write(queued_file.read())
mgg.queue_store.delete_file(queued_filepath)
diff --git a/mediagoblin/storage.py b/mediagoblin/storage.py
index 5d6faa4c..0e50938f 100644
--- a/mediagoblin/storage.py
+++ b/mediagoblin/storage.py
@@ -19,6 +19,7 @@ import re
import shutil
import urlparse
import uuid
+import cloudfiles
from werkzeug.utils import secure_filename
@@ -161,6 +162,61 @@ class StorageInterface(object):
dest_file.write(source_file.read())
+class CloudFilesStorage(StorageInterface):
+ def __init__(self, **kwargs):
+ self.param_container = kwargs.get('cloudfiles_container')
+ self.param_user = kwargs.get('cloudfiles_user')
+ self.param_api_key = kwargs.get('cloudfiles_api_key')
+ self.param_host = kwargs.get('cloudfiles_host')
+ self.param_use_servicenet = kwargs.get('cloudfiles_use_servicenet')
+
+ if not self.param_host:
+ print('No CloudFiles host URL specified, defaulting to Rackspace US')
+
+ self.connection = cloudfiles.get_connection(
+ username=self.param_user,
+ api_key=self.param_api_key,
+ servicenet=True if self.param_use_servicenet == 'true' or \
+ self.param_use_servicenet == True else False)
+
+ if not self.param_container in [self.connection.get_container(self.param_container)]:
+ self.container = self.connection.create_container(self.param_container)
+ self.container.make_public(
+ ttl=60 * 60 * 2)
+ else:
+ self.container = self.connection.get_container(self.param_container)
+
+ def _resolve_filepath(self, filepath):
+ return '-'.join(
+ clean_listy_filepath(filepath))
+
+ def file_exists(self, filepath):
+ try:
+ object = self.container.get_object(
+ self._resolve_filepath(filepath))
+ return True
+ except cloudfiles.errors.NoSuchObject:
+ return False
+
+ def get_file(self, filepath, mode='r'):
+ try:
+ obj = self.container.get_object(
+ self._resolve_filepath(filepath))
+ except cloudfiles.errors.NoSuchObject:
+ obj = self.container.create_object(
+ self._resolve_filepath(filepath))
+
+ return obj
+
+ def delete_file(self, filepath):
+ # TODO: Also delete unused directories if empty (safely, with
+ # checks to avoid race conditions).
+ self.container.delete_object(filepath)
+
+ def file_url(self, filepath):
+ return self.get_file(filepath).public_uri()
+
+
class BasicFileStorage(StorageInterface):
"""
Basic local filesystem implementation of storage API
diff --git a/setup.py b/setup.py
index 3508f5f0..6043ad41 100644
--- a/setup.py
+++ b/setup.py
@@ -44,6 +44,7 @@ setup(
'webtest',
'ConfigObj',
'Markdown',
+ 'python-cloudfiles',
## For now we're expecting that users will install this from
## their package managers.
# 'lxml',