aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJoar Wandborg <git@wandborg.com>2011-06-12 14:37:49 +0200
committerJoar Wandborg <git@wandborg.com>2011-06-12 14:37:49 +0200
commita48014d67a5185c645ac876d365b4cb85cbb6b1f (patch)
treedf9a459c0f246f1b189bb338b1b3470d87ff3f02
parent44e2da2fe60a3b8765d0fef5a9ce0c3e5997dd01 (diff)
parent68ffb13690fa0c364c514ce253364f928e50841c (diff)
downloadmediagoblin-a48014d67a5185c645ac876d365b4cb85cbb6b1f.tar.lz
mediagoblin-a48014d67a5185c645ac876d365b4cb85cbb6b1f.tar.xz
mediagoblin-a48014d67a5185c645ac876d365b4cb85cbb6b1f.zip
Merge branch 'master' of http://git.gitorious.org/mediagoblin/mediagoblin
-rw-r--r--mediagoblin/app.py10
-rw-r--r--mediagoblin/celery_setup/from_celery.py9
-rw-r--r--mediagoblin/process_media/__init__.py28
-rw-r--r--mediagoblin/storage.py42
-rw-r--r--mediagoblin/tests/test_storage.py45
-rw-r--r--mediagoblin/tests/test_workbench.py96
-rw-r--r--mediagoblin/workbench.py135
7 files changed, 348 insertions, 17 deletions
diff --git a/mediagoblin/app.py b/mediagoblin/app.py
index e5949531..5d594039 100644
--- a/mediagoblin/app.py
+++ b/mediagoblin/app.py
@@ -25,6 +25,7 @@ from mediagoblin import routing, util, storage, staticdirect
from mediagoblin.db.open import setup_connection_and_db_from_config
from mediagoblin.globals import setup_globals
from mediagoblin.celery_setup import setup_celery_from_config
+from mediagoblin.workbench import WorkbenchManager, DEFAULT_WORKBENCH_DIR
class Error(Exception): pass
@@ -39,7 +40,8 @@ class MediaGoblinApp(object):
public_store, queue_store,
staticdirector,
email_sender_address, email_debug_mode,
- user_template_path=None):
+ user_template_path=None,
+ workbench_path=DEFAULT_WORKBENCH_DIR):
# Get the template environment
self.template_loader = util.get_jinja_loader(user_template_path)
@@ -66,7 +68,8 @@ class MediaGoblinApp(object):
db_connection=connection,
database=self.db,
public_store=self.public_store,
- queue_store=self.queue_store)
+ queue_store=self.queue_store,
+ workbench_manager=WorkbenchManager(workbench_path))
def __call__(self, environ, start_response):
request = Request(environ)
@@ -154,6 +157,7 @@ def paste_app_factory(global_config, **app_config):
email_sender_address=app_config.get(
'email_sender_address', 'notice@mediagoblin.example.org'),
email_debug_mode=asbool(app_config.get('email_debug_mode')),
- user_template_path=app_config.get('local_templates'))
+ user_template_path=app_config.get('local_templates'),
+ workbench_path=app_config.get('workbench_path', DEFAULT_WORKBENCH_DIR))
return mgoblin_app
diff --git a/mediagoblin/celery_setup/from_celery.py b/mediagoblin/celery_setup/from_celery.py
index 0669e80c..5fa9ba76 100644
--- a/mediagoblin/celery_setup/from_celery.py
+++ b/mediagoblin/celery_setup/from_celery.py
@@ -23,7 +23,7 @@ from mediagoblin import storage
from mediagoblin.db.open import setup_connection_and_db_from_config
from mediagoblin.celery_setup import setup_celery_from_config
from mediagoblin.globals import setup_globals
-from mediagoblin import globals as mgoblin_globals
+from mediagoblin.workbench import WorkbenchManager, DEFAULT_WORKBENCH_DIR
OUR_MODULENAME = 'mediagoblin.celery_setup.from_celery'
@@ -76,6 +76,10 @@ def setup_self(setup_globals_func=setup_globals):
queue_store = storage.storage_system_from_paste_config(
mgoblin_section, 'queuestore')
+ workbench_manager = WorkbenchManager(
+ mgoblin_section.get(
+ 'workbench_path', DEFAULT_WORKBENCH_DIR))
+
setup_globals_func(
db_connection=connection,
database=db,
@@ -84,7 +88,8 @@ def setup_self(setup_globals_func=setup_globals):
email_sender_address=mgoblin_section.get(
'email_sender_address',
'notice@mediagoblin.example.org'),
- queue_store=queue_store)
+ queue_store=queue_store,
+ workbench_manager=workbench_manager)
if os.environ['CELERY_CONFIG_MODULE'] == OUR_MODULENAME:
diff --git a/mediagoblin/process_media/__init__.py b/mediagoblin/process_media/__init__.py
index 4f06a686..531eb16d 100644
--- a/mediagoblin/process_media/__init__.py
+++ b/mediagoblin/process_media/__init__.py
@@ -18,7 +18,7 @@ import Image
from mediagoblin.db.util import ObjectId
from celery.task import task
-from mediagoblin.globals import database, queue_store, public_store
+from mediagoblin import globals as mg_globals
THUMB_SIZE = 200, 200
@@ -26,40 +26,50 @@ THUMB_SIZE = 200, 200
@task
def process_media_initial(media_id):
- entry = database.MediaEntry.one(
+ workbench = mg_globals.workbench_manager.create_workbench()
+
+ entry = mg_globals.database.MediaEntry.one(
{'_id': ObjectId(media_id)})
queued_filepath = entry['queued_media_file']
- queued_file = queue_store.get_file(queued_filepath, 'r')
+ queued_filename = mg_globals.workbench_manager.localized_file(
+ workbench, mg_globals.queue_store, queued_filepath,
+ 'source')
+
+ queued_file = file(queued_filename, 'r')
with queued_file:
thumb = Image.open(queued_file)
thumb.thumbnail(THUMB_SIZE, Image.ANTIALIAS)
- thumb_filepath = public_store.get_unique_filepath(
+ thumb_filepath = mg_globals.public_store.get_unique_filepath(
['media_entries',
unicode(entry['_id']),
'thumbnail.jpg'])
- with public_store.get_file(thumb_filepath, 'w') as thumb_file:
+ thumb_file = mg_globals.public_store.get_file(thumb_filepath, 'w')
+ with thumb_file:
thumb.save(thumb_file, "JPEG")
# we have to re-read because unlike PIL, not everything reads
# things in string representation :)
- queued_file = queue_store.get_file(queued_filepath, 'rb')
+ queued_file = file(queued_filename, 'rb')
with queued_file:
- main_filepath = public_store.get_unique_filepath(
+ main_filepath = mg_globals.public_store.get_unique_filepath(
['media_entries',
unicode(entry['_id']),
queued_filepath[-1]])
- with public_store.get_file(main_filepath, 'wb') as main_file:
+ with mg_globals.public_store.get_file(main_filepath, 'wb') as main_file:
main_file.write(queued_file.read())
- queue_store.delete_file(queued_filepath)
+ mg_globals.queue_store.delete_file(queued_filepath)
media_files_dict = entry.setdefault('media_files', {})
media_files_dict['thumb'] = thumb_filepath
media_files_dict['main'] = main_filepath
entry['state'] = u'processed'
entry.save()
+
+ # clean up workbench
+ mg_globals.workbench_manager.destroy_workbench(workbench)
diff --git a/mediagoblin/storage.py b/mediagoblin/storage.py
index 5d7e70d6..ba6ac017 100644
--- a/mediagoblin/storage.py
+++ b/mediagoblin/storage.py
@@ -16,6 +16,7 @@
import os
import re
+import shutil
import urlparse
import uuid
@@ -60,6 +61,9 @@ class StorageInterface(object):
StorageInterface.
"""
+ # Whether this file store is on the local filesystem.
+ local_storage = False
+
def __raise_not_implemented(self):
"""
Raise a warning about some component not implemented by a
@@ -127,12 +131,43 @@ class StorageInterface(object):
else:
return filepath
+ def get_local_path(self, filepath):
+ """
+ If this is a local_storage implementation, give us a link to
+ the local filesystem reference to this file.
+
+ >>> storage_handler.get_local_path(['foo', 'bar', 'baz.jpg'])
+ u'/path/to/mounting/foo/bar/baz.jpg'
+ """
+ # Subclasses should override this method, if applicable.
+ self.__raise_not_implemented()
+
+ def copy_locally(self, filepath, dest_path):
+ """
+ Copy this file locally.
+
+ A basic working method for this is provided that should
+ function both for local_storage systems and remote storge
+ systems, but if more efficient systems for copying locally
+ apply to your system, override this method with something more
+ appropriate.
+ """
+ if self.local_storage:
+ shutil.copy(
+ self.get_local_path(filepath), dest_path)
+ else:
+ with self.get_file(filepath, 'rb') as source_file:
+ with file(dest_path, 'wb') as dest_file:
+ dest_file.write(source_file.read())
+
class BasicFileStorage(StorageInterface):
"""
Basic local filesystem implementation of storage API
"""
+ local_storage = True
+
def __init__(self, base_dir, base_url=None, **kwargs):
"""
Keyword arguments:
@@ -177,6 +212,9 @@ class BasicFileStorage(StorageInterface):
self.base_url,
'/'.join(clean_listy_filepath(filepath)))
+ def get_local_path(self, filepath):
+ return self._resolve_filepath(filepath)
+
###########
# Utilities
@@ -187,7 +225,7 @@ def clean_listy_filepath(listy_filepath):
Take a listy filepath (like ['dir1', 'dir2', 'filename.jpg']) and
clean out any nastiness from it.
- For example:
+
>>> clean_listy_filepath([u'/dir1/', u'foo/../nasty', u'linooks.jpg'])
[u'dir1', u'foo_.._nasty', u'linooks.jpg']
@@ -253,3 +291,5 @@ def storage_system_from_paste_config(paste_config, storage_prefix):
storage_class = util.import_component(storage_class)
return storage_class(**config_params)
+
+
diff --git a/mediagoblin/tests/test_storage.py b/mediagoblin/tests/test_storage.py
index 61dd5dca..55b66e84 100644
--- a/mediagoblin/tests/test_storage.py
+++ b/mediagoblin/tests/test_storage.py
@@ -52,6 +52,11 @@ class FakeStorageSystem():
self.foobie = foobie
self.blech = blech
+class FakeRemoteStorage(storage.BasicFileStorage):
+ # Theoretically despite this, all the methods should work but it
+ # should force copying to the workbench
+ local_storage = False
+
def test_storage_system_from_paste_config():
this_storage = storage.storage_system_from_paste_config(
@@ -81,9 +86,12 @@ def test_storage_system_from_paste_config():
# Basic file storage tests
##########################
-def get_tmp_filestorage(mount_url=None):
+def get_tmp_filestorage(mount_url=None, fake_remote=False):
tmpdir = tempfile.mkdtemp()
- this_storage = storage.BasicFileStorage(tmpdir, mount_url)
+ if fake_remote:
+ this_storage = FakeRemoteStorage(tmpdir, mount_url)
+ else:
+ this_storage = storage.BasicFileStorage(tmpdir, mount_url)
return tmpdir, this_storage
@@ -214,3 +222,36 @@ def test_basic_storage_url_for_file():
['dir1', 'dir2', 'filename.txt'])
expected = 'http://media.example.org/ourmedia/dir1/dir2/filename.txt'
assert result == expected
+
+
+def test_basic_storage_get_local_path():
+ tmpdir, this_storage = get_tmp_filestorage()
+
+ result = this_storage.get_local_path(
+ ['dir1', 'dir2', 'filename.txt'])
+
+ expected = os.path.join(
+ tmpdir, 'dir1/dir2/filename.txt')
+
+ assert result == expected
+
+
+def test_basic_storage_is_local():
+ tmpdir, this_storage = get_tmp_filestorage()
+ assert this_storage.local_storage is True
+
+
+def test_basic_storage_copy_locally():
+ tmpdir, this_storage = get_tmp_filestorage()
+
+ dest_tmpdir = tempfile.mkdtemp()
+
+ filepath = ['dir1', 'dir2', 'ourfile.txt']
+ with this_storage.get_file(filepath, 'w') as our_file:
+ our_file.write('Testing this file')
+
+ new_file_dest = os.path.join(dest_tmpdir, 'file2.txt')
+
+ this_storage.copy_locally(filepath, new_file_dest)
+
+ assert file(new_file_dest).read() == 'Testing this file'
diff --git a/mediagoblin/tests/test_workbench.py b/mediagoblin/tests/test_workbench.py
new file mode 100644
index 00000000..89f2ef33
--- /dev/null
+++ b/mediagoblin/tests/test_workbench.py
@@ -0,0 +1,96 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011 Free Software Foundation, Inc
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import tempfile
+
+from nose.tools import assert_raises
+
+from mediagoblin import workbench
+from mediagoblin.tests.test_storage import get_tmp_filestorage
+
+
+class TestWorkbench(object):
+ def setUp(self):
+ self.workbench_manager = workbench.WorkbenchManager(
+ os.path.join(tempfile.gettempdir(), u'mgoblin_workbench_testing'))
+
+ def test_create_workbench(self):
+ workbench = self.workbench_manager.create_workbench()
+ assert os.path.isdir(workbench)
+ assert workbench.startswith(self.workbench_manager.base_workbench_dir)
+
+ def test_destroy_workbench(self):
+ # kill a workbench
+ this_workbench = self.workbench_manager.create_workbench()
+ tmpfile = file(os.path.join(this_workbench, 'temp.txt'), 'w')
+ with tmpfile:
+ tmpfile.write('lollerskates')
+
+ assert os.path.exists(os.path.join(this_workbench, 'temp.txt'))
+
+ self.workbench_manager.destroy_workbench(this_workbench)
+ assert not os.path.exists(os.path.join(this_workbench, 'temp.txt'))
+ assert not os.path.exists(this_workbench)
+
+ # make sure we can't kill other stuff though
+ dont_kill_this = tempfile.mkdtemp()
+
+ assert_raises(
+ workbench.WorkbenchOutsideScope,
+ self.workbench_manager.destroy_workbench,
+ dont_kill_this)
+
+ def test_localized_file(self):
+ tmpdir, this_storage = get_tmp_filestorage()
+ this_workbench = self.workbench_manager.create_workbench()
+
+ # Write a brand new file
+ filepath = ['dir1', 'dir2', 'ourfile.txt']
+
+ with this_storage.get_file(filepath, 'w') as our_file:
+ our_file.write('Our file')
+
+ # with a local file storage
+ filename = self.workbench_manager.localized_file(
+ this_workbench, this_storage, filepath)
+ assert filename == os.path.join(
+ tmpdir, 'dir1/dir2/ourfile.txt')
+
+ # with a fake remote file storage
+ tmpdir, this_storage = get_tmp_filestorage(fake_remote=True)
+
+ # ... write a brand new file, again ;)
+ with this_storage.get_file(filepath, 'w') as our_file:
+ our_file.write('Our file')
+
+ filename = self.workbench_manager.localized_file(
+ this_workbench, this_storage, filepath)
+ assert filename == os.path.join(
+ this_workbench, 'ourfile.txt')
+
+ # fake remote file storage, filename_if_copying set
+ filename = self.workbench_manager.localized_file(
+ this_workbench, this_storage, filepath, 'thisfile')
+ assert filename == os.path.join(
+ this_workbench, 'thisfile.txt')
+
+ # fake remote file storage, filename_if_copying set,
+ # keep_extension_if_copying set to false
+ filename = self.workbench_manager.localized_file(
+ this_workbench, this_storage, filepath, 'thisfile.text', False)
+ assert filename == os.path.join(
+ this_workbench, 'thisfile.text')
diff --git a/mediagoblin/workbench.py b/mediagoblin/workbench.py
new file mode 100644
index 00000000..d7252623
--- /dev/null
+++ b/mediagoblin/workbench.py
@@ -0,0 +1,135 @@
+# GNU MediaGoblin -- federated, autonomous media hosting
+# Copyright (C) 2011 Free Software Foundation, Inc
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+import shutil
+import tempfile
+
+
+DEFAULT_WORKBENCH_DIR = os.path.join(
+ tempfile.gettempdir(), u'mgoblin_workbench')
+
+
+# Exception(s)
+# ------------
+
+class WorkbenchOutsideScope(Exception):
+ """
+ Raised when a workbench is outside a WorkbenchManager scope.
+ """
+ pass
+
+
+# Actual workbench stuff
+# ----------------------
+
+class WorkbenchManager(object):
+ """
+ A system for generating and destroying workbenches.
+
+ Workbenches are actually just subdirectories of a temporary storage space
+ for during the processing stage.
+ """
+
+ def __init__(self, base_workbench_dir):
+ self.base_workbench_dir = os.path.abspath(base_workbench_dir)
+ if not os.path.exists(self.base_workbench_dir):
+ os.makedirs(self.base_workbench_dir)
+
+ def create_workbench(self):
+ """
+ Create and return the path to a new workbench (directory).
+ """
+ return tempfile.mkdtemp(dir=self.base_workbench_dir)
+
+ def destroy_workbench(self, workbench):
+ """
+ Destroy this workbench! Deletes the directory and all its contents!
+
+ Makes sure the workbench actually belongs to this manager though.
+ """
+ # just in case
+ workbench = os.path.abspath(workbench)
+
+ if not workbench.startswith(self.base_workbench_dir):
+ raise WorkbenchOutsideScope(
+ "Can't destroy workbench outside the base workbench dir")
+
+ shutil.rmtree(workbench)
+
+ def localized_file(self, workbench, storage, filepath,
+ filename_if_copying=None,
+ keep_extension_if_copying=True):
+ """
+ Possibly localize the file from this storage system (for read-only
+ purposes, modifications should be written to a new file.).
+
+ If the file is already local, just return the absolute filename of that
+ local file. Otherwise, copy the file locally to the workbench, and
+ return the absolute path of the new file.
+
+ If it is copying locally, we might want to require a filename like
+ "source.jpg" to ensure that we won't conflict with other filenames in
+ our workbench... if that's the case, make sure filename_if_copying is
+ set to something like 'source.jpg'. Relatedly, if you set
+ keep_extension_if_copying, you don't have to set an extension on
+ filename_if_copying yourself, it'll be set for you (assuming such an
+ extension can be extacted from the filename in the filepath).
+
+ Returns:
+ localized_filename
+
+ Examples:
+ >>> wb_manager.localized_file(
+ ... '/our/workbench/subdir', local_storage,
+ ... ['path', 'to', 'foobar.jpg'])
+ u'/local/storage/path/to/foobar.jpg'
+
+ >>> wb_manager.localized_file(
+ ... '/our/workbench/subdir', remote_storage,
+ ... ['path', 'to', 'foobar.jpg'])
+ '/our/workbench/subdir/foobar.jpg'
+
+ >>> wb_manager.localized_file(
+ ... '/our/workbench/subdir', remote_storage,
+ ... ['path', 'to', 'foobar.jpg'], 'source.jpeg', False)
+ '/our/workbench/subdir/foobar.jpeg'
+
+ >>> wb_manager.localized_file(
+ ... '/our/workbench/subdir', remote_storage,
+ ... ['path', 'to', 'foobar.jpg'], 'source', True)
+ '/our/workbench/subdir/foobar.jpg'
+ """
+ if storage.local_storage:
+ return storage.get_local_path(filepath)
+ else:
+ if filename_if_copying is None:
+ dest_filename = filepath[-1]
+ else:
+ orig_filename, orig_ext = os.path.splitext(filepath[-1])
+ if keep_extension_if_copying and orig_ext:
+ dest_filename = filename_if_copying + orig_ext
+ else:
+ dest_filename = filename_if_copying
+
+ full_dest_filename = os.path.join(
+ workbench, dest_filename)
+
+ # copy it over
+ storage.copy_locally(
+ filepath, full_dest_filename)
+
+ return full_dest_filename