diff options
author | Joar Wandborg <git@wandborg.com> | 2011-06-12 14:37:49 +0200 |
---|---|---|
committer | Joar Wandborg <git@wandborg.com> | 2011-06-12 14:37:49 +0200 |
commit | a48014d67a5185c645ac876d365b4cb85cbb6b1f (patch) | |
tree | df9a459c0f246f1b189bb338b1b3470d87ff3f02 | |
parent | 44e2da2fe60a3b8765d0fef5a9ce0c3e5997dd01 (diff) | |
parent | 68ffb13690fa0c364c514ce253364f928e50841c (diff) | |
download | mediagoblin-a48014d67a5185c645ac876d365b4cb85cbb6b1f.tar.lz mediagoblin-a48014d67a5185c645ac876d365b4cb85cbb6b1f.tar.xz mediagoblin-a48014d67a5185c645ac876d365b4cb85cbb6b1f.zip |
Merge branch 'master' of http://git.gitorious.org/mediagoblin/mediagoblin
-rw-r--r-- | mediagoblin/app.py | 10 | ||||
-rw-r--r-- | mediagoblin/celery_setup/from_celery.py | 9 | ||||
-rw-r--r-- | mediagoblin/process_media/__init__.py | 28 | ||||
-rw-r--r-- | mediagoblin/storage.py | 42 | ||||
-rw-r--r-- | mediagoblin/tests/test_storage.py | 45 | ||||
-rw-r--r-- | mediagoblin/tests/test_workbench.py | 96 | ||||
-rw-r--r-- | mediagoblin/workbench.py | 135 |
7 files changed, 348 insertions, 17 deletions
diff --git a/mediagoblin/app.py b/mediagoblin/app.py index e5949531..5d594039 100644 --- a/mediagoblin/app.py +++ b/mediagoblin/app.py @@ -25,6 +25,7 @@ from mediagoblin import routing, util, storage, staticdirect from mediagoblin.db.open import setup_connection_and_db_from_config from mediagoblin.globals import setup_globals from mediagoblin.celery_setup import setup_celery_from_config +from mediagoblin.workbench import WorkbenchManager, DEFAULT_WORKBENCH_DIR class Error(Exception): pass @@ -39,7 +40,8 @@ class MediaGoblinApp(object): public_store, queue_store, staticdirector, email_sender_address, email_debug_mode, - user_template_path=None): + user_template_path=None, + workbench_path=DEFAULT_WORKBENCH_DIR): # Get the template environment self.template_loader = util.get_jinja_loader(user_template_path) @@ -66,7 +68,8 @@ class MediaGoblinApp(object): db_connection=connection, database=self.db, public_store=self.public_store, - queue_store=self.queue_store) + queue_store=self.queue_store, + workbench_manager=WorkbenchManager(workbench_path)) def __call__(self, environ, start_response): request = Request(environ) @@ -154,6 +157,7 @@ def paste_app_factory(global_config, **app_config): email_sender_address=app_config.get( 'email_sender_address', 'notice@mediagoblin.example.org'), email_debug_mode=asbool(app_config.get('email_debug_mode')), - user_template_path=app_config.get('local_templates')) + user_template_path=app_config.get('local_templates'), + workbench_path=app_config.get('workbench_path', DEFAULT_WORKBENCH_DIR)) return mgoblin_app diff --git a/mediagoblin/celery_setup/from_celery.py b/mediagoblin/celery_setup/from_celery.py index 0669e80c..5fa9ba76 100644 --- a/mediagoblin/celery_setup/from_celery.py +++ b/mediagoblin/celery_setup/from_celery.py @@ -23,7 +23,7 @@ from mediagoblin import storage from mediagoblin.db.open import setup_connection_and_db_from_config from mediagoblin.celery_setup import setup_celery_from_config from mediagoblin.globals import setup_globals -from mediagoblin import globals as mgoblin_globals +from mediagoblin.workbench import WorkbenchManager, DEFAULT_WORKBENCH_DIR OUR_MODULENAME = 'mediagoblin.celery_setup.from_celery' @@ -76,6 +76,10 @@ def setup_self(setup_globals_func=setup_globals): queue_store = storage.storage_system_from_paste_config( mgoblin_section, 'queuestore') + workbench_manager = WorkbenchManager( + mgoblin_section.get( + 'workbench_path', DEFAULT_WORKBENCH_DIR)) + setup_globals_func( db_connection=connection, database=db, @@ -84,7 +88,8 @@ def setup_self(setup_globals_func=setup_globals): email_sender_address=mgoblin_section.get( 'email_sender_address', 'notice@mediagoblin.example.org'), - queue_store=queue_store) + queue_store=queue_store, + workbench_manager=workbench_manager) if os.environ['CELERY_CONFIG_MODULE'] == OUR_MODULENAME: diff --git a/mediagoblin/process_media/__init__.py b/mediagoblin/process_media/__init__.py index 4f06a686..531eb16d 100644 --- a/mediagoblin/process_media/__init__.py +++ b/mediagoblin/process_media/__init__.py @@ -18,7 +18,7 @@ import Image from mediagoblin.db.util import ObjectId from celery.task import task -from mediagoblin.globals import database, queue_store, public_store +from mediagoblin import globals as mg_globals THUMB_SIZE = 200, 200 @@ -26,40 +26,50 @@ THUMB_SIZE = 200, 200 @task def process_media_initial(media_id): - entry = database.MediaEntry.one( + workbench = mg_globals.workbench_manager.create_workbench() + + entry = mg_globals.database.MediaEntry.one( {'_id': ObjectId(media_id)}) queued_filepath = entry['queued_media_file'] - queued_file = queue_store.get_file(queued_filepath, 'r') + queued_filename = mg_globals.workbench_manager.localized_file( + workbench, mg_globals.queue_store, queued_filepath, + 'source') + + queued_file = file(queued_filename, 'r') with queued_file: thumb = Image.open(queued_file) thumb.thumbnail(THUMB_SIZE, Image.ANTIALIAS) - thumb_filepath = public_store.get_unique_filepath( + thumb_filepath = mg_globals.public_store.get_unique_filepath( ['media_entries', unicode(entry['_id']), 'thumbnail.jpg']) - with public_store.get_file(thumb_filepath, 'w') as thumb_file: + thumb_file = mg_globals.public_store.get_file(thumb_filepath, 'w') + with thumb_file: thumb.save(thumb_file, "JPEG") # we have to re-read because unlike PIL, not everything reads # things in string representation :) - queued_file = queue_store.get_file(queued_filepath, 'rb') + queued_file = file(queued_filename, 'rb') with queued_file: - main_filepath = public_store.get_unique_filepath( + main_filepath = mg_globals.public_store.get_unique_filepath( ['media_entries', unicode(entry['_id']), queued_filepath[-1]]) - with public_store.get_file(main_filepath, 'wb') as main_file: + with mg_globals.public_store.get_file(main_filepath, 'wb') as main_file: main_file.write(queued_file.read()) - queue_store.delete_file(queued_filepath) + mg_globals.queue_store.delete_file(queued_filepath) media_files_dict = entry.setdefault('media_files', {}) media_files_dict['thumb'] = thumb_filepath media_files_dict['main'] = main_filepath entry['state'] = u'processed' entry.save() + + # clean up workbench + mg_globals.workbench_manager.destroy_workbench(workbench) diff --git a/mediagoblin/storage.py b/mediagoblin/storage.py index 5d7e70d6..ba6ac017 100644 --- a/mediagoblin/storage.py +++ b/mediagoblin/storage.py @@ -16,6 +16,7 @@ import os import re +import shutil import urlparse import uuid @@ -60,6 +61,9 @@ class StorageInterface(object): StorageInterface. """ + # Whether this file store is on the local filesystem. + local_storage = False + def __raise_not_implemented(self): """ Raise a warning about some component not implemented by a @@ -127,12 +131,43 @@ class StorageInterface(object): else: return filepath + def get_local_path(self, filepath): + """ + If this is a local_storage implementation, give us a link to + the local filesystem reference to this file. + + >>> storage_handler.get_local_path(['foo', 'bar', 'baz.jpg']) + u'/path/to/mounting/foo/bar/baz.jpg' + """ + # Subclasses should override this method, if applicable. + self.__raise_not_implemented() + + def copy_locally(self, filepath, dest_path): + """ + Copy this file locally. + + A basic working method for this is provided that should + function both for local_storage systems and remote storge + systems, but if more efficient systems for copying locally + apply to your system, override this method with something more + appropriate. + """ + if self.local_storage: + shutil.copy( + self.get_local_path(filepath), dest_path) + else: + with self.get_file(filepath, 'rb') as source_file: + with file(dest_path, 'wb') as dest_file: + dest_file.write(source_file.read()) + class BasicFileStorage(StorageInterface): """ Basic local filesystem implementation of storage API """ + local_storage = True + def __init__(self, base_dir, base_url=None, **kwargs): """ Keyword arguments: @@ -177,6 +212,9 @@ class BasicFileStorage(StorageInterface): self.base_url, '/'.join(clean_listy_filepath(filepath))) + def get_local_path(self, filepath): + return self._resolve_filepath(filepath) + ########### # Utilities @@ -187,7 +225,7 @@ def clean_listy_filepath(listy_filepath): Take a listy filepath (like ['dir1', 'dir2', 'filename.jpg']) and clean out any nastiness from it. - For example: + >>> clean_listy_filepath([u'/dir1/', u'foo/../nasty', u'linooks.jpg']) [u'dir1', u'foo_.._nasty', u'linooks.jpg'] @@ -253,3 +291,5 @@ def storage_system_from_paste_config(paste_config, storage_prefix): storage_class = util.import_component(storage_class) return storage_class(**config_params) + + diff --git a/mediagoblin/tests/test_storage.py b/mediagoblin/tests/test_storage.py index 61dd5dca..55b66e84 100644 --- a/mediagoblin/tests/test_storage.py +++ b/mediagoblin/tests/test_storage.py @@ -52,6 +52,11 @@ class FakeStorageSystem(): self.foobie = foobie self.blech = blech +class FakeRemoteStorage(storage.BasicFileStorage): + # Theoretically despite this, all the methods should work but it + # should force copying to the workbench + local_storage = False + def test_storage_system_from_paste_config(): this_storage = storage.storage_system_from_paste_config( @@ -81,9 +86,12 @@ def test_storage_system_from_paste_config(): # Basic file storage tests ########################## -def get_tmp_filestorage(mount_url=None): +def get_tmp_filestorage(mount_url=None, fake_remote=False): tmpdir = tempfile.mkdtemp() - this_storage = storage.BasicFileStorage(tmpdir, mount_url) + if fake_remote: + this_storage = FakeRemoteStorage(tmpdir, mount_url) + else: + this_storage = storage.BasicFileStorage(tmpdir, mount_url) return tmpdir, this_storage @@ -214,3 +222,36 @@ def test_basic_storage_url_for_file(): ['dir1', 'dir2', 'filename.txt']) expected = 'http://media.example.org/ourmedia/dir1/dir2/filename.txt' assert result == expected + + +def test_basic_storage_get_local_path(): + tmpdir, this_storage = get_tmp_filestorage() + + result = this_storage.get_local_path( + ['dir1', 'dir2', 'filename.txt']) + + expected = os.path.join( + tmpdir, 'dir1/dir2/filename.txt') + + assert result == expected + + +def test_basic_storage_is_local(): + tmpdir, this_storage = get_tmp_filestorage() + assert this_storage.local_storage is True + + +def test_basic_storage_copy_locally(): + tmpdir, this_storage = get_tmp_filestorage() + + dest_tmpdir = tempfile.mkdtemp() + + filepath = ['dir1', 'dir2', 'ourfile.txt'] + with this_storage.get_file(filepath, 'w') as our_file: + our_file.write('Testing this file') + + new_file_dest = os.path.join(dest_tmpdir, 'file2.txt') + + this_storage.copy_locally(filepath, new_file_dest) + + assert file(new_file_dest).read() == 'Testing this file' diff --git a/mediagoblin/tests/test_workbench.py b/mediagoblin/tests/test_workbench.py new file mode 100644 index 00000000..89f2ef33 --- /dev/null +++ b/mediagoblin/tests/test_workbench.py @@ -0,0 +1,96 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import tempfile + +from nose.tools import assert_raises + +from mediagoblin import workbench +from mediagoblin.tests.test_storage import get_tmp_filestorage + + +class TestWorkbench(object): + def setUp(self): + self.workbench_manager = workbench.WorkbenchManager( + os.path.join(tempfile.gettempdir(), u'mgoblin_workbench_testing')) + + def test_create_workbench(self): + workbench = self.workbench_manager.create_workbench() + assert os.path.isdir(workbench) + assert workbench.startswith(self.workbench_manager.base_workbench_dir) + + def test_destroy_workbench(self): + # kill a workbench + this_workbench = self.workbench_manager.create_workbench() + tmpfile = file(os.path.join(this_workbench, 'temp.txt'), 'w') + with tmpfile: + tmpfile.write('lollerskates') + + assert os.path.exists(os.path.join(this_workbench, 'temp.txt')) + + self.workbench_manager.destroy_workbench(this_workbench) + assert not os.path.exists(os.path.join(this_workbench, 'temp.txt')) + assert not os.path.exists(this_workbench) + + # make sure we can't kill other stuff though + dont_kill_this = tempfile.mkdtemp() + + assert_raises( + workbench.WorkbenchOutsideScope, + self.workbench_manager.destroy_workbench, + dont_kill_this) + + def test_localized_file(self): + tmpdir, this_storage = get_tmp_filestorage() + this_workbench = self.workbench_manager.create_workbench() + + # Write a brand new file + filepath = ['dir1', 'dir2', 'ourfile.txt'] + + with this_storage.get_file(filepath, 'w') as our_file: + our_file.write('Our file') + + # with a local file storage + filename = self.workbench_manager.localized_file( + this_workbench, this_storage, filepath) + assert filename == os.path.join( + tmpdir, 'dir1/dir2/ourfile.txt') + + # with a fake remote file storage + tmpdir, this_storage = get_tmp_filestorage(fake_remote=True) + + # ... write a brand new file, again ;) + with this_storage.get_file(filepath, 'w') as our_file: + our_file.write('Our file') + + filename = self.workbench_manager.localized_file( + this_workbench, this_storage, filepath) + assert filename == os.path.join( + this_workbench, 'ourfile.txt') + + # fake remote file storage, filename_if_copying set + filename = self.workbench_manager.localized_file( + this_workbench, this_storage, filepath, 'thisfile') + assert filename == os.path.join( + this_workbench, 'thisfile.txt') + + # fake remote file storage, filename_if_copying set, + # keep_extension_if_copying set to false + filename = self.workbench_manager.localized_file( + this_workbench, this_storage, filepath, 'thisfile.text', False) + assert filename == os.path.join( + this_workbench, 'thisfile.text') diff --git a/mediagoblin/workbench.py b/mediagoblin/workbench.py new file mode 100644 index 00000000..d7252623 --- /dev/null +++ b/mediagoblin/workbench.py @@ -0,0 +1,135 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import shutil +import tempfile + + +DEFAULT_WORKBENCH_DIR = os.path.join( + tempfile.gettempdir(), u'mgoblin_workbench') + + +# Exception(s) +# ------------ + +class WorkbenchOutsideScope(Exception): + """ + Raised when a workbench is outside a WorkbenchManager scope. + """ + pass + + +# Actual workbench stuff +# ---------------------- + +class WorkbenchManager(object): + """ + A system for generating and destroying workbenches. + + Workbenches are actually just subdirectories of a temporary storage space + for during the processing stage. + """ + + def __init__(self, base_workbench_dir): + self.base_workbench_dir = os.path.abspath(base_workbench_dir) + if not os.path.exists(self.base_workbench_dir): + os.makedirs(self.base_workbench_dir) + + def create_workbench(self): + """ + Create and return the path to a new workbench (directory). + """ + return tempfile.mkdtemp(dir=self.base_workbench_dir) + + def destroy_workbench(self, workbench): + """ + Destroy this workbench! Deletes the directory and all its contents! + + Makes sure the workbench actually belongs to this manager though. + """ + # just in case + workbench = os.path.abspath(workbench) + + if not workbench.startswith(self.base_workbench_dir): + raise WorkbenchOutsideScope( + "Can't destroy workbench outside the base workbench dir") + + shutil.rmtree(workbench) + + def localized_file(self, workbench, storage, filepath, + filename_if_copying=None, + keep_extension_if_copying=True): + """ + Possibly localize the file from this storage system (for read-only + purposes, modifications should be written to a new file.). + + If the file is already local, just return the absolute filename of that + local file. Otherwise, copy the file locally to the workbench, and + return the absolute path of the new file. + + If it is copying locally, we might want to require a filename like + "source.jpg" to ensure that we won't conflict with other filenames in + our workbench... if that's the case, make sure filename_if_copying is + set to something like 'source.jpg'. Relatedly, if you set + keep_extension_if_copying, you don't have to set an extension on + filename_if_copying yourself, it'll be set for you (assuming such an + extension can be extacted from the filename in the filepath). + + Returns: + localized_filename + + Examples: + >>> wb_manager.localized_file( + ... '/our/workbench/subdir', local_storage, + ... ['path', 'to', 'foobar.jpg']) + u'/local/storage/path/to/foobar.jpg' + + >>> wb_manager.localized_file( + ... '/our/workbench/subdir', remote_storage, + ... ['path', 'to', 'foobar.jpg']) + '/our/workbench/subdir/foobar.jpg' + + >>> wb_manager.localized_file( + ... '/our/workbench/subdir', remote_storage, + ... ['path', 'to', 'foobar.jpg'], 'source.jpeg', False) + '/our/workbench/subdir/foobar.jpeg' + + >>> wb_manager.localized_file( + ... '/our/workbench/subdir', remote_storage, + ... ['path', 'to', 'foobar.jpg'], 'source', True) + '/our/workbench/subdir/foobar.jpg' + """ + if storage.local_storage: + return storage.get_local_path(filepath) + else: + if filename_if_copying is None: + dest_filename = filepath[-1] + else: + orig_filename, orig_ext = os.path.splitext(filepath[-1]) + if keep_extension_if_copying and orig_ext: + dest_filename = filename_if_copying + orig_ext + else: + dest_filename = filename_if_copying + + full_dest_filename = os.path.join( + workbench, dest_filename) + + # copy it over + storage.copy_locally( + filepath, full_dest_filename) + + return full_dest_filename |