diff options
Diffstat (limited to 'mediagoblin/storage')
| -rw-r--r-- | mediagoblin/storage/__init__.py | 14 | ||||
| -rw-r--r-- | mediagoblin/storage/cloudfiles.py | 25 | ||||
| -rw-r--r-- | mediagoblin/storage/filestorage.py | 19 | ||||
| -rw-r--r-- | mediagoblin/storage/mountstorage.py | 10 |
4 files changed, 33 insertions, 35 deletions
diff --git a/mediagoblin/storage/__init__.py b/mediagoblin/storage/__init__.py index 51b46c07..14f13bd3 100644 --- a/mediagoblin/storage/__init__.py +++ b/mediagoblin/storage/__init__.py @@ -14,9 +14,13 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +from __future__ import absolute_import + import shutil import uuid +import six + from werkzeug.utils import secure_filename from mediagoblin.tools import common @@ -174,7 +178,7 @@ class StorageInterface(object): shutil.copy(self.get_local_path(filepath), dest_path) else: with self.get_file(filepath, 'rb') as source_file: - with file(dest_path, 'wb') as dest_file: + with open(dest_path, 'wb') as dest_file: # Copy from remote storage in 4M chunks shutil.copyfileobj(source_file, dest_file, length=4*1048576) @@ -187,7 +191,7 @@ class StorageInterface(object): your storage system. """ with self.get_file(filepath, 'wb') as dest_file: - with file(filename, 'rb') as source_file: + with open(filename, 'rb') as source_file: # Copy to storage system in 4M chunks shutil.copyfileobj(source_file, dest_file, length=4*1048576) @@ -220,7 +224,7 @@ def clean_listy_filepath(listy_filepath): A cleaned list of unicode objects. """ cleaned_filepath = [ - unicode(secure_filename(filepath)) + six.text_type(secure_filename(filepath)) for filepath in listy_filepath] if u'' in cleaned_filepath: @@ -257,7 +261,7 @@ def storage_system_from_config(config_section): """ # This construct is needed, because dict(config) does # not replace the variables in the config items. - config_params = dict(config_section.iteritems()) + config_params = dict(six.iteritems(config_section)) if 'storage_class' in config_params: storage_class = config_params['storage_class'] @@ -268,4 +272,4 @@ def storage_system_from_config(config_section): storage_class = common.import_component(storage_class) return storage_class(**config_params) -import filestorage +from . import filestorage diff --git a/mediagoblin/storage/cloudfiles.py b/mediagoblin/storage/cloudfiles.py index 47c81ad6..61665ea0 100644 --- a/mediagoblin/storage/cloudfiles.py +++ b/mediagoblin/storage/cloudfiles.py @@ -143,7 +143,7 @@ class CloudFilesStorage(StorageInterface): """ # Override this method, using the "stream" iterator for efficient streaming with self.get_file(filepath, 'rb') as source_file: - with file(dest_path, 'wb') as dest_file: + with open(dest_path, 'wb') as dest_file: for data in source_file: dest_file.write(data) @@ -164,7 +164,7 @@ class CloudFilesStorage(StorageInterface): # TODO: Fixing write() still seems worthwhile though. _log.debug('Sending {0} to cloudfiles...'.format(filepath)) with self.get_file(filepath, 'wb') as dest_file: - with file(filename, 'rb') as source_file: + with open(filename, 'rb') as source_file: # Copy to storage system in 4096 byte chunks dest_file.send(source_file) @@ -193,27 +193,6 @@ class CloudFilesStorageObjectWrapper(): return self.storage_object.read(*args, **kwargs) def write(self, data, *args, **kwargs): - """ - write data to the cloudfiles storage object - - The original motivation for this wrapper is to ensure - that buffered writing to a cloudfiles storage object does not overwrite - any preexisting data. - - Currently this method does not support any write modes except "append". - However if we should need it it would be easy implement. - """ - _log.warn( - '{0}.write() has bad performance! Use .send instead for now'\ - .format(self.__class__.__name__)) - - if self.storage_object.size and type(data) == str: - _log.debug('{0} is > 0 in size, appending data'.format( - self.storage_object.name)) - data = self.read() + data - - _log.debug('Writing {0}'.format( - self.storage_object.name)) self.storage_object.write(data, *args, **kwargs) def send(self, *args, **kw): diff --git a/mediagoblin/storage/filestorage.py b/mediagoblin/storage/filestorage.py index 29b8383b..89f43276 100644 --- a/mediagoblin/storage/filestorage.py +++ b/mediagoblin/storage/filestorage.py @@ -14,14 +14,25 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +import io +import os +import shutil + +import six.moves.urllib.parse as urlparse + from mediagoblin.storage import ( StorageInterface, clean_listy_filepath, NoWebServing) -import os -import shutil -import urlparse +class FileObjectAwareFile(io.FileIO): + def write(self, data): + if hasattr(data, 'read'): + # We can call data.read(). It means that the data is a file-like + # object, which should be saved RAM-friendly way + shutil.copyfileobj(data, self) + else: + super(FileObjectAwareFile, self).write(data) class BasicFileStorage(StorageInterface): @@ -59,7 +70,7 @@ class BasicFileStorage(StorageInterface): os.makedirs(directory) # Grab and return the file in the mode specified - return open(self._resolve_filepath(filepath), mode) + return FileObjectAwareFile(self._resolve_filepath(filepath), mode) def delete_file(self, filepath): """Delete file at filepath diff --git a/mediagoblin/storage/mountstorage.py b/mediagoblin/storage/mountstorage.py index dffc619b..a829db31 100644 --- a/mediagoblin/storage/mountstorage.py +++ b/mediagoblin/storage/mountstorage.py @@ -14,6 +14,10 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +from __future__ import print_function + +import six + from mediagoblin.storage import StorageInterface, clean_listy_filepath @@ -59,9 +63,9 @@ class MountStorage(StorageInterface): """ new_ent = clean_listy_filepath(dirpath) - print "Mounting:", repr(new_ent) + print("Mounting:", repr(new_ent)) already, rem_1, table, rem_2 = self._resolve_to_backend(new_ent, True) - print "===", repr(already), repr(rem_1), repr(rem_2), len(table) + print("===", repr(already), repr(rem_1), repr(rem_2), len(table)) assert (len(rem_2) > 0) or (None not in table), \ "That path is already mounted" @@ -120,7 +124,7 @@ class MountStorage(StorageInterface): v = table.get(None) if v: res.append(" " * len(indent) + repr(indent) + ": " + repr(v)) - for k, v in table.iteritems(): + for k, v in six.iteritems(table): if k == None: continue res.append(" " * len(indent) + repr(k) + ":") |