diff options
| -rw-r--r-- | mediagoblin/config_spec.ini | 6 | ||||
| -rw-r--r-- | mediagoblin/db/migrations.py | 20 | ||||
| -rw-r--r-- | mediagoblin/db/models.py | 3 | ||||
| -rw-r--r-- | mediagoblin/static/js/file_size.js | 45 | ||||
| -rw-r--r-- | mediagoblin/storage/__init__.py | 7 | ||||
| -rw-r--r-- | mediagoblin/storage/cloudfiles.py | 6 | ||||
| -rw-r--r-- | mediagoblin/storage/filestorage.py | 3 | ||||
| -rw-r--r-- | mediagoblin/submit/forms.py | 52 | ||||
| -rw-r--r-- | mediagoblin/submit/views.py | 77 | ||||
| -rw-r--r-- | mediagoblin/templates/mediagoblin/submit/start.html | 5 | ||||
| -rw-r--r-- | mediagoblin/tests/resources.py | 2 | ||||
| -rw-r--r-- | mediagoblin/tests/test_mgoblin_app.ini | 4 | ||||
| -rw-r--r-- | mediagoblin/tests/test_submission.py | 112 | ||||
| -rw-r--r-- | mediagoblin/tests/test_submission/COPYING.txt | 5 | ||||
| -rw-r--r-- | mediagoblin/tests/test_submission/big.png | bin | 0 -> 2212445 bytes | |||
| -rw-r--r-- | mediagoblin/tests/test_submission/medium.png | bin | 0 -> 1796336 bytes | |||
| -rw-r--r-- | mediagoblin/user_pages/views.py | 5 |
17 files changed, 315 insertions, 37 deletions
diff --git a/mediagoblin/config_spec.ini b/mediagoblin/config_spec.ini index 43e1898c..6f318d64 100644 --- a/mediagoblin/config_spec.ini +++ b/mediagoblin/config_spec.ini @@ -75,6 +75,12 @@ theme = string() plugin_web_path = string(default="/plugin_static/") plugin_linked_assets_dir = string(default="%(here)s/user_dev/plugin_static/") +# Default user upload limit (in Mb) +upload_limit = integer(default=None) + +# Max file size (in Mb) +max_file_size = integer(default=None) + [jinja2] # Jinja2 supports more directives than the minimum required by mediagoblin. # This setting allows users creating custom templates to specify a list of diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py index 62fb7e8d..e2a0bf26 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/migrations.py @@ -474,3 +474,23 @@ def wants_notifications(db): col.create(user_table) db.commit() + + +@RegisterMigration(16, MIGRATIONS) +def upload_limits(db): + """Add user upload limit columns""" + metadata = MetaData(bind=db.bind) + + user_table = inspect_table(metadata, 'core__users') + media_entry_table = inspect_table(metadata, 'core__media_entries') + + col = Column('uploaded', Integer, default=0) + col.create(user_table) + + col = Column('upload_limit', Integer) + col.create(user_table) + + col = Column('file_size', Integer, default=0) + col.create(media_entry_table) + + db.commit() diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 4341e086..a2675678 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -74,6 +74,8 @@ class User(Base, UserMixin): is_admin = Column(Boolean, default=False, nullable=False) url = Column(Unicode) bio = Column(UnicodeText) # ?? + uploaded = Column(Integer, default=0) + upload_limit = Column(Integer) ## TODO # plugin data would be in a separate model @@ -190,6 +192,7 @@ class MediaEntry(Base, MediaEntryMixin): # or use sqlalchemy.types.Enum? license = Column(Unicode) collected = Column(Integer, default=0) + file_size = Column(Integer, default=0) fail_error = Column(Unicode) fail_metadata = Column(JSONEncoded) diff --git a/mediagoblin/static/js/file_size.js b/mediagoblin/static/js/file_size.js new file mode 100644 index 00000000..2238ef89 --- /dev/null +++ b/mediagoblin/static/js/file_size.js @@ -0,0 +1,45 @@ +/** + * GNU MediaGoblin -- federated, autonomous media hosting + * Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +$(document).ready(function(){ + var file = document.getElementById('file'); + var uploaded = parseInt(document.getElementById('uploaded').value); + var upload_limit = parseInt(document.getElementById('upload_limit').value); + var max_file_size = parseInt(document.getElementById('max_file_size').value); + + file.onchange = function() { + var file_size = file.files[0].size / (1024.0 * 1024); + + if (file_size >= max_file_size) { + $('#file').after('<p id="file_size_error" class="form_field_error">Sorry, the file size is too big.</p>'); + } + else if (document.getElementById('file_size_error')) { + $('#file_size_error').hide(); + } + + if (upload_limit) { + if ( uploaded + file_size >= upload_limit) { + $('#file').after('<p id="upload_limit_error" class="form_field_error">Sorry, uploading this file will put you over your upload limit.</p>'); + } + else if (document.getElementById('upload_limit_error')) { + $('#upload_limit_error').hide(); + console.log(file_size >= max_file_size); + } + } + }; +}); diff --git a/mediagoblin/storage/__init__.py b/mediagoblin/storage/__init__.py index bbe134a7..51b46c07 100644 --- a/mediagoblin/storage/__init__.py +++ b/mediagoblin/storage/__init__.py @@ -191,6 +191,13 @@ class StorageInterface(object): # Copy to storage system in 4M chunks shutil.copyfileobj(source_file, dest_file, length=4*1048576) + def get_file_size(self, filepath): + """ + Return the size of the file in bytes. + """ + # Subclasses should override this method. + self.__raise_not_implemented() + ########### # Utilities diff --git a/mediagoblin/storage/cloudfiles.py b/mediagoblin/storage/cloudfiles.py index 250f06d4..47c81ad6 100644 --- a/mediagoblin/storage/cloudfiles.py +++ b/mediagoblin/storage/cloudfiles.py @@ -168,6 +168,12 @@ class CloudFilesStorage(StorageInterface): # Copy to storage system in 4096 byte chunks dest_file.send(source_file) + def get_file_size(self, filepath): + """Returns the file size in bytes""" + obj = self.container.get_object( + self._resolve_filepath(filepath)) + return obj.total_bytes + class CloudFilesStorageObjectWrapper(): """ Wrapper for python-cloudfiles's cloudfiles.storage_object.Object diff --git a/mediagoblin/storage/filestorage.py b/mediagoblin/storage/filestorage.py index 3d6e0753..29b8383b 100644 --- a/mediagoblin/storage/filestorage.py +++ b/mediagoblin/storage/filestorage.py @@ -111,3 +111,6 @@ class BasicFileStorage(StorageInterface): os.makedirs(directory) # This uses chunked copying of 16kb buffers (Py2.7): shutil.copy(filename, self.get_local_path(filepath)) + + def get_file_size(self, filepath): + return os.stat(self._resolve_filepath(filepath)).st_size diff --git a/mediagoblin/submit/forms.py b/mediagoblin/submit/forms.py index e9bd93fd..e2264645 100644 --- a/mediagoblin/submit/forms.py +++ b/mediagoblin/submit/forms.py @@ -17,30 +17,44 @@ import wtforms +from mediagoblin import mg_globals from mediagoblin.tools.text import tag_length_validator from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ from mediagoblin.tools.licenses import licenses_as_choices -class SubmitStartForm(wtforms.Form): - file = wtforms.FileField(_('File')) - title = wtforms.TextField( - _('Title'), - [wtforms.validators.Length(min=0, max=500)]) - description = wtforms.TextAreaField( - _('Description of this work'), - description=_("""You can use - <a href="http://daringfireball.net/projects/markdown/basics"> - Markdown</a> for formatting.""")) - tags = wtforms.TextField( - _('Tags'), - [tag_length_validator], - description=_( - "Separate tags by commas.")) - license = wtforms.SelectField( - _('License'), - [wtforms.validators.Optional(),], - choices=licenses_as_choices()) +def get_submit_start_form(form, **kwargs): + max_file_size = kwargs.get('max_file_size') + desc = None + if max_file_size: + desc = _('Max file size: {0} mb'.format(max_file_size)) + + class SubmitStartForm(wtforms.Form): + file = wtforms.FileField( + _('File'), + description=desc) + title = wtforms.TextField( + _('Title'), + [wtforms.validators.Length(min=0, max=500)]) + description = wtforms.TextAreaField( + _('Description of this work'), + description=_("""You can use + <a href="http://daringfireball.net/projects/markdown/basics"> + Markdown</a> for formatting.""")) + tags = wtforms.TextField( + _('Tags'), + [tag_length_validator], + description=_( + "Separate tags by commas.")) + license = wtforms.SelectField( + _('License'), + [wtforms.validators.Optional(),], + choices=licenses_as_choices()) + max_file_size = wtforms.HiddenField('') + upload_limit = wtforms.HiddenField('') + uploaded = wtforms.HiddenField('') + + return SubmitStartForm(form, **kwargs) class AddCollectionForm(wtforms.Form): title = wtforms.TextField( diff --git a/mediagoblin/submit/views.py b/mediagoblin/submit/views.py index 6bb95ecb..7f7dee33 100644 --- a/mediagoblin/submit/views.py +++ b/mediagoblin/submit/views.py @@ -43,8 +43,28 @@ def submit_start(request): """ First view for submitting a file. """ - submit_form = submit_forms.SubmitStartForm(request.form, - license=request.user.license_preference) + user = request.user + if user.upload_limit >= 0: + upload_limit = user.upload_limit + else: + upload_limit = mg_globals.app_config.get('upload_limit', None) + + if upload_limit and user.uploaded >= upload_limit: + messages.add_message( + request, + messages.WARNING, + _('Sorry, you have reached your upload limit.')) + return redirect(request, "mediagoblin.user_pages.user_home", + user=request.user.username) + + max_file_size = mg_globals.app_config.get('max_file_size', None) + + submit_form = submit_forms.get_submit_start_form( + request.form, + license=request.user.license_preference, + max_file_size=max_file_size, + upload_limit=upload_limit, + uploaded=user.uploaded) if request.method == 'POST' and submit_form.validate(): if not check_file_field(request, 'file'): @@ -86,24 +106,49 @@ def submit_start(request): with queue_file: queue_file.write(request.files['file'].stream.read()) - # Save now so we have this data before kicking off processing - entry.save() + # Get file size and round to 2 decimal places + file_size = request.app.queue_store.get_file_size( + entry.queued_media_file) / (1024.0 * 1024) + file_size = float('{0:.2f}'.format(file_size)) + + error = False + + # Check if file size is over the limit + if max_file_size and file_size >= max_file_size: + submit_form.file.errors.append( + _(u'Sorry, the file size is too big.')) + error = True + + # Check if user is over upload limit + if upload_limit and (user.uploaded + file_size) >= upload_limit: + submit_form.file.errors.append( + _('Sorry, uploading this file will put you over your' + ' upload limit.')) + error = True + + if not error: + user.uploaded = user.uploaded + file_size + user.save() + + entry.file_size = file_size - # Pass off to async processing - # - # (... don't change entry after this point to avoid race - # conditions with changes to the document via processing code) - feed_url = request.urlgen( - 'mediagoblin.user_pages.atom_feed', - qualified=True, user=request.user.username) - run_process_media(entry, feed_url) + # Save now so we have this data before kicking off processing + entry.save() - add_message(request, SUCCESS, _('Woohoo! Submitted!')) + # Pass off to processing + # + # (... don't change entry after this point to avoid race + # conditions with changes to the document via processing code) + feed_url = request.urlgen( + 'mediagoblin.user_pages.atom_feed', + qualified=True, user=request.user.username) + run_process_media(entry, feed_url) + add_message(request, SUCCESS, _('Woohoo! Submitted!')) - add_comment_subscription(request.user, entry) + add_comment_subscription(request.user, entry) - return redirect(request, "mediagoblin.user_pages.user_home", - user=request.user.username) + return redirect(request, "mediagoblin.user_pages.user_home", + user=user.username) except Exception as e: ''' This section is intended to catch exceptions raised in diff --git a/mediagoblin/templates/mediagoblin/submit/start.html b/mediagoblin/templates/mediagoblin/submit/start.html index aa390f56..d335d742 100644 --- a/mediagoblin/templates/mediagoblin/submit/start.html +++ b/mediagoblin/templates/mediagoblin/submit/start.html @@ -19,6 +19,11 @@ {% import "/mediagoblin/utils/wtforms.html" as wtforms_util %} +{% block mediagoblin_head %} + <script type="text/javascript" + src="{{ request.staticdirect('/js/file_size.js') }}"></script> +{% endblock %} + {% block title -%} {% trans %}Add your media{% endtrans %} — {{ super() }} {%- endblock %} diff --git a/mediagoblin/tests/resources.py b/mediagoblin/tests/resources.py index f7b3037d..480f6d9a 100644 --- a/mediagoblin/tests/resources.py +++ b/mediagoblin/tests/resources.py @@ -29,6 +29,8 @@ EVIL_JPG = resource('evil.jpg') EVIL_PNG = resource('evil.png') BIG_BLUE = resource('bigblue.png') GOOD_PDF = resource('good.pdf') +MED_PNG = resource('medium.png') +BIG_PNG = resource('big.png') def resource_exif(f): diff --git a/mediagoblin/tests/test_mgoblin_app.ini b/mediagoblin/tests/test_mgoblin_app.ini index da0dffb9..4cd3d9b6 100644 --- a/mediagoblin/tests/test_mgoblin_app.ini +++ b/mediagoblin/tests/test_mgoblin_app.ini @@ -13,6 +13,10 @@ tags_max_length = 50 # So we can start to test attachments: allow_attachments = True +upload_limit = 500 + +max_file_size = 2 + [storage:publicstore] base_dir = %(here)s/user_dev/media/public base_url = /mgoblin_media/ diff --git a/mediagoblin/tests/test_submission.py b/mediagoblin/tests/test_submission.py index ac941063..7f4e8086 100644 --- a/mediagoblin/tests/test_submission.py +++ b/mediagoblin/tests/test_submission.py @@ -24,13 +24,14 @@ import pytest from mediagoblin.tests.tools import fixture_add_user from mediagoblin import mg_globals -from mediagoblin.db.models import MediaEntry +from mediagoblin.db.models import MediaEntry, User +from mediagoblin.db.base import Session from mediagoblin.tools import template from mediagoblin.media_types.image import ImageMediaManager from mediagoblin.media_types.pdf.processing import check_prerequisites as pdf_check_prerequisites from .resources import GOOD_JPG, GOOD_PNG, EVIL_FILE, EVIL_JPG, EVIL_PNG, \ - BIG_BLUE, GOOD_PDF, GPS_JPG + BIG_BLUE, GOOD_PDF, GPS_JPG, MED_PNG, BIG_PNG GOOD_TAG_STRING = u'yin,yang' BAD_TAG_STRING = unicode('rage,' + 'f' * 26 + 'u' * 26) @@ -107,9 +108,38 @@ class TestSubmission: self.logout() self.test_app.get(url) + def user_upload_limits(self, uploaded=None, upload_limit=None): + if uploaded: + self.test_user.uploaded = uploaded + if upload_limit: + self.test_user.upload_limit = upload_limit + + self.test_user.save() + + # Reload + self.test_user = User.query.filter_by( + username=self.test_user.username + ).first() + + # ... and detach from session: + Session.expunge(self.test_user) + def test_normal_jpg(self): + # User uploaded should be 0 + assert self.test_user.uploaded == 0 + self.check_normal_upload(u'Normal upload 1', GOOD_JPG) + # User uploaded should be the same as GOOD_JPG size in Mb + file_size = os.stat(GOOD_JPG).st_size / (1024.0 * 1024) + file_size = float('{0:.2f}'.format(file_size)) + + # Reload user + self.test_user = User.query.filter_by( + username=self.test_user.username + ).first() + assert self.test_user.uploaded == file_size + def test_normal_png(self): self.check_normal_upload(u'Normal upload 2', GOOD_PNG) @@ -121,6 +151,75 @@ class TestSubmission: self.check_url(response, '/u/{0}/'.format(self.test_user.username)) assert 'mediagoblin/user_pages/user.html' in context + def test_default_upload_limits(self): + self.user_upload_limits(uploaded=500) + + # User uploaded should be 500 + assert self.test_user.uploaded == 500 + + response, context = self.do_post({'title': u'Normal upload 4'}, + do_follow=True, + **self.upload_data(GOOD_JPG)) + self.check_url(response, '/u/{0}/'.format(self.test_user.username)) + assert 'mediagoblin/user_pages/user.html' in context + + # Reload user + self.test_user = User.query.filter_by( + username=self.test_user.username + ).first() + + # Shouldn't have uploaded + assert self.test_user.uploaded == 500 + + def test_user_upload_limit(self): + self.user_upload_limits(uploaded=25, upload_limit=25) + + # User uploaded should be 25 + assert self.test_user.uploaded == 25 + + response, context = self.do_post({'title': u'Normal upload 5'}, + do_follow=True, + **self.upload_data(GOOD_JPG)) + self.check_url(response, '/u/{0}/'.format(self.test_user.username)) + assert 'mediagoblin/user_pages/user.html' in context + + # Reload user + self.test_user = User.query.filter_by( + username=self.test_user.username + ).first() + + # Shouldn't have uploaded + assert self.test_user.uploaded == 25 + + def test_user_under_limit(self): + self.user_upload_limits(uploaded=499) + + # User uploaded should be 499 + assert self.test_user.uploaded == 499 + + response, context = self.do_post({'title': u'Normal upload 6'}, + do_follow=False, + **self.upload_data(MED_PNG)) + form = context['mediagoblin/submit/start.html']['submit_form'] + assert form.file.errors == [u'Sorry, uploading this file will put you' + ' over your upload limit.'] + + # Reload user + self.test_user = User.query.filter_by( + username=self.test_user.username + ).first() + + # Shouldn't have uploaded + assert self.test_user.uploaded == 499 + + def test_big_file(self): + response, context = self.do_post({'title': u'Normal upload 7'}, + do_follow=False, + **self.upload_data(BIG_PNG)) + + form = context['mediagoblin/submit/start.html']['submit_form'] + assert form.file.errors == [u'Sorry, the file size is too big.'] + def check_media(self, request, find_data, count=None): media = MediaEntry.query.filter_by(**find_data) if count is not None: @@ -155,6 +254,7 @@ class TestSubmission: 'ffffffffffffffffffffffffffuuuuuuuuuuuuuuuuuuuuuuuuuu'] def test_delete(self): + self.user_upload_limits(uploaded=50) response, request = self.do_post({'title': u'Balanced Goblin'}, *REQUEST_CONTEXT, do_follow=True, **self.upload_data(GOOD_JPG)) @@ -199,6 +299,14 @@ class TestSubmission: self.check_media(request, {'id': media_id}, 0) self.check_comments(request, media_id, 0) + # Reload user + self.test_user = User.query.filter_by( + username = self.test_user.username + ).first() + + # Check that user.uploaded is the same as before the upload + assert self.test_user.uploaded == 50 + def test_evil_file(self): # Test non-suppoerted file with non-supported extension # ----------------------------------------------------- diff --git a/mediagoblin/tests/test_submission/COPYING.txt b/mediagoblin/tests/test_submission/COPYING.txt new file mode 100644 index 00000000..3818aae4 --- /dev/null +++ b/mediagoblin/tests/test_submission/COPYING.txt @@ -0,0 +1,5 @@ +Images located in this directory tree are released under a GPLv3 license +and CC BY-SA 3.0 license. To the extent possible under law, the author(s) +have dedicated all copyright and related and neighboring rights to these +files to the public domain worldwide. These files are distributed without +any warranty. diff --git a/mediagoblin/tests/test_submission/big.png b/mediagoblin/tests/test_submission/big.png Binary files differnew file mode 100644 index 00000000..a284cfda --- /dev/null +++ b/mediagoblin/tests/test_submission/big.png diff --git a/mediagoblin/tests/test_submission/medium.png b/mediagoblin/tests/test_submission/medium.png Binary files differnew file mode 100644 index 00000000..e8b9ca00 --- /dev/null +++ b/mediagoblin/tests/test_submission/medium.png diff --git a/mediagoblin/user_pages/views.py b/mediagoblin/user_pages/views.py index 49691a29..5eac0fe4 100644 --- a/mediagoblin/user_pages/views.py +++ b/mediagoblin/user_pages/views.py @@ -296,6 +296,11 @@ def media_confirm_delete(request, media): if request.method == 'POST' and form.validate(): if form.confirm.data is True: username = media.get_uploader.username + + media.get_uploader.uploaded = media.get_uploader.uploaded - \ + media.file_size + media.get_uploader.save() + # Delete MediaEntry and all related files, comments etc. media.delete() messages.add_message( |
