From bdd2242155d3192615740661ce52f6fb960d1a05 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Wed, 12 Jun 2013 12:02:11 -0700 Subject: added user upload limits --- mediagoblin/config_spec.ini | 3 +++ mediagoblin/db/migrations.py | 24 ++++++++++++++++++++++-- mediagoblin/db/models.py | 3 +++ mediagoblin/storage/__init__.py | 7 +++++++ mediagoblin/storage/cloudfiles.py | 6 ++++++ mediagoblin/storage/filestorage.py | 3 +++ mediagoblin/submit/views.py | 34 ++++++++++++++++++++++++++++++++++ mediagoblin/user_pages/views.py | 4 ++++ 8 files changed, 82 insertions(+), 2 deletions(-) (limited to 'mediagoblin') diff --git a/mediagoblin/config_spec.ini b/mediagoblin/config_spec.ini index 8f03509d..596e4388 100644 --- a/mediagoblin/config_spec.ini +++ b/mediagoblin/config_spec.ini @@ -75,6 +75,9 @@ theme = string() plugin_web_path = string(default="/plugin_static/") plugin_linked_assets_dir = string(default="%(here)s/user_dev/plugin_static/") +# User upload limit (in Mbs) +upload_limit = integer(default=500) + [jinja2] # Jinja2 supports more directives than the minimum required by mediagoblin. # This setting allows users creating custom templates to specify a list of diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py index 374ab4c8..b3dca72c 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/migrations.py @@ -425,7 +425,7 @@ class RequestToken_v0(declarative_base()): callback = Column(Unicode, nullable=False, default=u"oob") created = Column(DateTime, nullable=False, default=datetime.datetime.now) updated = Column(DateTime, nullable=False, default=datetime.datetime.now) - + class AccessToken_v0(declarative_base()): """ Model for representing the access tokens @@ -438,7 +438,7 @@ class AccessToken_v0(declarative_base()): request_token = Column(Unicode, ForeignKey(RequestToken_v0.token)) created = Column(DateTime, nullable=False, default=datetime.datetime.now) updated = Column(DateTime, nullable=False, default=datetime.datetime.now) - + class NonceTimestamp_v0(declarative_base()): """ @@ -460,3 +460,23 @@ def create_oauth1_tables(db): NonceTimestamp_v0.__table__.create(db.bind) db.commit() + + +@RegisterMigration(15, MIGRATIONS) +def upload_limits(db): + """Add user upload limit columns""" + metadata = MetaData(bind=db.bind) + + user_table = inspect_table(metadata, 'core__users') + media_entry_table = inspect_table(metadata, 'core__media_entries') + + col = Column('uploaded', Integer, default=0) + col.create(user_table) + + col = Column('upload_limit', Integer) + col.create(user_table) + + col = Column('file_size', Integer, default=0) + col.create(media_entry_table) + + db.commit() diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 9cb39ff4..697ee326 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -73,6 +73,8 @@ class User(Base, UserMixin): is_admin = Column(Boolean, default=False, nullable=False) url = Column(Unicode) bio = Column(UnicodeText) # ?? + uploaded = Column(Integer, default=0) + upload_limit = Column(Integer) ## TODO # plugin data would be in a separate model @@ -189,6 +191,7 @@ class MediaEntry(Base, MediaEntryMixin): # or use sqlalchemy.types.Enum? license = Column(Unicode) collected = Column(Integer, default=0) + file_size = Column(Integer, default=0) fail_error = Column(Unicode) fail_metadata = Column(JSONEncoded) diff --git a/mediagoblin/storage/__init__.py b/mediagoblin/storage/__init__.py index bbe134a7..51b46c07 100644 --- a/mediagoblin/storage/__init__.py +++ b/mediagoblin/storage/__init__.py @@ -191,6 +191,13 @@ class StorageInterface(object): # Copy to storage system in 4M chunks shutil.copyfileobj(source_file, dest_file, length=4*1048576) + def get_file_size(self, filepath): + """ + Return the size of the file in bytes. + """ + # Subclasses should override this method. + self.__raise_not_implemented() + ########### # Utilities diff --git a/mediagoblin/storage/cloudfiles.py b/mediagoblin/storage/cloudfiles.py index 250f06d4..47c81ad6 100644 --- a/mediagoblin/storage/cloudfiles.py +++ b/mediagoblin/storage/cloudfiles.py @@ -168,6 +168,12 @@ class CloudFilesStorage(StorageInterface): # Copy to storage system in 4096 byte chunks dest_file.send(source_file) + def get_file_size(self, filepath): + """Returns the file size in bytes""" + obj = self.container.get_object( + self._resolve_filepath(filepath)) + return obj.total_bytes + class CloudFilesStorageObjectWrapper(): """ Wrapper for python-cloudfiles's cloudfiles.storage_object.Object diff --git a/mediagoblin/storage/filestorage.py b/mediagoblin/storage/filestorage.py index 3d6e0753..29b8383b 100644 --- a/mediagoblin/storage/filestorage.py +++ b/mediagoblin/storage/filestorage.py @@ -111,3 +111,6 @@ class BasicFileStorage(StorageInterface): os.makedirs(directory) # This uses chunked copying of 16kb buffers (Py2.7): shutil.copy(filename, self.get_local_path(filepath)) + + def get_file_size(self, filepath): + return os.stat(self._resolve_filepath(filepath)).st_size diff --git a/mediagoblin/submit/views.py b/mediagoblin/submit/views.py index 6bb95ecb..d5f05cb7 100644 --- a/mediagoblin/submit/views.py +++ b/mediagoblin/submit/views.py @@ -43,6 +43,20 @@ def submit_start(request): """ First view for submitting a file. """ + user = request.user + if user.upload_limit: + upload_limit = user.upload_limit + else: + upload_limit = mg_globals.app_config['upload_limit'] + + if user.uploaded >= upload_limit: + messages.add_message( + request, + messages.WARNING, + _('Sorry, you have reached your upload limit.')) + return redirect( + request, '/u/{0}'.format(user.username)) + submit_form = submit_forms.SubmitStartForm(request.form, license=request.user.license_preference) @@ -86,6 +100,26 @@ def submit_start(request): with queue_file: queue_file.write(request.files['file'].stream.read()) + # Get file size an round to 2 decimal places + file_size = request.app.queue_store.get_file_size( + entry.queued_media_file) / (1024.0 * 1024) + file_size = float('{0:.2f}'.format(file_size)) + + # Check if over upload limit + if (user.uploaded + file_size) >= upload_limit: + messages.add_message( + request, + messages.WARNING, + _('Sorry, uploading this file will put you over your' + ' upload limit.')) + return redirect( + request, '/u/{0}'.format(user.username)) + + user.uploaded = user.uploaded + file_size + user.save() + + entry.file_size = file_size + # Save now so we have this data before kicking off processing entry.save() diff --git a/mediagoblin/user_pages/views.py b/mediagoblin/user_pages/views.py index 49691a29..3b411fe0 100644 --- a/mediagoblin/user_pages/views.py +++ b/mediagoblin/user_pages/views.py @@ -296,6 +296,10 @@ def media_confirm_delete(request, media): if request.method == 'POST' and form.validate(): if form.confirm.data is True: username = media.get_uploader.username + + request.user.uploaded = request.user.uploaded - media.file_size + request.user.save() + # Delete MediaEntry and all related files, comments etc. media.delete() messages.add_message( -- cgit v1.2.3 From c3cce7564a0a949c2835a948e0203e489063dfbd Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Wed, 12 Jun 2013 13:56:43 -0700 Subject: added tests --- mediagoblin/submit/views.py | 14 ++++---- mediagoblin/tests/test_submission.py | 68 +++++++++++++++++++++++++++++++++++- 2 files changed, 74 insertions(+), 8 deletions(-) (limited to 'mediagoblin') diff --git a/mediagoblin/submit/views.py b/mediagoblin/submit/views.py index d5f05cb7..2b76abf7 100644 --- a/mediagoblin/submit/views.py +++ b/mediagoblin/submit/views.py @@ -44,7 +44,7 @@ def submit_start(request): First view for submitting a file. """ user = request.user - if user.upload_limit: + if user.upload_limit >= 0: upload_limit = user.upload_limit else: upload_limit = mg_globals.app_config['upload_limit'] @@ -54,8 +54,8 @@ def submit_start(request): request, messages.WARNING, _('Sorry, you have reached your upload limit.')) - return redirect( - request, '/u/{0}'.format(user.username)) + return redirect(request, "mediagoblin.user_pages.user_home", + user=request.user.username) submit_form = submit_forms.SubmitStartForm(request.form, license=request.user.license_preference) @@ -105,15 +105,15 @@ def submit_start(request): entry.queued_media_file) / (1024.0 * 1024) file_size = float('{0:.2f}'.format(file_size)) - # Check if over upload limit + # Check if user is over upload limit if (user.uploaded + file_size) >= upload_limit: messages.add_message( request, messages.WARNING, _('Sorry, uploading this file will put you over your' ' upload limit.')) - return redirect( - request, '/u/{0}'.format(user.username)) + return redirect(request, "mediagoblin.user_pages.user_home", + user=user.username) user.uploaded = user.uploaded + file_size user.save() @@ -137,7 +137,7 @@ def submit_start(request): add_comment_subscription(request.user, entry) return redirect(request, "mediagoblin.user_pages.user_home", - user=request.user.username) + user=user.username) except Exception as e: ''' This section is intended to catch exceptions raised in diff --git a/mediagoblin/tests/test_submission.py b/mediagoblin/tests/test_submission.py index ac941063..fdc322dc 100644 --- a/mediagoblin/tests/test_submission.py +++ b/mediagoblin/tests/test_submission.py @@ -24,7 +24,8 @@ import pytest from mediagoblin.tests.tools import fixture_add_user from mediagoblin import mg_globals -from mediagoblin.db.models import MediaEntry +from mediagoblin.db.models import MediaEntry, User +from mediagoblin.db.base import Session from mediagoblin.tools import template from mediagoblin.media_types.image import ImageMediaManager from mediagoblin.media_types.pdf.processing import check_prerequisites as pdf_check_prerequisites @@ -107,9 +108,38 @@ class TestSubmission: self.logout() self.test_app.get(url) + def user_upload_limits(self, uploaded=None, upload_limit=None): + if uploaded: + self.test_user.uploaded = uploaded + if upload_limit: + self.test_user.upload_limit = upload_limit + + self.test_user.save() + + # Reload + self.test_user = User.query.filter_by( + username=self.test_user.username + ).first() + + # ... and detach from session: + Session.expunge(self.test_user) + def test_normal_jpg(self): + # User uploaded should be 0 + assert self.test_user.uploaded == 0 + self.check_normal_upload(u'Normal upload 1', GOOD_JPG) + # User uploaded should be the same as GOOD_JPG size in Mb + file_size = os.stat(GOOD_JPG).st_size / (1024.0 * 1024) + file_size = float('{0:.2f}'.format(file_size)) + + # Reload user + self.test_user = User.query.filter_by( + username=self.test_user.username + ).first() + assert self.test_user.uploaded == file_size + def test_normal_png(self): self.check_normal_upload(u'Normal upload 2', GOOD_PNG) @@ -121,6 +151,33 @@ class TestSubmission: self.check_url(response, '/u/{0}/'.format(self.test_user.username)) assert 'mediagoblin/user_pages/user.html' in context + def test_default_upload_limits(self): + self.user_upload_limits(uploaded=500) + + response, context = self.do_post({'title': u'Normal upload 4'}, + do_follow=True, + **self.upload_data(GOOD_JPG)) + self.check_url(response, '/u/{0}/'.format(self.test_user.username)) + assert 'mediagoblin/user_pages/user.html' in context + + def test_user_upload_limit(self): + self.user_upload_limits(uploaded=25, upload_limit=25) + + response, context = self.do_post({'title': u'Normal upload 4'}, + do_follow=True, + **self.upload_data(GOOD_JPG)) + self.check_url(response, '/u/{0}/'.format(self.test_user.username)) + assert 'mediagoblin/user_pages/user.html' in context + + def test_user_under_limit(self): + self.user_upload_limits(uploaded=499) + + response, context = self.do_post({'title': u'Normal upload 4'}, + do_follow=True, + **self.upload_data(GOOD_JPG)) + self.check_url(response, '/u/{0}/'.format(self.test_user.username)) + assert 'mediagoblin/user_pages/user.html' in context + def check_media(self, request, find_data, count=None): media = MediaEntry.query.filter_by(**find_data) if count is not None: @@ -155,6 +212,7 @@ class TestSubmission: 'ffffffffffffffffffffffffffuuuuuuuuuuuuuuuuuuuuuuuuuu'] def test_delete(self): + self.user_upload_limits(uploaded=50) response, request = self.do_post({'title': u'Balanced Goblin'}, *REQUEST_CONTEXT, do_follow=True, **self.upload_data(GOOD_JPG)) @@ -199,6 +257,14 @@ class TestSubmission: self.check_media(request, {'id': media_id}, 0) self.check_comments(request, media_id, 0) + # Reload user + self.test_user = User.query.filter_by( + username = self.test_user.username + ).first() + + # Check that user.uploaded is the same as before the upload + assert self.test_user.uploaded == 50 + def test_evil_file(self): # Test non-suppoerted file with non-supported extension # ----------------------------------------------------- -- cgit v1.2.3 From 150bee3f8fd5dff59843610982592dfa0f6e411a Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Wed, 12 Jun 2013 14:03:09 -0700 Subject: typos --- mediagoblin/config_spec.ini | 2 +- mediagoblin/submit/views.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'mediagoblin') diff --git a/mediagoblin/config_spec.ini b/mediagoblin/config_spec.ini index 596e4388..1b9945c5 100644 --- a/mediagoblin/config_spec.ini +++ b/mediagoblin/config_spec.ini @@ -75,7 +75,7 @@ theme = string() plugin_web_path = string(default="/plugin_static/") plugin_linked_assets_dir = string(default="%(here)s/user_dev/plugin_static/") -# User upload limit (in Mbs) +# User upload limit (in Mb) upload_limit = integer(default=500) [jinja2] diff --git a/mediagoblin/submit/views.py b/mediagoblin/submit/views.py index 2b76abf7..cba9c8b4 100644 --- a/mediagoblin/submit/views.py +++ b/mediagoblin/submit/views.py @@ -112,7 +112,7 @@ def submit_start(request): messages.WARNING, _('Sorry, uploading this file will put you over your' ' upload limit.')) - return redirect(request, "mediagoblin.user_pages.user_home", + return redirect(request, "mediagoblin.submit.start", user=user.username) user.uploaded = user.uploaded + file_size -- cgit v1.2.3 From 91a52878cfca2187943481e6fc1d376fb65cae00 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 13 Jun 2013 16:41:13 -0700 Subject: made no upload limit the default --- mediagoblin/config_spec.ini | 4 ++-- mediagoblin/submit/views.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'mediagoblin') diff --git a/mediagoblin/config_spec.ini b/mediagoblin/config_spec.ini index 1b9945c5..acefa5cc 100644 --- a/mediagoblin/config_spec.ini +++ b/mediagoblin/config_spec.ini @@ -75,8 +75,8 @@ theme = string() plugin_web_path = string(default="/plugin_static/") plugin_linked_assets_dir = string(default="%(here)s/user_dev/plugin_static/") -# User upload limit (in Mb) -upload_limit = integer(default=500) +# Default user upload limit (in Mb) +#upload_limit = integer(default=500) [jinja2] # Jinja2 supports more directives than the minimum required by mediagoblin. diff --git a/mediagoblin/submit/views.py b/mediagoblin/submit/views.py index cba9c8b4..1a67486b 100644 --- a/mediagoblin/submit/views.py +++ b/mediagoblin/submit/views.py @@ -47,9 +47,9 @@ def submit_start(request): if user.upload_limit >= 0: upload_limit = user.upload_limit else: - upload_limit = mg_globals.app_config['upload_limit'] + upload_limit = mg_globals.app_config.get('upload_limit', None) - if user.uploaded >= upload_limit: + if upload_limit and user.uploaded >= upload_limit: messages.add_message( request, messages.WARNING, @@ -106,7 +106,7 @@ def submit_start(request): file_size = float('{0:.2f}'.format(file_size)) # Check if user is over upload limit - if (user.uploaded + file_size) >= upload_limit: + if upload_limit and (user.uploaded + file_size) >= upload_limit: messages.add_message( request, messages.WARNING, -- cgit v1.2.3 From 001a50a85085a9cd6c713a7934f16c9b69b8760b Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 13 Jun 2013 17:17:33 -0700 Subject: fixed tests and defaults --- mediagoblin/config_spec.ini | 2 +- mediagoblin/submit/views.py | 4 +-- mediagoblin/tests/resources.py | 1 + mediagoblin/tests/test_mgoblin_app.ini | 2 ++ mediagoblin/tests/test_submission.py | 46 +++++++++++++++++++++++---- mediagoblin/tests/test_submission/medium.png | Bin 0 -> 1796336 bytes 6 files changed, 45 insertions(+), 10 deletions(-) create mode 100644 mediagoblin/tests/test_submission/medium.png (limited to 'mediagoblin') diff --git a/mediagoblin/config_spec.ini b/mediagoblin/config_spec.ini index acefa5cc..be6132f6 100644 --- a/mediagoblin/config_spec.ini +++ b/mediagoblin/config_spec.ini @@ -76,7 +76,7 @@ plugin_web_path = string(default="/plugin_static/") plugin_linked_assets_dir = string(default="%(here)s/user_dev/plugin_static/") # Default user upload limit (in Mb) -#upload_limit = integer(default=500) +upload_limit = integer(default=None) [jinja2] # Jinja2 supports more directives than the minimum required by mediagoblin. diff --git a/mediagoblin/submit/views.py b/mediagoblin/submit/views.py index 1a67486b..7382c43d 100644 --- a/mediagoblin/submit/views.py +++ b/mediagoblin/submit/views.py @@ -107,9 +107,7 @@ def submit_start(request): # Check if user is over upload limit if upload_limit and (user.uploaded + file_size) >= upload_limit: - messages.add_message( - request, - messages.WARNING, + submit_form.file.errors.append( _('Sorry, uploading this file will put you over your' ' upload limit.')) return redirect(request, "mediagoblin.submit.start", diff --git a/mediagoblin/tests/resources.py b/mediagoblin/tests/resources.py index f7b3037d..4260df93 100644 --- a/mediagoblin/tests/resources.py +++ b/mediagoblin/tests/resources.py @@ -29,6 +29,7 @@ EVIL_JPG = resource('evil.jpg') EVIL_PNG = resource('evil.png') BIG_BLUE = resource('bigblue.png') GOOD_PDF = resource('good.pdf') +MED_PNG = resource('medium.png') def resource_exif(f): diff --git a/mediagoblin/tests/test_mgoblin_app.ini b/mediagoblin/tests/test_mgoblin_app.ini index da0dffb9..a2855d7a 100644 --- a/mediagoblin/tests/test_mgoblin_app.ini +++ b/mediagoblin/tests/test_mgoblin_app.ini @@ -13,6 +13,8 @@ tags_max_length = 50 # So we can start to test attachments: allow_attachments = True +upload_limit = 500 + [storage:publicstore] base_dir = %(here)s/user_dev/media/public base_url = /mgoblin_media/ diff --git a/mediagoblin/tests/test_submission.py b/mediagoblin/tests/test_submission.py index fdc322dc..e45ed36b 100644 --- a/mediagoblin/tests/test_submission.py +++ b/mediagoblin/tests/test_submission.py @@ -31,7 +31,7 @@ from mediagoblin.media_types.image import ImageMediaManager from mediagoblin.media_types.pdf.processing import check_prerequisites as pdf_check_prerequisites from .resources import GOOD_JPG, GOOD_PNG, EVIL_FILE, EVIL_JPG, EVIL_PNG, \ - BIG_BLUE, GOOD_PDF, GPS_JPG + BIG_BLUE, GOOD_PDF, GPS_JPG, MED_PNG GOOD_TAG_STRING = u'yin,yang' BAD_TAG_STRING = unicode('rage,' + 'f' * 26 + 'u' * 26) @@ -154,29 +154,63 @@ class TestSubmission: def test_default_upload_limits(self): self.user_upload_limits(uploaded=500) + # User uploaded should be 500 + assert self.test_user.uploaded == 500 + response, context = self.do_post({'title': u'Normal upload 4'}, do_follow=True, **self.upload_data(GOOD_JPG)) self.check_url(response, '/u/{0}/'.format(self.test_user.username)) assert 'mediagoblin/user_pages/user.html' in context + # Reload user + self.test_user = User.query.filter_by( + username=self.test_user.username + ).first() + + # Shouldn't have uploaded + assert self.test_user.uploaded == 500 + def test_user_upload_limit(self): self.user_upload_limits(uploaded=25, upload_limit=25) + # User uploaded should be 25 + assert self.test_user.uploaded == 25 + response, context = self.do_post({'title': u'Normal upload 4'}, do_follow=True, **self.upload_data(GOOD_JPG)) self.check_url(response, '/u/{0}/'.format(self.test_user.username)) assert 'mediagoblin/user_pages/user.html' in context + # Reload user + self.test_user = User.query.filter_by( + username=self.test_user.username + ).first() + + # Shouldn't have uploaded + assert self.test_user.uploaded == 25 + def test_user_under_limit(self): self.user_upload_limits(uploaded=499) - response, context = self.do_post({'title': u'Normal upload 4'}, - do_follow=True, - **self.upload_data(GOOD_JPG)) - self.check_url(response, '/u/{0}/'.format(self.test_user.username)) - assert 'mediagoblin/user_pages/user.html' in context + # User uploaded should be 499 + assert self.test_user.uploaded == 499 + + response, context = self.do_post({'title': u'Normal upload 6'}, + do_follow=False, + **self.upload_data(MED_PNG)) + form = context['mediagoblin/submit/start.html']['submit_form'] + assert form.file.errors == [u'Sorry, uploading this file will put you' + ' over your upload limit.'] + + # Reload user + self.test_user = User.query.filter_by( + username=self.test_user.username + ).first() + + # Shouldn't have uploaded + assert self.test_user.uploaded == 499 def check_media(self, request, find_data, count=None): media = MediaEntry.query.filter_by(**find_data) diff --git a/mediagoblin/tests/test_submission/medium.png b/mediagoblin/tests/test_submission/medium.png new file mode 100644 index 00000000..e8b9ca00 Binary files /dev/null and b/mediagoblin/tests/test_submission/medium.png differ -- cgit v1.2.3 From cc4457f4075e61ea62c422ada24901a6b8f6992d Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 13 Jun 2013 17:17:33 -0700 Subject: fixed tests and defaults --- mediagoblin/tests/test_submission.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'mediagoblin') diff --git a/mediagoblin/tests/test_submission.py b/mediagoblin/tests/test_submission.py index e45ed36b..f6ecd58a 100644 --- a/mediagoblin/tests/test_submission.py +++ b/mediagoblin/tests/test_submission.py @@ -212,6 +212,14 @@ class TestSubmission: # Shouldn't have uploaded assert self.test_user.uploaded == 499 + # Reload user + self.test_user = User.query.filter_by( + username=self.test_user.username + ).first() + + # Shouldn't have uploaded + assert self.test_user.uploaded == 499 + def check_media(self, request, find_data, count=None): media = MediaEntry.query.filter_by(**find_data) if count is not None: -- cgit v1.2.3 From 2f74de492e293fa5ba82bb65d291ba51834d5d85 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 13 Jun 2013 18:10:38 -0700 Subject: fixed tests --- mediagoblin/tests/test_submission.py | 8 -------- mediagoblin/tests/test_submission/big.png | Bin 0 -> 2212445 bytes 2 files changed, 8 deletions(-) create mode 100644 mediagoblin/tests/test_submission/big.png (limited to 'mediagoblin') diff --git a/mediagoblin/tests/test_submission.py b/mediagoblin/tests/test_submission.py index f6ecd58a..e45ed36b 100644 --- a/mediagoblin/tests/test_submission.py +++ b/mediagoblin/tests/test_submission.py @@ -212,14 +212,6 @@ class TestSubmission: # Shouldn't have uploaded assert self.test_user.uploaded == 499 - # Reload user - self.test_user = User.query.filter_by( - username=self.test_user.username - ).first() - - # Shouldn't have uploaded - assert self.test_user.uploaded == 499 - def check_media(self, request, find_data, count=None): media = MediaEntry.query.filter_by(**find_data) if count is not None: diff --git a/mediagoblin/tests/test_submission/big.png b/mediagoblin/tests/test_submission/big.png new file mode 100644 index 00000000..a284cfda Binary files /dev/null and b/mediagoblin/tests/test_submission/big.png differ -- cgit v1.2.3 From ecb45128229cfe22e29589237bdd59b7cc965181 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 13 Jun 2013 16:37:42 -0700 Subject: max file size --- mediagoblin/config_spec.ini | 3 +++ mediagoblin/submit/forms.py | 49 +++++++++++++++++++++++++----------------- mediagoblin/submit/views.py | 52 ++++++++++++++++++++++++++------------------- 3 files changed, 63 insertions(+), 41 deletions(-) (limited to 'mediagoblin') diff --git a/mediagoblin/config_spec.ini b/mediagoblin/config_spec.ini index be6132f6..535af4c1 100644 --- a/mediagoblin/config_spec.ini +++ b/mediagoblin/config_spec.ini @@ -78,6 +78,9 @@ plugin_linked_assets_dir = string(default="%(here)s/user_dev/plugin_static/") # Default user upload limit (in Mb) upload_limit = integer(default=None) +# Max file size (in Mb) +max_file_size = integer(default=5000) + [jinja2] # Jinja2 supports more directives than the minimum required by mediagoblin. # This setting allows users creating custom templates to specify a list of diff --git a/mediagoblin/submit/forms.py b/mediagoblin/submit/forms.py index e9bd93fd..0a3b4e4d 100644 --- a/mediagoblin/submit/forms.py +++ b/mediagoblin/submit/forms.py @@ -17,30 +17,41 @@ import wtforms +from mediagoblin import mg_globals from mediagoblin.tools.text import tag_length_validator from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ from mediagoblin.tools.licenses import licenses_as_choices -class SubmitStartForm(wtforms.Form): - file = wtforms.FileField(_('File')) - title = wtforms.TextField( - _('Title'), - [wtforms.validators.Length(min=0, max=500)]) - description = wtforms.TextAreaField( - _('Description of this work'), - description=_("""You can use - - Markdown for formatting.""")) - tags = wtforms.TextField( - _('Tags'), - [tag_length_validator], - description=_( - "Separate tags by commas.")) - license = wtforms.SelectField( - _('License'), - [wtforms.validators.Optional(),], - choices=licenses_as_choices()) +def get_submit_start_form(form, **kwargs): + max_file_size = mg_globals.app_config.get('max_file_size', None) + desc = None + if max_file_size: + desc = _('Max file size: {0} mb'.format(max_file_size)) + + class SubmitStartForm(wtforms.Form): + file = wtforms.FileField( + _('File'), + description=desc) + title = wtforms.TextField( + _('Title'), + [wtforms.validators.Length(min=0, max=500)]) + description = wtforms.TextAreaField( + _('Description of this work'), + description=_("""You can use + + Markdown for formatting.""")) + tags = wtforms.TextField( + _('Tags'), + [tag_length_validator], + description=_( + "Separate tags by commas.")) + license = wtforms.SelectField( + _('License'), + [wtforms.validators.Optional(),], + choices=licenses_as_choices()) + + return SubmitStartForm(form, **kwargs) class AddCollectionForm(wtforms.Form): title = wtforms.TextField( diff --git a/mediagoblin/submit/views.py b/mediagoblin/submit/views.py index 7382c43d..7608f509 100644 --- a/mediagoblin/submit/views.py +++ b/mediagoblin/submit/views.py @@ -57,7 +57,7 @@ def submit_start(request): return redirect(request, "mediagoblin.user_pages.user_home", user=request.user.username) - submit_form = submit_forms.SubmitStartForm(request.form, + submit_form = submit_forms.get_submit_start_form(request.form, license=request.user.license_preference) if request.method == 'POST' and submit_form.validate(): @@ -105,32 +105,40 @@ def submit_start(request): entry.queued_media_file) / (1024.0 * 1024) file_size = float('{0:.2f}'.format(file_size)) + error = False + + # Check if file size is over the limit + max_file_size = mg_globals.app_config.get('max_file_size', None) + if max_file_size and file_size >= max_file_size: + submit_form.file.errors.append( + _(u'Sorry, the file size is too big.')) + error = True + # Check if user is over upload limit if upload_limit and (user.uploaded + file_size) >= upload_limit: submit_form.file.errors.append( _('Sorry, uploading this file will put you over your' ' upload limit.')) - return redirect(request, "mediagoblin.submit.start", - user=user.username) - - user.uploaded = user.uploaded + file_size - user.save() - - entry.file_size = file_size - - # Save now so we have this data before kicking off processing - entry.save() - - # Pass off to async processing - # - # (... don't change entry after this point to avoid race - # conditions with changes to the document via processing code) - feed_url = request.urlgen( - 'mediagoblin.user_pages.atom_feed', - qualified=True, user=request.user.username) - run_process_media(entry, feed_url) - - add_message(request, SUCCESS, _('Woohoo! Submitted!')) + error = True + + if not error: + user.uploaded = user.uploaded + file_size + user.save() + + entry.file_size = file_size + + # Save now so we have this data before kicking off processing + entry.save() + + # Pass off to processing + # + # (... don't change entry after this point to avoid race + # conditions with changes to the document via processing code) + feed_url = request.urlgen( + 'mediagoblin.user_pages.atom_feed', + qualified=True, user=request.user.username) + run_process_media(entry, feed_url) + add_message(request, SUCCESS, _('Woohoo! Submitted!')) add_comment_subscription(request.user, entry) -- cgit v1.2.3 From 1cb84a3632cbf67bf9be41fcbcc7fe1d30cbf52c Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 13 Jun 2013 17:17:33 -0700 Subject: fixed tests and defaults --- mediagoblin/tests/test_submission.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'mediagoblin') diff --git a/mediagoblin/tests/test_submission.py b/mediagoblin/tests/test_submission.py index e45ed36b..f6ecd58a 100644 --- a/mediagoblin/tests/test_submission.py +++ b/mediagoblin/tests/test_submission.py @@ -212,6 +212,14 @@ class TestSubmission: # Shouldn't have uploaded assert self.test_user.uploaded == 499 + # Reload user + self.test_user = User.query.filter_by( + username=self.test_user.username + ).first() + + # Shouldn't have uploaded + assert self.test_user.uploaded == 499 + def check_media(self, request, find_data, count=None): media = MediaEntry.query.filter_by(**find_data) if count is not None: -- cgit v1.2.3 From 53cf5b4556d7327e67fdea1f91f8f349433289a0 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 13 Jun 2013 17:52:13 -0700 Subject: added tests --- mediagoblin/config_spec.ini | 2 +- mediagoblin/tests/resources.py | 2 ++ mediagoblin/tests/test_mgoblin_app.ini | 2 ++ mediagoblin/tests/test_submission.py | 16 ++++++++-------- 4 files changed, 13 insertions(+), 9 deletions(-) (limited to 'mediagoblin') diff --git a/mediagoblin/config_spec.ini b/mediagoblin/config_spec.ini index 535af4c1..1e4ee083 100644 --- a/mediagoblin/config_spec.ini +++ b/mediagoblin/config_spec.ini @@ -79,7 +79,7 @@ plugin_linked_assets_dir = string(default="%(here)s/user_dev/plugin_static/") upload_limit = integer(default=None) # Max file size (in Mb) -max_file_size = integer(default=5000) +max_file_size = integer(default=None) [jinja2] # Jinja2 supports more directives than the minimum required by mediagoblin. diff --git a/mediagoblin/tests/resources.py b/mediagoblin/tests/resources.py index 4260df93..63789a2e 100644 --- a/mediagoblin/tests/resources.py +++ b/mediagoblin/tests/resources.py @@ -30,6 +30,8 @@ EVIL_PNG = resource('evil.png') BIG_BLUE = resource('bigblue.png') GOOD_PDF = resource('good.pdf') MED_PNG = resource('medium.png') +BIG_PNG = resource('big.png') +MED_PNG = resource('medium.png') def resource_exif(f): diff --git a/mediagoblin/tests/test_mgoblin_app.ini b/mediagoblin/tests/test_mgoblin_app.ini index a2855d7a..4cd3d9b6 100644 --- a/mediagoblin/tests/test_mgoblin_app.ini +++ b/mediagoblin/tests/test_mgoblin_app.ini @@ -15,6 +15,8 @@ allow_attachments = True upload_limit = 500 +max_file_size = 2 + [storage:publicstore] base_dir = %(here)s/user_dev/media/public base_url = /mgoblin_media/ diff --git a/mediagoblin/tests/test_submission.py b/mediagoblin/tests/test_submission.py index f6ecd58a..7f4e8086 100644 --- a/mediagoblin/tests/test_submission.py +++ b/mediagoblin/tests/test_submission.py @@ -31,7 +31,7 @@ from mediagoblin.media_types.image import ImageMediaManager from mediagoblin.media_types.pdf.processing import check_prerequisites as pdf_check_prerequisites from .resources import GOOD_JPG, GOOD_PNG, EVIL_FILE, EVIL_JPG, EVIL_PNG, \ - BIG_BLUE, GOOD_PDF, GPS_JPG, MED_PNG + BIG_BLUE, GOOD_PDF, GPS_JPG, MED_PNG, BIG_PNG GOOD_TAG_STRING = u'yin,yang' BAD_TAG_STRING = unicode('rage,' + 'f' * 26 + 'u' * 26) @@ -177,7 +177,7 @@ class TestSubmission: # User uploaded should be 25 assert self.test_user.uploaded == 25 - response, context = self.do_post({'title': u'Normal upload 4'}, + response, context = self.do_post({'title': u'Normal upload 5'}, do_follow=True, **self.upload_data(GOOD_JPG)) self.check_url(response, '/u/{0}/'.format(self.test_user.username)) @@ -212,13 +212,13 @@ class TestSubmission: # Shouldn't have uploaded assert self.test_user.uploaded == 499 - # Reload user - self.test_user = User.query.filter_by( - username=self.test_user.username - ).first() + def test_big_file(self): + response, context = self.do_post({'title': u'Normal upload 7'}, + do_follow=False, + **self.upload_data(BIG_PNG)) - # Shouldn't have uploaded - assert self.test_user.uploaded == 499 + form = context['mediagoblin/submit/start.html']['submit_form'] + assert form.file.errors == [u'Sorry, the file size is too big.'] def check_media(self, request, find_data, count=None): media = MediaEntry.query.filter_by(**find_data) -- cgit v1.2.3 From 7707172e397d4c2be7a19aa300865ba3c5a07d2d Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 13 Jun 2013 18:28:38 -0700 Subject: copying.txt --- mediagoblin/tests/test_submission/COPYING.txt | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 mediagoblin/tests/test_submission/COPYING.txt (limited to 'mediagoblin') diff --git a/mediagoblin/tests/test_submission/COPYING.txt b/mediagoblin/tests/test_submission/COPYING.txt new file mode 100644 index 00000000..3818aae4 --- /dev/null +++ b/mediagoblin/tests/test_submission/COPYING.txt @@ -0,0 +1,5 @@ +Images located in this directory tree are released under a GPLv3 license +and CC BY-SA 3.0 license. To the extent possible under law, the author(s) +have dedicated all copyright and related and neighboring rights to these +files to the public domain worldwide. These files are distributed without +any warranty. -- cgit v1.2.3 From e1ad18cc6deefa1552038e9908d09243a97528f0 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 13 Jun 2013 18:34:42 -0700 Subject: duplicate --- mediagoblin/tests/resources.py | 1 - 1 file changed, 1 deletion(-) (limited to 'mediagoblin') diff --git a/mediagoblin/tests/resources.py b/mediagoblin/tests/resources.py index 63789a2e..480f6d9a 100644 --- a/mediagoblin/tests/resources.py +++ b/mediagoblin/tests/resources.py @@ -31,7 +31,6 @@ BIG_BLUE = resource('bigblue.png') GOOD_PDF = resource('good.pdf') MED_PNG = resource('medium.png') BIG_PNG = resource('big.png') -MED_PNG = resource('medium.png') def resource_exif(f): -- cgit v1.2.3 From a80ea747027d895b8c6951130fbbb7a166e9162c Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 27 Jun 2013 10:29:24 -0700 Subject: fix after rebase --- mediagoblin/submit/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'mediagoblin') diff --git a/mediagoblin/submit/views.py b/mediagoblin/submit/views.py index 7608f509..d07065a6 100644 --- a/mediagoblin/submit/views.py +++ b/mediagoblin/submit/views.py @@ -140,9 +140,9 @@ def submit_start(request): run_process_media(entry, feed_url) add_message(request, SUCCESS, _('Woohoo! Submitted!')) - add_comment_subscription(request.user, entry) + add_comment_subscription(request.user, entry) - return redirect(request, "mediagoblin.user_pages.user_home", + return redirect(request, "mediagoblin.user_pages.user_home", user=user.username) except Exception as e: ''' -- cgit v1.2.3 From 2188925bab741bcb45bab13261c5ca89719b7a04 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 27 Jun 2013 11:18:31 -0700 Subject: javascript limit validation --- mediagoblin/static/js/file_size.js | 45 ++++++++++++++++++++++ mediagoblin/submit/forms.py | 5 ++- mediagoblin/submit/views.py | 11 ++++-- .../templates/mediagoblin/submit/start.html | 5 +++ 4 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 mediagoblin/static/js/file_size.js (limited to 'mediagoblin') diff --git a/mediagoblin/static/js/file_size.js b/mediagoblin/static/js/file_size.js new file mode 100644 index 00000000..2238ef89 --- /dev/null +++ b/mediagoblin/static/js/file_size.js @@ -0,0 +1,45 @@ +/** + * GNU MediaGoblin -- federated, autonomous media hosting + * Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +$(document).ready(function(){ + var file = document.getElementById('file'); + var uploaded = parseInt(document.getElementById('uploaded').value); + var upload_limit = parseInt(document.getElementById('upload_limit').value); + var max_file_size = parseInt(document.getElementById('max_file_size').value); + + file.onchange = function() { + var file_size = file.files[0].size / (1024.0 * 1024); + + if (file_size >= max_file_size) { + $('#file').after('

Sorry, the file size is too big.

'); + } + else if (document.getElementById('file_size_error')) { + $('#file_size_error').hide(); + } + + if (upload_limit) { + if ( uploaded + file_size >= upload_limit) { + $('#file').after('

Sorry, uploading this file will put you over your upload limit.

'); + } + else if (document.getElementById('upload_limit_error')) { + $('#upload_limit_error').hide(); + console.log(file_size >= max_file_size); + } + } + }; +}); diff --git a/mediagoblin/submit/forms.py b/mediagoblin/submit/forms.py index 0a3b4e4d..e2264645 100644 --- a/mediagoblin/submit/forms.py +++ b/mediagoblin/submit/forms.py @@ -24,7 +24,7 @@ from mediagoblin.tools.licenses import licenses_as_choices def get_submit_start_form(form, **kwargs): - max_file_size = mg_globals.app_config.get('max_file_size', None) + max_file_size = kwargs.get('max_file_size') desc = None if max_file_size: desc = _('Max file size: {0} mb'.format(max_file_size)) @@ -50,6 +50,9 @@ def get_submit_start_form(form, **kwargs): _('License'), [wtforms.validators.Optional(),], choices=licenses_as_choices()) + max_file_size = wtforms.HiddenField('') + upload_limit = wtforms.HiddenField('') + uploaded = wtforms.HiddenField('') return SubmitStartForm(form, **kwargs) diff --git a/mediagoblin/submit/views.py b/mediagoblin/submit/views.py index d07065a6..3b853697 100644 --- a/mediagoblin/submit/views.py +++ b/mediagoblin/submit/views.py @@ -57,8 +57,14 @@ def submit_start(request): return redirect(request, "mediagoblin.user_pages.user_home", user=request.user.username) - submit_form = submit_forms.get_submit_start_form(request.form, - license=request.user.license_preference) + max_file_size = mg_globals.app_config.get('max_file_size', None) + + submit_form = submit_forms.get_submit_start_form( + request.form, + license=request.user.license_preference, + max_file_size=max_file_size, + upload_limit=upload_limit, + uploaded=user.uploaded) if request.method == 'POST' and submit_form.validate(): if not check_file_field(request, 'file'): @@ -108,7 +114,6 @@ def submit_start(request): error = False # Check if file size is over the limit - max_file_size = mg_globals.app_config.get('max_file_size', None) if max_file_size and file_size >= max_file_size: submit_form.file.errors.append( _(u'Sorry, the file size is too big.')) diff --git a/mediagoblin/templates/mediagoblin/submit/start.html b/mediagoblin/templates/mediagoblin/submit/start.html index aa390f56..d335d742 100644 --- a/mediagoblin/templates/mediagoblin/submit/start.html +++ b/mediagoblin/templates/mediagoblin/submit/start.html @@ -19,6 +19,11 @@ {% import "/mediagoblin/utils/wtforms.html" as wtforms_util %} +{% block mediagoblin_head %} + +{% endblock %} + {% block title -%} {% trans %}Add your media{% endtrans %} — {{ super() }} {%- endblock %} -- cgit v1.2.3 From 0b95003cd4055745494cf10c89ccbfa6ac04c0e4 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Sun, 30 Jun 2013 20:51:23 -0700 Subject: use media.get_uploader and fix typo --- mediagoblin/submit/views.py | 2 +- mediagoblin/user_pages/views.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'mediagoblin') diff --git a/mediagoblin/submit/views.py b/mediagoblin/submit/views.py index 3b853697..7f7dee33 100644 --- a/mediagoblin/submit/views.py +++ b/mediagoblin/submit/views.py @@ -106,7 +106,7 @@ def submit_start(request): with queue_file: queue_file.write(request.files['file'].stream.read()) - # Get file size an round to 2 decimal places + # Get file size and round to 2 decimal places file_size = request.app.queue_store.get_file_size( entry.queued_media_file) / (1024.0 * 1024) file_size = float('{0:.2f}'.format(file_size)) diff --git a/mediagoblin/user_pages/views.py b/mediagoblin/user_pages/views.py index 3b411fe0..5eac0fe4 100644 --- a/mediagoblin/user_pages/views.py +++ b/mediagoblin/user_pages/views.py @@ -297,8 +297,9 @@ def media_confirm_delete(request, media): if form.confirm.data is True: username = media.get_uploader.username - request.user.uploaded = request.user.uploaded - media.file_size - request.user.save() + media.get_uploader.uploaded = media.get_uploader.uploaded - \ + media.file_size + media.get_uploader.save() # Delete MediaEntry and all related files, comments etc. media.delete() -- cgit v1.2.3 From 1fef79f4f884c9e534b6f892cd2e448fc5e483eb Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Fri, 13 Sep 2013 10:16:07 -0500 Subject: Fix pagination for certain request.GET data This didn't work at all nicely with MultiDict objects in various circumstances and could possibly break pagination. This fix handles that! This commit sponsored by Alessandro Francolini. Thank you! --- mediagoblin/tools/pagination.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'mediagoblin') diff --git a/mediagoblin/tools/pagination.py b/mediagoblin/tools/pagination.py index d0f08c94..855878e0 100644 --- a/mediagoblin/tools/pagination.py +++ b/mediagoblin/tools/pagination.py @@ -18,7 +18,7 @@ import urllib import copy from math import ceil, floor from itertools import izip, count - +from werkzeug.datastructures import MultiDict PAGINATION_DEFAULT_PER_PAGE = 30 @@ -98,7 +98,11 @@ class Pagination(object): """ Get a page url by adding a page= parameter to the base url """ - new_get_params = dict(get_params) or {} + if isinstance(get_params, MultiDict): + new_get_params = get_params.to_dict() + else: + new_get_params = dict(get_params) or {} + new_get_params['page'] = page_no return "%s?%s" % ( base_url, urllib.urlencode(new_get_params)) -- cgit v1.2.3 From a7fc4ecf40196e64753c6aa283eb669adb15e942 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sun, 15 Sep 2013 09:46:01 -0500 Subject: Updating the video.js that we use to avoid an XSS attack. Yikes! ;\ This commit sponsored by Daniel Valentine. Thank you! --- .../mediagoblin/media_displays/video.html | 23 +++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) (limited to 'mediagoblin') diff --git a/mediagoblin/templates/mediagoblin/media_displays/video.html b/mediagoblin/templates/mediagoblin/media_displays/video.html index e35169bf..95271d78 100644 --- a/mediagoblin/templates/mediagoblin/media_displays/video.html +++ b/mediagoblin/templates/mediagoblin/media_displays/video.html @@ -21,9 +21,26 @@ {% block mediagoblin_head -%} {{ super() }} - + {# Sadly commented out till we can get the mediagoblin skin ported over + # to the newest video.js release ;\ #} + {# + + #} + + + {%- endblock %} {% block mediagoblin_media %} @@ -31,7 +48,7 @@