diff options
31 files changed, 725 insertions, 106 deletions
diff --git a/docs/hackinghowto.rst b/docs/hackinghowto.rst index a56498bb..a9aadb62 100644 --- a/docs/hackinghowto.rst +++ b/docs/hackinghowto.rst @@ -152,7 +152,7 @@ Running the test suite Run:: - ./bin/nosetests + CELERY_CONFIG_MODULE=mediagoblin.celery_setup.from_tests ./bin/nosetests Running a shell diff --git a/mediagoblin/app.py b/mediagoblin/app.py index e5949531..a1c6b512 100644 --- a/mediagoblin/app.py +++ b/mediagoblin/app.py @@ -23,8 +23,9 @@ from webob import Request, exc from mediagoblin import routing, util, storage, staticdirect from mediagoblin.db.open import setup_connection_and_db_from_config -from mediagoblin.globals import setup_globals +from mediagoblin.mg_globals import setup_globals from mediagoblin.celery_setup import setup_celery_from_config +from mediagoblin.workbench import WorkbenchManager, DEFAULT_WORKBENCH_DIR class Error(Exception): pass @@ -39,7 +40,8 @@ class MediaGoblinApp(object): public_store, queue_store, staticdirector, email_sender_address, email_debug_mode, - user_template_path=None): + user_template_path=None, + workbench_path=DEFAULT_WORKBENCH_DIR): # Get the template environment self.template_loader = util.get_jinja_loader(user_template_path) @@ -66,7 +68,8 @@ class MediaGoblinApp(object): db_connection=connection, database=self.db, public_store=self.public_store, - queue_store=self.queue_store) + queue_store=self.queue_store, + workbench_manager=WorkbenchManager(workbench_path)) def __call__(self, environ, start_response): request = Request(environ) @@ -154,6 +157,7 @@ def paste_app_factory(global_config, **app_config): email_sender_address=app_config.get( 'email_sender_address', 'notice@mediagoblin.example.org'), email_debug_mode=asbool(app_config.get('email_debug_mode')), - user_template_path=app_config.get('local_templates')) + user_template_path=app_config.get('local_templates'), + workbench_path=app_config.get('workbench_path', DEFAULT_WORKBENCH_DIR)) return mgoblin_app diff --git a/mediagoblin/auth/forms.py b/mediagoblin/auth/forms.py index db8aaceb..7bc0aeb1 100644 --- a/mediagoblin/auth/forms.py +++ b/mediagoblin/auth/forms.py @@ -27,7 +27,9 @@ class RegistrationForm(wtforms.Form): 'Password', [wtforms.validators.Required(), wtforms.validators.Length(min=6, max=30), - wtforms.validators.EqualTo('confirm_password')]) + wtforms.validators.EqualTo( + 'confirm_password', + 'Passwords must match.')]) confirm_password = wtforms.PasswordField( 'Confirm password', [wtforms.validators.Required()]) diff --git a/mediagoblin/auth/lib.py b/mediagoblin/auth/lib.py index dc5f9941..08bbdd16 100644 --- a/mediagoblin/auth/lib.py +++ b/mediagoblin/auth/lib.py @@ -19,8 +19,8 @@ import random import bcrypt -from mediagoblin.util import send_email -from mediagoblin import globals as mgoblin_globals +from mediagoblin.util import send_email, render_template +from mediagoblin import mg_globals def bcrypt_check_password(raw_pass, stored_hash, extra_salt=None): @@ -89,6 +89,10 @@ def fake_login_attempt(): randplus_stored_hash == randplus_hashed_pass +EMAIL_VERIFICATION_TEMPLATE = ( + u"http://{host}{uri}?" + u"userid={userid}&token={verification_key}") + def send_verification_email(user, request): """ Send the verification email to users to activate their accounts. @@ -97,13 +101,18 @@ def send_verification_email(user, request): - user: a user object - request: the request """ - - email_template = request.template_env.get_template( - 'mediagoblin/auth/verification_email.txt') + rendered_email = render_template( + request, 'mediagoblin/auth/verification_email.txt', + {'username': user['username'], + 'verification_url': EMAIL_VERIFICATION_TEMPLATE.format( + host=request.host, + uri=request.urlgen('mediagoblin.auth.verify_email'), + userid=unicode(user['_id']), + verification_key=user['verification_key'])}) # TODO: There is no error handling in place send_email( - mgoblin_globals.email_sender_address, + mg_globals.email_sender_address, [user['email']], # TODO # Due to the distributed nature of GNU MediaGoblin, we should @@ -111,10 +120,4 @@ def send_verification_email(user, request): # specific GNU MediaGoblin instance in the subject line. For # example "GNU MediaGoblin @ Wandborg - [...]". 'GNU MediaGoblin - Verify your email!', - email_template.render( - username=user['username'], - verification_url='http://{host}{uri}?userid={userid}&token={verification_key}'.format( - host=request.host, - uri=request.urlgen('mediagoblin.auth.verify_email'), - userid=unicode(user['_id']), - verification_key=user['verification_key']))) + rendered_email) diff --git a/mediagoblin/auth/routing.py b/mediagoblin/auth/routing.py index a8909fbb..46c585d2 100644 --- a/mediagoblin/auth/routing.py +++ b/mediagoblin/auth/routing.py @@ -20,7 +20,8 @@ auth_routes = [ Route('mediagoblin.auth.register', '/register/', controller='mediagoblin.auth.views:register'), Route('mediagoblin.auth.register_success', '/register/success/', - controller='mediagoblin.auth.views:register_success'), + template='mediagoblin/auth/register_success.html', + controller='mediagoblin.views:simple_template_render'), Route('mediagoblin.auth.login', '/login/', controller='mediagoblin.auth.views:login'), Route('mediagoblin.auth.logout', '/logout/', @@ -28,9 +29,11 @@ auth_routes = [ Route('mediagoblin.auth.verify_email', '/verify_email/', controller='mediagoblin.auth.views:verify_email'), Route('mediagoblin.auth.verify_email_notice', '/verification_required/', - controller='mediagoblin.auth.views:verify_email_notice'), + template='mediagoblin/auth/verification_needed.html', + controller='mediagoblin.views:simple_template_render'), Route('mediagoblin.auth.resend_verification', '/resend_verification/', controller='mediagoblin.auth.views:resend_activation'), Route('mediagoblin.auth.resend_verification_success', '/resend_verification_success/', - controller='mediagoblin.auth.views:resend_activation_success')] + template='mediagoblin/auth/resent_verification_email.html', + controller='mediagoblin.views:simple_template_render')] diff --git a/mediagoblin/auth/views.py b/mediagoblin/auth/views.py index 36d23e53..1d00f382 100644 --- a/mediagoblin/auth/views.py +++ b/mediagoblin/auth/views.py @@ -56,16 +56,12 @@ def register(request): return redirect(request, "mediagoblin.auth.register_success") - return render_to_response(request, + return render_to_response( + request, 'mediagoblin/auth/register.html', {'register_form': register_form}) -def register_success(request): - return render_to_response(request, - 'mediagoblin/auth/register_success.html', {}) - - def login(request): """ MediaGoblin login view. @@ -96,7 +92,8 @@ def login(request): auth_lib.fake_login_attempt() login_failed = True - return render_to_response(request, + return render_to_response( + request, 'mediagoblin/auth/login.html', {'login_form': login_form, 'next': request.GET.get('next') or request.POST.get('next'), @@ -132,21 +129,12 @@ def verify_email(request): else: verification_successful = False - return render_to_response(request, + return render_to_response( + request, 'mediagoblin/auth/verify_email.html', {'user': user, 'verification_successful': verification_successful}) -def verify_email_notice(request): - """ - Verify warning view. - - When the user tries to do some action that requires their account - to be verified beforehand, this view is called upon! - """ - return render_to_response(request, - 'mediagoblin/auth/verification_needed.html', {}) - def resend_activation(request): """ @@ -154,15 +142,9 @@ def resend_activation(request): Resend the activation email. """ - request.user['verification_key'] = unicode(uuid.uuid4()) request.user.save() send_verification_email(request.user, request) return redirect(request, 'mediagoblin.auth.resend_verification_success') - - -def resend_activation_success(request): - return render_to_response(request, - 'mediagoblin/auth/resent_verification_email.html', {}) diff --git a/mediagoblin/celery_setup/from_celery.py b/mediagoblin/celery_setup/from_celery.py index 0669e80c..c8ccebc8 100644 --- a/mediagoblin/celery_setup/from_celery.py +++ b/mediagoblin/celery_setup/from_celery.py @@ -22,14 +22,14 @@ from paste.deploy.converters import asbool from mediagoblin import storage from mediagoblin.db.open import setup_connection_and_db_from_config from mediagoblin.celery_setup import setup_celery_from_config -from mediagoblin.globals import setup_globals -from mediagoblin import globals as mgoblin_globals +from mediagoblin.mg_globals import setup_globals +from mediagoblin.workbench import WorkbenchManager, DEFAULT_WORKBENCH_DIR -OUR_MODULENAME = 'mediagoblin.celery_setup.from_celery' +OUR_MODULENAME = __name__ -def setup_self(setup_globals_func=setup_globals): +def setup_self(): """ Transform this module into a celery config module by reading the mediagoblin config file. Set the environment variable @@ -76,7 +76,11 @@ def setup_self(setup_globals_func=setup_globals): queue_store = storage.storage_system_from_paste_config( mgoblin_section, 'queuestore') - setup_globals_func( + workbench_manager = WorkbenchManager( + mgoblin_section.get( + 'workbench_path', DEFAULT_WORKBENCH_DIR)) + + setup_globals( db_connection=connection, database=db, public_store=public_store, @@ -84,7 +88,8 @@ def setup_self(setup_globals_func=setup_globals): email_sender_address=mgoblin_section.get( 'email_sender_address', 'notice@mediagoblin.example.org'), - queue_store=queue_store) + queue_store=queue_store, + workbench_manager=workbench_manager) if os.environ['CELERY_CONFIG_MODULE'] == OUR_MODULENAME: diff --git a/mediagoblin/celery_setup/from_tests.py b/mediagoblin/celery_setup/from_tests.py index fe7d7314..70814075 100644 --- a/mediagoblin/celery_setup/from_tests.py +++ b/mediagoblin/celery_setup/from_tests.py @@ -19,13 +19,12 @@ import os from mediagoblin.tests.tools import TEST_APP_CONFIG from mediagoblin import util from mediagoblin.celery_setup import setup_celery_from_config -from mediagoblin.globals import setup_globals -OUR_MODULENAME = 'mediagoblin.celery_setup.from_tests' +OUR_MODULENAME = __name__ -def setup_self(setup_globals_func=setup_globals): +def setup_self(): """ Set up celery for testing's sake, which just needs to set up celery and celery only. diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py index d035b15b..f1f625b7 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/migrations.py @@ -16,8 +16,6 @@ from mongokit import DocumentMigration -from mediagoblin import globals as mediagoblin_globals - class MediaEntryMigration(DocumentMigration): def allmigration01_uploader_to_reference(self): diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 3da97a49..d77cf619 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -20,7 +20,7 @@ from mongokit import Document, Set from mediagoblin import util from mediagoblin.auth import lib as auth_lib -from mediagoblin import globals as mediagoblin_globals +from mediagoblin import mg_globals from mediagoblin.db import migrations from mediagoblin.db.util import ObjectId @@ -114,7 +114,7 @@ class MediaEntry(Document): def generate_slug(self): self['slug'] = util.slugify(self['title']) - duplicate = mediagoblin_globals.database.media_entries.find_one( + duplicate = mg_globals.database.media_entries.find_one( {'slug': self['slug']}) if duplicate: diff --git a/mediagoblin/edit/views.py b/mediagoblin/edit/views.py index 04b73567..c5f0f435 100644 --- a/mediagoblin/edit/views.py +++ b/mediagoblin/edit/views.py @@ -54,7 +54,8 @@ def edit_media(request, media): return redirect(request, "mediagoblin.user_pages.media_home", user=media.uploader()['username'], media=media['slug']) - return render_to_response(request, + return render_to_response( + request, 'mediagoblin/edit/edit.html', {'media': media, 'form': form}) diff --git a/mediagoblin/gmg_commands/migrate.py b/mediagoblin/gmg_commands/migrate.py index e04fb343..3ce25701 100644 --- a/mediagoblin/gmg_commands/migrate.py +++ b/mediagoblin/gmg_commands/migrate.py @@ -17,7 +17,6 @@ from mediagoblin.db import migrations from mediagoblin.gmg_commands import util as commands_util -from mediagoblin import globals as mgoblin_globals def migrate_parser_setup(subparser): diff --git a/mediagoblin/gmg_commands/shell.py b/mediagoblin/gmg_commands/shell.py index 9c0259de..16caf398 100644 --- a/mediagoblin/gmg_commands/shell.py +++ b/mediagoblin/gmg_commands/shell.py @@ -17,7 +17,7 @@ import code -from mediagoblin import globals as mgoblin_globals +from mediagoblin import mg_globals from mediagoblin.gmg_commands import util as commands_util @@ -35,7 +35,7 @@ GNU MediaGoblin shell! ---------------------- Available vars: - mgoblin_app: instantiated mediagoblin application - - mgoblin_globals: mediagoblin.globals + - mg_globals: mediagoblin.globals - db: database instance """ @@ -50,5 +50,5 @@ def shell(args): banner=SHELL_BANNER, local={ 'mgoblin_app': mgoblin_app, - 'mgoblin_globals': mgoblin_globals, - 'db': mgoblin_globals.database}) + 'mg_globals': mg_globals, + 'db': mg_globals.database}) diff --git a/mediagoblin/globals.py b/mediagoblin/mg_globals.py index 80d1f01d..2fca3c0a 100644 --- a/mediagoblin/globals.py +++ b/mediagoblin/mg_globals.py @@ -27,7 +27,7 @@ translations = gettext.find( def setup_globals(**kwargs): - from mediagoblin import globals as mg_globals + from mediagoblin import mg_globals for key, value in kwargs.iteritems(): setattr(mg_globals, key, value) diff --git a/mediagoblin/process_media/__init__.py b/mediagoblin/process_media/__init__.py index 097b4375..a2d843e9 100644 --- a/mediagoblin/process_media/__init__.py +++ b/mediagoblin/process_media/__init__.py @@ -18,14 +18,14 @@ import Image from mediagoblin.db.util import ObjectId from celery.task import task -from mediagoblin.globals import database, queue_store, public_store +from mediagoblin import mg_globals as mgg THUMB_SIZE = 200, 200 def create_pub_filepath(entry, filename): - return public_store.get_unique_filepath( + return mgg.public_store.get_unique_filepath( ['media_entries', unicode(entry['_id']), filename]) @@ -33,35 +33,48 @@ def create_pub_filepath(entry, filename): @task def process_media_initial(media_id): - entry = database.MediaEntry.one( + workbench = mgg.workbench_manager.create_workbench() + + entry = mgg.database.MediaEntry.one( {'_id': ObjectId(media_id)}) queued_filepath = entry['queued_media_file'] - queued_file = queue_store.get_file(queued_filepath, 'r') + queued_filename = mgg.workbench_manager.localized_file( + workbench, mgg.queue_store, queued_filepath, + 'source') + + queued_file = file(queued_filename, 'r') with queued_file: thumb = Image.open(queued_file) thumb.thumbnail(THUMB_SIZE, Image.ANTIALIAS) + # ensure color mode is compatible with jpg + if thumb.mode != "RGB": + thumb = thumb.convert("RGB") thumb_filepath = create_pub_filepath(entry, 'thumbnail.jpg') - with public_store.get_file(thumb_filepath, 'w') as thumb_file: + thumb_file = mgg.public_store.get_file(thumb_filepath, 'w') + with thumb_file: thumb.save(thumb_file, "JPEG") # we have to re-read because unlike PIL, not everything reads # things in string representation :) - queued_file = queue_store.get_file(queued_filepath, 'rb') + queued_file = file(queued_filename, 'rb') with queued_file: main_filepath = create_pub_filepath(entry, queued_filepath[-1]) - with public_store.get_file(main_filepath, 'wb') as main_file: + with mgg.public_store.get_file(main_filepath, 'wb') as main_file: main_file.write(queued_file.read()) - queue_store.delete_file(queued_filepath) + mgg.queue_store.delete_file(queued_filepath) entry['queued_media_file'] = [] media_files_dict = entry.setdefault('media_files', {}) media_files_dict['thumb'] = thumb_filepath media_files_dict['main'] = main_filepath entry['state'] = u'processed' entry.save() + + # clean up workbench + mgg.workbench_manager.destroy_workbench(workbench) diff --git a/mediagoblin/storage.py b/mediagoblin/storage.py index 5d7e70d6..ba6ac017 100644 --- a/mediagoblin/storage.py +++ b/mediagoblin/storage.py @@ -16,6 +16,7 @@ import os import re +import shutil import urlparse import uuid @@ -60,6 +61,9 @@ class StorageInterface(object): StorageInterface. """ + # Whether this file store is on the local filesystem. + local_storage = False + def __raise_not_implemented(self): """ Raise a warning about some component not implemented by a @@ -127,12 +131,43 @@ class StorageInterface(object): else: return filepath + def get_local_path(self, filepath): + """ + If this is a local_storage implementation, give us a link to + the local filesystem reference to this file. + + >>> storage_handler.get_local_path(['foo', 'bar', 'baz.jpg']) + u'/path/to/mounting/foo/bar/baz.jpg' + """ + # Subclasses should override this method, if applicable. + self.__raise_not_implemented() + + def copy_locally(self, filepath, dest_path): + """ + Copy this file locally. + + A basic working method for this is provided that should + function both for local_storage systems and remote storge + systems, but if more efficient systems for copying locally + apply to your system, override this method with something more + appropriate. + """ + if self.local_storage: + shutil.copy( + self.get_local_path(filepath), dest_path) + else: + with self.get_file(filepath, 'rb') as source_file: + with file(dest_path, 'wb') as dest_file: + dest_file.write(source_file.read()) + class BasicFileStorage(StorageInterface): """ Basic local filesystem implementation of storage API """ + local_storage = True + def __init__(self, base_dir, base_url=None, **kwargs): """ Keyword arguments: @@ -177,6 +212,9 @@ class BasicFileStorage(StorageInterface): self.base_url, '/'.join(clean_listy_filepath(filepath))) + def get_local_path(self, filepath): + return self._resolve_filepath(filepath) + ########### # Utilities @@ -187,7 +225,7 @@ def clean_listy_filepath(listy_filepath): Take a listy filepath (like ['dir1', 'dir2', 'filename.jpg']) and clean out any nastiness from it. - For example: + >>> clean_listy_filepath([u'/dir1/', u'foo/../nasty', u'linooks.jpg']) [u'dir1', u'foo_.._nasty', u'linooks.jpg'] @@ -253,3 +291,5 @@ def storage_system_from_paste_config(paste_config, storage_prefix): storage_class = util.import_component(storage_class) return storage_class(**config_params) + + diff --git a/mediagoblin/submit/routing.py b/mediagoblin/submit/routing.py index cff28acb..3edbab70 100644 --- a/mediagoblin/submit/routing.py +++ b/mediagoblin/submit/routing.py @@ -20,5 +20,5 @@ submit_routes = [ Route('mediagoblin.submit.start', '/', controller='mediagoblin.submit.views:submit_start'), Route('mediagoblin.submit.success', '/success/', - controller='mediagoblin.submit.views:submit_success'), - ] + template='mediagoblin/submit/success.html', + controller='mediagoblin.views:simple_template_render')] diff --git a/mediagoblin/submit/security.py b/mediagoblin/submit/security.py new file mode 100644 index 00000000..b2cb6d88 --- /dev/null +++ b/mediagoblin/submit/security.py @@ -0,0 +1,26 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from mimetypes import guess_type + + +ALLOWED = ['image/jpeg', 'image/png', 'image/tiff', 'image/gif'] + +def check_filetype(posted_file): + if not guess_type(posted_file.filename)[0] in ALLOWED: + return False + + return True diff --git a/mediagoblin/submit/views.py b/mediagoblin/submit/views.py index d4ecc75a..e9b5c37e 100644 --- a/mediagoblin/submit/views.py +++ b/mediagoblin/submit/views.py @@ -21,7 +21,7 @@ from werkzeug.utils import secure_filename from mediagoblin.util import render_to_response, redirect from mediagoblin.decorators import require_active_login -from mediagoblin.submit import forms as submit_forms +from mediagoblin.submit import forms as submit_forms, security from mediagoblin.process_media import process_media_initial @@ -38,6 +38,9 @@ def submit_start(request): and request.POST['file'].file): submit_form.file.errors.append( u'You must provide a file.') + elif not security.check_filetype(request.POST['file']): + submit_form.file.errors.append( + u'The file doesn\'t seem to be an image!') else: filename = request.POST['file'].filename @@ -77,11 +80,12 @@ def submit_start(request): return redirect(request, "mediagoblin.submit.success") - return render_to_response(request, + return render_to_response( + request, 'mediagoblin/submit/start.html', {'submit_form': submit_form}) def submit_success(request): - return render_to_response(request, - 'mediagoblin/submit/success.html', {}) + return render_to_response( + request, 'mediagoblin/submit/success.html', {}) diff --git a/mediagoblin/templates/mediagoblin/utils/pagination.html b/mediagoblin/templates/mediagoblin/utils/pagination.html index 62e8af91..2be0b92e 100644 --- a/mediagoblin/templates/mediagoblin/utils/pagination.html +++ b/mediagoblin/templates/mediagoblin/utils/pagination.html @@ -18,16 +18,16 @@ {# only display if {{pagination}} is defined #} {% if pagination %} - <div class=pagination> + <div class="pagination"> {% if pagination.has_prev %} - <a href={{ pagination.get_page_url(request, pagination.page-1) }}>« Prev</> + <a href="{{ pagination.get_page_url(request, pagination.page-1) }}">« Prev</> {% endif %} {%- for page in pagination.iter_pages() %} {% if page %} {% if page != pagination.page %} - <a href={{ pagination.get_page_url(request, page) }}>{{ page }}</a> + <a href="{{ pagination.get_page_url(request, page) }}">{{ page }}</a> {% else %} <strong>{{ page }}</strong> {% endif %} @@ -37,7 +37,7 @@ {%- endfor %} {% if pagination.has_next %} - <a href={{ pagination.get_page_url(request, pagination.page+1) }}>Next »</a> + <a href="{{ pagination.get_page_url(request, pagination.page + 1) }}">Next »</a> {% endif %} </div> {% endif %} diff --git a/mediagoblin/tests/__init__.py b/mediagoblin/tests/__init__.py index c129cbf8..e9e2a59a 100644 --- a/mediagoblin/tests/__init__.py +++ b/mediagoblin/tests/__init__.py @@ -13,3 +13,14 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + +from mediagoblin import mg_globals + + +def setup_package(): + pass + +def teardown_package(): + print "Killing db ..." + mg_globals.db_connection.drop_database(mg_globals.database.name) + print "... done" diff --git a/mediagoblin/tests/test_auth.py b/mediagoblin/tests/test_auth.py index 94ce6bba..3d569093 100644 --- a/mediagoblin/tests/test_auth.py +++ b/mediagoblin/tests/test_auth.py @@ -14,8 +14,14 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +import urlparse + +from nose.tools import assert_equal from mediagoblin.auth import lib as auth_lib +from mediagoblin.tests.tools import setup_fresh_app +from mediagoblin import mg_globals +from mediagoblin import util ######################## @@ -57,3 +63,169 @@ def test_bcrypt_gen_password_hash(): pw, hashed_pw, '3><7R45417') assert not auth_lib.bcrypt_check_password( 'notthepassword', hashed_pw, '3><7R45417') + + +@setup_fresh_app +def test_register_views(test_app): + """ + Massive test function that all our registration-related views all work. + """ + # Test doing a simple GET on the page + # ----------------------------------- + + test_app.get('/auth/register/') + # Make sure it rendered with the appropriate template + assert util.TEMPLATE_TEST_CONTEXT.has_key( + 'mediagoblin/auth/register.html') + + # Try to register without providing anything, should error + # -------------------------------------------------------- + + util.clear_test_template_context() + test_app.post( + '/auth/register/', {}) + context = util.TEMPLATE_TEST_CONTEXT['mediagoblin/auth/register.html'] + form = context['register_form'] + assert form.username.errors == [u'This field is required.'] + assert form.password.errors == [u'This field is required.'] + assert form.confirm_password.errors == [u'This field is required.'] + assert form.email.errors == [u'This field is required.'] + + # Try to register with fields that are known to be invalid + # -------------------------------------------------------- + + ## too short + util.clear_test_template_context() + test_app.post( + '/auth/register/', { + 'username': 'l', + 'password': 'o', + 'confirm_password': 'o', + 'email': 'l'}) + context = util.TEMPLATE_TEST_CONTEXT['mediagoblin/auth/register.html'] + form = context['register_form'] + + assert form.username.errors == [ + u'Field must be between 3 and 30 characters long.'] + assert form.password.errors == [ + u'Field must be between 6 and 30 characters long.'] + + ## bad form + util.clear_test_template_context() + test_app.post( + '/auth/register/', { + 'username': '@_@', + 'email': 'lollerskates'}) + context = util.TEMPLATE_TEST_CONTEXT['mediagoblin/auth/register.html'] + form = context['register_form'] + + assert form.username.errors == [ + u'Invalid input.'] + assert form.email.errors == [ + u'Invalid email address.'] + + ## mismatching passwords + util.clear_test_template_context() + test_app.post( + '/auth/register/', { + 'password': 'herpderp', + 'confirm_password': 'derpherp'}) + context = util.TEMPLATE_TEST_CONTEXT['mediagoblin/auth/register.html'] + form = context['register_form'] + + assert form.password.errors == [ + u'Passwords must match.'] + + ## At this point there should be no users in the database ;) + assert not mg_globals.database.User.find().count() + + # Successful register + # ------------------- + util.clear_test_template_context() + response = test_app.post( + '/auth/register/', { + 'username': 'happygirl', + 'password': 'iamsohappy', + 'confirm_password': 'iamsohappy', + 'email': 'happygrrl@example.org'}) + response.follow() + + ## Did we redirect to the proper page? Use the right template? + assert_equal( + urlparse.urlsplit(response.location)[2], + '/auth/register/success/') + assert util.TEMPLATE_TEST_CONTEXT.has_key( + 'mediagoblin/auth/register_success.html') + + ## Make sure user is in place + new_user = mg_globals.database.User.find_one( + {'username': 'happygirl'}) + assert new_user + assert new_user['status'] == u'needs_email_verification' + assert new_user['email_verified'] == False + + ## Make sure we get email confirmation, and try verifying + assert len(util.EMAIL_TEST_INBOX) == 1 + message = util.EMAIL_TEST_INBOX.pop() + assert message['To'] == 'happygrrl@example.org' + email_context = util.TEMPLATE_TEST_CONTEXT[ + 'mediagoblin/auth/verification_email.txt'] + assert email_context['verification_url'] in message.get_payload(decode=True) + + path = urlparse.urlsplit(email_context['verification_url'])[2] + get_params = urlparse.urlsplit(email_context['verification_url'])[3] + assert path == u'/auth/verify_email/' + parsed_get_params = urlparse.parse_qs(get_params) + + ### user should have these same parameters + assert parsed_get_params['userid'] == [ + unicode(new_user['_id'])] + assert parsed_get_params['token'] == [ + new_user['verification_key']] + + ## Try verifying with bs verification key, shouldn't work + util.clear_test_template_context() + test_app.get( + "/auth/verify_email/?userid=%s&token=total_bs" % unicode( + new_user['_id'])) + context = util.TEMPLATE_TEST_CONTEXT[ + 'mediagoblin/auth/verify_email.html'] + assert context['verification_successful'] == False + new_user = mg_globals.database.User.find_one( + {'username': 'happygirl'}) + assert new_user + assert new_user['status'] == u'needs_email_verification' + assert new_user['email_verified'] == False + + ## Verify the email activation works + util.clear_test_template_context() + test_app.get("%s?%s" % (path, get_params)) + context = util.TEMPLATE_TEST_CONTEXT[ + 'mediagoblin/auth/verify_email.html'] + assert context['verification_successful'] == True + new_user = mg_globals.database.User.find_one( + {'username': 'happygirl'}) + assert new_user + assert new_user['status'] == u'active' + assert new_user['email_verified'] == True + + ## TODO: Try logging in + + # Uniqueness checks + # ----------------- + ## We shouldn't be able to register with that user twice + util.clear_test_template_context() + response = test_app.post( + '/auth/register/', { + 'username': 'happygirl', + 'password': 'iamsohappy2', + 'confirm_password': 'iamsohappy2', + 'email': 'happygrrl2@example.org'}) + + context = util.TEMPLATE_TEST_CONTEXT[ + 'mediagoblin/auth/register.html'] + form = context['register_form'] + assert form.username.errors == [ + u'Sorry, a user with that name already exists.'] + + ## TODO: Also check for double instances of an email address? diff --git a/mediagoblin/tests/test_globals.py b/mediagoblin/tests/test_globals.py index 6d2e01da..59d217f3 100644 --- a/mediagoblin/tests/test_globals.py +++ b/mediagoblin/tests/test_globals.py @@ -14,7 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from mediagoblin import globals as mg_globals +from mediagoblin import mg_globals def test_setup_globals(): mg_globals.setup_globals( diff --git a/mediagoblin/tests/test_storage.py b/mediagoblin/tests/test_storage.py index 61dd5dca..55b66e84 100644 --- a/mediagoblin/tests/test_storage.py +++ b/mediagoblin/tests/test_storage.py @@ -52,6 +52,11 @@ class FakeStorageSystem(): self.foobie = foobie self.blech = blech +class FakeRemoteStorage(storage.BasicFileStorage): + # Theoretically despite this, all the methods should work but it + # should force copying to the workbench + local_storage = False + def test_storage_system_from_paste_config(): this_storage = storage.storage_system_from_paste_config( @@ -81,9 +86,12 @@ def test_storage_system_from_paste_config(): # Basic file storage tests ########################## -def get_tmp_filestorage(mount_url=None): +def get_tmp_filestorage(mount_url=None, fake_remote=False): tmpdir = tempfile.mkdtemp() - this_storage = storage.BasicFileStorage(tmpdir, mount_url) + if fake_remote: + this_storage = FakeRemoteStorage(tmpdir, mount_url) + else: + this_storage = storage.BasicFileStorage(tmpdir, mount_url) return tmpdir, this_storage @@ -214,3 +222,36 @@ def test_basic_storage_url_for_file(): ['dir1', 'dir2', 'filename.txt']) expected = 'http://media.example.org/ourmedia/dir1/dir2/filename.txt' assert result == expected + + +def test_basic_storage_get_local_path(): + tmpdir, this_storage = get_tmp_filestorage() + + result = this_storage.get_local_path( + ['dir1', 'dir2', 'filename.txt']) + + expected = os.path.join( + tmpdir, 'dir1/dir2/filename.txt') + + assert result == expected + + +def test_basic_storage_is_local(): + tmpdir, this_storage = get_tmp_filestorage() + assert this_storage.local_storage is True + + +def test_basic_storage_copy_locally(): + tmpdir, this_storage = get_tmp_filestorage() + + dest_tmpdir = tempfile.mkdtemp() + + filepath = ['dir1', 'dir2', 'ourfile.txt'] + with this_storage.get_file(filepath, 'w') as our_file: + our_file.write('Testing this file') + + new_file_dest = os.path.join(dest_tmpdir, 'file2.txt') + + this_storage.copy_locally(filepath, new_file_dest) + + assert file(new_file_dest).read() == 'Testing this file' diff --git a/mediagoblin/tests/test_tests.py b/mediagoblin/tests/test_tests.py new file mode 100644 index 00000000..8ac7f0a4 --- /dev/null +++ b/mediagoblin/tests/test_tests.py @@ -0,0 +1,38 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from mediagoblin.tests.tools import get_test_app + +from mediagoblin import mg_globals + + +def test_get_test_app_wipes_db(): + """ + Make sure we get a fresh database on every wipe :) + """ + get_test_app() + assert mg_globals.database.User.find().count() == 0 + + new_user = mg_globals.database.User() + new_user['username'] = u'lolcat' + new_user['email'] = u'lol@cats.example.org' + new_user['pw_hash'] = u'pretend_this_is_a_hash' + new_user.save() + assert mg_globals.database.User.find().count() == 1 + + get_test_app() + + assert mg_globals.database.User.find().count() == 0 diff --git a/mediagoblin/tests/test_workbench.py b/mediagoblin/tests/test_workbench.py new file mode 100644 index 00000000..89f2ef33 --- /dev/null +++ b/mediagoblin/tests/test_workbench.py @@ -0,0 +1,96 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import tempfile + +from nose.tools import assert_raises + +from mediagoblin import workbench +from mediagoblin.tests.test_storage import get_tmp_filestorage + + +class TestWorkbench(object): + def setUp(self): + self.workbench_manager = workbench.WorkbenchManager( + os.path.join(tempfile.gettempdir(), u'mgoblin_workbench_testing')) + + def test_create_workbench(self): + workbench = self.workbench_manager.create_workbench() + assert os.path.isdir(workbench) + assert workbench.startswith(self.workbench_manager.base_workbench_dir) + + def test_destroy_workbench(self): + # kill a workbench + this_workbench = self.workbench_manager.create_workbench() + tmpfile = file(os.path.join(this_workbench, 'temp.txt'), 'w') + with tmpfile: + tmpfile.write('lollerskates') + + assert os.path.exists(os.path.join(this_workbench, 'temp.txt')) + + self.workbench_manager.destroy_workbench(this_workbench) + assert not os.path.exists(os.path.join(this_workbench, 'temp.txt')) + assert not os.path.exists(this_workbench) + + # make sure we can't kill other stuff though + dont_kill_this = tempfile.mkdtemp() + + assert_raises( + workbench.WorkbenchOutsideScope, + self.workbench_manager.destroy_workbench, + dont_kill_this) + + def test_localized_file(self): + tmpdir, this_storage = get_tmp_filestorage() + this_workbench = self.workbench_manager.create_workbench() + + # Write a brand new file + filepath = ['dir1', 'dir2', 'ourfile.txt'] + + with this_storage.get_file(filepath, 'w') as our_file: + our_file.write('Our file') + + # with a local file storage + filename = self.workbench_manager.localized_file( + this_workbench, this_storage, filepath) + assert filename == os.path.join( + tmpdir, 'dir1/dir2/ourfile.txt') + + # with a fake remote file storage + tmpdir, this_storage = get_tmp_filestorage(fake_remote=True) + + # ... write a brand new file, again ;) + with this_storage.get_file(filepath, 'w') as our_file: + our_file.write('Our file') + + filename = self.workbench_manager.localized_file( + this_workbench, this_storage, filepath) + assert filename == os.path.join( + this_workbench, 'ourfile.txt') + + # fake remote file storage, filename_if_copying set + filename = self.workbench_manager.localized_file( + this_workbench, this_storage, filepath, 'thisfile') + assert filename == os.path.join( + this_workbench, 'thisfile.txt') + + # fake remote file storage, filename_if_copying set, + # keep_extension_if_copying set to false + filename = self.workbench_manager.localized_file( + this_workbench, this_storage, filepath, 'thisfile.text', False) + assert filename == os.path.join( + this_workbench, 'thisfile.text') diff --git a/mediagoblin/tests/tools.py b/mediagoblin/tests/tools.py index 70b74b89..342b54b7 100644 --- a/mediagoblin/tests/tools.py +++ b/mediagoblin/tests/tools.py @@ -18,10 +18,11 @@ import pkg_resources import os, shutil -from paste.deploy import appconfig +from paste.deploy import appconfig, loadapp from webtest import TestApp -from mediagoblin import app +from mediagoblin import util +from mediagoblin.decorators import _make_safe from mediagoblin.db.open import setup_connection_and_db_from_config @@ -88,7 +89,21 @@ def get_test_app(dump_old_app=True): # TODO: Drop and recreate indexes # setup app and return - test_app = app.paste_app_factory( - config.global_conf, **config.local_conf) + test_app = loadapp( + 'config:' + TEST_APP_CONFIG) return TestApp(test_app) + + +def setup_fresh_app(func): + """ + Decorator to setup a fresh test application for this function. + + Cleans out test buckets and passes in a new, fresh test_app. + """ + def wrapper(*args, **kwargs): + test_app = get_test_app() + util.clear_test_buckets() + return func(test_app, *args, **kwargs) + + return _make_safe(wrapper, func) diff --git a/mediagoblin/user_pages/views.py b/mediagoblin/user_pages/views.py index e2fbcc80..323c3e54 100644 --- a/mediagoblin/user_pages/views.py +++ b/mediagoblin/user_pages/views.py @@ -42,7 +42,8 @@ def user_home(request, page): if media_entries == None: return exc.HTTPNotFound() - return render_to_response(request, + return render_to_response( + request, 'mediagoblin/user_pages/user.html', {'user': user, 'media_entries': media_entries, @@ -52,7 +53,8 @@ def user_home(request, page): @get_user_media_entry def media_home(request, media): """'Homepage' of a MediaEntry()""" - return render_to_response(request, + return render_to_response( + request, 'mediagoblin/user_pages/media.html', {'media': media}) diff --git a/mediagoblin/util.py b/mediagoblin/util.py index a0a09adf..f29f8570 100644 --- a/mediagoblin/util.py +++ b/mediagoblin/util.py @@ -31,7 +31,7 @@ import translitcodec from paste.deploy.loadwsgi import NicerConfigParser from webob import Response, exc -from mediagoblin import globals as mgoblin_globals +from mediagoblin import mg_globals from mediagoblin.db.util import ObjectId @@ -44,6 +44,26 @@ def _activate_testing(): TESTS_ENABLED = True +def clear_test_buckets(): + """ + We store some things for testing purposes that should be cleared + when we want a "clean slate" of information for our next round of + tests. Call this function to wipe all that stuff clean. + + Also wipes out some other things we might redefine during testing, + like the jinja envs. + """ + global SETUP_JINJA_ENVS + SETUP_JINJA_ENVS = {} + + global EMAIL_TEST_INBOX + global EMAIL_TEST_MBOX_INBOX + EMAIL_TEST_INBOX = [] + EMAIL_TEST_MBOX_INBOX = [] + + clear_test_template_context() + + def get_jinja_loader(user_template_path=None): """ Set up the Jinja template loaders, possibly allowing for user @@ -82,8 +102,8 @@ def get_jinja_env(template_loader, locale): extensions=['jinja2.ext.i18n']) template_env.install_gettext_callables( - mgoblin_globals.translations.gettext, - mgoblin_globals.translations.ngettext) + mg_globals.translations.gettext, + mg_globals.translations.ngettext) if exists(locale): SETUP_JINJA_ENVS[locale] = template_env @@ -95,7 +115,7 @@ def get_jinja_env(template_loader, locale): TEMPLATE_TEST_CONTEXT = {} -def render_template(request, template, context): +def render_template(request, template_path, context): """ Render a template with context. @@ -103,12 +123,12 @@ def render_template(request, template, context): Also stores the context if we're doing unit tests. Helpful! """ template = request.template_env.get_template( - template) + template_path) context['request'] = request rendered = template.render(context) if TESTS_ENABLED: - TEMPLATE_TEST_CONTEXT[template] = context + TEMPLATE_TEST_CONTEXT[template_path] = context return rendered @@ -244,9 +264,9 @@ def send_email(from_addr, to_addrs, subject, message_body): - message_body: email body text """ # TODO: make a mock mhost if testing is enabled - if TESTS_ENABLED or mgoblin_globals.email_debug_mode: + if TESTS_ENABLED or mg_globals.email_debug_mode: mhost = FakeMhost() - elif not mgoblin_globals.email_debug_mode: + elif not mg_globals.email_debug_mode: mhost = smtplib.SMTP() mhost.connect() @@ -259,7 +279,7 @@ def send_email(from_addr, to_addrs, subject, message_body): if TESTS_ENABLED: EMAIL_TEST_INBOX.append(message) - if getattr(mgoblin_globals, 'email_debug_mode', False): + if getattr(mg_globals, 'email_debug_mode', False): print u"===== Email =====" print u"From address: %s" % message['From'] print u"To addresses: %s" % message['To'] @@ -373,7 +393,7 @@ def setup_gettext(locale): if exists(locale): SETUP_GETTEXTS[locale] = this_gettext - mgoblin_globals.setup_globals( + mg_globals.setup_globals( translations=this_gettext) diff --git a/mediagoblin/views.py b/mediagoblin/views.py index 22673d59..5b6d9773 100644 --- a/mediagoblin/views.py +++ b/mediagoblin/views.py @@ -21,6 +21,16 @@ def root_view(request): media_entries = request.db.MediaEntry.find( {u'state': u'processed'}).sort('created', DESCENDING) - return render_to_response(request, - 'mediagoblin/root.html', + return render_to_response( + request, 'mediagoblin/root.html', {'media_entries': media_entries}) + + +def simple_template_render(request): + """ + A view for absolutely simple template rendering. + Just make sure 'template' is in the matchdict! + """ + template_name = request.matchdict['template'] + return render_to_response( + request, template_name, {}) diff --git a/mediagoblin/workbench.py b/mediagoblin/workbench.py new file mode 100644 index 00000000..d7252623 --- /dev/null +++ b/mediagoblin/workbench.py @@ -0,0 +1,135 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import shutil +import tempfile + + +DEFAULT_WORKBENCH_DIR = os.path.join( + tempfile.gettempdir(), u'mgoblin_workbench') + + +# Exception(s) +# ------------ + +class WorkbenchOutsideScope(Exception): + """ + Raised when a workbench is outside a WorkbenchManager scope. + """ + pass + + +# Actual workbench stuff +# ---------------------- + +class WorkbenchManager(object): + """ + A system for generating and destroying workbenches. + + Workbenches are actually just subdirectories of a temporary storage space + for during the processing stage. + """ + + def __init__(self, base_workbench_dir): + self.base_workbench_dir = os.path.abspath(base_workbench_dir) + if not os.path.exists(self.base_workbench_dir): + os.makedirs(self.base_workbench_dir) + + def create_workbench(self): + """ + Create and return the path to a new workbench (directory). + """ + return tempfile.mkdtemp(dir=self.base_workbench_dir) + + def destroy_workbench(self, workbench): + """ + Destroy this workbench! Deletes the directory and all its contents! + + Makes sure the workbench actually belongs to this manager though. + """ + # just in case + workbench = os.path.abspath(workbench) + + if not workbench.startswith(self.base_workbench_dir): + raise WorkbenchOutsideScope( + "Can't destroy workbench outside the base workbench dir") + + shutil.rmtree(workbench) + + def localized_file(self, workbench, storage, filepath, + filename_if_copying=None, + keep_extension_if_copying=True): + """ + Possibly localize the file from this storage system (for read-only + purposes, modifications should be written to a new file.). + + If the file is already local, just return the absolute filename of that + local file. Otherwise, copy the file locally to the workbench, and + return the absolute path of the new file. + + If it is copying locally, we might want to require a filename like + "source.jpg" to ensure that we won't conflict with other filenames in + our workbench... if that's the case, make sure filename_if_copying is + set to something like 'source.jpg'. Relatedly, if you set + keep_extension_if_copying, you don't have to set an extension on + filename_if_copying yourself, it'll be set for you (assuming such an + extension can be extacted from the filename in the filepath). + + Returns: + localized_filename + + Examples: + >>> wb_manager.localized_file( + ... '/our/workbench/subdir', local_storage, + ... ['path', 'to', 'foobar.jpg']) + u'/local/storage/path/to/foobar.jpg' + + >>> wb_manager.localized_file( + ... '/our/workbench/subdir', remote_storage, + ... ['path', 'to', 'foobar.jpg']) + '/our/workbench/subdir/foobar.jpg' + + >>> wb_manager.localized_file( + ... '/our/workbench/subdir', remote_storage, + ... ['path', 'to', 'foobar.jpg'], 'source.jpeg', False) + '/our/workbench/subdir/foobar.jpeg' + + >>> wb_manager.localized_file( + ... '/our/workbench/subdir', remote_storage, + ... ['path', 'to', 'foobar.jpg'], 'source', True) + '/our/workbench/subdir/foobar.jpg' + """ + if storage.local_storage: + return storage.get_local_path(filepath) + else: + if filename_if_copying is None: + dest_filename = filepath[-1] + else: + orig_filename, orig_ext = os.path.splitext(filepath[-1]) + if keep_extension_if_copying and orig_ext: + dest_filename = filename_if_copying + orig_ext + else: + dest_filename = filename_if_copying + + full_dest_filename = os.path.join( + workbench, dest_filename) + + # copy it over + storage.copy_locally( + filepath, full_dest_filename) + + return full_dest_filename |