diff options
27 files changed, 356 insertions, 128 deletions
diff --git a/docs/source/devel/codebase.rst b/docs/source/devel/codebase.rst index cd46242c..9718a097 100644 --- a/docs/source/devel/codebase.rst +++ b/docs/source/devel/codebase.rst @@ -142,8 +142,8 @@ Software Stack * `werkzeug <http://werkzeug.pocoo.org/>`_: nice abstraction layer from HTTP requests, responses and WSGI bits - * `Beaker <http://beaker.groovie.org/>`_: for handling sessions and - caching + * `itsdangerous <http://pythonhosted.org/itsdangerous/>`_: + for handling sessions * `Jinja2 <http://jinja.pocoo.org/docs/>`_: the templating engine diff --git a/docs/source/siteadmin/deploying.rst b/docs/source/siteadmin/deploying.rst index 9b2324ae..77e60037 100644 --- a/docs/source/siteadmin/deploying.rst +++ b/docs/source/siteadmin/deploying.rst @@ -185,6 +185,11 @@ flup:: ./bin/easy_install flup +(Sometimes this breaks because flup's site is flakey. If it does for +you, try):: + + ./bin/easy_install https://pypi.python.org/pypi/flup/1.0.3.dev-20110405 + This concludes the initial configuration of the development environment. In the future, when you update your codebase, you should also run:: diff --git a/extlib/README b/extlib/README index c690beac..a2cc6ec0 100644 --- a/extlib/README +++ b/extlib/README @@ -17,7 +17,7 @@ unwittingly interfere with other software that depends on the canonical release versions of those same libraries! Forking upstream software for trivial reasons makes us bad citizens in -the Open Source community and adds unnecessary heartache for our +the Free Software community and adds unnecessary heartache for our users. Don't make us "that" project. diff --git a/mediagoblin/app.py b/mediagoblin/app.py index bb6be4d4..1137c0d7 100644 --- a/mediagoblin/app.py +++ b/mediagoblin/app.py @@ -25,7 +25,7 @@ from werkzeug.exceptions import HTTPException from werkzeug.routing import RequestRedirect from mediagoblin import meddleware, __version__ -from mediagoblin.tools import common, translate, template +from mediagoblin.tools import common, session, translate, template from mediagoblin.tools.response import render_http_exception from mediagoblin.tools.theme import register_themes from mediagoblin.tools import request as mg_request @@ -34,8 +34,9 @@ from mediagoblin.init.celery import setup_celery_from_config from mediagoblin.init.plugins import setup_plugins from mediagoblin.init import (get_jinja_loader, get_staticdirector, setup_global_and_app_config, setup_locales, setup_workbench, setup_database, - setup_storage, setup_beaker_cache) + setup_storage) from mediagoblin.tools.pluginapi import PluginManager +from mediagoblin.tools.crypto import setup_crypto _log = logging.getLogger(__name__) @@ -66,10 +67,15 @@ class MediaGoblinApp(object): # Open and setup the config global_config, app_config = setup_global_and_app_config(config_path) + setup_crypto() + ########################################## # Setup other connections / useful objects ########################################## + # Setup Session Manager, not needed in celery + self.session_manager = session.SessionManager() + # load all available locales setup_locales() @@ -100,9 +106,6 @@ class MediaGoblinApp(object): # set up staticdirector tool self.staticdirector = get_staticdirector(app_config) - # set up caching - self.cache = setup_beaker_cache() - # Setup celery, if appropriate if setup_celery and not app_config.get('celery_setup_elsewhere'): if os.environ.get('CELERY_ALWAYS_EAGER', 'false').lower() == 'true': @@ -157,7 +160,8 @@ class MediaGoblinApp(object): ## Attach utilities to the request object # Do we really want to load this via middleware? Maybe? - request.session = request.environ['beaker.session'] + session_manager = self.session_manager + request.session = session_manager.load_session_from_cookie(request) # Attach self as request.app # Also attach a few utilities from request.app for convenience? request.app = self @@ -226,6 +230,9 @@ class MediaGoblinApp(object): response = render_http_exeption( request, e, e.get_description(environ)) + session_manager.save_session_to_cookie(request.session, + request, response) + return response(environ, start_response) def __call__(self, environ, start_response): diff --git a/mediagoblin/config_spec.ini b/mediagoblin/config_spec.ini index 44f6a68f..e830e863 100644 --- a/mediagoblin/config_spec.ini +++ b/mediagoblin/config_spec.ini @@ -14,6 +14,9 @@ sql_engine = string(default="sqlite:///%(here)s/mediagoblin.db") # Where temporary files used in processing and etc are kept workbench_path = string(default="%(here)s/user_dev/media/workbench") +# Where to store cryptographic sensible data +crypto_path = string(default="%(here)s/user_dev/crypto") + # Where mediagoblin-builtin static assets are kept direct_remote_path = string(default="/mgoblin_static/") @@ -122,11 +125,6 @@ spectrogram_fft_size = integer(default=4096) [media_type:mediagoblin.media_types.ascii] thumbnail_font = string(default=None) -[beaker.cache] -type = string(default="file") -data_dir = string(default="%(here)s/user_dev/beaker/cache/data") -lock_dir = string(default="%(here)s/user_dev/beaker/cache/lock") - [celery] # default result stuff diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 2f58503f..fcfd0f61 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -172,8 +172,7 @@ class MediaEntry(Base, MediaEntryMixin): order_col = MediaComment.created if not ascending: order_col = desc(order_col) - return MediaComment.query.filter_by( - media_entry=self.id).order_by(order_col) + return self.all_comments.order_by(order_col) def url_to_prev(self, urlgen): """get the next 'newer' entry by this user""" @@ -238,9 +237,7 @@ class MediaEntry(Base, MediaEntryMixin): :param del_orphan_tags: True/false if we delete unused Tags too :param commit: True/False if this should end the db transaction""" # User's CollectionItems are automatically deleted via "cascade". - # Delete all the associated comments - for comment in self.get_comments(): - comment.delete(commit=False) + # Comments on this Media are deleted by cascade, hopefully. # Delete all related files/attachments try: @@ -385,13 +382,22 @@ class MediaComment(Base, MediaCommentMixin): content = Column(UnicodeText, nullable=False) # Cascade: Comments are owned by their creator. So do the full thing. - # lazy=dynamic: People might post a *lot* of comments, so make - # the "posted_comments" a query-like thing. + # lazy=dynamic: People might post a *lot* of comments, + # so make the "posted_comments" a query-like thing. get_author = relationship(User, backref=backref("posted_comments", lazy="dynamic", cascade="all, delete-orphan")) + # Cascade: Comments are somewhat owned by their MediaEntry. + # So do the full thing. + # lazy=dynamic: MediaEntries might have many comments, + # so make the "all_comments" a query-like thing. + get_media_entry = relationship(MediaEntry, + backref=backref("all_comments", + lazy="dynamic", + cascade="all, delete-orphan")) + class Collection(Base, CollectionMixin): """An 'album' or 'set' of media by a user. diff --git a/mediagoblin/init/__init__.py b/mediagoblin/init/__init__.py index 7c832442..d16027db 100644 --- a/mediagoblin/init/__init__.py +++ b/mediagoblin/init/__init__.py @@ -14,8 +14,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from beaker.cache import CacheManager -from beaker.util import parse_cache_config_options import jinja2 from mediagoblin.tools import staticdirect @@ -146,16 +144,3 @@ def setup_workbench(): workbench_manager = WorkbenchManager(app_config['workbench_path']) setup_globals(workbench_manager=workbench_manager) - - -def setup_beaker_cache(): - """ - Setup the Beaker Cache manager. - """ - cache_config = mg_globals.global_config['beaker.cache'] - cache_config = dict( - [(u'cache.%s' % key, value) - for key, value in cache_config.iteritems()]) - cache = CacheManager(**parse_cache_config_options(cache_config)) - setup_globals(cache=cache) - return cache diff --git a/mediagoblin/media_types/ascii/processing.py b/mediagoblin/media_types/ascii/processing.py index 382cd015..309aab0a 100644 --- a/mediagoblin/media_types/ascii/processing.py +++ b/mediagoblin/media_types/ascii/processing.py @@ -127,8 +127,14 @@ def process_ascii(proc_state): 'ascii', 'xmlcharrefreplace')) - mgg.queue_store.delete_file(queued_filepath) + # Remove queued media file from storage and database. + # queued_filepath is in the task_id directory which should + # be removed too, but fail if the directory is not empty to be on + # the super-safe side. + mgg.queue_store.delete_file(queued_filepath) # rm file + mgg.queue_store.delete_dir(queued_filepath[:-1]) # rm dir entry.queued_media_file = [] + media_files_dict = entry.setdefault('media_files', {}) media_files_dict['thumb'] = thumb_filepath media_files_dict['unicode'] = unicode_filepath diff --git a/mediagoblin/media_types/audio/processing.py b/mediagoblin/media_types/audio/processing.py index 5dffcaf9..101b83e5 100644 --- a/mediagoblin/media_types/audio/processing.py +++ b/mediagoblin/media_types/audio/processing.py @@ -147,4 +147,10 @@ def process_audio(proc_state): else: entry.media_files['thumb'] = ['fake', 'thumb', 'path.jpg'] - mgg.queue_store.delete_file(queued_filepath) + # Remove queued media file from storage and database. + # queued_filepath is in the task_id directory which should + # be removed too, but fail if the directory is not empty to be on + # the super-safe side. + mgg.queue_store.delete_file(queued_filepath) # rm file + mgg.queue_store.delete_dir(queued_filepath[:-1]) # rm dir + entry.queued_media_file = [] diff --git a/mediagoblin/media_types/stl/processing.py b/mediagoblin/media_types/stl/processing.py index 77744ac5..e41df395 100644 --- a/mediagoblin/media_types/stl/processing.py +++ b/mediagoblin/media_types/stl/processing.py @@ -165,8 +165,12 @@ def process_stl(proc_state): with open(queued_filename, 'rb') as queued_file: model_file.write(queued_file.read()) - # Remove queued media file from storage and database - mgg.queue_store.delete_file(queued_filepath) + # Remove queued media file from storage and database. + # queued_filepath is in the task_id directory which should + # be removed too, but fail if the directory is not empty to be on + # the super-safe side. + mgg.queue_store.delete_file(queued_filepath) # rm file + mgg.queue_store.delete_dir(queued_filepath[:-1]) # rm dir entry.queued_media_file = [] # Insert media file information into database diff --git a/mediagoblin/mg_globals.py b/mediagoblin/mg_globals.py index e4b94bdc..26ed66fa 100644 --- a/mediagoblin/mg_globals.py +++ b/mediagoblin/mg_globals.py @@ -29,9 +29,6 @@ import threading # SQL database engine database = None -# beaker's cache manager -cache = None - # should be the same as the public_store = None queue_store = None diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py index 02462567..a1fd3fb7 100644 --- a/mediagoblin/processing/__init__.py +++ b/mediagoblin/processing/__init__.py @@ -111,8 +111,13 @@ class ProcessingState(object): self.entry.media_files[keyname] = target_filepath def delete_queue_file(self): + # Remove queued media file from storage and database. + # queued_filepath is in the task_id directory which should + # be removed too, but fail if the directory is not empty to be on + # the super-safe side. queued_filepath = self.entry.queued_media_file - mgg.queue_store.delete_file(queued_filepath) + mgg.queue_store.delete_file(queued_filepath) # rm file + mgg.queue_store.delete_dir(queued_filepath[:-1]) # rm dir self.entry.queued_media_file = [] diff --git a/mediagoblin/storage/__init__.py b/mediagoblin/storage/__init__.py index 5c1d7d36..bbe134a7 100644 --- a/mediagoblin/storage/__init__.py +++ b/mediagoblin/storage/__init__.py @@ -101,10 +101,20 @@ class StorageInterface(object): def delete_file(self, filepath): """ - Delete or dereference the file at filepath. + Delete or dereference the file (not directory) at filepath. + """ + # Subclasses should override this method. + self.__raise_not_implemented() + + def delete_dir(self, dirpath, recursive=False): + """Delete the directory at dirpath + + :param recursive: Usually, a directory must not contain any + files for the delete to succeed. If True, containing files + and subdirectories within dirpath will be recursively + deleted. - This might need to delete directories, buckets, whatever, for - cleanliness. (Be sure to avoid race conditions on that though) + :returns: True in case of success, False otherwise. """ # Subclasses should override this method. self.__raise_not_implemented() diff --git a/mediagoblin/storage/filestorage.py b/mediagoblin/storage/filestorage.py index ef786b61..3d6e0753 100644 --- a/mediagoblin/storage/filestorage.py +++ b/mediagoblin/storage/filestorage.py @@ -62,10 +62,32 @@ class BasicFileStorage(StorageInterface): return open(self._resolve_filepath(filepath), mode) def delete_file(self, filepath): - # TODO: Also delete unused directories if empty (safely, with - # checks to avoid race conditions). + """Delete file at filepath + + Raises OSError in case filepath is a directory.""" + #TODO: log error os.remove(self._resolve_filepath(filepath)) + def delete_dir(self, dirpath, recursive=False): + """returns True on succes, False on failure""" + + dirpath = self._resolve_filepath(dirpath) + + # Shortcut the default and simple case of nonempty=F, recursive=F + if recursive: + try: + shutil.rmtree(dirpath) + except OSError as e: + #TODO: log something here + return False + else: # recursively delete everything + try: + os.rmdir(dirpath) + except OSError as e: + #TODO: log something here + return False + return True + def file_url(self, filepath): if not self.base_url: raise NoWebServing( diff --git a/mediagoblin/tests/test_cache.py b/mediagoblin/tests/test_cache.py deleted file mode 100644 index 403173cd..00000000 --- a/mediagoblin/tests/test_cache.py +++ /dev/null @@ -1,50 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - - -from mediagoblin import mg_globals - - -DATA_TO_CACHE = { - 'herp': 'derp', - 'lol': 'cats'} - - -def _get_some_data(key): - """ - Stuid function that makes use of some caching. - """ - some_data_cache = mg_globals.cache.get_cache('sum_data') - if some_data_cache.has_key(key): - return some_data_cache.get(key) - - value = DATA_TO_CACHE.get(key) - some_data_cache.put(key, value) - return value - - -def test_cache_working(test_app): - some_data_cache = mg_globals.cache.get_cache('sum_data') - assert not some_data_cache.has_key('herp') - assert _get_some_data('herp') == 'derp' - assert some_data_cache.get('herp') == 'derp' - # should get the same value again - assert _get_some_data('herp') == 'derp' - - # now we force-change it, but the function should use the cached - # version - some_data_cache.put('herp', 'pred') - assert _get_some_data('herp') == 'pred' diff --git a/mediagoblin/tests/test_mgoblin_app.ini b/mediagoblin/tests/test_mgoblin_app.ini index 42d3785a..b78abe64 100644 --- a/mediagoblin/tests/test_mgoblin_app.ini +++ b/mediagoblin/tests/test_mgoblin_app.ini @@ -23,10 +23,6 @@ base_url = /mgoblin_media/ [storage:queuestore] base_dir = %(here)s/test_user_dev/media/queue -[beaker.cache] -data_dir = %(here)s/test_user_dev/beaker/cache/data -lock_dir = %(here)s/test_user_dev/beaker/cache/lock - [celery] CELERY_ALWAYS_EAGER = true CELERY_RESULT_DBURI = "sqlite:///%(here)s/test_user_dev/celery.db" diff --git a/mediagoblin/tests/test_paste.ini b/mediagoblin/tests/test_paste.ini index 875b4f65..91ecbb84 100644 --- a/mediagoblin/tests/test_paste.ini +++ b/mediagoblin/tests/test_paste.ini @@ -9,7 +9,6 @@ use = egg:Paste#urlmap [app:mediagoblin] use = egg:mediagoblin#app -filter-with = beaker config = %(here)s/mediagoblin.ini [app:publicstore_serve] @@ -20,14 +19,6 @@ document_root = %(here)s/test_user_dev/media/public use = egg:Paste#static document_root = %(here)s/mediagoblin/static/ -[filter:beaker] -use = egg:Beaker#beaker_session -cache_dir = %(here)s/test_user_dev/beaker -beaker.session.key = mediagoblin -# beaker.session.secret = somesupersecret -beaker.session.data_dir = %(here)s/test_user_dev/beaker/sessions/data -beaker.session.lock_dir = %(here)s/test_user_dev/beaker/sessions/lock - [celery] CELERY_ALWAYS_EAGER = true diff --git a/mediagoblin/tests/test_session.py b/mediagoblin/tests/test_session.py new file mode 100644 index 00000000..78d790eb --- /dev/null +++ b/mediagoblin/tests/test_session.py @@ -0,0 +1,30 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from mediagoblin.tools import session + +def test_session(): + sess = session.Session() + assert not sess + assert not sess.is_updated() + sess['user_id'] = 27 + assert sess + assert not sess.is_updated() + sess.save() + assert sess.is_updated() + sess.delete() + assert not sess + assert sess.is_updated() diff --git a/mediagoblin/tests/test_storage.py b/mediagoblin/tests/test_storage.py index 294fec7d..749f7b07 100644 --- a/mediagoblin/tests/test_storage.py +++ b/mediagoblin/tests/test_storage.py @@ -87,7 +87,7 @@ def test_storage_system_from_config(): ########################## def get_tmp_filestorage(mount_url=None, fake_remote=False): - tmpdir = tempfile.mkdtemp() + tmpdir = tempfile.mkdtemp(prefix="test_gmg_storage") if fake_remote: this_storage = FakeRemoteStorage(tmpdir, mount_url) else: @@ -111,6 +111,8 @@ def test_basic_storage__resolve_filepath(): this_storage._resolve_filepath, ['../../', 'etc', 'passwd']) + os.rmdir(tmpdir) + def test_basic_storage_file_exists(): tmpdir, this_storage = get_tmp_filestorage() @@ -124,6 +126,8 @@ def test_basic_storage_file_exists(): assert not this_storage.file_exists(['dir1', 'dir2', 'thisfile.lol']) assert not this_storage.file_exists(['dnedir1', 'dnedir2', 'somefile.lol']) + this_storage.delete_file(['dir1', 'dir2', 'filename.txt']) + def test_basic_storage_get_unique_filepath(): tmpdir, this_storage = get_tmp_filestorage() @@ -144,6 +148,8 @@ def test_basic_storage_get_unique_filepath(): assert len(new_filename) > len('filename.txt') assert new_filename == secure_filename(new_filename) + os.remove(filename) + def test_basic_storage_get_file(): tmpdir, this_storage = get_tmp_filestorage() @@ -180,6 +186,10 @@ def test_basic_storage_get_file(): with this_storage.get_file(['testydir', 'testyfile.txt']) as testyfile: assert testyfile.read() == 'testy file! so testy.' + this_storage.delete_file(filepath) + this_storage.delete_file(new_filepath) + this_storage.delete_file(['testydir', 'testyfile.txt']) + def test_basic_storage_delete_file(): tmpdir, this_storage = get_tmp_filestorage() @@ -207,6 +217,7 @@ def test_basic_storage_url_for_file(): storage.NoWebServing, this_storage.file_url, ['dir1', 'dir2', 'filename.txt']) + os.rmdir(tmpdir) # base_url without domain tmpdir, this_storage = get_tmp_filestorage('/media/') @@ -214,6 +225,7 @@ def test_basic_storage_url_for_file(): ['dir1', 'dir2', 'filename.txt']) expected = '/media/dir1/dir2/filename.txt' assert result == expected + os.rmdir(tmpdir) # base_url with domain tmpdir, this_storage = get_tmp_filestorage( @@ -222,6 +234,7 @@ def test_basic_storage_url_for_file(): ['dir1', 'dir2', 'filename.txt']) expected = 'http://media.example.org/ourmedia/dir1/dir2/filename.txt' assert result == expected + os.rmdir(tmpdir) def test_basic_storage_get_local_path(): @@ -235,10 +248,13 @@ def test_basic_storage_get_local_path(): assert result == expected + os.rmdir(tmpdir) + def test_basic_storage_is_local(): tmpdir, this_storage = get_tmp_filestorage() assert this_storage.local_storage is True + os.rmdir(tmpdir) def test_basic_storage_copy_locally(): @@ -253,9 +269,13 @@ def test_basic_storage_copy_locally(): new_file_dest = os.path.join(dest_tmpdir, 'file2.txt') this_storage.copy_locally(filepath, new_file_dest) + this_storage.delete_file(filepath) assert file(new_file_dest).read() == 'Testing this file' + os.remove(new_file_dest) + os.rmdir(dest_tmpdir) + def _test_copy_local_to_storage_works(tmpdir, this_storage): local_filename = tempfile.mktemp() @@ -265,10 +285,14 @@ def _test_copy_local_to_storage_works(tmpdir, this_storage): this_storage.copy_local_to_storage( local_filename, ['dir1', 'dir2', 'copiedto.txt']) + os.remove(local_filename) + assert file( os.path.join(tmpdir, 'dir1/dir2/copiedto.txt'), 'r').read() == 'haha' + this_storage.delete_file(['dir1', 'dir2', 'copiedto.txt']) + def test_basic_storage_copy_local_to_storage(): tmpdir, this_storage = get_tmp_filestorage() diff --git a/mediagoblin/tests/test_workbench.py b/mediagoblin/tests/test_workbench.py index 3b2fc2c6..9cd49671 100644 --- a/mediagoblin/tests/test_workbench.py +++ b/mediagoblin/tests/test_workbench.py @@ -26,8 +26,13 @@ from mediagoblin.tests.test_storage import get_tmp_filestorage class TestWorkbench(object): def setup(self): + self.workbench_base = tempfile.mkdtemp(prefix='gmg_workbench_testing') self.workbench_manager = workbench.WorkbenchManager( - os.path.join(tempfile.gettempdir(), u'mgoblin_workbench_testing')) + self.workbench_base) + + def teardown(self): + # If the workbench is empty, this should work. + os.rmdir(self.workbench_base) def test_create_workbench(self): workbench = self.workbench_manager.create() @@ -70,6 +75,7 @@ class TestWorkbench(object): filename = this_workbench.localized_file(this_storage, filepath) assert filename == os.path.join( tmpdir, 'dir1/dir2/ourfile.txt') + this_storage.delete_file(filepath) # with a fake remote file storage tmpdir, this_storage = get_tmp_filestorage(fake_remote=True) @@ -95,6 +101,9 @@ class TestWorkbench(object): assert filename == os.path.join( this_workbench.dir, 'thisfile.text') + this_storage.delete_file(filepath) + this_workbench.destroy() + def test_workbench_decorator(self): """Test @get_workbench decorator and automatic cleanup""" # The decorator needs mg_globals.workbench_manager diff --git a/mediagoblin/tests/tools.py b/mediagoblin/tests/tools.py index f7025715..a0498a6e 100644 --- a/mediagoblin/tests/tools.py +++ b/mediagoblin/tests/tools.py @@ -45,9 +45,7 @@ TEST_USER_DEV = pkg_resources.resource_filename( 'mediagoblin.tests', 'test_user_dev') -USER_DEV_DIRECTORIES_TO_SETUP = [ - 'media/public', 'media/queue', - 'beaker/sessions/data', 'beaker/sessions/lock'] +USER_DEV_DIRECTORIES_TO_SETUP = ['media/public', 'media/queue'] BAD_CELERY_MESSAGE = """\ Sorry, you *absolutely* must run tests with the diff --git a/mediagoblin/tools/crypto.py b/mediagoblin/tools/crypto.py new file mode 100644 index 00000000..1379d21b --- /dev/null +++ b/mediagoblin/tools/crypto.py @@ -0,0 +1,113 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2013 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import errno +import itsdangerous +import logging +import os.path +import random +import tempfile +from mediagoblin import mg_globals + +_log = logging.getLogger(__name__) + + +# Use the system (hardware-based) random number generator if it exists. +# -- this optimization is lifted from Django +try: + getrandbits = random.SystemRandom().getrandbits +except AttributeError: + getrandbits = random.getrandbits + + +__itsda_secret = None + + +def load_key(filename): + global __itsda_secret + key_file = open(filename) + try: + __itsda_secret = key_file.read() + finally: + key_file.close() + + +def create_key(key_dir, key_filepath): + global __itsda_secret + old_umask = os.umask(077) + key_file = None + try: + if not os.path.isdir(key_dir): + os.makedirs(key_dir) + _log.info("Created %s", key_dir) + key = str(getrandbits(192)) + key_file = tempfile.NamedTemporaryFile(dir=key_dir, suffix='.bin', + delete=False) + key_file.write(key) + key_file.flush() + os.rename(key_file.name, key_filepath) + key_file.close() + finally: + os.umask(old_umask) + if (key_file is not None) and (not key_file.closed): + key_file.close() + os.unlink(key_file.name) + __itsda_secret = key + _log.info("Saved new key for It's Dangerous") + + +def setup_crypto(): + global __itsda_secret + key_dir = mg_globals.app_config["crypto_path"] + key_filepath = os.path.join(key_dir, 'itsdangeroussecret.bin') + try: + load_key(key_filepath) + except IOError, error: + if error.errno != errno.ENOENT: + raise + create_key(key_dir, key_filepath) + + +def get_timed_signer_url(namespace): + """ + This gives a basic signing/verifying object. + + The namespace makes sure signed tokens can't be used in + a different area. Like using a forgot-password-token as + a session cookie. + + Basic usage: + + .. code-block:: python + + _signer = None + TOKEN_VALID_DAYS = 10 + def setup(): + global _signer + _signer = get_timed_signer_url("session cookie") + def create_token(obj): + return _signer.dumps(obj) + def parse_token(token): + # This might raise an exception in case + # of an invalid token, or an expired token. + return _signer.loads(token, max_age=TOKEN_VALID_DAYS*24*3600) + + For more details see + http://pythonhosted.org/itsdangerous/#itsdangerous.URLSafeTimedSerializer + """ + assert __itsda_secret is not None + return itsdangerous.URLSafeTimedSerializer(__itsda_secret, + salt=namespace) diff --git a/mediagoblin/tools/request.py b/mediagoblin/tools/request.py index bc67b96f..ee342eae 100644 --- a/mediagoblin/tools/request.py +++ b/mediagoblin/tools/request.py @@ -35,4 +35,4 @@ def setup_user_in_request(request): # Something's wrong... this user doesn't exist? Invalidate # this session. _log.warn("Killing session for user id %r", request.session['user_id']) - request.session.invalidate() + request.session.delete() diff --git a/mediagoblin/tools/session.py b/mediagoblin/tools/session.py new file mode 100644 index 00000000..fdc32523 --- /dev/null +++ b/mediagoblin/tools/session.py @@ -0,0 +1,68 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2013 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import itsdangerous +import logging + +import crypto + +_log = logging.getLogger(__name__) + +class Session(dict): + def __init__(self, *args, **kwargs): + self.send_new_cookie = False + dict.__init__(self, *args, **kwargs) + + def save(self): + self.send_new_cookie = True + + def is_updated(self): + return self.send_new_cookie + + def delete(self): + self.clear() + self.save() + + +class SessionManager(object): + def __init__(self, cookie_name='MGSession', namespace=None): + if namespace is None: + namespace = cookie_name + self.signer = crypto.get_timed_signer_url(namespace) + self.cookie_name = cookie_name + + def load_session_from_cookie(self, request): + cookie = request.cookies.get(self.cookie_name) + if not cookie: + return Session() + ### FIXME: Future cookie-blacklisting code + # m = BadCookie.query.filter_by(cookie = cookie) + # if m: + # _log.warn("Bad cookie received: %s", m.reason) + # raise BadRequest() + try: + return Session(self.signer.loads(cookie)) + except itsdangerous.BadData: + return Session() + + def save_session_to_cookie(self, session, request, response): + if not session.is_updated(): + return + elif not session: + response.delete_cookie(self.cookie_name) + else: + response.set_cookie(self.cookie_name, self.signer.dumps(session), + httponly=True) @@ -17,7 +17,6 @@ use = egg:Paste#urlmap [app:mediagoblin] use = egg:mediagoblin#app -filter-with = beaker config = %(here)s/mediagoblin_local.ini %(here)s/mediagoblin.ini [loggers] @@ -57,14 +56,6 @@ use = egg:Paste#static document_root = %(here)s/user_dev/theme_static/ cache_max_age = 86400 -[filter:beaker] -use = egg:Beaker#beaker_session -cache_dir = %(here)s/user_dev/beaker -beaker.session.key = mediagoblin -# beaker.session.secret = somesupersecret -beaker.session.data_dir = %(here)s/user_dev/beaker/sessions/data -beaker.session.lock_dir = %(here)s/user_dev/beaker/sessions/lock - [filter:errors] use = egg:mediagoblin#errors debug = false diff --git a/runtests.sh b/runtests.sh index cd53da2d..382e2fa6 100755 --- a/runtests.sh +++ b/runtests.sh @@ -49,9 +49,16 @@ echo "+ CELERY_CONFIG_MODULE=$CELERY_CONFIG_MODULE" # will try to read all directories, and this turns into a mess! need_arg=1 +ignore_next=0 for i in "$@" do + if [ "$ignore_next" = 1 ] + then + ignore_next=0 + continue + fi case "$i" in + -n) ignore_next=1;; -*) ;; *) need_arg=0; break ;; esac @@ -43,7 +43,6 @@ setup( install_requires=[ 'setuptools', 'PasteScript', - 'beaker', 'wtforms', 'py-bcrypt', 'pytest', @@ -61,6 +60,7 @@ setup( 'sqlalchemy>=0.7.0', 'sqlalchemy-migrate', 'mock', + 'itsdangerous', ## This is optional! # 'translitcodec', ## For now we're expecting that users will install this from |