diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | mediagoblin/db/sql_switch.py | 1 | ||||
| -rw-r--r-- | mediagoblin/gmg_commands/dbupdate.py | 8 | ||||
| -rw-r--r-- | mediagoblin/init/celery/__init__.py | 55 | ||||
| -rw-r--r-- | mediagoblin/tests/__init__.py | 14 | ||||
| -rw-r--r-- | mediagoblin/tests/test_auth.py | 1 | ||||
| -rw-r--r-- | mediagoblin/tests/test_mgoblin_app.ini | 6 | ||||
| -rw-r--r-- | mediagoblin/tests/test_submission.py | 28 | ||||
| -rw-r--r-- | mediagoblin/tests/tools.py | 33 |
9 files changed, 94 insertions, 53 deletions
@@ -20,7 +20,6 @@ /celery.db /kombu.db /server-log.txt -/mediagoblin/db/sql_switch.py # Tests /mediagoblin/tests/user_dev/ diff --git a/mediagoblin/db/sql_switch.py b/mediagoblin/db/sql_switch.py new file mode 100644 index 00000000..571adbdb --- /dev/null +++ b/mediagoblin/db/sql_switch.py @@ -0,0 +1 @@ +use_sql = True diff --git a/mediagoblin/gmg_commands/dbupdate.py b/mediagoblin/gmg_commands/dbupdate.py index 27698170..5415b997 100644 --- a/mediagoblin/gmg_commands/dbupdate.py +++ b/mediagoblin/gmg_commands/dbupdate.py @@ -65,14 +65,13 @@ def gather_database_data(media_types): return managed_dbdata -def dbupdate(args): +def run_dbupdate(app_config): """ Initialize or migrate the database as specified by the config file. Will also initialize or migrate all extensions (media types, and in the future, plugins) """ - globa_config, app_config = setup_global_and_app_config(args.conf_file) # Gather information from all media managers / projects dbdatas = gather_database_data(app_config['media_types']) @@ -87,3 +86,8 @@ def dbupdate(args): for dbdata in dbdatas: migration_manager = dbdata.make_migration_manager(Session()) migration_manager.init_or_migrate() + + +def dbupdate(args): + global_config, app_config = setup_global_and_app_config(args.conf_file) + run_dbupdate(app_config) diff --git a/mediagoblin/init/celery/__init__.py b/mediagoblin/init/celery/__init__.py index 67b020c3..fc595ea7 100644 --- a/mediagoblin/init/celery/__init__.py +++ b/mediagoblin/init/celery/__init__.py @@ -17,28 +17,18 @@ import os import sys +from celery import Celery + MANDATORY_CELERY_IMPORTS = ['mediagoblin.processing.task'] DEFAULT_SETTINGS_MODULE = 'mediagoblin.init.celery.dummy_settings_module' -def setup_celery_from_config(app_config, global_config, - settings_module=DEFAULT_SETTINGS_MODULE, - force_celery_always_eager=False, - set_environ=True): +def get_celery_settings_dict(app_config, global_config, + force_celery_always_eager=False): """ - Take a mediagoblin app config and try to set up a celery settings - module from this. - - Args: - - app_config: the application config section - - global_config: the entire ConfigObj loaded config, all sections - - settings_module: the module to populate, as a string - - force_celery_always_eager: whether or not to force celery into - always eager mode; good for development and small installs - - set_environ: if set, this will CELERY_CONFIG_MODULE to the - settings_module + Get a celery settings dictionary from reading the config """ if 'celery' in global_config: celery_conf = global_config['celery'] @@ -61,6 +51,41 @@ def setup_celery_from_config(app_config, global_config, celery_settings['CELERY_ALWAYS_EAGER'] = True celery_settings['CELERY_EAGER_PROPAGATES_EXCEPTIONS'] = True + return celery_settings + + +def setup_celery_app(app_config, global_config, + settings_module=DEFAULT_SETTINGS_MODULE, + force_celery_always_eager=False): + """ + Setup celery without using terrible setup-celery-module hacks. + """ + celery_settings = get_celery_settings_dict( + app_config, global_config, force_celery_always_eager) + celery_app = Celery() + celery_app.config_from_object(celery_settings) + + +def setup_celery_from_config(app_config, global_config, + settings_module=DEFAULT_SETTINGS_MODULE, + force_celery_always_eager=False, + set_environ=True): + """ + Take a mediagoblin app config and try to set up a celery settings + module from this. + + Args: + - app_config: the application config section + - global_config: the entire ConfigObj loaded config, all sections + - settings_module: the module to populate, as a string + - force_celery_always_eager: whether or not to force celery into + always eager mode; good for development and small installs + - set_environ: if set, this will CELERY_CONFIG_MODULE to the + settings_module + """ + celery_settings = get_celery_settings_dict( + app_config, global_config, force_celery_always_eager) + __import__(settings_module) this_module = sys.modules[settings_module] diff --git a/mediagoblin/tests/__init__.py b/mediagoblin/tests/__init__.py index 15a5add0..4e84914a 100644 --- a/mediagoblin/tests/__init__.py +++ b/mediagoblin/tests/__init__.py @@ -14,10 +14,12 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from mediagoblin import mg_globals +import os +import shutil +from mediagoblin import mg_globals from mediagoblin.tests.tools import ( - MEDIAGOBLIN_TEST_DB_NAME, suicide_if_bad_celery_environ) + TEST_USER_DEV, suicide_if_bad_celery_environ) def setup_package(): @@ -25,8 +27,6 @@ def setup_package(): def teardown_package(): - if ((mg_globals.db_connection - and mg_globals.database.name == MEDIAGOBLIN_TEST_DB_NAME)): - print "Killing db ..." - mg_globals.db_connection.drop_database(MEDIAGOBLIN_TEST_DB_NAME) - print "... done" + # Remove and reinstall user_dev directories + if os.path.exists(TEST_USER_DEV): + shutil.rmtree(TEST_USER_DEV) diff --git a/mediagoblin/tests/test_auth.py b/mediagoblin/tests/test_auth.py index 3a33c66c..8f988af3 100644 --- a/mediagoblin/tests/test_auth.py +++ b/mediagoblin/tests/test_auth.py @@ -269,6 +269,7 @@ def test_register_views(test_app): ## Try using an expired token to change password, shouldn't work template.clear_test_template_context() + new_user = mg_globals.database.User.find_one({'username': 'happygirl'}) real_token_expiration = new_user.fp_token_expire new_user.fp_token_expire = datetime.datetime.now() new_user.save() diff --git a/mediagoblin/tests/test_mgoblin_app.ini b/mediagoblin/tests/test_mgoblin_app.ini index 01bf0972..3b979ff7 100644 --- a/mediagoblin/tests/test_mgoblin_app.ini +++ b/mediagoblin/tests/test_mgoblin_app.ini @@ -2,7 +2,9 @@ direct_remote_path = /test_static/ email_sender_address = "notice@mediagoblin.example.org" email_debug_mode = true -db_name = __mediagoblin_tests__ + +# TODO: Switch to using an in-memory database +sql_engine = "sqlite:///%(here)s/test_user_dev/mediagoblin.db" # tag parsing tags_max_length = 50 @@ -27,3 +29,5 @@ lock_dir = %(here)s/test_user_dev/beaker/cache/lock [celery] CELERY_ALWAYS_EAGER = true +CELERY_RESULT_DBURI = "sqlite:///%(here)s/test_user_dev/celery.db" +BROKER_HOST = "sqlite:///%(here)s/test_user_dev/kombu.db" diff --git a/mediagoblin/tests/test_submission.py b/mediagoblin/tests/test_submission.py index b9b7388e..8d1b3745 100644 --- a/mediagoblin/tests/test_submission.py +++ b/mediagoblin/tests/test_submission.py @@ -81,8 +81,8 @@ class TestSubmission: def upload_data(self, filename): return {'upload_files': [('file', filename)]} - def check_comments(self, request, media, count): - comments = request.db.MediaComment.find({'media_entry': media._id}) + def check_comments(self, request, media_id, count): + comments = request.db.MediaComment.find({'media_entry': media_id}) assert_equal(count, len(list(comments))) def test_missing_fields(self): @@ -134,9 +134,11 @@ class TestSubmission: *REQUEST_CONTEXT, do_follow=True, **self.upload_data(GOOD_JPG)) media = self.check_media(request, {'title': 'Balanced Goblin'}, 1) - assert_equal(media.tags, - [{'name': u'yin', 'slug': u'yin'}, - {'name': u'yang', 'slug': u'yang'}]) + assert media.tags[0]['name'] == u'yin' + assert media.tags[0]['slug'] == u'yin' + + assert media.tags[1]['name'] == u'yang' + assert media.tags[1]['slug'] == u'yang' # Test tags that are too long # --------------- @@ -154,31 +156,33 @@ class TestSubmission: *REQUEST_CONTEXT, do_follow=True, **self.upload_data(GOOD_JPG)) media = self.check_media(request, {'title': 'Balanced Goblin'}, 1) + media_id = media.id # Add a comment, so we can test for its deletion later. - self.check_comments(request, media, 0) + self.check_comments(request, media_id, 0) comment_url = request.urlgen( 'mediagoblin.user_pages.media_post_comment', - user=self.test_user.username, media=media._id) + user=self.test_user.username, media=media_id) response = self.do_post({'comment_content': 'i love this test'}, url=comment_url, do_follow=True)[0] - self.check_comments(request, media, 1) + self.check_comments(request, media_id, 1) # Do not confirm deletion # --------------------------------------------------- delete_url = request.urlgen( 'mediagoblin.user_pages.media_confirm_delete', - user=self.test_user.username, media=media._id) + user=self.test_user.username, media=media_id) # Empty data means don't confirm response = self.do_post({}, do_follow=True, url=delete_url)[0] media = self.check_media(request, {'title': 'Balanced Goblin'}, 1) + media_id = media.id # Confirm deletion # --------------------------------------------------- response, request = self.do_post({'confirm': 'y'}, *REQUEST_CONTEXT, do_follow=True, url=delete_url) - self.check_media(request, {'_id': media._id}, 0) - self.check_comments(request, media, 0) + self.check_media(request, {'_id': media_id}, 0) + self.check_comments(request, media_id, 0) def test_evil_file(self): # Test non-suppoerted file with non-supported extension @@ -246,7 +250,7 @@ class TestSubmission: filename = resource_filename( 'mediagoblin.tests', os.path.join('test_user_dev/media/public', - *media['media_files'].get(key, []))) + *media.media_files.get(key, []))) assert_true(filename.endswith('_' + basename)) # Is it smaller than the last processed image we looked at? size = os.stat(filename).st_size diff --git a/mediagoblin/tests/tools.py b/mediagoblin/tests/tools.py index 7cf355b0..5b4e3746 100644 --- a/mediagoblin/tests/tools.py +++ b/mediagoblin/tests/tools.py @@ -26,8 +26,11 @@ from mediagoblin.tools import testing from mediagoblin.init.config import read_mediagoblin_config from mediagoblin.decorators import _make_safe from mediagoblin.db.open import setup_connection_and_db_from_config +from mediagoblin.db.sql.base import Session from mediagoblin.meddleware import BaseMeddleware from mediagoblin.auth.lib import bcrypt_gen_password_hash +from mediagoblin.gmg_commands.dbupdate import run_dbupdate +from mediagoblin.init.celery import setup_celery_app MEDIAGOBLIN_TEST_DB_NAME = u'__mediagoblin_tests__' @@ -125,26 +128,19 @@ def get_test_app(dump_old_app=True): global_config, validation_result = read_mediagoblin_config(TEST_APP_CONFIG) app_config = global_config['mediagoblin'] - # Wipe database - # @@: For now we're dropping collections, but we could also just - # collection.remove() ? - connection, db = setup_connection_and_db_from_config(app_config) - assert db.name == MEDIAGOBLIN_TEST_DB_NAME - - collections_to_wipe = [ - collection - for collection in db.collection_names() - if not collection.startswith('system.')] - - for collection in collections_to_wipe: - db.drop_collection(collection) - - # TODO: Drop and recreate indexes + # Run database setup/migrations + run_dbupdate(app_config) # setup app and return test_app = loadapp( 'config:' + TEST_SERVER_CONFIG) + Session.rollback() + Session.remove() + + # Re-setup celery + setup_celery_app(app_config, global_config) + # Insert the TestingMeddleware, which can do some # sanity checks on every request/response. # Doing it this way is probably not the cleanest way. @@ -216,4 +212,11 @@ def fixture_add_user(username = u'chris', password = 'toast', test_user.save() + # Reload + test_user = mg_globals.database.User.find_one({'username': username}) + + # ... and detach from session: + from mediagoblin.db.sql.base import Session + Session.expunge(test_user) + return test_user |