diff options
| -rw-r--r-- | mediagoblin/db/indexes.py | 118 | ||||
| -rw-r--r-- | mediagoblin/db/models.py | 5 | ||||
| -rw-r--r-- | mediagoblin/db/util.py | 80 | ||||
| -rw-r--r-- | mediagoblin/gmg_commands/migrate.py | 20 | ||||
| -rw-r--r-- | mediagoblin/messages.py | 34 | ||||
| -rw-r--r-- | mediagoblin/static/css/base.css | 41 | ||||
| -rw-r--r-- | mediagoblin/templates/mediagoblin/base.html | 2 | ||||
| -rw-r--r-- | mediagoblin/templates/mediagoblin/utils/messages.html | 32 | ||||
| -rw-r--r-- | mediagoblin/tests/test_messages.py | 44 | ||||
| -rw-r--r-- | mediagoblin/util.py | 5 |
10 files changed, 374 insertions, 7 deletions
diff --git a/mediagoblin/db/indexes.py b/mediagoblin/db/indexes.py new file mode 100644 index 00000000..bbcceb6d --- /dev/null +++ b/mediagoblin/db/indexes.py @@ -0,0 +1,118 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +""" +Indexes for the local database. + +To add new indexes +------------------ + +Indexes are recorded in the following format: + +ACTIVE_INDEXES = { + 'collection_name': { + 'identifier': { # key identifier used for possibly deprecating later + 'index': [index_foo_goes_here]}} + +... and anything else being parameters to the create_index function +(including unique=True, etc) + +Current indexes must be registered in ACTIVE_INDEXES... deprecated +indexes should be marked in DEPRECATED_INDEXES. + +Remember, ordering of compound indexes MATTERS. Read below for more. + +REQUIRED READING: + - http://kylebanker.com/blog/2010/09/21/the-joy-of-mongodb-indexes/ + + - http://www.mongodb.org/display/DOCS/Indexes + - http://www.mongodb.org/display/DOCS/Indexing+Advice+and+FAQ + + +To remove deprecated indexes +---------------------------- + +Removing deprecated indexes is easier, just do: + +INACTIVE_INDEXES = { + 'collection_name': [ + 'deprecated_index_identifier1', 'deprecated_index_identifier2']} + +... etc. + +If an index has been deprecated that identifier should NEVER BE USED +AGAIN. Eg, if you previously had 'awesomepants_unique', you shouldn't +use 'awesomepants_unique' again, you should create a totally new name +or at worst use 'awesomepants_unique2'. +""" + +from pymongo import ASCENDING, DESCENDING + + +################ +# Active indexes +################ +ACTIVE_INDEXES = {} + +# MediaEntry indexes +# ------------------ + +MEDIAENTRY_INDEXES = { + 'uploader_slug_unique': { + # Matching an object to an uploader + slug. + # MediaEntries are unique on these two combined, eg: + # /u/${myuser}/m/${myslugname}/ + 'index': [('uploader', ASCENDING), + ('slug', ASCENDING)], + 'unique': True}, + + 'created': { + # A global index for all media entries created, in descending + # order. This is used for the site's frontpage. + 'index': [('created', DESCENDING)]}, + + 'uploader_created': { + # Indexing on uploaders and when media entries are created. + # Used for showing a user gallery, etc. + 'index': [('uploader', ASCENDING), + ('created', DESCENDING)]}} + + +ACTIVE_INDEXES['media_entries'] = MEDIAENTRY_INDEXES + + +# User indexes +# ------------ + +USER_INDEXES = { + 'username_unique': { + # Index usernames, and make sure they're unique. + # ... I guess we might need to adjust this once we're federated :) + 'index': 'username', + 'unique': True}, + 'created': { + # All most recently created users + 'index': 'created'}} + + +ACTIVE_INDEXES['users'] = USER_INDEXES + + +#################### +# Deprecated indexes +#################### + +DEPRECATED_INDEXES = {} diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 5196dede..bf825a23 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -109,11 +109,6 @@ class MediaEntry(Document): migration_handler = migrations.MediaEntryMigration - indexes = [ - # Referene uniqueness of slugs by uploader - {'fields': ['uploader', 'slug'], - 'unique': True}] - def main_mediafile(self): pass diff --git a/mediagoblin/db/util.py b/mediagoblin/db/util.py index 470da531..46f899f7 100644 --- a/mediagoblin/db/util.py +++ b/mediagoblin/db/util.py @@ -14,8 +14,88 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +""" +Utilities for database operations. + +Some note on migration and indexing tools: + +We store information about what the state of the database is in the +'mediagoblin' document of the 'app_metadata' collection. Keys in that +document relevant to here: + + - 'migration_number': The integer representing the current state of + the migrations +""" + +import copy # Imports that other modules might use from pymongo import DESCENDING from pymongo.errors import InvalidId from mongokit import ObjectId + +from mediagoblin.db.indexes import ACTIVE_INDEXES, DEPRECATED_INDEXES + + +def add_new_indexes(database, active_indexes=ACTIVE_INDEXES): + """ + Add any new indexes to the database. + + Args: + - database: pymongo or mongokit database instance. + - active_indexes: indexes to possibly add in the pattern of: + {'collection_name': { + 'identifier': { + 'index': [index_foo_goes_here], + 'unique': True}} + where 'index' is the index to add and all other options are + arguments for collection.create_index. + + Returns: + A list of indexes added in form ('collection', 'index_name') + """ + indexes_added = [] + + for collection_name, indexes in active_indexes.iteritems(): + collection = database[collection_name] + collection_indexes = collection.index_information().keys() + + for index_name, index_data in indexes.iteritems(): + if not index_name in collection_indexes: + # Get a copy actually so we don't modify the actual + # structure + index_data = copy.copy(index_data) + index = index_data.pop('index') + collection.create_index( + index, name=index_name, **index_data) + + indexes_added.append((collection_name, index_name)) + + return indexes_added + + +def remove_deprecated_indexes(database, deprecated_indexes=DEPRECATED_INDEXES): + """ + Remove any deprecated indexes from the database. + + Args: + - database: pymongo or mongokit database instance. + - deprecated_indexes: the indexes to deprecate in the pattern of: + {'collection': ['index_identifier1', 'index_identifier2']} + + Returns: + A list of indexes removed in form ('collection', 'index_name') + """ + indexes_removed = [] + + for collection_name, index_names in deprecated_indexes.iteritems(): + collection = database[collection_name] + collection_indexes = collection.index_information().keys() + + for index_name in index_names: + if index_name in collection_indexes: + collection.drop_index(index_name) + + indexes_removed.append((collection_name, index_name)) + + return indexes_removed diff --git a/mediagoblin/gmg_commands/migrate.py b/mediagoblin/gmg_commands/migrate.py index 9e01d51c..ab1a267b 100644 --- a/mediagoblin/gmg_commands/migrate.py +++ b/mediagoblin/gmg_commands/migrate.py @@ -16,6 +16,7 @@ from mediagoblin.db import migrations +from mediagoblin.db import util as db_util from mediagoblin.gmg_commands import util as commands_util @@ -27,8 +28,17 @@ def migrate_parser_setup(subparser): def migrate(args): mgoblin_app = commands_util.setup_app(args) - print "Applying migrations..." + # Clear old indexes + print "== Clearing old indexes... ==" + removed_indexes = db_util.remove_deprecated_indexes(mgoblin_app.db) + + for collection, index_name in removed_indexes: + print "Removed index '%s' in collection '%s'" % ( + index_name, collection) + + # Migrate + print "== Applying migrations... ==" for model_name in migrations.MIGRATE_CLASSES: model = getattr(mgoblin_app.db, model_name) @@ -38,4 +48,10 @@ def migrate(args): migration = model.migration_handler(model) migration.migrate_all(collection=model.collection) - print "... done." + # Add new indexes + print "== Adding new indexes... ==" + new_indexes = db_util.add_new_indexes(mgoblin_app.db) + + for collection, index_name in new_indexes: + print "Added index '%s' to collection '%s'" % ( + index_name, collection) diff --git a/mediagoblin/messages.py b/mediagoblin/messages.py new file mode 100644 index 00000000..afe6ee7e --- /dev/null +++ b/mediagoblin/messages.py @@ -0,0 +1,34 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +DEBUG = 'debug' +INFO = 'info' +SUCCESS = 'success' +WARNING = 'warning' +ERROR = 'error' + +def add_message(request, level, text): + messages = request.session.setdefault('messages', []) + messages.append({'level': level, 'text': text}) + request.session.save() + +def fetch_messages(request, clear_from_session=True): + messages = request.session.get('messages') + if messages and clear_from_session: + # Save that we removed the messages from the session + request.session['messages'] = [] + request.session.save() + return messages diff --git a/mediagoblin/static/css/base.css b/mediagoblin/static/css/base.css index e39770d5..55410bca 100644 --- a/mediagoblin/static/css/base.css +++ b/mediagoblin/static/css/base.css @@ -73,6 +73,47 @@ label { padding-bottom:74px; } +ul.mediagoblin_messages { + list-style:none inside; + color:#393932; + margin:2px; + padding:2px; +} + +ul.mediagoblin_messages li { + background-color:#d4d4d4; + border-style:solid; + border-width:3px; + border-color:#959595; + margin:5px; + padding:8px; +} + +ul.mediagoblin_messages li.message_success { + background-color: #88d486; + border-color: #5bba59; +} + +ul.mediagoblin_messages li.message_warning { + background-color: #d4c686; + border-color: #baa959; +} + +ul.mediagoblin_messages li.message_error { + background-color: #d48686; + border-color: #ba5959; +} + +ul.mediagoblin_messages li.message_info { + background-color: #86b9d4; + border-color: #5998ba; +} + +ul.mediagoblin_messages li.message_debug { + background-color: #aa86d4; + border-color: #8659ba; +} + a.mediagoblin_logo { width:34px; height:25px; diff --git a/mediagoblin/templates/mediagoblin/base.html b/mediagoblin/templates/mediagoblin/base.html index 2b47a14c..58de7325 100644 --- a/mediagoblin/templates/mediagoblin/base.html +++ b/mediagoblin/templates/mediagoblin/base.html @@ -51,6 +51,8 @@ </div> </div> {% endblock %} + + {% include "mediagoblin/utils/messages.html" %} <div class="container_12 mediagoblin_content"> <div class="grid_12"> diff --git a/mediagoblin/templates/mediagoblin/utils/messages.html b/mediagoblin/templates/mediagoblin/utils/messages.html new file mode 100644 index 00000000..52d03daa --- /dev/null +++ b/mediagoblin/templates/mediagoblin/utils/messages.html @@ -0,0 +1,32 @@ +{# +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +#} + +{# Display any queued messages #} +{% set messages = fetch_messages(request) %} +{% if messages %} +<div class="container_12 mediagoblin_messages"> + <div class="grid_12"> + <ul class="mediagoblin_messages"> + {% for msg in messages %} + <li class="message_{{ msg.level }}">{{ msg.text }}</li> + {% endfor %} + </ul> + </div> +</div> +{% endif %} + diff --git a/mediagoblin/tests/test_messages.py b/mediagoblin/tests/test_messages.py new file mode 100644 index 00000000..4cd9381a --- /dev/null +++ b/mediagoblin/tests/test_messages.py @@ -0,0 +1,44 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from mediagoblin.messages import fetch_messages, add_message +from mediagoblin.tests.tools import setup_fresh_app +from mediagoblin import util + + +@setup_fresh_app +def test_messages(test_app): + """ + Added messages should show up in the request.session, + fetched messages should be the same as the added ones, + and fetching should clear the message list. + """ + # Aquire a request object + test_app.get('/') + context = util.TEMPLATE_TEST_CONTEXT['mediagoblin/root.html'] + request = context['request'] + + # The message queue should be empty + assert request.session.get('messages', []) == [] + + # Adding a message should modify the session accordingly + add_message(request, 'herp_derp', 'First!') + test_msg_queue = [{'text': 'First!', 'level': 'herp_derp'}] + assert request.session['messages'] == test_msg_queue + + # fetch_messages should return and empty the queue + assert fetch_messages(request) == test_msg_queue + assert request.session.get('messages') == [] diff --git a/mediagoblin/util.py b/mediagoblin/util.py index 91fbee0a..a20e87c4 100644 --- a/mediagoblin/util.py +++ b/mediagoblin/util.py @@ -32,6 +32,7 @@ from lxml.html.clean import Cleaner import markdown from mediagoblin import mg_globals +from mediagoblin import messages from mediagoblin.db.util import ObjectId TESTS_ENABLED = False @@ -104,6 +105,10 @@ def get_jinja_env(template_loader, locale): mg_globals.translations.gettext, mg_globals.translations.ngettext) + # All templates will know how to ... + # ... fetch all waiting messages and remove them from the queue + template_env.globals['fetch_messages'] = messages.fetch_messages + if exists(locale): SETUP_JINJA_ENVS[locale] = template_env |