diff options
| -rw-r--r-- | docs/source/index.rst | 1 | ||||
| -rw-r--r-- | docs/source/plugindocs/raven.rst | 2 | ||||
| -rw-r--r-- | docs/source/siteadmin/production-deployments.rst | 47 | ||||
| -rw-r--r-- | mediagoblin/app.py | 3 | ||||
| -rw-r--r-- | mediagoblin/db/migration_tools.py | 8 | ||||
| -rw-r--r-- | mediagoblin/decorators.py | 2 | ||||
| -rw-r--r-- | mediagoblin/init/celery/__init__.py | 4 | ||||
| -rw-r--r-- | mediagoblin/init/celery/from_celery.py | 5 | ||||
| -rw-r--r-- | mediagoblin/plugins/oauth/forms.py | 7 | ||||
| -rw-r--r-- | mediagoblin/plugins/oauth/templates/oauth/authorize.html | 2 | ||||
| -rw-r--r-- | mediagoblin/plugins/raven/README.rst | 15 | ||||
| -rw-r--r-- | mediagoblin/plugins/raven/__init__.py | 92 | ||||
| -rw-r--r-- | mediagoblin/storage/cloudfiles.py | 58 | ||||
| -rw-r--r-- | mediagoblin/templates/mediagoblin/edit/edit_account.html | 9 | ||||
| -rw-r--r-- | mediagoblin/user_pages/views.py | 6 |
15 files changed, 203 insertions, 58 deletions
diff --git a/docs/source/index.rst b/docs/source/index.rst index abd891a0..9124b1c1 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -58,6 +58,7 @@ Part 2: Core plugin documentation plugindocs/sampleplugin plugindocs/oauth plugindocs/trim_whitespace + plugindocs/raven Part 3: Plugin Writer's Guide diff --git a/docs/source/plugindocs/raven.rst b/docs/source/plugindocs/raven.rst new file mode 100644 index 00000000..71e284d0 --- /dev/null +++ b/docs/source/plugindocs/raven.rst @@ -0,0 +1,2 @@ +.. _raven-setup: Set up the raven plugin +.. include:: ../../../mediagoblin/plugins/raven/README.rst diff --git a/docs/source/siteadmin/production-deployments.rst b/docs/source/siteadmin/production-deployments.rst index 0ed5ac6a..3e9431c9 100644 --- a/docs/source/siteadmin/production-deployments.rst +++ b/docs/source/siteadmin/production-deployments.rst @@ -77,51 +77,16 @@ Modify your existing MediaGoblin and application init scripts, if necessary, to prevent them from starting their own ``celeryd`` processes. -Monitor exceptions ------------------- - -This is an example config using raven_ to report exceptions and -:py:mod:`logging` messages to a sentry_ instance - -.. _raven: http://raven.readthedocs.org/ -.. _sentry: https://github.com/getsentry - -.. code-block:: ini - - [pipeline:main] - pipeline = - errors - raven - routing - - [loggers] - keys = root, sentry - - [handlers] - keys = console, sentry - - [formatters] - keys = generic +.. _sentry: - [logger_root] - level = INFO - handlers = console, sentry +Set up sentry to monitor exceptions +----------------------------------- - [logger_sentry] - level = WARN - handlers = console - qualname = sentry.errors - propagate = 0 +We have a plugin for `raven`_ integration, see the ":doc:`/plugindocs/raven`" +documentation. - [handler_sentry] - class = raven.handlers.logging.SentryHandler - args = ('http://public:secret@example.com/1',) - level = WARNING - formatter = generic +.. _`raven`: http://raven.readthedocs.org - [filter:raven] - use = egg:raven#raven - dsn = http://71727ea2c69043e4bbcd793bb0115cd4:e9cedccb32d9482d81f99eeca8b1ad30@sentry.talka.tv/3 .. _init-script: diff --git a/mediagoblin/app.py b/mediagoblin/app.py index 607d599b..bb6be4d4 100644 --- a/mediagoblin/app.py +++ b/mediagoblin/app.py @@ -253,4 +253,7 @@ def paste_app_factory(global_config, **app_config): mgoblin_app = MediaGoblinApp(mediagoblin_config) + for callable_hook in PluginManager().get_hook_callables('wrap_wsgi'): + mgoblin_app = callable_hook(mgoblin_app) + return mgoblin_app diff --git a/mediagoblin/db/migration_tools.py b/mediagoblin/db/migration_tools.py index e5380a3b..c0c7e998 100644 --- a/mediagoblin/db/migration_tools.py +++ b/mediagoblin/db/migration_tools.py @@ -17,6 +17,9 @@ from mediagoblin.tools.common import simple_printer from sqlalchemy import Table +class TableAlreadyExists(Exception): + pass + class MigrationManager(object): """ @@ -128,7 +131,10 @@ class MigrationManager(object): # sanity check before we proceed, none of these should be created for model in self.models: # Maybe in the future just print out a "Yikes!" or something? - assert not model.__table__.exists(self.session.bind) + if model.__table__.exists(self.session.bind): + raise TableAlreadyExists( + u"Intended to create table '%s' but it already exists" % + model.__table__.name) self.migration_model.metadata.create_all( self.session.bind, diff --git a/mediagoblin/decorators.py b/mediagoblin/decorators.py index fbf7b188..f3535fcf 100644 --- a/mediagoblin/decorators.py +++ b/mediagoblin/decorators.py @@ -43,7 +43,7 @@ def require_active_login(controller): request.url) return redirect(request, 'mediagoblin.auth.login', - next=url_quote(next_url)) + next=next_url) return controller(request, *args, **kwargs) diff --git a/mediagoblin/init/celery/__init__.py b/mediagoblin/init/celery/__init__.py index fc595ea7..8d7a41bd 100644 --- a/mediagoblin/init/celery/__init__.py +++ b/mediagoblin/init/celery/__init__.py @@ -18,6 +18,7 @@ import os import sys from celery import Celery +from mediagoblin.tools.pluginapi import PluginManager MANDATORY_CELERY_IMPORTS = ['mediagoblin.processing.task'] @@ -65,6 +66,9 @@ def setup_celery_app(app_config, global_config, celery_app = Celery() celery_app.config_from_object(celery_settings) + for callable_hook in PluginManager().get_hook_callables('celery_setup'): + callable_hook(celery_app) + def setup_celery_from_config(app_config, global_config, settings_module=DEFAULT_SETTINGS_MODULE, diff --git a/mediagoblin/init/celery/from_celery.py b/mediagoblin/init/celery/from_celery.py index 5c99ddff..8a794abb 100644 --- a/mediagoblin/init/celery/from_celery.py +++ b/mediagoblin/init/celery/from_celery.py @@ -22,6 +22,7 @@ from celery.signals import setup_logging from mediagoblin import app, mg_globals from mediagoblin.init.celery import setup_celery_from_config +from mediagoblin.tools.pluginapi import PluginManager OUR_MODULENAME = __name__ @@ -46,6 +47,10 @@ def setup_logging_from_paste_ini(loglevel, **kw): logging.config.fileConfig(logging_conf_file) + for callable_hook in \ + PluginManager().get_hook_callables('celery_logging_setup'): + callable_hook() + setup_logging.connect(setup_logging_from_paste_ini) diff --git a/mediagoblin/plugins/oauth/forms.py b/mediagoblin/plugins/oauth/forms.py index 2a956dad..d0a4e9b8 100644 --- a/mediagoblin/plugins/oauth/forms.py +++ b/mediagoblin/plugins/oauth/forms.py @@ -23,10 +23,9 @@ from mediagoblin.tools.translate import fake_ugettext_passthrough as _ class AuthorizationForm(wtforms.Form): - client_id = wtforms.HiddenField(_(u'Client ID'), - [wtforms.validators.Required()]) - next = wtforms.HiddenField(_(u'Next URL'), - [wtforms.validators.Required()]) + client_id = wtforms.HiddenField(u'', + validators=[wtforms.validators.Required()]) + next = wtforms.HiddenField(u'', validators=[wtforms.validators.Required()]) allow = wtforms.SubmitField(_(u'Allow')) deny = wtforms.SubmitField(_(u'Deny')) diff --git a/mediagoblin/plugins/oauth/templates/oauth/authorize.html b/mediagoblin/plugins/oauth/templates/oauth/authorize.html index 647fa41f..8a00c925 100644 --- a/mediagoblin/plugins/oauth/templates/oauth/authorize.html +++ b/mediagoblin/plugins/oauth/templates/oauth/authorize.html @@ -16,7 +16,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. -#} {% extends "mediagoblin/base.html" %} -{% import "/mediagoblin/utils/wtforms.html" as wtforms_util %} +{% import "mediagoblin/utils/wtforms.html" as wtforms_util %} {% block mediagoblin_content %} <form action="{{ request.urlgen('mediagoblin.plugins.oauth.authorize_client') }}" diff --git a/mediagoblin/plugins/raven/README.rst b/mediagoblin/plugins/raven/README.rst new file mode 100644 index 00000000..de5fd20d --- /dev/null +++ b/mediagoblin/plugins/raven/README.rst @@ -0,0 +1,15 @@ +============== + raven plugin +============== + +.. _raven-setup: + +Set up the raven plugin +======================= + +1. Add the following to your MediaGoblin .ini file in the ``[plugins]`` section:: + + [[mediagoblin.plugins.raven]] + sentry_dsn = <YOUR SENTRY DSN> + # Logging is very high-volume, set to 0 if you want to turn off logging + setup_logging = 1 diff --git a/mediagoblin/plugins/raven/__init__.py b/mediagoblin/plugins/raven/__init__.py new file mode 100644 index 00000000..8cfaed0a --- /dev/null +++ b/mediagoblin/plugins/raven/__init__.py @@ -0,0 +1,92 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import logging + +from mediagoblin.tools import pluginapi + +_log = logging.getLogger(__name__) + + +def get_client(): + from raven import Client + config = pluginapi.get_config('mediagoblin.plugins.raven') + + sentry_dsn = config.get('sentry_dsn') + + client = None + + if sentry_dsn: + _log.info('Setting up raven from plugin config: {0}'.format( + sentry_dsn)) + client = Client(sentry_dsn) + elif os.environ.get('SENTRY_DSN'): + _log.info('Setting up raven from SENTRY_DSN environment variable: {0}'\ + .format(os.environ.get('SENTRY_DSN'))) + client = Client() # Implicitly looks for SENTRY_DSN + + if not client: + _log.error('Could not set up client, missing sentry DSN') + return None + + return client + + +def setup_celery(): + from raven.contrib.celery import register_signal + + client = get_client() + + register_signal(client) + + +def setup_logging(): + config = pluginapi.get_config('mediagoblin.plugins.raven') + + conf_setup_logging = False + if config.get('setup_logging'): + conf_setup_logging = bool(int(config.get('setup_logging'))) + + if not conf_setup_logging: + return + + from raven.handlers.logging import SentryHandler + from raven.conf import setup_logging + + client = get_client() + + _log.info('Setting up raven logging handler') + + setup_logging(SentryHandler(client)) + + +def wrap_wsgi(app): + from raven.middleware import Sentry + + client = get_client() + + _log.info('Attaching raven middleware...') + + return Sentry(app, client) + + +hooks = { + 'setup': setup_logging, + 'wrap_wsgi': wrap_wsgi, + 'celery_logging_setup': setup_logging, + 'celery_setup': setup_celery, + } diff --git a/mediagoblin/storage/cloudfiles.py b/mediagoblin/storage/cloudfiles.py index 1b5a6363..b6e57c91 100644 --- a/mediagoblin/storage/cloudfiles.py +++ b/mediagoblin/storage/cloudfiles.py @@ -131,6 +131,43 @@ class CloudFilesStorage(StorageInterface): self._resolve_filepath(filepath)]) + def copy_locally(self, filepath, dest_path): + """ + Copy this file locally. + + A basic working method for this is provided that should + function both for local_storage systems and remote storge + systems, but if more efficient systems for copying locally + apply to your system, override this method with something more + appropriate. + """ + # Override this method, using the "stream" iterator for efficient streaming + with self.get_file(filepath, 'rb') as source_file: + with file(dest_path, 'wb') as dest_file: + for data in source_file: + dest_file.write(data) + + def copy_local_to_storage(self, filename, filepath): + """ + Copy this file from locally to the storage system. + + This is kind of the opposite of copy_locally. It's likely you + could override this method with something more appropriate to + your storage system. + """ + # It seems that (our implementation of) cloudfiles.write() takes + # all existing data and appends write(data) to it, sending the + # full monty over the wire everytime. This would of course + # absolutely kill chunked writes with some O(1^n) performance + # and bandwidth usage. So, override this method and use the + # Cloudfile's "send" interface instead. + # TODO: Fixing write() still seems worthwhile though. + _log.debug('Sending {0} to cloudfiles...'.format(filepath)) + with self.get_file(filepath, 'wb') as dest_file: + with file(filename, 'rb') as source_file: + # Copy to storage system in 4096 byte chunks + dest_file.send(source_file) + class CloudFilesStorageObjectWrapper(): """ Wrapper for python-cloudfiles's cloudfiles.storage_object.Object @@ -160,6 +197,10 @@ class CloudFilesStorageObjectWrapper(): Currently this method does not support any write modes except "append". However if we should need it it would be easy implement. """ + _log.warn( + '{0}.write() has bad performance! Use .send instead for now'\ + .format(self.__class__.__name__)) + if self.storage_object.size and type(data) == str: _log.debug('{0} is > 0 in size, appending data'.format( self.storage_object.name)) @@ -169,9 +210,12 @@ class CloudFilesStorageObjectWrapper(): self.storage_object.name)) self.storage_object.write(data, *args, **kwargs) + def send(self, *args, **kw): + self.storage_object.send(*args, **kw) + def close(self): """ - Not implemented. + Not sure we need anything here. """ pass @@ -188,3 +232,15 @@ class CloudFilesStorageObjectWrapper(): see self.__enter__() """ self.close() + + + def __iter__(self, **kwargs): + """Make CloudFile an iterator, yielding 8192 bytes by default + + This returns a generator object that can be used to getting the + object's content in a memory efficient way. + + Warning: The HTTP response is only complete after this generator + has raised a StopIteration. No other methods can be called until + this has occurred.""" + return self.storage_object.stream(**kwargs) diff --git a/mediagoblin/templates/mediagoblin/edit/edit_account.html b/mediagoblin/templates/mediagoblin/edit/edit_account.html index 3f508af4..7fe2c031 100644 --- a/mediagoblin/templates/mediagoblin/edit/edit_account.html +++ b/mediagoblin/templates/mediagoblin/edit/edit_account.html @@ -15,7 +15,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. #} -{% extends "mediagoblin/base.html" %} +{%- extends "mediagoblin/base.html" %} {% import "/mediagoblin/utils/wtforms.html" as wtforms_util %} @@ -39,7 +39,7 @@ <h1> {%- trans username=user.username -%} Changing {{ username }}'s account settings - {%- endtrans %} + {%- endtrans -%} </h1> {{ wtforms_util.render_field_div(form.old_password) }} {{ wtforms_util.render_field_div(form.new_password) }} @@ -47,10 +47,7 @@ <p>{{ form.wants_comment_notification }} {{ wtforms_util.render_label(form.wants_comment_notification) }}</p> </div> - <div class="form_field_input"> - <p>{{ form.license_preference }} - {{ wtforms_util.render_label(form.license_preference) }}</p> - </div> + {{- wtforms_util.render_field_div(form.license_preference) }} <div class="form_submit_buttons"> <input type="submit" value="{% trans %}Save changes{% endtrans %}" class="button_form" /> {{ csrf_token }} diff --git a/mediagoblin/user_pages/views.py b/mediagoblin/user_pages/views.py index 69d7defb..dc562084 100644 --- a/mediagoblin/user_pages/views.py +++ b/mediagoblin/user_pages/views.py @@ -227,7 +227,8 @@ def media_collect(request, media): # Otherwise, use the collection selected from the drop-down else: collection = Collection.query.filter_by( - id=request.form.get('collection')).first() + id=form.collection.data, + creator=request.user.id).first() # Make sure the user actually selected a collection if not collection: @@ -236,7 +237,7 @@ def media_collect(request, media): _('You have to select or add a collection')) return redirect(request, "mediagoblin.user_pages.media_collect", user=media.get_uploader.username, - media=media.id) + media_id=media.id) # Check whether media already exists in collection @@ -250,7 +251,6 @@ def media_collect(request, media): collection_item = request.db.CollectionItem() collection_item.collection = collection.id collection_item.media_entry = media.id - collection_item.author = request.user.id collection_item.note = request.form['note'] collection_item.save() |