From 2cfffd5ed8c054bb60c27ede4e69667f97d12b09 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Tue, 15 Jan 2013 14:41:30 +0100 Subject: Make PuSHing the Pubhubsubbub server an async task (#436, #585) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Notifying the PuSH servers had 3 problems.  1) it was done immediately after sending of the processing task to celery. So if celery was run in a separate process we would notify the PuSH servers before the new media was processed/ visible. (#436) 2) Notification code was called in submit/views.py, so submitting via the API never resulted in notifications. (#585) 3) If Notifying the PuSH server failed, we would never retry. The solution was to make the PuSH notification an asynchronous subtask. This way: 1) it will only be called once async processing has finished, 2) it is in the main processing code path, so even API calls will result in notifications, and 3) We retry 3 times in case of failure before giving up. If the server is in a separate process, we will wait 3x 2 minutes before retrying the notification. The only downside is that the celery server needs to have access to the internet to ping the PuSH server. If that is a problem, we need to make the task belong to a special group of celery servers that has access to the internet. As a side effect, I believe I removed the limitation that prevented us from upgrading celery. Signed-off-by: Sebastian Spaeth --- mediagoblin/submit/views.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'mediagoblin/submit/views.py') diff --git a/mediagoblin/submit/views.py b/mediagoblin/submit/views.py index 2d609b31..145b9f5e 100644 --- a/mediagoblin/submit/views.py +++ b/mediagoblin/submit/views.py @@ -32,8 +32,7 @@ from mediagoblin.submit import forms as submit_forms from mediagoblin.messages import add_message, SUCCESS from mediagoblin.media_types import sniff_media, \ InvalidFileType, FileTypeNotSupported -from mediagoblin.submit.lib import handle_push_urls, run_process_media, \ - prepare_queue_task +from mediagoblin.submit.lib import run_process_media, prepare_queue_task @require_active_login @@ -90,10 +89,7 @@ def submit_start(request): # # (... don't change entry after this point to avoid race # conditions with changes to the document via processing code) - run_process_media(entry) - - handle_push_urls(request) - + run_process_media(entry, request) add_message(request, SUCCESS, _('Woohoo! Submitted!')) return redirect(request, "mediagoblin.user_pages.user_home", -- cgit v1.2.3