From f67611fb485b5a84cedc62b73beb1e551e8cb934 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Wed, 7 Aug 2013 16:15:27 -0700 Subject: Leave slug empty until we are sure media processing was successful. Patch submitted by LotusEcho --- mediagoblin/processing/task.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'mediagoblin/processing') diff --git a/mediagoblin/processing/task.py b/mediagoblin/processing/task.py index 9af192ed..ba2323b2 100644 --- a/mediagoblin/processing/task.py +++ b/mediagoblin/processing/task.py @@ -94,6 +94,8 @@ class ProcessMedia(task.Task): # We set the state to processed and save the entry here so there's # no need to save at the end of the processing stage, probably ;) entry.state = u'processed' + #Generate the slug here rather than earlier when it could have failed. + entry.generate_slug() entry.save() # Notify the PuSH servers as async task -- cgit v1.2.3 From b26edfb944a02bbb3b1a326c536c10fce6e7bb03 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 8 Aug 2013 15:49:33 -0700 Subject: Revert "Leave slug empty until we are sure media processing was successful." This reverts commit f67611fb485b5a84cedc62b73beb1e551e8cb934. For some reason, generating a slug here throws an integrity error during a query when there is a duplicate slug. --- mediagoblin/processing/task.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'mediagoblin/processing') diff --git a/mediagoblin/processing/task.py b/mediagoblin/processing/task.py index ba2323b2..9af192ed 100644 --- a/mediagoblin/processing/task.py +++ b/mediagoblin/processing/task.py @@ -94,8 +94,6 @@ class ProcessMedia(task.Task): # We set the state to processed and save the entry here so there's # no need to save at the end of the processing stage, probably ;) entry.state = u'processed' - #Generate the slug here rather than earlier when it could have failed. - entry.generate_slug() entry.save() # Notify the PuSH servers as async task -- cgit v1.2.3 From bf2dafd1a04ef8050ebf08bb512862a1592998c0 Mon Sep 17 00:00:00 2001 From: Sebastian Spaeth Date: Thu, 20 Dec 2012 13:42:37 +0100 Subject: Tweak Celery Task - Make sure Exceptions are pickleable (not sure if this was not the case but this is the pattern as documented in the celery docs. - Don't create a task_id in the GMG code, but save the one implicitely created by celery. - Don't create a task-id directory per upload. Just store queued uploads in a single directory (this is the most controversial change and might need discussion!!!) Signed-off-by: Sebastian Spaeth --- mediagoblin/processing/__init__.py | 7 ++++--- mediagoblin/processing/task.py | 22 ++++++++++------------ 2 files changed, 14 insertions(+), 15 deletions(-) (limited to 'mediagoblin/processing') diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py index f3a85940..ae3652cf 100644 --- a/mediagoblin/processing/__init__.py +++ b/mediagoblin/processing/__init__.py @@ -181,9 +181,10 @@ class BaseProcessingFail(Exception): return u"%s:%s" % ( self.__class__.__module__, self.__class__.__name__) - def __init__(self, **metadata): - self.metadata = metadata or {} - + def __init__(self, *args, **kwargs): + # next line is REQUIRED to have pickable exceptions if you want + # to be able to pass in custom arguments (see celery docs) + Exception.__init__(self, *args, **metadata) class BadMediaFail(BaseProcessingFail): """ diff --git a/mediagoblin/processing/task.py b/mediagoblin/processing/task.py index 9af192ed..550906d0 100644 --- a/mediagoblin/processing/task.py +++ b/mediagoblin/processing/task.py @@ -18,11 +18,13 @@ import logging import urllib import urllib2 -from celery import registry, task +#TODO: newer celeries use from celery import Task. Change when we upgrade +from celery.task import Task +from celery.registry import tasks from mediagoblin import mg_globals as mgg -from mediagoblin.db.models import MediaEntry -from . import mark_entry_failed, BaseProcessingFail, ProcessingState +from mediagoblin.db.sql.models import MediaEntry +from mediagoblin.processing import mark_entry_failed, BaseProcessingFail from mediagoblin.tools.processing import json_processing_callback _log = logging.getLogger(__name__) @@ -63,12 +65,10 @@ def handle_push_urls(feed_url): ################################ # Media processing initial steps ################################ +class ProcessMedia(Task): + track_started=True -class ProcessMedia(task.Task): - """ - Pass this entry off for processing. - """ - def run(self, media_id, feed_url): + def run(self, media_id): """ Pass the media entry off to the appropriate processing function (for now just process_image...) @@ -81,8 +81,8 @@ class ProcessMedia(task.Task): # Try to process, and handle expected errors. try: entry.state = u'processing' + entry.queued_task_id = self.request.id entry.save() - _log.debug('Processing {0}'.format(entry)) proc_state = ProcessingState(entry) @@ -140,6 +140,4 @@ class ProcessMedia(task.Task): entry = mgg.database.MediaEntry.query.filter_by(id=entry_id).first() json_processing_callback(entry) -# Register the task -process_media = registry.tasks[ProcessMedia.name] - +tasks.register(ProcessMedia) -- cgit v1.2.3 From b505952508f717e0d4f59d24c87a54ef42673c3f Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Mon, 19 Aug 2013 12:58:00 -0700 Subject: -update to latest master - have mg generate task_id remove --- mediagoblin/processing/__init__.py | 6 ++---- mediagoblin/processing/task.py | 17 ++++++++++------- 2 files changed, 12 insertions(+), 11 deletions(-) (limited to 'mediagoblin/processing') diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py index ae3652cf..454eb09b 100644 --- a/mediagoblin/processing/__init__.py +++ b/mediagoblin/processing/__init__.py @@ -181,10 +181,8 @@ class BaseProcessingFail(Exception): return u"%s:%s" % ( self.__class__.__module__, self.__class__.__name__) - def __init__(self, *args, **kwargs): - # next line is REQUIRED to have pickable exceptions if you want - # to be able to pass in custom arguments (see celery docs) - Exception.__init__(self, *args, **metadata) + def __init__(self, **metadata): + self.metadata = metadata or {} class BadMediaFail(BaseProcessingFail): """ diff --git a/mediagoblin/processing/task.py b/mediagoblin/processing/task.py index 550906d0..bb09daec 100644 --- a/mediagoblin/processing/task.py +++ b/mediagoblin/processing/task.py @@ -18,13 +18,13 @@ import logging import urllib import urllib2 -#TODO: newer celeries use from celery import Task. Change when we upgrade -from celery.task import Task +import celery from celery.registry import tasks from mediagoblin import mg_globals as mgg -from mediagoblin.db.sql.models import MediaEntry -from mediagoblin.processing import mark_entry_failed, BaseProcessingFail +from mediagoblin.db.models import MediaEntry +from mediagoblin.processing import (mark_entry_failed, BaseProcessingFail, + ProcessingState) from mediagoblin.tools.processing import json_processing_callback _log = logging.getLogger(__name__) @@ -32,7 +32,7 @@ logging.basicConfig() _log.setLevel(logging.DEBUG) -@task.task(default_retry_delay=2 * 60) +@celery.task(default_retry_delay=2 * 60) def handle_push_urls(feed_url): """Subtask, notifying the PuSH servers of new content @@ -62,10 +62,14 @@ def handle_push_urls(feed_url): 'Giving up.'.format(feed_url)) return False + ################################ # Media processing initial steps ################################ -class ProcessMedia(Task): +class ProcessMedia(celery.Task): + """ + Pass this entry off for processing. + """ track_started=True def run(self, media_id): @@ -81,7 +85,6 @@ class ProcessMedia(Task): # Try to process, and handle expected errors. try: entry.state = u'processing' - entry.queued_task_id = self.request.id entry.save() _log.debug('Processing {0}'.format(entry)) -- cgit v1.2.3 From d4ae4c9f971b5795f1697f443b359d3155a16013 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Mon, 19 Aug 2013 14:57:00 -0700 Subject: - need self.metadata with BaseProcessingFail - pass feed_url into ProcessMedia run() --- mediagoblin/processing/task.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'mediagoblin/processing') diff --git a/mediagoblin/processing/task.py b/mediagoblin/processing/task.py index bb09daec..05cac844 100644 --- a/mediagoblin/processing/task.py +++ b/mediagoblin/processing/task.py @@ -72,7 +72,7 @@ class ProcessMedia(celery.Task): """ track_started=True - def run(self, media_id): + def run(self, media_id, feed_url): """ Pass the media entry off to the appropriate processing function (for now just process_image...) @@ -86,6 +86,7 @@ class ProcessMedia(celery.Task): try: entry.state = u'processing' entry.save() + _log.debug('Processing {0}'.format(entry)) proc_state = ProcessingState(entry) -- cgit v1.2.3