From 9a2c66ca9ef763fa68dc09a483c02fe2ee02d78f Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Fri, 2 Aug 2013 11:40:41 -0700 Subject: added image reprocessing --- mediagoblin/processing/task.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'mediagoblin/processing/task.py') diff --git a/mediagoblin/processing/task.py b/mediagoblin/processing/task.py index 9af192ed..c0dfb9b4 100644 --- a/mediagoblin/processing/task.py +++ b/mediagoblin/processing/task.py @@ -68,13 +68,15 @@ class ProcessMedia(task.Task): """ Pass this entry off for processing. """ - def run(self, media_id, feed_url): + def run(self, media_id, feed_url, reprocess_info=None): """ Pass the media entry off to the appropriate processing function (for now just process_image...) :param feed_url: The feed URL that the PuSH server needs to be updated for. + :param reprocess: A dict containing all of the necessary reprocessing + info for the media_type. """ entry = MediaEntry.query.get(media_id) @@ -89,7 +91,7 @@ class ProcessMedia(task.Task): with mgg.workbench_manager.create() as workbench: proc_state.set_workbench(workbench) # run the processing code - entry.media_manager.processor(proc_state) + entry.media_manager.processor(proc_state, reprocess_info) # We set the state to processed and save the entry here so there's # no need to save at the end of the processing stage, probably ;) -- cgit v1.2.3 From 3e9faf85da1ee2971e9ff2fde12b192ea470d806 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Fri, 2 Aug 2013 15:12:07 -0700 Subject: added comments and did a little refactoring. not sure if it is actually any clearer though --- mediagoblin/processing/task.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'mediagoblin/processing/task.py') diff --git a/mediagoblin/processing/task.py b/mediagoblin/processing/task.py index c0dfb9b4..36ee31fd 100644 --- a/mediagoblin/processing/task.py +++ b/mediagoblin/processing/task.py @@ -89,9 +89,17 @@ class ProcessMedia(task.Task): proc_state = ProcessingState(entry) with mgg.workbench_manager.create() as workbench: + proc_state.set_workbench(workbench) - # run the processing code - entry.media_manager.processor(proc_state, reprocess_info) + processor = entry.media_manager.processor(proc_state) + + # If we have reprocess_info, let's reprocess + if reprocess_info: + processor.reprocess(reprocess_info) + + # Run initial processing + else: + processor.initial_processing() # We set the state to processed and save the entry here so there's # no need to save at the end of the processing stage, probably ;) -- cgit v1.2.3 From 77ea4c9bd1e8372fb7206596ca5125738033ced5 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sun, 11 Aug 2013 14:34:45 -0500 Subject: Updating to the point where we can allllmost run with the new reprocessing code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit sponsored by Odin Hørthe Omdal. Thank you! --- mediagoblin/processing/task.py | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) (limited to 'mediagoblin/processing/task.py') diff --git a/mediagoblin/processing/task.py b/mediagoblin/processing/task.py index 36ee31fd..240be4e5 100644 --- a/mediagoblin/processing/task.py +++ b/mediagoblin/processing/task.py @@ -21,9 +21,9 @@ import urllib2 from celery import registry, task from mediagoblin import mg_globals as mgg -from mediagoblin.db.models import MediaEntry -from . import mark_entry_failed, BaseProcessingFail, ProcessingState +from . import mark_entry_failed, BaseProcessingFail from mediagoblin.tools.processing import json_processing_callback +from mediagoblin.processing import get_entry_and_manager _log = logging.getLogger(__name__) logging.basicConfig() @@ -68,7 +68,7 @@ class ProcessMedia(task.Task): """ Pass this entry off for processing. """ - def run(self, media_id, feed_url, reprocess_info=None): + def run(self, media_id, feed_url, reprocess_action, reprocess_info=None): """ Pass the media entry off to the appropriate processing function (for now just process_image...) @@ -78,28 +78,20 @@ class ProcessMedia(task.Task): :param reprocess: A dict containing all of the necessary reprocessing info for the media_type. """ - entry = MediaEntry.query.get(media_id) + reprocess_info = reprocess_info or {} + entry, manager = get_entry_and_manager(media_id) # Try to process, and handle expected errors. try: + processor_class = manager.get_processor(reprocess_action, entry) + entry.state = u'processing' entry.save() _log.debug('Processing {0}'.format(entry)) - proc_state = ProcessingState(entry) - with mgg.workbench_manager.create() as workbench: - - proc_state.set_workbench(workbench) - processor = entry.media_manager.processor(proc_state) - - # If we have reprocess_info, let's reprocess - if reprocess_info: - processor.reprocess(reprocess_info) - - # Run initial processing - else: - processor.initial_processing() + with processor_class(manager, entry) as processor: + processor.process(**reprocess_info) # We set the state to processed and save the entry here so there's # no need to save at the end of the processing stage, probably ;) -- cgit v1.2.3 From 55cfa3406390732173195bb920bf3f86bd1ce9f4 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sun, 11 Aug 2013 15:22:43 -0500 Subject: Renaming the processing manager stuff to be less ambiguous. BONUS COMMIT to Ben Finney and the Free Software Melbourne crew. :) IRONY: Initially I committed this as "media manager". --- mediagoblin/processing/task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'mediagoblin/processing/task.py') diff --git a/mediagoblin/processing/task.py b/mediagoblin/processing/task.py index 240be4e5..397514d0 100644 --- a/mediagoblin/processing/task.py +++ b/mediagoblin/processing/task.py @@ -23,7 +23,7 @@ from celery import registry, task from mediagoblin import mg_globals as mgg from . import mark_entry_failed, BaseProcessingFail from mediagoblin.tools.processing import json_processing_callback -from mediagoblin.processing import get_entry_and_manager +from mediagoblin.processing import get_entry_and_processing_manager _log = logging.getLogger(__name__) logging.basicConfig() @@ -79,7 +79,7 @@ class ProcessMedia(task.Task): info for the media_type. """ reprocess_info = reprocess_info or {} - entry, manager = get_entry_and_manager(media_id) + entry, manager = get_entry_and_processing_manager(media_id) # Try to process, and handle expected errors. try: -- cgit v1.2.3 From 22479c39a0fff75208309e437f5fdaf57730cf0e Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Mon, 12 Aug 2013 08:22:14 -0500 Subject: Record the original state of the media entry in the processor This allows our processor to make some informed decisions based on the state by still having access to the original state. This commit sponsored by William Rico. Thank you! --- mediagoblin/processing/task.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'mediagoblin/processing/task.py') diff --git a/mediagoblin/processing/task.py b/mediagoblin/processing/task.py index 397514d0..d3770588 100644 --- a/mediagoblin/processing/task.py +++ b/mediagoblin/processing/task.py @@ -85,12 +85,14 @@ class ProcessMedia(task.Task): try: processor_class = manager.get_processor(reprocess_action, entry) - entry.state = u'processing' - entry.save() + with processor_class(manager, entry) as processor: + # Initial state change has to be here because + # the entry.state gets recorded on processor_class init + entry.state = u'processing' + entry.save() - _log.debug('Processing {0}'.format(entry)) + _log.debug('Processing {0}'.format(entry)) - with processor_class(manager, entry) as processor: processor.process(**reprocess_info) # We set the state to processed and save the entry here so there's -- cgit v1.2.3 From 7d3fda06b03691601bc08b5d88baf1da1c3f83fc Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Wed, 14 Aug 2013 14:54:10 -0700 Subject: catch processing exceptions and if entry_orig_state is processed, then ignore the exception --- mediagoblin/processing/task.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'mediagoblin/processing/task.py') diff --git a/mediagoblin/processing/task.py b/mediagoblin/processing/task.py index d3770588..df44dd7a 100644 --- a/mediagoblin/processing/task.py +++ b/mediagoblin/processing/task.py @@ -93,7 +93,18 @@ class ProcessMedia(task.Task): _log.debug('Processing {0}'.format(entry)) - processor.process(**reprocess_info) + try: + processor.process(**reprocess_info) + except Exception as exc: + if processor.entry_orig_state == 'processed': + _log.error( + 'Entry {0} failed to process due to the following' + ' error: {1}'.format(entry.id, exc)) + _log.info( + 'Setting entry.state back to "processed"') + pass + else: + raise # We set the state to processed and save the entry here so there's # no need to save at the end of the processing stage, probably ;) -- cgit v1.2.3