From 9a2c66ca9ef763fa68dc09a483c02fe2ee02d78f Mon Sep 17 00:00:00 2001
From: Rodney Ewing <ewing.rj@gmail.com>
Date: Fri, 2 Aug 2013 11:40:41 -0700
Subject: added image reprocessing

---
 mediagoblin/processing/__init__.py | 17 +++++++++++++++++
 mediagoblin/processing/task.py     |  6 ++++--
 2 files changed, 21 insertions(+), 2 deletions(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index f3a85940..bbe9f364 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -87,6 +87,7 @@ class ProcessingState(object):
         self.entry = entry
         self.workbench = None
         self.queued_filename = None
+        self.reprocess_filename = None
 
     def set_workbench(self, wb):
         self.workbench = wb
@@ -128,6 +129,22 @@ class ProcessingState(object):
         mgg.queue_store.delete_dir(queued_filepath[:-1])  # rm dir
         self.entry.queued_media_file = []
 
+    def get_reprocess_filename(self):
+        """
+        Get the filename to use during reprocessing
+        """
+        # Currently only returns the original file, but eventually will return
+        # the highest quality file if the original doesn't exist
+        if self.reprocess_filename is not None:
+            return self.reprocess_filename
+
+        reprocess_filepath = self.entry.media_files['original'][2]
+        reprocess_filename = self.workbench.local_file(
+            mgg.public_store, reprocess_filepath,
+            'original')
+        self.reprocess_filename = reprocess_filename
+        return reprocess_filename
+
 
 def mark_entry_failed(entry_id, exc):
     """
diff --git a/mediagoblin/processing/task.py b/mediagoblin/processing/task.py
index 9af192ed..c0dfb9b4 100644
--- a/mediagoblin/processing/task.py
+++ b/mediagoblin/processing/task.py
@@ -68,13 +68,15 @@ class ProcessMedia(task.Task):
     """
     Pass this entry off for processing.
     """
-    def run(self, media_id, feed_url):
+    def run(self, media_id, feed_url, reprocess_info=None):
         """
         Pass the media entry off to the appropriate processing function
         (for now just process_image...)
 
         :param feed_url: The feed URL that the PuSH server needs to be
             updated for.
+        :param reprocess: A dict containing all of the necessary reprocessing
+            info for the media_type.
         """
         entry = MediaEntry.query.get(media_id)
 
@@ -89,7 +91,7 @@ class ProcessMedia(task.Task):
             with mgg.workbench_manager.create() as workbench:
                 proc_state.set_workbench(workbench)
                 # run the processing code
-                entry.media_manager.processor(proc_state)
+                entry.media_manager.processor(proc_state, reprocess_info)
 
             # We set the state to processed and save the entry here so there's
             # no need to save at the end of the processing stage, probably ;)
-- 
cgit v1.2.3


From 49db7785797a251ee408c62c0954ccd71af9d088 Mon Sep 17 00:00:00 2001
From: Rodney Ewing <ewing.rj@gmail.com>
Date: Fri, 2 Aug 2013 13:18:35 -0700
Subject: very rough working version of image reprocessing

---
 mediagoblin/processing/__init__.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index bbe9f364..13c677eb 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -138,10 +138,10 @@ class ProcessingState(object):
         if self.reprocess_filename is not None:
             return self.reprocess_filename
 
-        reprocess_filepath = self.entry.media_files['original'][2]
-        reprocess_filename = self.workbench.local_file(
+        reprocess_filepath = self.entry.media_files['original']
+        reprocess_filename = self.workbench.localized_file(
             mgg.public_store, reprocess_filepath,
-            'original')
+            'source')
         self.reprocess_filename = reprocess_filename
         return reprocess_filename
 
-- 
cgit v1.2.3


From 3e9faf85da1ee2971e9ff2fde12b192ea470d806 Mon Sep 17 00:00:00 2001
From: Rodney Ewing <ewing.rj@gmail.com>
Date: Fri, 2 Aug 2013 15:12:07 -0700
Subject: added comments and did a little refactoring. not sure if it is
 actually any clearer though

---
 mediagoblin/processing/task.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/task.py b/mediagoblin/processing/task.py
index c0dfb9b4..36ee31fd 100644
--- a/mediagoblin/processing/task.py
+++ b/mediagoblin/processing/task.py
@@ -89,9 +89,17 @@ class ProcessMedia(task.Task):
 
             proc_state = ProcessingState(entry)
             with mgg.workbench_manager.create() as workbench:
+
                 proc_state.set_workbench(workbench)
-                # run the processing code
-                entry.media_manager.processor(proc_state, reprocess_info)
+                processor = entry.media_manager.processor(proc_state)
+
+                # If we have reprocess_info, let's reprocess
+                if reprocess_info:
+                    processor.reprocess(reprocess_info)
+
+                # Run initial processing
+                else:
+                    processor.initial_processing()
 
             # We set the state to processed and save the entry here so there's
             # no need to save at the end of the processing stage, probably ;)
-- 
cgit v1.2.3


From 45b20dce1ac5a8d9fb045faf67e796a8092f65e4 Mon Sep 17 00:00:00 2001
From: Rodney Ewing <ewing.rj@gmail.com>
Date: Fri, 2 Aug 2013 15:20:59 -0700
Subject: change get_queued_filename to get_orig_filename and modified function

---
 mediagoblin/processing/__init__.py | 48 +++++++++++++++++---------------------
 1 file changed, 21 insertions(+), 27 deletions(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index 13c677eb..60565e09 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -86,27 +86,37 @@ class ProcessingState(object):
     def __init__(self, entry):
         self.entry = entry
         self.workbench = None
-        self.queued_filename = None
-        self.reprocess_filename = None
+        self.orig_filename = None
 
     def set_workbench(self, wb):
         self.workbench = wb
 
-    def get_queued_filename(self):
+    def get_orig_filename(self):
         """
         Get the a filename for the original, on local storage
+
+        If the media entry has a queued_media_file, use that, otherwise
+        use the original.
+
+        In the future, this will return the highest quality file available
+        if neither the original or queued file are available
         """
-        if self.queued_filename is not None:
-            return self.queued_filename
-        queued_filepath = self.entry.queued_media_file
-        queued_filename = self.workbench.localized_file(
-            mgg.queue_store, queued_filepath,
+        if self.orig_filename is not None:
+            return self.orig_filename
+
+        if self.entry.queued_media_file:
+            orig_filepath = self.entry.queued_media_file
+        else:
+            orig_filepath = self.entry.media_files['original']
+
+        orig_filename = self.workbench.localized_file(
+            mgg.queue_store, orig_filepath,
             'source')
-        self.queued_filename = queued_filename
-        return queued_filename
+        self.orig_filename = orig_filename
+        return orig_filename
 
     def copy_original(self, target_name, keyname=u"original"):
-        self.store_public(keyname, self.get_queued_filename(), target_name)
+        self.store_public(keyname, self.get_orig_filename(), target_name)
 
     def store_public(self, keyname, local_file, target_name=None):
         if target_name is None:
@@ -129,22 +139,6 @@ class ProcessingState(object):
         mgg.queue_store.delete_dir(queued_filepath[:-1])  # rm dir
         self.entry.queued_media_file = []
 
-    def get_reprocess_filename(self):
-        """
-        Get the filename to use during reprocessing
-        """
-        # Currently only returns the original file, but eventually will return
-        # the highest quality file if the original doesn't exist
-        if self.reprocess_filename is not None:
-            return self.reprocess_filename
-
-        reprocess_filepath = self.entry.media_files['original']
-        reprocess_filename = self.workbench.localized_file(
-            mgg.public_store, reprocess_filepath,
-            'source')
-        self.reprocess_filename = reprocess_filename
-        return reprocess_filename
-
 
 def mark_entry_failed(entry_id, exc):
     """
-- 
cgit v1.2.3


From c541fb71f7f92ce13783400cf9b22083f38ae189 Mon Sep 17 00:00:00 2001
From: Rodney Ewing <ewing.rj@gmail.com>
Date: Tue, 6 Aug 2013 10:48:26 -0700
Subject: fix storage paramater in get_orig_filename(), fix __init__ for
 ProceessImage, better description for --size flag

---
 mediagoblin/processing/__init__.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index 60565e09..5ce9281b 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -106,11 +106,13 @@ class ProcessingState(object):
 
         if self.entry.queued_media_file:
             orig_filepath = self.entry.queued_media_file
+            storage = mgg.queue_store
         else:
             orig_filepath = self.entry.media_files['original']
+            storage = mgg.public_store
 
         orig_filename = self.workbench.localized_file(
-            mgg.queue_store, orig_filepath,
+            storage, orig_filepath,
             'source')
         self.orig_filename = orig_filename
         return orig_filename
-- 
cgit v1.2.3


From 14565fb72022e015ee9ba64cf087befb33516b71 Mon Sep 17 00:00:00 2001
From: Christopher Allan Webber <cwebber@dustycloud.org>
Date: Wed, 7 Aug 2013 15:01:46 -0500
Subject: started coding basics of new processing code

---
 mediagoblin/processing/__init__.py | 52 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index 5ce9281b..95f346d2 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -74,6 +74,58 @@ class FilenameBuilder(object):
                              ext=self.ext)
 
 
+
+
+class MediaProcessor(object):
+    # You MUST override this in the child MediaProcessor!
+    name = None
+
+    def __init__(self, manager):
+        self.manager = manager
+
+    def media_is_eligibile(self, media_entry):
+        raise NotImplementedError
+
+    def process(self):
+        raise NotImplementedError
+
+    def generate_parser(self):
+        raise NotImplementedError
+
+
+class ProcessingManager(object):
+    def __init__(self, entry):
+        self.entry = entry
+        # May merge these two classes soon....
+        self.state = ProcessingState(entry)
+
+        # Dict of all MediaProcessors of this media type
+        self.processors = {}
+
+    def add_processor(self, processor):
+        """
+        Add a processor class to this media type
+        """
+        name = processor.name
+        if name is None:
+            raise AttributeError("Processor class's .name attribute not set")
+        
+        self.processors[name] = processor
+
+    def list_eligible_processors(self):
+        """
+        List all processors that this media entry is eligible to be processed
+        for.
+        """
+        return [
+            processor
+            for processor in self.processors.keys()
+            if processor.media_is_eligible(self.entry)]
+
+    def process(self, processor):
+        pass
+
+
 class ProcessingState(object):
     """
     The first and only argument to the "processor" of a media type
-- 
cgit v1.2.3


From 274a0f67fd9c36fe01950f2547425fb115c59aff Mon Sep 17 00:00:00 2001
From: Christopher Allan Webber <cwebber@dustycloud.org>
Date: Wed, 7 Aug 2013 17:07:19 -0500
Subject: Documentation for the MediaProcessor

---
 mediagoblin/processing/__init__.py | 58 +++++++++++++++++++++++++++++++++++---
 1 file changed, 54 insertions(+), 4 deletions(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index 95f346d2..41028fbb 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -75,25 +75,75 @@ class FilenameBuilder(object):
 
 
 
-
 class MediaProcessor(object):
+    """A particular processor for this media type.
+
+    While the ProcessingManager handles all types of MediaProcessing
+    possible for a particular media type, a MediaProcessor can be
+    thought of as a *particular* processing action for a media type.
+    For example, you may have separate MediaProcessors for:
+
+    - initial_processing: the intial processing of a media
+    - gen_thumb: generate a thumbnail
+    - resize: resize an image
+    - transcode: transcode a video
+
+    ... etc.  
+
+    Some information on producing a new MediaProcessor for your media type:
+
+    - You *must* supply a name attribute.  This must be a class level
+      attribute, and a string.  This will be used to determine the
+      subcommand of your process
+    - It's recommended that you supply a class level description
+      attribute.
+    - Supply a media_is_eligible classmethod.  This will be used to
+      determine whether or not a media entry is eligible to use this
+      processor type.  See the method documentation for details.
+    - To give "./bin/gmg reprocess run" abilities to this media type,
+      supply both gnerate_parser and parser_to_request classmethods.
+    - The process method will be what actually processes your media.
+    """
     # You MUST override this in the child MediaProcessor!
     name = None
 
+    # Optional, but will be used in various places to describe the
+    # action this MediaProcessor provides
+    description = None
+
     def __init__(self, manager):
         self.manager = manager
 
-    def media_is_eligibile(self, media_entry):
+    def process(self, **kwargs):
+        """
+        Actually process this media entry.
+        """
         raise NotImplementedError
 
-    def process(self):
+    @classmethod
+    def media_is_eligibile(self, media_entry):
         raise NotImplementedError
 
+    ###############################
+    # Command line interface things
+    ###############################
+
+    @classmethod
     def generate_parser(self):
         raise NotImplementedError
 
+    @classmethod
+    def parser_to_request(self, parser):
+        raise NotImplementedError
+
+    ##########################################
+    # THE FUTURE: web interface things here :)
+    ##########################################
+
 
 class ProcessingManager(object):
+    """
+    """
     def __init__(self, entry):
         self.entry = entry
         # May merge these two classes soon....
@@ -122,7 +172,7 @@ class ProcessingManager(object):
             for processor in self.processors.keys()
             if processor.media_is_eligible(self.entry)]
 
-    def process(self, processor):
+    def process(self, directive, request):
         pass
 
 
-- 
cgit v1.2.3


From e4bdc9091c392bf31d40bcb5ae12b92e17a6cb2a Mon Sep 17 00:00:00 2001
From: Christopher Allan Webber <cwebber@dustycloud.org>
Date: Thu, 8 Aug 2013 13:53:28 -0500
Subject: More steps towards a working reprocessing system.

Fleshing out the base classes and setting up some docstrings.  Not
everything is totally clear yet, but I think it's on a good track, and
getting clearer.

This commit sponsored by Ben Finney, on behalf of Free Software Melbourne.
Thank you all!
---
 mediagoblin/processing/__init__.py | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index 41028fbb..66ef2a53 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -114,6 +114,10 @@ class MediaProcessor(object):
     def __init__(self, manager):
         self.manager = manager
 
+        # Should be initialized at time of processing, at least
+        self.workbench = None
+
+    # @with_workbench
     def process(self, **kwargs):
         """
         Actually process this media entry.
@@ -142,13 +146,12 @@ class MediaProcessor(object):
 
 
 class ProcessingManager(object):
-    """
-    """
-    def __init__(self, entry):
-        self.entry = entry
-        # May merge these two classes soon....
-        self.state = ProcessingState(entry)
+    """Manages all the processing actions available for a media type
 
+    Specific processing actions, MediaProcessor subclasses, are added
+    to the ProcessingManager.
+    """
+    def __init__(self):
         # Dict of all MediaProcessors of this media type
         self.processors = {}
 
@@ -162,7 +165,7 @@ class ProcessingManager(object):
         
         self.processors[name] = processor
 
-    def list_eligible_processors(self):
+    def list_eligible_processors(self, entry):
         """
         List all processors that this media entry is eligible to be processed
         for.
@@ -170,9 +173,16 @@ class ProcessingManager(object):
         return [
             processor
             for processor in self.processors.keys()
-            if processor.media_is_eligible(self.entry)]
+            if processor.media_is_eligible(entry)]
+
+    def gen_process_request_via_cli(self, subparser):
+        # Got to figure out what actually goes here before I can write this properly
+        pass
 
-    def process(self, directive, request):
+    def process(self, entry, directive, request):
+        """
+        Process a media entry.
+        """
         pass
 
 
-- 
cgit v1.2.3


From 85ead8ac3cf59aeee12ddd3b33ecfeb03c3aa946 Mon Sep 17 00:00:00 2001
From: Christopher Allan Webber <cwebber@dustycloud.org>
Date: Fri, 9 Aug 2013 12:13:53 -0500
Subject: "initial" reprocessing subcommand now works!
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

We are on our way now to a working reprocessing system under this
redesign!

This commit sponsored by Bjarni Rúnar Einarsson.  Thank you!
---
 mediagoblin/processing/__init__.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index 66ef2a53..95622b9d 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -14,6 +14,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from collections import OrderedDict
 import logging
 import os
 
@@ -153,7 +154,7 @@ class ProcessingManager(object):
     """
     def __init__(self):
         # Dict of all MediaProcessors of this media type
-        self.processors = {}
+        self.processors = OrderedDict()
 
     def add_processor(self, processor):
         """
@@ -172,9 +173,12 @@ class ProcessingManager(object):
         """
         return [
             processor
-            for processor in self.processors.keys()
+            for processor in self.processors.values()
             if processor.media_is_eligible(entry)]
 
+    def list_all_processors(self):
+        return self.processors.values()
+
     def gen_process_request_via_cli(self, subparser):
         # Got to figure out what actually goes here before I can write this properly
         pass
-- 
cgit v1.2.3


From 55a10fef0ae97cb33c8393a7a25487c2666b4cf1 Mon Sep 17 00:00:00 2001
From: Christopher Allan Webber <cwebber@dustycloud.org>
Date: Fri, 9 Aug 2013 13:56:23 -0500
Subject: `gmg reprocess available --action-help` now tells you processor
 arguments!

Every reprocessing action possible can inform you of its command line
argument stuff!  Is that awesome or what?
---
 mediagoblin/processing/__init__.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index 95622b9d..9e77d2b2 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -126,7 +126,7 @@ class MediaProcessor(object):
         raise NotImplementedError
 
     @classmethod
-    def media_is_eligibile(self, media_entry):
+    def media_is_eligibile(cls, media_entry):
         raise NotImplementedError
 
     ###############################
@@ -134,11 +134,11 @@ class MediaProcessor(object):
     ###############################
 
     @classmethod
-    def generate_parser(self):
+    def generate_parser(cls):
         raise NotImplementedError
 
     @classmethod
-    def parser_to_request(self, parser):
+    def parser_to_request(cls, parser):
         raise NotImplementedError
 
     ##########################################
-- 
cgit v1.2.3


From 4ba5bdd96ef3703d8da216ca3dd92f080214f164 Mon Sep 17 00:00:00 2001
From: Christopher Allan Webber <cwebber@dustycloud.org>
Date: Fri, 9 Aug 2013 16:12:06 -0500
Subject: Steps toward working "run" reprocessing command.

This commit sponsored by Philippe Casteleyn.  Thank you!
---
 mediagoblin/processing/__init__.py | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index 9e77d2b2..6ef203cb 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -126,7 +126,7 @@ class MediaProcessor(object):
         raise NotImplementedError
 
     @classmethod
-    def media_is_eligibile(cls, media_entry):
+    def media_is_eligible(cls, media_entry):
         raise NotImplementedError
 
     ###############################
@@ -146,6 +146,11 @@ class MediaProcessor(object):
     ##########################################
 
 
+class ProcessingKeyError(Exception): pass
+class ProcessorDoesNotExist(ProcessingKeyError): pass
+class ProcessorNotEligible(ProcessingKeyError): pass
+
+
 class ProcessingManager(object):
     """Manages all the processing actions available for a media type
 
@@ -183,6 +188,25 @@ class ProcessingManager(object):
         # Got to figure out what actually goes here before I can write this properly
         pass
 
+    def get_processor(self, key, entry=None):
+        """
+        Get the processor with this key.
+
+        If entry supplied, make sure this entry is actually compatible;
+        otherwise raise error.
+        """
+        try:
+            processor = self.processors[key]
+        except KeyError:
+            raise ProcessorDoesNotExist(
+                "'%s' processor does not exist for this media type" % key)
+
+        if entry and not processor.media_is_eligible(entry):
+            raise ProcessorNotEligible(
+                "This entry is not eligible for processor with name '%s'" % key)
+
+        return processor
+
     def process(self, entry, directive, request):
         """
         Process a media entry.
-- 
cgit v1.2.3


From d1e9913b71a6f3b7bb41f5eee1051093b92fcd8a Mon Sep 17 00:00:00 2001
From: Christopher Allan Webber <cwebber@dustycloud.org>
Date: Fri, 9 Aug 2013 17:30:52 -0500
Subject: Should be enough to get to the point where you can actually
 initialize a processing command now.

However, it doesn't celery task-ify it...

This commit sponsored by Catalin Cosovanu.  Thank you!
---
 mediagoblin/processing/__init__.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index 6ef203cb..1c8f7202 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -112,8 +112,9 @@ class MediaProcessor(object):
     # action this MediaProcessor provides
     description = None
 
-    def __init__(self, manager):
+    def __init__(self, manager, media_entry):
         self.manager = manager
+        self.media_entry = media_entry
 
         # Should be initialized at time of processing, at least
         self.workbench = None
@@ -138,7 +139,7 @@ class MediaProcessor(object):
         raise NotImplementedError
 
     @classmethod
-    def parser_to_request(cls, parser):
+    def args_to_request(cls, args):
         raise NotImplementedError
 
     ##########################################
@@ -214,6 +215,17 @@ class ProcessingManager(object):
         pass
 
 
+def request_from_args(args, which_args):
+    """
+    Generate a request from the values of some argparse parsed args
+    """
+    request = {}
+    for arg in which_args:
+        request[arg] = getattr(args, arg)
+
+    return request
+
+
 class ProcessingState(object):
     """
     The first and only argument to the "processor" of a media type
-- 
cgit v1.2.3


From 77ea4c9bd1e8372fb7206596ca5125738033ced5 Mon Sep 17 00:00:00 2001
From: Christopher Allan Webber <cwebber@dustycloud.org>
Date: Sun, 11 Aug 2013 14:34:45 -0500
Subject: Updating to the point where we can allllmost run with the new
 reprocessing code
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This commit sponsored by Odin Hørthe Omdal.  Thank you!
---
 mediagoblin/processing/__init__.py | 40 +++++++++++++++++++++++++++++++++++---
 mediagoblin/processing/task.py     | 26 +++++++++----------------
 2 files changed, 46 insertions(+), 20 deletions(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index 1c8f7202..b668baa7 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -18,9 +18,10 @@ from collections import OrderedDict
 import logging
 import os
 
-from mediagoblin.db.util import atomic_update
 from mediagoblin import mg_globals as mgg
-
+from mediagoblin.db.util import atomic_update
+from mediagoblin.db.models import MediaEntry
+from mediagoblin.tools.pluginapi import hook_handle
 from mediagoblin.tools.translate import lazy_pass_to_ugettext as _
 
 _log = logging.getLogger(__name__)
@@ -208,7 +209,7 @@ class ProcessingManager(object):
 
         return processor
 
-    def process(self, entry, directive, request):
+    def process_from_args(self, entry, reprocess_command, request):
         """
         Process a media entry.
         """
@@ -226,6 +227,39 @@ def request_from_args(args, which_args):
     return request
 
 
+class MediaEntryNotFound(Exception): pass
+
+
+def get_manager_for_type(media_type):
+    """
+    Get the appropriate media manager for this type
+    """
+    manager_class = hook_handle(('reprocess_manager', media_type))
+    manager = manager_class()
+
+    return manager
+
+
+def get_entry_and_manager(media_id):
+    """
+    Get a MediaEntry, its media type, and its manager all in one go.
+
+    Returns a tuple of: `(entry, media_type, media_manager)`
+    """
+    entry = MediaEntry.query.filter_by(id=media_id).first()
+    if entry is None:
+        raise MediaEntryNotFound("Can't find media with id '%s'" % media_id)
+
+    manager = get_manager_for_type(entry.media_type)
+
+    return entry, manager
+
+
+################################################
+# TODO: This ProcessingState is OUTDATED,
+#   and needs to be refactored into other tools!
+################################################
+
 class ProcessingState(object):
     """
     The first and only argument to the "processor" of a media type
diff --git a/mediagoblin/processing/task.py b/mediagoblin/processing/task.py
index 36ee31fd..240be4e5 100644
--- a/mediagoblin/processing/task.py
+++ b/mediagoblin/processing/task.py
@@ -21,9 +21,9 @@ import urllib2
 from celery import registry, task
 
 from mediagoblin import mg_globals as mgg
-from mediagoblin.db.models import MediaEntry
-from . import mark_entry_failed, BaseProcessingFail, ProcessingState
+from . import mark_entry_failed, BaseProcessingFail
 from mediagoblin.tools.processing import json_processing_callback
+from mediagoblin.processing import get_entry_and_manager
 
 _log = logging.getLogger(__name__)
 logging.basicConfig()
@@ -68,7 +68,7 @@ class ProcessMedia(task.Task):
     """
     Pass this entry off for processing.
     """
-    def run(self, media_id, feed_url, reprocess_info=None):
+    def run(self, media_id, feed_url, reprocess_action, reprocess_info=None):
         """
         Pass the media entry off to the appropriate processing function
         (for now just process_image...)
@@ -78,28 +78,20 @@ class ProcessMedia(task.Task):
         :param reprocess: A dict containing all of the necessary reprocessing
             info for the media_type.
         """
-        entry = MediaEntry.query.get(media_id)
+        reprocess_info = reprocess_info or {}
+        entry, manager = get_entry_and_manager(media_id)
 
         # Try to process, and handle expected errors.
         try:
+            processor_class = manager.get_processor(reprocess_action, entry)
+
             entry.state = u'processing'
             entry.save()
 
             _log.debug('Processing {0}'.format(entry))
 
-            proc_state = ProcessingState(entry)
-            with mgg.workbench_manager.create() as workbench:
-
-                proc_state.set_workbench(workbench)
-                processor = entry.media_manager.processor(proc_state)
-
-                # If we have reprocess_info, let's reprocess
-                if reprocess_info:
-                    processor.reprocess(reprocess_info)
-
-                # Run initial processing
-                else:
-                    processor.initial_processing()
+            with processor_class(manager, entry) as processor:
+                processor.process(**reprocess_info)
 
             # We set the state to processed and save the entry here so there's
             # no need to save at the end of the processing stage, probably ;)
-- 
cgit v1.2.3


From 55cfa3406390732173195bb920bf3f86bd1ce9f4 Mon Sep 17 00:00:00 2001
From: Christopher Allan Webber <cwebber@dustycloud.org>
Date: Sun, 11 Aug 2013 15:22:43 -0500
Subject: Renaming the processing manager stuff to be less ambiguous.

BONUS COMMIT to Ben Finney and the Free Software Melbourne crew. :)

IRONY: Initially I committed this as "media manager".
---
 mediagoblin/processing/__init__.py | 13 ++++++++++---
 mediagoblin/processing/task.py     |  4 ++--
 2 files changed, 12 insertions(+), 5 deletions(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index b668baa7..02dba2f9 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -120,6 +120,13 @@ class MediaProcessor(object):
         # Should be initialized at time of processing, at least
         self.workbench = None
 
+    def __enter__(self):
+        self.workbench = mgg.workbench_manager.create()
+
+    def __exit__(self, *args):
+        self.workbench.destroy()
+        self.workbench = None
+
     # @with_workbench
     def process(self, **kwargs):
         """
@@ -230,7 +237,7 @@ def request_from_args(args, which_args):
 class MediaEntryNotFound(Exception): pass
 
 
-def get_manager_for_type(media_type):
+def get_processing_manager_for_type(media_type):
     """
     Get the appropriate media manager for this type
     """
@@ -240,7 +247,7 @@ def get_manager_for_type(media_type):
     return manager
 
 
-def get_entry_and_manager(media_id):
+def get_entry_and_processing_manager(media_id):
     """
     Get a MediaEntry, its media type, and its manager all in one go.
 
@@ -250,7 +257,7 @@ def get_entry_and_manager(media_id):
     if entry is None:
         raise MediaEntryNotFound("Can't find media with id '%s'" % media_id)
 
-    manager = get_manager_for_type(entry.media_type)
+    manager = get_processing_manager_for_type(entry.media_type)
 
     return entry, manager
 
diff --git a/mediagoblin/processing/task.py b/mediagoblin/processing/task.py
index 240be4e5..397514d0 100644
--- a/mediagoblin/processing/task.py
+++ b/mediagoblin/processing/task.py
@@ -23,7 +23,7 @@ from celery import registry, task
 from mediagoblin import mg_globals as mgg
 from . import mark_entry_failed, BaseProcessingFail
 from mediagoblin.tools.processing import json_processing_callback
-from mediagoblin.processing import get_entry_and_manager
+from mediagoblin.processing import get_entry_and_processing_manager
 
 _log = logging.getLogger(__name__)
 logging.basicConfig()
@@ -79,7 +79,7 @@ class ProcessMedia(task.Task):
             info for the media_type.
         """
         reprocess_info = reprocess_info or {}
-        entry, manager = get_entry_and_manager(media_id)
+        entry, manager = get_entry_and_processing_manager(media_id)
 
         # Try to process, and handle expected errors.
         try:
-- 
cgit v1.2.3


From 22479c39a0fff75208309e437f5fdaf57730cf0e Mon Sep 17 00:00:00 2001
From: Christopher Allan Webber <cwebber@dustycloud.org>
Date: Mon, 12 Aug 2013 08:22:14 -0500
Subject: Record the original state of the media entry in the processor

This allows our processor to make some informed decisions based on the
state by still having access to the original state.

This commit sponsored by William Rico.  Thank you!
---
 mediagoblin/processing/__init__.py |  1 +
 mediagoblin/processing/task.py     | 10 ++++++----
 2 files changed, 7 insertions(+), 4 deletions(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index 02dba2f9..47f0b84e 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -116,6 +116,7 @@ class MediaProcessor(object):
     def __init__(self, manager, media_entry):
         self.manager = manager
         self.media_entry = media_entry
+        self.entry_orig_state = media_entry.state
 
         # Should be initialized at time of processing, at least
         self.workbench = None
diff --git a/mediagoblin/processing/task.py b/mediagoblin/processing/task.py
index 397514d0..d3770588 100644
--- a/mediagoblin/processing/task.py
+++ b/mediagoblin/processing/task.py
@@ -85,12 +85,14 @@ class ProcessMedia(task.Task):
         try:
             processor_class = manager.get_processor(reprocess_action, entry)
 
-            entry.state = u'processing'
-            entry.save()
+            with processor_class(manager, entry) as processor:
+                # Initial state change has to be here because
+                # the entry.state gets recorded on processor_class init
+                entry.state = u'processing'
+                entry.save()
 
-            _log.debug('Processing {0}'.format(entry))
+                _log.debug('Processing {0}'.format(entry))
 
-            with processor_class(manager, entry) as processor:
                 processor.process(**reprocess_info)
 
             # We set the state to processed and save the entry here so there's
-- 
cgit v1.2.3


From eb372949a13c67962e7460e2411f389ff87d2661 Mon Sep 17 00:00:00 2001
From: Christopher Allan Webber <cwebber@dustycloud.org>
Date: Mon, 12 Aug 2013 08:57:56 -0500
Subject: Factored the get_orig_filename from processing state and put it to
 use.

This commit sponsored by Vincent Demeester.  Thank you!
---
 mediagoblin/processing/__init__.py | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index 47f0b84e..9466aec6 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -372,6 +372,40 @@ def mark_entry_failed(entry_id, exc):
              u'fail_metadata': {}})
 
 
+###############################################################################
+# refactoring procstate stuff here
+
+
+def get_orig_filename(entry, workbench):
+    """
+    Get the a filename for the original, on local storage
+
+    If the media entry has a queued_media_file, use that, otherwise
+    use the original.
+
+    In the future, this will return the highest quality file available
+    if neither the original or queued file are available by checking
+    some ordered list of preferred keys.
+    """
+    if entry.queued_media_file:
+        orig_filepath = entry.queued_media_file
+        storage = mgg.queue_store
+    else:
+        orig_filepath = entry.media_files['original']
+        storage = mgg.public_store
+
+    orig_filename = workbench.localized_file(
+        storage, orig_filepath,
+        'source')
+
+    return orig_filename
+
+
+# end refactoring
+###############################################################################
+
+
+
 class BaseProcessingFail(Exception):
     """
     Base exception that all other processing failure messages should
-- 
cgit v1.2.3


From 5fd239fa581780134e5d5f6547bb2f50f139e30d Mon Sep 17 00:00:00 2001
From: Christopher Allan Webber <cwebber@dustycloud.org>
Date: Mon, 12 Aug 2013 10:40:07 -0500
Subject: Theoretically the last steps to get reprocessing working for initial
 & images
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Haven't tested it yet though :)

This commit sponsored by Samuel Bächler.  Thank you!
---
 mediagoblin/processing/__init__.py | 44 +++++++++++++++++++++++++++++---------
 1 file changed, 34 insertions(+), 10 deletions(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index 9466aec6..bfb78780 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -113,10 +113,10 @@ class MediaProcessor(object):
     # action this MediaProcessor provides
     description = None
 
-    def __init__(self, manager, media_entry):
+    def __init__(self, manager, entry):
         self.manager = manager
-        self.media_entry = media_entry
-        self.entry_orig_state = media_entry.state
+        self.entry = entry
+        self.entry_orig_state = entry.state
 
         # Should be initialized at time of processing, at least
         self.workbench = None
@@ -136,7 +136,7 @@ class MediaProcessor(object):
         raise NotImplementedError
 
     @classmethod
-    def media_is_eligible(cls, media_entry):
+    def media_is_eligible(cls, entry):
         raise NotImplementedError
 
     ###############################
@@ -155,6 +155,20 @@ class MediaProcessor(object):
     # THE FUTURE: web interface things here :)
     ##########################################
 
+    #####################
+    # Some common "steps"
+    #####################
+
+    def delete_queue_file(self):
+        # Remove queued media file from storage and database.
+        # queued_filepath is in the task_id directory which should
+        # be removed too, but fail if the directory is not empty to be on
+        # the super-safe side.
+        queued_filepath = self.entry.queued_media_file
+        mgg.queue_store.delete_file(queued_filepath)      # rm file
+        mgg.queue_store.delete_dir(queued_filepath[:-1])  # rm dir
+        self.entry.queued_media_file = []
+
 
 class ProcessingKeyError(Exception): pass
 class ProcessorDoesNotExist(ProcessingKeyError): pass
@@ -217,12 +231,6 @@ class ProcessingManager(object):
 
         return processor
 
-    def process_from_args(self, entry, reprocess_command, request):
-        """
-        Process a media entry.
-        """
-        pass
-
 
 def request_from_args(args, which_args):
     """
@@ -401,6 +409,22 @@ def get_orig_filename(entry, workbench):
     return orig_filename
 
 
+def store_public(entry, keyname, local_file, target_name=None):
+    if target_name is None:
+        target_name = os.path.basename(local_file)
+    target_filepath = create_pub_filepath(entry, target_name)
+    if keyname in entry.media_files:
+        _log.warn("store_public: keyname %r already used for file %r, "
+                  "replacing with %r", keyname,
+                  entry.media_files[keyname], target_filepath)
+    mgg.public_store.copy_local_to_storage(local_file, target_filepath)
+    entry.media_files[keyname] = target_filepath
+
+
+def copy_original(entry, orig_filename, target_name, keyname=u"original"):
+    store_public(entry, keyname, orig_filename, target_name)
+
+
 # end refactoring
 ###############################################################################
 
-- 
cgit v1.2.3


From 7a85bf985db67482a56e8987e28a6139b5e087fd Mon Sep 17 00:00:00 2001
From: Christopher Allan Webber <cwebber@dustycloud.org>
Date: Mon, 12 Aug 2013 10:48:07 -0500
Subject: Fixing the MediaProcessor context manager so it actually, you know,
 works :)

This commit sponsored by Mikiya Okuno.  Thank you!
---
 mediagoblin/processing/__init__.py | 1 +
 1 file changed, 1 insertion(+)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index bfb78780..9493091b 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -123,6 +123,7 @@ class MediaProcessor(object):
 
     def __enter__(self):
         self.workbench = mgg.workbench_manager.create()
+        return self
 
     def __exit__(self, *args):
         self.workbench.destroy()
-- 
cgit v1.2.3


From 98d1fa3beddfc602c541fe7f538ca882ad6c7e9c Mon Sep 17 00:00:00 2001
From: Christopher Allan Webber <cwebber@dustycloud.org>
Date: Mon, 12 Aug 2013 11:00:15 -0500
Subject: Fixing normal submission of media (well for images anyway)

---
 mediagoblin/processing/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index 9493091b..684ffe04 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -223,6 +223,8 @@ class ProcessingManager(object):
         try:
             processor = self.processors[key]
         except KeyError:
+            import pdb
+            pdb.set_trace()
             raise ProcessorDoesNotExist(
                 "'%s' processor does not exist for this media type" % key)
 
-- 
cgit v1.2.3


From 583501415acce95ff458e5cf12733d1b61332e0e Mon Sep 17 00:00:00 2001
From: Rodney Ewing <ewing.rj@gmail.com>
Date: Mon, 12 Aug 2013 11:37:59 -0700
Subject: Add image resizer and some cleanup of old code

---
 mediagoblin/processing/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index 684ffe04..d5ec1fba 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -90,7 +90,7 @@ class MediaProcessor(object):
     - resize: resize an image
     - transcode: transcode a video
 
-    ... etc.  
+    ... etc.
 
     Some information on producing a new MediaProcessor for your media type:
 
@@ -193,7 +193,7 @@ class ProcessingManager(object):
         name = processor.name
         if name is None:
             raise AttributeError("Processor class's .name attribute not set")
-        
+
         self.processors[name] = processor
 
     def list_eligible_processors(self, entry):
-- 
cgit v1.2.3


From fb56676bf49de8e25487b938dc9a56f8440086f5 Mon Sep 17 00:00:00 2001
From: Rodney Ewing <ewing.rj@gmail.com>
Date: Mon, 12 Aug 2013 11:55:00 -0700
Subject: delete existing file in store_public

---
 mediagoblin/processing/__init__.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index d5ec1fba..aadee78b 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -412,7 +412,8 @@ def get_orig_filename(entry, workbench):
     return orig_filename
 
 
-def store_public(entry, keyname, local_file, target_name=None):
+def store_public(entry, keyname, local_file, target_name=None,
+                 delete_if_exists=True):
     if target_name is None:
         target_name = os.path.basename(local_file)
     target_filepath = create_pub_filepath(entry, target_name)
@@ -420,6 +421,8 @@ def store_public(entry, keyname, local_file, target_name=None):
         _log.warn("store_public: keyname %r already used for file %r, "
                   "replacing with %r", keyname,
                   entry.media_files[keyname], target_filepath)
+        if delete_if_exists:
+            mgg.public_store.delete_file(entry.media_files[keyname])
     mgg.public_store.copy_local_to_storage(local_file, target_filepath)
     entry.media_files[keyname] = target_filepath
 
-- 
cgit v1.2.3


From 455f71d24c7d5e3163b1cc25682161fe1c7f7cc6 Mon Sep 17 00:00:00 2001
From: Rodney Ewing <ewing.rj@gmail.com>
Date: Mon, 12 Aug 2013 11:57:47 -0700
Subject: remove ProcessingState

---
 mediagoblin/processing/__init__.py | 82 --------------------------------------
 1 file changed, 82 deletions(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index aadee78b..19e88199 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -274,79 +274,6 @@ def get_entry_and_processing_manager(media_id):
     return entry, manager
 
 
-################################################
-# TODO: This ProcessingState is OUTDATED,
-#   and needs to be refactored into other tools!
-################################################
-
-class ProcessingState(object):
-    """
-    The first and only argument to the "processor" of a media type
-
-    This could be thought of as a "request" to the processor
-    function. It has the main info for the request (media entry)
-    and a bunch of tools for the request on it.
-    It can get more fancy without impacting old media types.
-    """
-    def __init__(self, entry):
-        self.entry = entry
-        self.workbench = None
-        self.orig_filename = None
-
-    def set_workbench(self, wb):
-        self.workbench = wb
-
-    def get_orig_filename(self):
-        """
-        Get the a filename for the original, on local storage
-
-        If the media entry has a queued_media_file, use that, otherwise
-        use the original.
-
-        In the future, this will return the highest quality file available
-        if neither the original or queued file are available
-        """
-        if self.orig_filename is not None:
-            return self.orig_filename
-
-        if self.entry.queued_media_file:
-            orig_filepath = self.entry.queued_media_file
-            storage = mgg.queue_store
-        else:
-            orig_filepath = self.entry.media_files['original']
-            storage = mgg.public_store
-
-        orig_filename = self.workbench.localized_file(
-            storage, orig_filepath,
-            'source')
-        self.orig_filename = orig_filename
-        return orig_filename
-
-    def copy_original(self, target_name, keyname=u"original"):
-        self.store_public(keyname, self.get_orig_filename(), target_name)
-
-    def store_public(self, keyname, local_file, target_name=None):
-        if target_name is None:
-            target_name = os.path.basename(local_file)
-        target_filepath = create_pub_filepath(self.entry, target_name)
-        if keyname in self.entry.media_files:
-            _log.warn("store_public: keyname %r already used for file %r, "
-                      "replacing with %r", keyname,
-                      self.entry.media_files[keyname], target_filepath)
-        mgg.public_store.copy_local_to_storage(local_file, target_filepath)
-        self.entry.media_files[keyname] = target_filepath
-
-    def delete_queue_file(self):
-        # Remove queued media file from storage and database.
-        # queued_filepath is in the task_id directory which should
-        # be removed too, but fail if the directory is not empty to be on
-        # the super-safe side.
-        queued_filepath = self.entry.queued_media_file
-        mgg.queue_store.delete_file(queued_filepath)      # rm file
-        mgg.queue_store.delete_dir(queued_filepath[:-1])  # rm dir
-        self.entry.queued_media_file = []
-
-
 def mark_entry_failed(entry_id, exc):
     """
     Mark a media entry as having failed in its conversion.
@@ -383,10 +310,6 @@ def mark_entry_failed(entry_id, exc):
              u'fail_metadata': {}})
 
 
-###############################################################################
-# refactoring procstate stuff here
-
-
 def get_orig_filename(entry, workbench):
     """
     Get the a filename for the original, on local storage
@@ -431,11 +354,6 @@ def copy_original(entry, orig_filename, target_name, keyname=u"original"):
     store_public(entry, keyname, orig_filename, target_name)
 
 
-# end refactoring
-###############################################################################
-
-
-
 class BaseProcessingFail(Exception):
     """
     Base exception that all other processing failure messages should
-- 
cgit v1.2.3


From 7584080bf7d7b2d74087d31ca781e1111c2024da Mon Sep 17 00:00:00 2001
From: Rodney Ewing <ewing.rj@gmail.com>
Date: Mon, 12 Aug 2013 14:28:03 -0700
Subject: add bulk_run, thumbs, and initial sub_commands

---
 mediagoblin/processing/__init__.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index 19e88199..1930a480 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -137,7 +137,7 @@ class MediaProcessor(object):
         raise NotImplementedError
 
     @classmethod
-    def media_is_eligible(cls, entry):
+    def media_is_eligible(cls, entry=None, state=None):
         raise NotImplementedError
 
     ###############################
@@ -204,7 +204,18 @@ class ProcessingManager(object):
         return [
             processor
             for processor in self.processors.values()
-            if processor.media_is_eligible(entry)]
+            if processor.media_is_eligible(entry=entry)]
+
+    def list_all_processors_by_state(self, state):
+        """
+        List all processors that this media state is eligible to be processed
+        for.
+        """
+        return [
+            processor
+            for processor in self.processors.values()
+            if processor.media_is_eligible(state=state)]
+
 
     def list_all_processors(self):
         return self.processors.values()
-- 
cgit v1.2.3


From 4e6013689beded08121c0d139565ffccbf3c0000 Mon Sep 17 00:00:00 2001
From: Rodney Ewing <ewing.rj@gmail.com>
Date: Mon, 12 Aug 2013 14:54:02 -0700
Subject: run initial processing on all failed entries

---
 mediagoblin/processing/__init__.py | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index 1930a480..0c13e807 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -174,6 +174,8 @@ class MediaProcessor(object):
 class ProcessingKeyError(Exception): pass
 class ProcessorDoesNotExist(ProcessingKeyError): pass
 class ProcessorNotEligible(ProcessingKeyError): pass
+class ProcessingManagerDoesNotExist(ProcessingKeyError): pass
+
 
 
 class ProcessingManager(object):
@@ -265,6 +267,9 @@ def get_processing_manager_for_type(media_type):
     Get the appropriate media manager for this type
     """
     manager_class = hook_handle(('reprocess_manager', media_type))
+    if not manager_class:
+        raise ProcessingManagerDoesNotExist(
+            "A processing manager does not exist for {0}".format(media_type))
     manager = manager_class()
 
     return manager
-- 
cgit v1.2.3


From 79f84d7e479f6b370709c6826c85070ab1996ea6 Mon Sep 17 00:00:00 2001
From: Rodney Ewing <ewing.rj@gmail.com>
Date: Wed, 14 Aug 2013 13:47:39 -0700
Subject: raise an error if the file failed to copy to public storage

catch copy_local_to_storage errors and raise PublicStoreFail, saving the keyname
---
 mediagoblin/processing/__init__.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index 0c13e807..e31b70bb 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -356,13 +356,24 @@ def store_public(entry, keyname, local_file, target_name=None,
     if target_name is None:
         target_name = os.path.basename(local_file)
     target_filepath = create_pub_filepath(entry, target_name)
+
     if keyname in entry.media_files:
         _log.warn("store_public: keyname %r already used for file %r, "
                   "replacing with %r", keyname,
                   entry.media_files[keyname], target_filepath)
         if delete_if_exists:
             mgg.public_store.delete_file(entry.media_files[keyname])
-    mgg.public_store.copy_local_to_storage(local_file, target_filepath)
+
+    try:
+        mgg.public_store.copy_local_to_storage(local_file, target_filepath)
+    except:
+        raise PublicStoreFail(keyname=keyname)
+
+    # raise an error if the file failed to copy
+    copied_filepath = mgg.public_store.get_local_path(target_filepath)
+    if not os.path.exists(copied_filepath):
+        raise PublicStoreFail(keyname=keyname)
+
     entry.media_files[keyname] = target_filepath
 
 
@@ -396,3 +407,10 @@ class BadMediaFail(BaseProcessingFail):
     for the media type specified.
     """
     general_message = _(u'Invalid file given for media type.')
+
+
+class PublicStoreFail(BaseProcessingFail):
+    """
+    Error that should be raised when copying to public store fails
+    """
+    general_message = _('Copying to public storage failed.')
-- 
cgit v1.2.3


From 7d3fda06b03691601bc08b5d88baf1da1c3f83fc Mon Sep 17 00:00:00 2001
From: Rodney Ewing <ewing.rj@gmail.com>
Date: Wed, 14 Aug 2013 14:54:10 -0700
Subject: catch processing exceptions and if entry_orig_state is processed,
 then ignore the exception

---
 mediagoblin/processing/task.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/task.py b/mediagoblin/processing/task.py
index d3770588..df44dd7a 100644
--- a/mediagoblin/processing/task.py
+++ b/mediagoblin/processing/task.py
@@ -93,7 +93,18 @@ class ProcessMedia(task.Task):
 
                 _log.debug('Processing {0}'.format(entry))
 
-                processor.process(**reprocess_info)
+                try:
+                    processor.process(**reprocess_info)
+                except Exception as exc:
+                    if processor.entry_orig_state == 'processed':
+                        _log.error(
+                            'Entry {0} failed to process due to the following'
+                            ' error: {1}'.format(entry.id, exc))
+                        _log.info(
+                            'Setting entry.state back to "processed"')
+                        pass
+                    else:
+                        raise
 
             # We set the state to processed and save the entry here so there's
             # no need to save at the end of the processing stage, probably ;)
-- 
cgit v1.2.3


From 882779f547f4bc20887e8af7d4973d5bbb8bf147 Mon Sep 17 00:00:00 2001
From: Rodney Ewing <ewing.rj@gmail.com>
Date: Wed, 14 Aug 2013 16:39:01 -0700
Subject: only try and delete queue file if it exists

---
 mediagoblin/processing/__init__.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index e31b70bb..e2415fd5 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -166,9 +166,10 @@ class MediaProcessor(object):
         # be removed too, but fail if the directory is not empty to be on
         # the super-safe side.
         queued_filepath = self.entry.queued_media_file
-        mgg.queue_store.delete_file(queued_filepath)      # rm file
-        mgg.queue_store.delete_dir(queued_filepath[:-1])  # rm dir
-        self.entry.queued_media_file = []
+        if queued_filepath:
+            mgg.queue_store.delete_file(queued_filepath)      # rm file
+            mgg.queue_store.delete_dir(queued_filepath[:-1])  # rm dir
+            self.entry.queued_media_file = []
 
 
 class ProcessingKeyError(Exception): pass
-- 
cgit v1.2.3


From 1cefccc7554a5df4c9bb126ef3b80b53f9e41cd7 Mon Sep 17 00:00:00 2001
From: Rodney Ewing <ewing.rj@gmail.com>
Date: Thu, 15 Aug 2013 08:54:09 -0700
Subject: refactor get_orig_filename to return an acceptable filename to the
 processor.

If there is an original video file and we skip transcoding, delete the webm_640 file
---
 mediagoblin/processing/__init__.py | 41 +++++++++++++++++++++++++-------------
 1 file changed, 27 insertions(+), 14 deletions(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index e2415fd5..746f4d8e 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -327,29 +327,34 @@ def mark_entry_failed(entry_id, exc):
              u'fail_metadata': {}})
 
 
-def get_orig_filename(entry, workbench):
+def get_process_filename(entry, workbench, acceptable_files):
     """
-    Get the a filename for the original, on local storage
+    Try and get the queued file if available, otherwise return the first file
+    in the acceptable_files that we have.
 
-    If the media entry has a queued_media_file, use that, otherwise
-    use the original.
-
-    In the future, this will return the highest quality file available
-    if neither the original or queued file are available by checking
-    some ordered list of preferred keys.
+    If no acceptable_files, raise ProcessFileNotFound
     """
     if entry.queued_media_file:
-        orig_filepath = entry.queued_media_file
+        filepath = entry.queued_media_file
         storage = mgg.queue_store
     else:
-        orig_filepath = entry.media_files['original']
-        storage = mgg.public_store
+        for keyname in acceptable_files:
+            if entry.media_files.get(keyname):
+                filepath = entry.media_files[keyname]
+                storage = mgg.public_store
+                break
+
+    if not filepath:
+        raise ProcessFileNotFound()
 
-    orig_filename = workbench.localized_file(
-        storage, orig_filepath,
+    filename = workbench.localized_file(
+        storage, filepath,
         'source')
 
-    return orig_filename
+    if not os.path.exists(filename):
+        raise ProcessFileNotFound()
+
+    return filename
 
 
 def store_public(entry, keyname, local_file, target_name=None,
@@ -415,3 +420,11 @@ class PublicStoreFail(BaseProcessingFail):
     Error that should be raised when copying to public store fails
     """
     general_message = _('Copying to public storage failed.')
+
+
+class ProcessFileNotFound(BaseProcessingFail):
+    """
+    Error that should be raised when an acceptable file for processing
+    is not found.
+    """
+    general_message = _(u'An acceptable processing file was not found')
-- 
cgit v1.2.3


From bf2dafd1a04ef8050ebf08bb512862a1592998c0 Mon Sep 17 00:00:00 2001
From: Sebastian Spaeth <Sebastian@SSpaeth.de>
Date: Thu, 20 Dec 2012 13:42:37 +0100
Subject: Tweak Celery Task

- Make sure Exceptions are pickleable (not sure if this was not the
  case but this is the pattern as documented in the celery docs.
- Don't create a task_id in the GMG code, but save the one
  implicitely created by celery.
- Don't create a task-id directory per upload. Just store queued uploads
  in a single directory (this is the most controversial change and might
  need discussion!!!)

Signed-off-by: Sebastian Spaeth <Sebastian@SSpaeth.de>
---
 mediagoblin/processing/__init__.py |  7 ++++---
 mediagoblin/processing/task.py     | 22 ++++++++++------------
 2 files changed, 14 insertions(+), 15 deletions(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index f3a85940..ae3652cf 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -181,9 +181,10 @@ class BaseProcessingFail(Exception):
         return u"%s:%s" % (
             self.__class__.__module__, self.__class__.__name__)
 
-    def __init__(self, **metadata):
-        self.metadata = metadata or {}
-
+    def __init__(self, *args, **kwargs):
+        # next line is REQUIRED to have pickable exceptions if you want
+        # to be able to pass in custom arguments (see celery docs)
+        Exception.__init__(self, *args, **metadata)
 
 class BadMediaFail(BaseProcessingFail):
     """
diff --git a/mediagoblin/processing/task.py b/mediagoblin/processing/task.py
index 9af192ed..550906d0 100644
--- a/mediagoblin/processing/task.py
+++ b/mediagoblin/processing/task.py
@@ -18,11 +18,13 @@ import logging
 import urllib
 import urllib2
 
-from celery import registry, task
+#TODO: newer celeries use from celery import Task. Change when we upgrade
+from celery.task import Task
+from celery.registry import tasks
 
 from mediagoblin import mg_globals as mgg
-from mediagoblin.db.models import MediaEntry
-from . import mark_entry_failed, BaseProcessingFail, ProcessingState
+from mediagoblin.db.sql.models import MediaEntry
+from mediagoblin.processing import mark_entry_failed, BaseProcessingFail
 from mediagoblin.tools.processing import json_processing_callback
 
 _log = logging.getLogger(__name__)
@@ -63,12 +65,10 @@ def handle_push_urls(feed_url):
 ################################
 # Media processing initial steps
 ################################
+class ProcessMedia(Task):
+    track_started=True
 
-class ProcessMedia(task.Task):
-    """
-    Pass this entry off for processing.
-    """
-    def run(self, media_id, feed_url):
+    def run(self, media_id):
         """
         Pass the media entry off to the appropriate processing function
         (for now just process_image...)
@@ -81,8 +81,8 @@ class ProcessMedia(task.Task):
         # Try to process, and handle expected errors.
         try:
             entry.state = u'processing'
+            entry.queued_task_id = self.request.id
             entry.save()
-
             _log.debug('Processing {0}'.format(entry))
 
             proc_state = ProcessingState(entry)
@@ -140,6 +140,4 @@ class ProcessMedia(task.Task):
         entry = mgg.database.MediaEntry.query.filter_by(id=entry_id).first()
         json_processing_callback(entry)
 
-# Register the task
-process_media = registry.tasks[ProcessMedia.name]
-
+tasks.register(ProcessMedia)
-- 
cgit v1.2.3


From b505952508f717e0d4f59d24c87a54ef42673c3f Mon Sep 17 00:00:00 2001
From: Rodney Ewing <ewing.rj@gmail.com>
Date: Mon, 19 Aug 2013 12:58:00 -0700
Subject:  -update to latest master - have mg generate task_id

remove
---
 mediagoblin/processing/__init__.py |  6 ++----
 mediagoblin/processing/task.py     | 17 ++++++++++-------
 2 files changed, 12 insertions(+), 11 deletions(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/__init__.py b/mediagoblin/processing/__init__.py
index ae3652cf..454eb09b 100644
--- a/mediagoblin/processing/__init__.py
+++ b/mediagoblin/processing/__init__.py
@@ -181,10 +181,8 @@ class BaseProcessingFail(Exception):
         return u"%s:%s" % (
             self.__class__.__module__, self.__class__.__name__)
 
-    def __init__(self, *args, **kwargs):
-        # next line is REQUIRED to have pickable exceptions if you want
-        # to be able to pass in custom arguments (see celery docs)
-        Exception.__init__(self, *args, **metadata)
+    def __init__(self, **metadata):
+        self.metadata = metadata or {}
 
 class BadMediaFail(BaseProcessingFail):
     """
diff --git a/mediagoblin/processing/task.py b/mediagoblin/processing/task.py
index 550906d0..bb09daec 100644
--- a/mediagoblin/processing/task.py
+++ b/mediagoblin/processing/task.py
@@ -18,13 +18,13 @@ import logging
 import urllib
 import urllib2
 
-#TODO: newer celeries use from celery import Task. Change when we upgrade
-from celery.task import Task
+import celery
 from celery.registry import tasks
 
 from mediagoblin import mg_globals as mgg
-from mediagoblin.db.sql.models import MediaEntry
-from mediagoblin.processing import mark_entry_failed, BaseProcessingFail
+from mediagoblin.db.models import MediaEntry
+from mediagoblin.processing import (mark_entry_failed, BaseProcessingFail,
+                                    ProcessingState)
 from mediagoblin.tools.processing import json_processing_callback
 
 _log = logging.getLogger(__name__)
@@ -32,7 +32,7 @@ logging.basicConfig()
 _log.setLevel(logging.DEBUG)
 
 
-@task.task(default_retry_delay=2 * 60)
+@celery.task(default_retry_delay=2 * 60)
 def handle_push_urls(feed_url):
     """Subtask, notifying the PuSH servers of new content
 
@@ -62,10 +62,14 @@ def handle_push_urls(feed_url):
                           'Giving up.'.format(feed_url))
                 return False
 
+
 ################################
 # Media processing initial steps
 ################################
-class ProcessMedia(Task):
+class ProcessMedia(celery.Task):
+    """
+    Pass this entry off for processing.
+    """
     track_started=True
 
     def run(self, media_id):
@@ -81,7 +85,6 @@ class ProcessMedia(Task):
         # Try to process, and handle expected errors.
         try:
             entry.state = u'processing'
-            entry.queued_task_id = self.request.id
             entry.save()
             _log.debug('Processing {0}'.format(entry))
 
-- 
cgit v1.2.3


From d4ae4c9f971b5795f1697f443b359d3155a16013 Mon Sep 17 00:00:00 2001
From: Rodney Ewing <ewing.rj@gmail.com>
Date: Mon, 19 Aug 2013 14:57:00 -0700
Subject: - need self.metadata with BaseProcessingFail - pass feed_url into
 ProcessMedia run()

---
 mediagoblin/processing/task.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'mediagoblin/processing')

diff --git a/mediagoblin/processing/task.py b/mediagoblin/processing/task.py
index bb09daec..05cac844 100644
--- a/mediagoblin/processing/task.py
+++ b/mediagoblin/processing/task.py
@@ -72,7 +72,7 @@ class ProcessMedia(celery.Task):
     """
     track_started=True
 
-    def run(self, media_id):
+    def run(self, media_id, feed_url):
         """
         Pass the media entry off to the appropriate processing function
         (for now just process_image...)
@@ -86,6 +86,7 @@ class ProcessMedia(celery.Task):
         try:
             entry.state = u'processing'
             entry.save()
+
             _log.debug('Processing {0}'.format(entry))
 
             proc_state = ProcessingState(entry)
-- 
cgit v1.2.3