From f7e1bfea7a5d53dc900c43ce84674d17d1c7d0e6 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Wed, 31 May 2017 01:21:34 +0530 Subject: media_types/video/util.py: Add accepted resolutions Add dict consisting of all the accepted resolutions with their dimensions. This should be eventually moved to config where the instance owner can modify it. --- mediagoblin/media_types/video/util.py | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/video/util.py b/mediagoblin/media_types/video/util.py index 8b65d839..1f5e907d 100644 --- a/mediagoblin/media_types/video/util.py +++ b/mediagoblin/media_types/video/util.py @@ -18,6 +18,15 @@ import logging from mediagoblin import mg_globals as mgg +ACCEPTED_RESOLUTIONS = { + '144p' : (256, 144), + '240p' : (352, 240), + '360p' : (480, 360), + '480p' : (858, 480), + '720p' : (1280, 720), + '1080p' : (1920, 1080), +} + _log = logging.getLogger(__name__) -- cgit v1.2.3 From 2963b0a18231aac5d5fda3ae2ecc55a0b5c08823 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Fri, 2 Jun 2017 01:55:42 +0530 Subject: media_types/video/processing.py: Move store_metadata Move store_metadata to self.common_setup() so that it's easy to parallelize self.transcode() later. --- mediagoblin/media_types/video/processing.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index ca3087a2..e6b4a0b2 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -178,6 +178,14 @@ class CommonVideoProcessor(MediaProcessor): self.transcoder = transcoders.VideoTranscoder() self.did_transcode = False + # Extract metadata and keep a record of it + self.metadata = transcoders.discover(self.process_filename) + + # metadata's stream info here is a DiscovererContainerInfo instance, + # it gets split into DiscovererAudioInfo and DiscovererVideoInfo; + # metadata itself has container-related data in tags, like video-codec + store_metadata(self.entry, self.metadata) + def copy_original(self): # If we didn't transcode, then we need to keep the original if not self.did_transcode or \ @@ -246,20 +254,12 @@ class CommonVideoProcessor(MediaProcessor): if self._skip_processing('webm_video', **file_metadata): return - # Extract metadata and keep a record of it - metadata = transcoders.discover(self.process_filename) - - # metadata's stream info here is a DiscovererContainerInfo instance, - # it gets split into DiscovererAudioInfo and DiscovererVideoInfo; - # metadata itself has container-related data in tags, like video-codec - store_metadata(self.entry, metadata) - - orig_dst_dimensions = (metadata.get_video_streams()[0].get_width(), - metadata.get_video_streams()[0].get_height()) + orig_dst_dimensions = (self.metadata.get_video_streams()[0].get_width(), + self.metadata.get_video_streams()[0].get_height()) # Figure out whether or not we need to transcode this video or # if we can skip it - if skip_transcode(metadata, medium_size): + if skip_transcode(self.metadata, medium_size): _log.debug('Skipping transcoding') dst_dimensions = orig_dst_dimensions -- cgit v1.2.3 From 81c59ef06b1e53fa0d9b177b5a8e7978d1b01186 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Mon, 5 Jun 2017 01:58:00 +0530 Subject: Add workflow method to VideoProcessingManager This commit makes sure the old celery call works perfectly when workflow method is introduced. --- mediagoblin/media_types/video/processing.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index e6b4a0b2..d3fa9d5f 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -28,6 +28,7 @@ from mediagoblin.processing import ( ProcessingManager, request_from_args, get_process_filename, store_public, copy_original) +from mediagoblin.processing.task import ProcessMedia from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ from mediagoblin.media_types import MissingComponents @@ -503,3 +504,8 @@ class VideoProcessingManager(ProcessingManager): self.add_processor(InitialProcessor) self.add_processor(Resizer) self.add_processor(Transcoder) + + def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + ProcessMedia().apply_async( + [entry.id, feed_url, reprocess_action, reprocess_info], {}, + task_id=entry.queued_task_id) -- cgit v1.2.3 From c62181f4bca1ae7f74d4bd8ec15615337d23ec9a Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Mon, 5 Jun 2017 02:11:56 +0530 Subject: Add workflow method to other media ProcessingManagers Add workflow method to all the media types other than video. Calls the old celery task as it is. --- mediagoblin/media_types/ascii/processing.py | 5 +++++ mediagoblin/media_types/audio/processing.py | 5 +++++ mediagoblin/media_types/image/processing.py | 6 ++++++ mediagoblin/media_types/pdf/processing.py | 5 +++++ mediagoblin/media_types/raw_image/processing.py | 5 +++++ mediagoblin/media_types/stl/processing.py | 5 +++++ 6 files changed, 31 insertions(+) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/ascii/processing.py b/mediagoblin/media_types/ascii/processing.py index 00d04e63..71ccc86e 100644 --- a/mediagoblin/media_types/ascii/processing.py +++ b/mediagoblin/media_types/ascii/processing.py @@ -273,3 +273,8 @@ class AsciiProcessingManager(ProcessingManager): super(AsciiProcessingManager, self).__init__() self.add_processor(InitialProcessor) self.add_processor(Resizer) + + def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + ProcessMedia().apply_async( + [entry.id, feed_url, reprocess_action, reprocess_info], {}, + task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/audio/processing.py b/mediagoblin/media_types/audio/processing.py index 427309de..a83d60f7 100644 --- a/mediagoblin/media_types/audio/processing.py +++ b/mediagoblin/media_types/audio/processing.py @@ -365,3 +365,8 @@ class AudioProcessingManager(ProcessingManager): self.add_processor(InitialProcessor) self.add_processor(Resizer) self.add_processor(Transcoder) + + def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + ProcessMedia().apply_async( + [entry.id, feed_url, reprocess_action, reprocess_info], {}, + task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/image/processing.py b/mediagoblin/media_types/image/processing.py index 14091d6e..42234eff 100644 --- a/mediagoblin/media_types/image/processing.py +++ b/mediagoblin/media_types/image/processing.py @@ -431,6 +431,12 @@ class ImageProcessingManager(ProcessingManager): self.add_processor(Resizer) self.add_processor(MetadataProcessing) + def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + ProcessMedia().apply_async( + [entry.id, feed_url, reprocess_action, reprocess_info], {}, + task_id=entry.queued_task_id) + + if __name__ == '__main__': import sys import pprint diff --git a/mediagoblin/media_types/pdf/processing.py b/mediagoblin/media_types/pdf/processing.py index ac4bab6d..d93b19bb 100644 --- a/mediagoblin/media_types/pdf/processing.py +++ b/mediagoblin/media_types/pdf/processing.py @@ -470,3 +470,8 @@ class PdfProcessingManager(ProcessingManager): super(PdfProcessingManager, self).__init__() self.add_processor(InitialProcessor) self.add_processor(Resizer) + + def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + ProcessMedia().apply_async( + [entry.id, feed_url, reprocess_action, reprocess_info], {}, + task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/raw_image/processing.py b/mediagoblin/media_types/raw_image/processing.py index 740ba2dd..a385d563 100644 --- a/mediagoblin/media_types/raw_image/processing.py +++ b/mediagoblin/media_types/raw_image/processing.py @@ -80,3 +80,8 @@ class RawImageProcessingManager(ProcessingManager): super(RawImageProcessingManager, self).__init__() self.add_processor(InitialRawProcessor) self.add_processor(Resizer) + + def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + ProcessMedia().apply_async( + [entry.id, feed_url, reprocess_action, reprocess_info], {}, + task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/stl/processing.py b/mediagoblin/media_types/stl/processing.py index 55764aeb..7f2f350d 100644 --- a/mediagoblin/media_types/stl/processing.py +++ b/mediagoblin/media_types/stl/processing.py @@ -368,3 +368,8 @@ class StlProcessingManager(ProcessingManager): super(StlProcessingManager, self).__init__() self.add_processor(InitialProcessor) self.add_processor(Resizer) + + def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + ProcessMedia().apply_async( + [entry.id, feed_url, reprocess_action, reprocess_info], {}, + task_id=entry.queued_task_id) -- cgit v1.2.3 From 16ef1164c15b13c20945890cbb5cc4a8e4e73852 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Wed, 7 Jun 2017 20:39:26 +0530 Subject: Modify CommonVideoPrcoessor to accomodate mulitple qualities A few trivial modifciations like replacing store_metadata, etc needed to accomodate multiple qualities. --- mediagoblin/media_types/video/processing.py | 84 +++++++++++++---------------- 1 file changed, 37 insertions(+), 47 deletions(-) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index d3fa9d5f..a7e2d20c 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -165,9 +165,10 @@ class CommonVideoProcessor(MediaProcessor): """ Provides a base for various video processing steps """ - acceptable_files = ['original', 'best_quality', 'webm_video'] + acceptable_files = ['original, best_quality', 'webm_144p', 'webm_360p', + 'webm_480p', 'webm_720p', 'webm_1080p', 'webm_video'] - def common_setup(self): + def common_setup(self, resolution=None): self.video_config = mgg \ .global_config['plugins'][MEDIA_TYPE] @@ -179,32 +180,23 @@ class CommonVideoProcessor(MediaProcessor): self.transcoder = transcoders.VideoTranscoder() self.did_transcode = False - # Extract metadata and keep a record of it - self.metadata = transcoders.discover(self.process_filename) - - # metadata's stream info here is a DiscovererContainerInfo instance, - # it gets split into DiscovererAudioInfo and DiscovererVideoInfo; - # metadata itself has container-related data in tags, like video-codec - store_metadata(self.entry, self.metadata) + if resolution: + self.curr_file = 'webm_' + str(resolution) + self.part_filename = (self.name_builder.fill('{basename}.' + + str(resolution) + '.webm')) + else: + self.curr_file = 'webm_video' + self.part_filename = self.name_builder.fill('{basename}.medium.webm') def copy_original(self): # If we didn't transcode, then we need to keep the original - if not self.did_transcode or \ - (self.video_config['keep_original'] and self.did_transcode): - copy_original( - self.entry, self.process_filename, - self.name_builder.fill('{basename}{ext}')) + raise NotImplementedError def _keep_best(self): """ If there is no original, keep the best file that we have """ - if not self.entry.media_files.get('best_quality'): - # Save the best quality file if no original? - if not self.entry.media_files.get('original') and \ - self.entry.media_files.get('webm_video'): - self.entry.media_files['best_quality'] = self.entry \ - .media_files['webm_video'] + raise NotImplementedError def _skip_processing(self, keyname, **kwargs): file_metadata = self.entry.get_file_metadata(keyname) @@ -213,7 +205,7 @@ class CommonVideoProcessor(MediaProcessor): return False skip = True - if keyname == 'webm_video': + if 'webm' in keyname: if kwargs.get('medium_size') != file_metadata.get('medium_size'): skip = False elif kwargs.get('vp8_quality') != file_metadata.get('vp8_quality'): @@ -233,8 +225,7 @@ class CommonVideoProcessor(MediaProcessor): def transcode(self, medium_size=None, vp8_quality=None, vp8_threads=None, vorbis_quality=None): progress_callback = ProgressCallback(self.entry) - tmp_dst = os.path.join(self.workbench.dir, - self.name_builder.fill('{basename}.medium.webm')) + tmp_dst = os.path.join(self.workbench.dir, self.part_filename) if not medium_size: medium_size = ( @@ -252,24 +243,23 @@ class CommonVideoProcessor(MediaProcessor): 'vp8_quality': vp8_quality, 'vorbis_quality': vorbis_quality} - if self._skip_processing('webm_video', **file_metadata): + if self._skip_processing(self.curr_file, **file_metadata): return - orig_dst_dimensions = (self.metadata.get_video_streams()[0].get_width(), - self.metadata.get_video_streams()[0].get_height()) + metadata = transcoders.discover(self.process_filename) + orig_dst_dimensions = (metadata.get_video_streams()[0].get_width(), + metadata.get_video_streams()[0].get_height()) # Figure out whether or not we need to transcode this video or # if we can skip it - if skip_transcode(self.metadata, medium_size): + if skip_transcode(metadata, medium_size): _log.debug('Skipping transcoding') - dst_dimensions = orig_dst_dimensions - # If there is an original and transcoded, delete the transcoded # since it must be of lower quality then the original if self.entry.media_files.get('original') and \ - self.entry.media_files.get('webm_video'): - self.entry.media_files['webm_video'].delete() + self.entry.media_files.get(self.curr_file): + self.entry.media_files[self.curr_file].delete() else: self.transcoder.transcode(self.process_filename, tmp_dst, @@ -279,27 +269,16 @@ class CommonVideoProcessor(MediaProcessor): progress_callback=progress_callback, dimensions=tuple(medium_size)) if self.transcoder.dst_data: - video_info = self.transcoder.dst_data.get_video_streams()[0] - dst_dimensions = (video_info.get_width(), - video_info.get_height()) - self._keep_best() - # Push transcoded video to public storage _log.debug('Saving medium...') store_public(self.entry, 'webm_video', tmp_dst, self.name_builder.fill('{basename}.medium.webm')) _log.debug('Saved medium') - self.entry.set_file_metadata('webm_video', **file_metadata) + # Is this the file_metadata that paroneayea was talking about? + self.entry.set_file_metadata(self.curr_file, **file_metadata) self.did_transcode = True - else: - dst_dimensions = orig_dst_dimensions - - # Save the width and height of the transcoded video - self.entry.media_data_init( - width=dst_dimensions[0], - height=dst_dimensions[1]) def generate_thumb(self, thumb_size=None): # Temporary file for the video thumbnail (cleaned up with workbench) @@ -331,6 +310,17 @@ class CommonVideoProcessor(MediaProcessor): self.entry.set_file_metadata('thumb', thumb_size=thumb_size) + def store_orig_metadata(self): + + # Extract metadata and keep a record of it + metadata = transcoders.discover(self.process_filename) + + # metadata's stream info here is a DiscovererContainerInfo instance, + # it gets split into DiscovererAudioInfo and DiscovererVideoInfo; + # metadata itself has container-related data in tags, like video-codec + store_metadata(self.entry, metadata) + + class InitialProcessor(CommonVideoProcessor): """ Initial processing steps for new video @@ -387,9 +377,9 @@ class InitialProcessor(CommonVideoProcessor): 'vorbis_quality', 'thumb_size']) def process(self, medium_size=None, vp8_threads=None, vp8_quality=None, - vorbis_quality=None, thumb_size=None): - self.common_setup() - + vorbis_quality=None, thumb_size=None, resolution=None): + self.common_setup(resolution=resolution) + self.store_orig_metadata() self.transcode(medium_size=medium_size, vp8_quality=vp8_quality, vp8_threads=vp8_threads, vorbis_quality=vorbis_quality) -- cgit v1.2.3 From 7cc9b6d1c7ab174485ddfbf8db63f14162e01d7b Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Wed, 7 Jun 2017 20:42:42 +0530 Subject: Add main_task, complimentary_task and cleanup --- mediagoblin/media_types/video/processing.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index a7e2d20c..4dee8d55 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -160,6 +160,31 @@ def store_metadata(media_entry, metadata): if len(stored_metadata): media_entry.media_data_init(orig_metadata=stored_metadata) +# ===================== + + +def main_task(**process_info): + processor = CommonVideoProcessor(process_info['manager'], process_info['entry']) + processor.common_setup(process_info['resolution']) + processor.transcode(medium_size=process_info['medium_size'], vp8_quality=process_info['vp8_quality'], + vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) + processor.generate_thumb(thumb_size=process_info['thumb_size']) + processor.store_orig_metadata() + + +def complimentary_task(**process_info): + processor = CommonVideoProcessor(process_info['manager'], process_info['entry']) + processor.common_setup(process_info['resolution']) + processor.transcode(medium_size=process_info['medium_size'], vp8_quality=process_info['vp8_quality'], + vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) + + +def processing_cleanup(**process_info): + processor = CommonVideoProcessor(process_info['manager'], process_info['entry']) + processor.delete_queue_file() + +# ===================== + class CommonVideoProcessor(MediaProcessor): """ -- cgit v1.2.3 From 9a27fa60a42cd39596cb8c4bb0331279b998bae7 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Sun, 11 Jun 2017 19:07:58 +0530 Subject: Add additional celery config settings Fixes older webm_video backward compatibilty issue. Add 'default' queue to be used from now. Add other necessary celery settings for priority. --- mediagoblin/media_types/video/processing.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index 4dee8d55..5cae42f5 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -18,6 +18,7 @@ import argparse import os.path import logging import datetime +import celery import six @@ -163,6 +164,7 @@ def store_metadata(media_entry, metadata): # ===================== +@celery.task() def main_task(**process_info): processor = CommonVideoProcessor(process_info['manager'], process_info['entry']) processor.common_setup(process_info['resolution']) @@ -172,6 +174,7 @@ def main_task(**process_info): processor.store_orig_metadata() +@celery.task() def complimentary_task(**process_info): processor = CommonVideoProcessor(process_info['manager'], process_info['entry']) processor.common_setup(process_info['resolution']) @@ -179,6 +182,7 @@ def complimentary_task(**process_info): vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) +@celery.task() def processing_cleanup(**process_info): processor = CommonVideoProcessor(process_info['manager'], process_info['entry']) processor.delete_queue_file() @@ -408,7 +412,6 @@ class InitialProcessor(CommonVideoProcessor): self.transcode(medium_size=medium_size, vp8_quality=vp8_quality, vp8_threads=vp8_threads, vorbis_quality=vorbis_quality) - self.copy_original() self.generate_thumb(thumb_size=thumb_size) self.delete_queue_file() -- cgit v1.2.3 From 25ecdec9971394064063db397232eb7f0e89fae3 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Mon, 12 Jun 2017 20:53:23 +0530 Subject: Add priority to the celery tasks Few more changes to be made before executing the tasks. Also #1 should be handled soon after this. --- mediagoblin/media_types/ascii/processing.py | 3 +- mediagoblin/media_types/audio/processing.py | 3 +- mediagoblin/media_types/image/processing.py | 3 +- mediagoblin/media_types/pdf/processing.py | 3 +- mediagoblin/media_types/raw_image/processing.py | 3 +- mediagoblin/media_types/stl/processing.py | 3 +- mediagoblin/media_types/video/processing.py | 40 +++++++++++++++++-------- 7 files changed, 39 insertions(+), 19 deletions(-) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/ascii/processing.py b/mediagoblin/media_types/ascii/processing.py index 71ccc86e..823dc4fd 100644 --- a/mediagoblin/media_types/ascii/processing.py +++ b/mediagoblin/media_types/ascii/processing.py @@ -274,7 +274,8 @@ class AsciiProcessingManager(ProcessingManager): self.add_processor(InitialProcessor) self.add_processor(Resizer) - def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + def workflow(self, entry, manager, feed_url, reprocess_action, + reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/audio/processing.py b/mediagoblin/media_types/audio/processing.py index a83d60f7..b74364bc 100644 --- a/mediagoblin/media_types/audio/processing.py +++ b/mediagoblin/media_types/audio/processing.py @@ -366,7 +366,8 @@ class AudioProcessingManager(ProcessingManager): self.add_processor(Resizer) self.add_processor(Transcoder) - def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + def workflow(self, entry, manager, feed_url, reprocess_action, + reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/image/processing.py b/mediagoblin/media_types/image/processing.py index 42234eff..a189fef3 100644 --- a/mediagoblin/media_types/image/processing.py +++ b/mediagoblin/media_types/image/processing.py @@ -431,7 +431,8 @@ class ImageProcessingManager(ProcessingManager): self.add_processor(Resizer) self.add_processor(MetadataProcessing) - def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + def workflow(self, entry, manager, feed_url, reprocess_action, + reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/pdf/processing.py b/mediagoblin/media_types/pdf/processing.py index d93b19bb..6a13c8e3 100644 --- a/mediagoblin/media_types/pdf/processing.py +++ b/mediagoblin/media_types/pdf/processing.py @@ -471,7 +471,8 @@ class PdfProcessingManager(ProcessingManager): self.add_processor(InitialProcessor) self.add_processor(Resizer) - def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + def workflow(self, entry, manager, feed_url, reprocess_action, + reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/raw_image/processing.py b/mediagoblin/media_types/raw_image/processing.py index a385d563..7f2d155a 100644 --- a/mediagoblin/media_types/raw_image/processing.py +++ b/mediagoblin/media_types/raw_image/processing.py @@ -81,7 +81,8 @@ class RawImageProcessingManager(ProcessingManager): self.add_processor(InitialRawProcessor) self.add_processor(Resizer) - def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + def workflow(self, entry, manager, feed_url, reprocess_action, + reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/stl/processing.py b/mediagoblin/media_types/stl/processing.py index 7f2f350d..9dd6d49b 100644 --- a/mediagoblin/media_types/stl/processing.py +++ b/mediagoblin/media_types/stl/processing.py @@ -369,7 +369,8 @@ class StlProcessingManager(ProcessingManager): self.add_processor(InitialProcessor) self.add_processor(Resizer) - def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + def workflow(self, entry, manager, feed_url, reprocess_action, + reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index 5cae42f5..d039c24b 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -22,6 +22,7 @@ import celery import six +from celery import group, chord from mediagoblin import mg_globals as mgg from mediagoblin.processing import ( FilenameBuilder, BaseProcessingFail, @@ -34,7 +35,7 @@ from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ from mediagoblin.media_types import MissingComponents from . import transcoders -from .util import skip_transcode +from .util import skip_transcode, ACCEPTED_RESOLUTIONS _log = logging.getLogger(__name__) _log.setLevel(logging.DEBUG) @@ -165,26 +166,26 @@ def store_metadata(media_entry, metadata): @celery.task() -def main_task(**process_info): +def main_task(resolution, medium_size, **process_info): processor = CommonVideoProcessor(process_info['manager'], process_info['entry']) - processor.common_setup(process_info['resolution']) - processor.transcode(medium_size=process_info['medium_size'], vp8_quality=process_info['vp8_quality'], + processor.common_setup(resolution) + processor.transcode(medium_size=medium_size, vp8_quality=process_info['vp8_quality'], vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) processor.generate_thumb(thumb_size=process_info['thumb_size']) processor.store_orig_metadata() @celery.task() -def complimentary_task(**process_info): +def complimentary_task(resolution, medium_size, **process_info): processor = CommonVideoProcessor(process_info['manager'], process_info['entry']) - processor.common_setup(process_info['resolution']) - processor.transcode(medium_size=process_info['medium_size'], vp8_quality=process_info['vp8_quality'], + processor.common_setup(resolution) + processor.transcode(medium_size=medium_size, vp8_quality=process_info['vp8_quality'], vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) @celery.task() -def processing_cleanup(**process_info): - processor = CommonVideoProcessor(process_info['manager'], process_info['entry']) +def processing_cleanup(entry, manager): + processor = CommonVideoProcessor(manager, entry) # is it manager, entry or entry, manager? processor.delete_queue_file() # ===================== @@ -523,7 +524,20 @@ class VideoProcessingManager(ProcessingManager): self.add_processor(Resizer) self.add_processor(Transcoder) - def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): - ProcessMedia().apply_async( - [entry.id, feed_url, reprocess_action, reprocess_info], {}, - task_id=entry.queued_task_id) + def workflow(self, entry, manager, feed_url, reprocess_action, + reprocess_info=None): + + reprocess_info['entry'] = entry.id # ? + reprocess_info['manager'] = manager # can celery serialize this? + + # Add args + + transcoding_tasks = group( + main_task.signature(queue='default', priority=5, immutable=True), + complimentary_task.signature(queue='default', priority=4, immutable=True), + complimentary_task.signature(queue='default', priority=3, immutable=True), + complimentary_task.signature(queue='default', priority=2, immutable=True) + complimentary_task.signature(queue='default', priority=1, immutable=True) + ) + + chord(transcoding_tasks)(processing_cleanup.signature(queue='default', immutable=True)) -- cgit v1.2.3 From bd011c940eeeddd060ccf921ad3519d20d77a015 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Mon, 12 Jun 2017 23:17:44 +0530 Subject: Few more changes to celery priority tasking Addition of arguments to the celery tasks. --- mediagoblin/media_types/video/processing.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index d039c24b..64cacb5f 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -185,7 +185,7 @@ def complimentary_task(resolution, medium_size, **process_info): @celery.task() def processing_cleanup(entry, manager): - processor = CommonVideoProcessor(manager, entry) # is it manager, entry or entry, manager? + processor = CommonVideoProcessor(manager, entry) processor.delete_queue_file() # ===================== @@ -527,17 +527,22 @@ class VideoProcessingManager(ProcessingManager): def workflow(self, entry, manager, feed_url, reprocess_action, reprocess_info=None): - reprocess_info['entry'] = entry.id # ? - reprocess_info['manager'] = manager # can celery serialize this? - - # Add args + reprocess_info['entry'] = entry + reprocess_info['manager'] = manager transcoding_tasks = group( - main_task.signature(queue='default', priority=5, immutable=True), - complimentary_task.signature(queue='default', priority=4, immutable=True), - complimentary_task.signature(queue='default', priority=3, immutable=True), - complimentary_task.signature(queue='default', priority=2, immutable=True) - complimentary_task.signature(queue='default', priority=1, immutable=True) + main_task.signature(args=('480p', ACCEPTED_RESOLUTIONS['480p']), + kwargs=reprocess_info, queue='default', + priority=5, immutable=True), + complimentary_task.signature(args=('360p', ACCEPTED_RESOLUTIONS['360p']), + kwargs=reprocess_info, queue='default', + priority=4, immutable=True), + complimentary_task.signature(args=('720p', ACCEPTED_RESOLUTIONS['720p']), + kwargs=reprocess_info, queue='default', + priority=3, immutable=True), ) - chord(transcoding_tasks)(processing_cleanup.signature(queue='default', immutable=True)) + cleanup_task = processing_cleanup.signature(args=(entry, manager), + queue='default', immutable=True) + + chord(transcoding_tasks)(cleanup_task) -- cgit v1.2.3 From d77eb56280f57e547294e29e6a1b2b4d46c15ac6 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Tue, 13 Jun 2017 01:43:43 +0530 Subject: Celery Priority testing with debug statements Error at this line: `self.entry.set_file_metadata(self.curr_file, **file_metadata)` Otherwise, celery part should work fine. --- mediagoblin/media_types/ascii/processing.py | 3 +- mediagoblin/media_types/audio/processing.py | 3 +- mediagoblin/media_types/image/processing.py | 3 +- mediagoblin/media_types/pdf/processing.py | 3 +- mediagoblin/media_types/raw_image/processing.py | 3 +- mediagoblin/media_types/stl/processing.py | 3 +- mediagoblin/media_types/video/processing.py | 101 ++++++++++++++++-------- 7 files changed, 75 insertions(+), 44 deletions(-) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/ascii/processing.py b/mediagoblin/media_types/ascii/processing.py index 823dc4fd..c9b47fb5 100644 --- a/mediagoblin/media_types/ascii/processing.py +++ b/mediagoblin/media_types/ascii/processing.py @@ -274,8 +274,7 @@ class AsciiProcessingManager(ProcessingManager): self.add_processor(InitialProcessor) self.add_processor(Resizer) - def workflow(self, entry, manager, feed_url, reprocess_action, - reprocess_info=None): + def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/audio/processing.py b/mediagoblin/media_types/audio/processing.py index b74364bc..15d0b0a7 100644 --- a/mediagoblin/media_types/audio/processing.py +++ b/mediagoblin/media_types/audio/processing.py @@ -366,8 +366,7 @@ class AudioProcessingManager(ProcessingManager): self.add_processor(Resizer) self.add_processor(Transcoder) - def workflow(self, entry, manager, feed_url, reprocess_action, - reprocess_info=None): + def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/image/processing.py b/mediagoblin/media_types/image/processing.py index a189fef3..7224a8fd 100644 --- a/mediagoblin/media_types/image/processing.py +++ b/mediagoblin/media_types/image/processing.py @@ -431,8 +431,7 @@ class ImageProcessingManager(ProcessingManager): self.add_processor(Resizer) self.add_processor(MetadataProcessing) - def workflow(self, entry, manager, feed_url, reprocess_action, - reprocess_info=None): + def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/pdf/processing.py b/mediagoblin/media_types/pdf/processing.py index 6a13c8e3..e6e6e0a9 100644 --- a/mediagoblin/media_types/pdf/processing.py +++ b/mediagoblin/media_types/pdf/processing.py @@ -471,8 +471,7 @@ class PdfProcessingManager(ProcessingManager): self.add_processor(InitialProcessor) self.add_processor(Resizer) - def workflow(self, entry, manager, feed_url, reprocess_action, - reprocess_info=None): + def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/raw_image/processing.py b/mediagoblin/media_types/raw_image/processing.py index 7f2d155a..4bfd9f3a 100644 --- a/mediagoblin/media_types/raw_image/processing.py +++ b/mediagoblin/media_types/raw_image/processing.py @@ -81,8 +81,7 @@ class RawImageProcessingManager(ProcessingManager): self.add_processor(InitialRawProcessor) self.add_processor(Resizer) - def workflow(self, entry, manager, feed_url, reprocess_action, - reprocess_info=None): + def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/stl/processing.py b/mediagoblin/media_types/stl/processing.py index 9dd6d49b..cd3ffd8c 100644 --- a/mediagoblin/media_types/stl/processing.py +++ b/mediagoblin/media_types/stl/processing.py @@ -369,8 +369,7 @@ class StlProcessingManager(ProcessingManager): self.add_processor(InitialProcessor) self.add_processor(Resizer) - def workflow(self, entry, manager, feed_url, reprocess_action, - reprocess_info=None): + def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): ProcessMedia().apply_async( [entry.id, feed_url, reprocess_action, reprocess_info], {}, task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index 64cacb5f..c3257c84 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -29,7 +29,7 @@ from mediagoblin.processing import ( ProgressCallback, MediaProcessor, ProcessingManager, request_from_args, get_process_filename, store_public, - copy_original) + copy_original, get_entry_and_processing_manager) from mediagoblin.processing.task import ProcessMedia from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ from mediagoblin.media_types import MissingComponents @@ -166,27 +166,35 @@ def store_metadata(media_entry, metadata): @celery.task() -def main_task(resolution, medium_size, **process_info): - processor = CommonVideoProcessor(process_info['manager'], process_info['entry']) - processor.common_setup(resolution) - processor.transcode(medium_size=medium_size, vp8_quality=process_info['vp8_quality'], - vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) - processor.generate_thumb(thumb_size=process_info['thumb_size']) - processor.store_orig_metadata() +def main_task(entry_id, resolution, medium_size, **process_info): + entry, manager = get_entry_and_processing_manager(entry_id) + print "\nEntered main_task\n" + with CommonVideoProcessor(manager, entry) as processor: + processor.common_setup(resolution) + processor.transcode(medium_size=tuple(medium_size), vp8_quality=process_info['vp8_quality'], + vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) + processor.generate_thumb(thumb_size=process_info['thumb_size']) + processor.store_orig_metadata() + print "\nExited main_task\n" @celery.task() -def complimentary_task(resolution, medium_size, **process_info): - processor = CommonVideoProcessor(process_info['manager'], process_info['entry']) - processor.common_setup(resolution) - processor.transcode(medium_size=medium_size, vp8_quality=process_info['vp8_quality'], - vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) +def complimentary_task(entry_id, resolution, medium_size, **process_info): + entry, manager = get_entry_and_processing_manager(entry_id) + print "\nEntered complimentary_task\n" + with CommonVideoProcessor(manager, entry) as processor: + processor.common_setup(resolution) + processor.transcode(medium_size=tuple(medium_size), vp8_quality=process_info['vp8_quality'], + vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) + print "\nExited complimentary_task\n" @celery.task() -def processing_cleanup(entry, manager): - processor = CommonVideoProcessor(manager, entry) - processor.delete_queue_file() +def processing_cleanup(entry_id): + entry, manager = get_entry_and_processing_manager(entry_id) + with CommonVideoProcessor(manager, entry) as processor: + processor.delete_queue_file() + print "\nDeleted queue_file\n" # ===================== @@ -206,7 +214,7 @@ class CommonVideoProcessor(MediaProcessor): self.process_filename = get_process_filename( self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) - + self.transcoder = transcoders.VideoTranscoder() self.did_transcode = False @@ -218,6 +226,8 @@ class CommonVideoProcessor(MediaProcessor): self.curr_file = 'webm_video' self.part_filename = self.name_builder.fill('{basename}.medium.webm') + print self.curr_file, ": Done common_setup()" + def copy_original(self): # If we didn't transcode, then we need to keep the original raise NotImplementedError @@ -254,6 +264,7 @@ class CommonVideoProcessor(MediaProcessor): def transcode(self, medium_size=None, vp8_quality=None, vp8_threads=None, vorbis_quality=None): + print self.curr_file, ": Enter transcode" progress_callback = ProgressCallback(self.entry) tmp_dst = os.path.join(self.workbench.dir, self.part_filename) @@ -292,25 +303,34 @@ class CommonVideoProcessor(MediaProcessor): self.entry.media_files[self.curr_file].delete() else: + print self.curr_file, ": ->1.1" + print type(medium_size) + medium_size = tuple(medium_size) + print type(medium_size) + print self.curr_file, ": ->1.2" self.transcoder.transcode(self.process_filename, tmp_dst, vp8_quality=vp8_quality, vp8_threads=vp8_threads, vorbis_quality=vorbis_quality, progress_callback=progress_callback, dimensions=tuple(medium_size)) + print self.curr_file, ": ->2" if self.transcoder.dst_data: + print self.curr_file, ": ->3" # Push transcoded video to public storage _log.debug('Saving medium...') - store_public(self.entry, 'webm_video', tmp_dst, - self.name_builder.fill('{basename}.medium.webm')) + store_public(self.entry, 'webm_video', tmp_dst, self.part_filename) _log.debug('Saved medium') + print self.curr_file, ": ->4" # Is this the file_metadata that paroneayea was talking about? self.entry.set_file_metadata(self.curr_file, **file_metadata) self.did_transcode = True + print self.curr_file, ": Done transcode()" def generate_thumb(self, thumb_size=None): + print self.curr_file, ": Enter generate_thumb()" # Temporary file for the video thumbnail (cleaned up with workbench) tmp_thumb = os.path.join(self.workbench.dir, self.name_builder.fill( @@ -339,9 +359,10 @@ class CommonVideoProcessor(MediaProcessor): self.name_builder.fill('{basename}.thumbnail.jpg')) self.entry.set_file_metadata('thumb', thumb_size=thumb_size) + print self.curr_file, ": Done generate_thumb()" def store_orig_metadata(self): - + print self.curr_file, ": 2" # Extract metadata and keep a record of it metadata = transcoders.discover(self.process_filename) @@ -524,25 +545,41 @@ class VideoProcessingManager(ProcessingManager): self.add_processor(Resizer) self.add_processor(Transcoder) - def workflow(self, entry, manager, feed_url, reprocess_action, - reprocess_info=None): + def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): - reprocess_info['entry'] = entry - reprocess_info['manager'] = manager + reprocess_info = reprocess_info or {} + if 'vp8_quality' not in reprocess_info: + reprocess_info['vp8_quality'] = None + if 'vorbis_quality' not in reprocess_info: + reprocess_info['vorbis_quality'] = None + if 'vp8_threads' not in reprocess_info: + reprocess_info['vp8_threads'] = None + if 'thumb_size' not in reprocess_info: + reprocess_info['thumb_size'] = None - transcoding_tasks = group( - main_task.signature(args=('480p', ACCEPTED_RESOLUTIONS['480p']), + transcoding_tasks = group([ + main_task.signature(args=(entry_id, '480p', ACCEPTED_RESOLUTIONS['480p']), kwargs=reprocess_info, queue='default', priority=5, immutable=True), - complimentary_task.signature(args=('360p', ACCEPTED_RESOLUTIONS['360p']), + ]) + + cleanup_task = processing_cleanup.signature(args=(entry_id,), + queue='default', immutable=True) + + """ + complimentary_task.signature(args=(entry_id, '360p', ACCEPTED_RESOLUTIONS['360p']), kwargs=reprocess_info, queue='default', priority=4, immutable=True), - complimentary_task.signature(args=('720p', ACCEPTED_RESOLUTIONS['720p']), + complimentary_task.signature(args=(entry_id, '720p', ACCEPTED_RESOLUTIONS['720p']), kwargs=reprocess_info, queue='default', priority=3, immutable=True), - ) + main_task.apply_async(args=(entry_id, '480p', ACCEPTED_RESOLUTIONS['480p']), + kwargs=reprocess_info, queue='default', + priority=5, immutable=True) + processing_cleanup.apply_async(args=(entry_id,), queue='default', immutable=True) + """ - cleanup_task = processing_cleanup.signature(args=(entry, manager), - queue='default', immutable=True) - chord(transcoding_tasks)(cleanup_task) + + # main_task(entry_id, '480p', ACCEPTED_RESOLUTIONS['480p'], **reprocess_info) + # processing_cleanup(entry_id) -- cgit v1.2.3 From 982fbde8e8bb6d0377e4e942a794e520774b6d72 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Tue, 13 Jun 2017 23:24:32 +0530 Subject: Individual resolution transcoding fixes Few minor changes which fix the errors. Although, the user still cannot view the video since the front end part needs to changed. Fixes #3 --- mediagoblin/media_types/video/processing.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index c3257c84..a59a7c0b 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -191,6 +191,7 @@ def complimentary_task(entry_id, resolution, medium_size, **process_info): @celery.task() def processing_cleanup(entry_id): + print "\nEnter processing_cleanup()\n" entry, manager = get_entry_and_processing_manager(entry_id) with CommonVideoProcessor(manager, entry) as processor: processor.delete_queue_file() @@ -303,11 +304,7 @@ class CommonVideoProcessor(MediaProcessor): self.entry.media_files[self.curr_file].delete() else: - print self.curr_file, ": ->1.1" - print type(medium_size) - medium_size = tuple(medium_size) - print type(medium_size) - print self.curr_file, ": ->1.2" + print self.curr_file, ": ->1" self.transcoder.transcode(self.process_filename, tmp_dst, vp8_quality=vp8_quality, vp8_threads=vp8_threads, @@ -319,7 +316,7 @@ class CommonVideoProcessor(MediaProcessor): print self.curr_file, ": ->3" # Push transcoded video to public storage _log.debug('Saving medium...') - store_public(self.entry, 'webm_video', tmp_dst, self.part_filename) + store_public(self.entry, self.curr_file, tmp_dst, self.part_filename) _log.debug('Saved medium') print self.curr_file, ": ->4" @@ -362,7 +359,7 @@ class CommonVideoProcessor(MediaProcessor): print self.curr_file, ": Done generate_thumb()" def store_orig_metadata(self): - print self.curr_file, ": 2" + print self.curr_file, ": Enter store_orig_metadata()" # Extract metadata and keep a record of it metadata = transcoders.discover(self.process_filename) @@ -370,6 +367,7 @@ class CommonVideoProcessor(MediaProcessor): # it gets split into DiscovererAudioInfo and DiscovererVideoInfo; # metadata itself has container-related data in tags, like video-codec store_metadata(self.entry, metadata) + print self.curr_file, ": Done store_orig_metadata()" class InitialProcessor(CommonVideoProcessor): -- cgit v1.2.3 From 869048dd15108ab379466a2df8f5fd4af1e4e068 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Tue, 13 Jun 2017 23:54:40 +0530 Subject: Add complimentary_tasks to the celery group Add a couple of complimentary_tasks to check if priority is working or not. --- mediagoblin/media_types/video/processing.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index a59a7c0b..586a0c85 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -191,7 +191,7 @@ def complimentary_task(entry_id, resolution, medium_size, **process_info): @celery.task() def processing_cleanup(entry_id): - print "\nEnter processing_cleanup()\n" + print "\nEntered processing_cleanup()\n" entry, manager = get_entry_and_processing_manager(entry_id) with CommonVideoProcessor(manager, entry) as processor: processor.delete_queue_file() @@ -559,18 +559,18 @@ class VideoProcessingManager(ProcessingManager): main_task.signature(args=(entry_id, '480p', ACCEPTED_RESOLUTIONS['480p']), kwargs=reprocess_info, queue='default', priority=5, immutable=True), - ]) - - cleanup_task = processing_cleanup.signature(args=(entry_id,), - queue='default', immutable=True) - - """ complimentary_task.signature(args=(entry_id, '360p', ACCEPTED_RESOLUTIONS['360p']), kwargs=reprocess_info, queue='default', priority=4, immutable=True), complimentary_task.signature(args=(entry_id, '720p', ACCEPTED_RESOLUTIONS['720p']), kwargs=reprocess_info, queue='default', priority=3, immutable=True), + ]) + + cleanup_task = processing_cleanup.signature(args=(entry_id,), + queue='default', immutable=True) + + """ main_task.apply_async(args=(entry_id, '480p', ACCEPTED_RESOLUTIONS['480p']), kwargs=reprocess_info, queue='default', priority=5, immutable=True) @@ -578,6 +578,3 @@ class VideoProcessingManager(ProcessingManager): """ chord(transcoding_tasks)(cleanup_task) - - # main_task(entry_id, '480p', ACCEPTED_RESOLUTIONS['480p'], **reprocess_info) - # processing_cleanup(entry_id) -- cgit v1.2.3 From 33d5ac6c4de44aa8d98a8cf759c79351f5a59885 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Sat, 17 Jun 2017 00:53:39 +0530 Subject: Simple hack to handle main workflow problem Remove redundunt workflow methods from the other media type's processing.py. Fixes #1 --- mediagoblin/media_types/ascii/processing.py | 5 ----- mediagoblin/media_types/audio/processing.py | 5 ----- mediagoblin/media_types/image/processing.py | 5 ----- mediagoblin/media_types/pdf/processing.py | 5 ----- mediagoblin/media_types/raw_image/processing.py | 5 ----- mediagoblin/media_types/stl/processing.py | 5 ----- mediagoblin/media_types/video/processing.py | 21 ++++++++------------- 7 files changed, 8 insertions(+), 43 deletions(-) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/ascii/processing.py b/mediagoblin/media_types/ascii/processing.py index c9b47fb5..00d04e63 100644 --- a/mediagoblin/media_types/ascii/processing.py +++ b/mediagoblin/media_types/ascii/processing.py @@ -273,8 +273,3 @@ class AsciiProcessingManager(ProcessingManager): super(AsciiProcessingManager, self).__init__() self.add_processor(InitialProcessor) self.add_processor(Resizer) - - def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): - ProcessMedia().apply_async( - [entry.id, feed_url, reprocess_action, reprocess_info], {}, - task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/audio/processing.py b/mediagoblin/media_types/audio/processing.py index 15d0b0a7..427309de 100644 --- a/mediagoblin/media_types/audio/processing.py +++ b/mediagoblin/media_types/audio/processing.py @@ -365,8 +365,3 @@ class AudioProcessingManager(ProcessingManager): self.add_processor(InitialProcessor) self.add_processor(Resizer) self.add_processor(Transcoder) - - def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): - ProcessMedia().apply_async( - [entry.id, feed_url, reprocess_action, reprocess_info], {}, - task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/image/processing.py b/mediagoblin/media_types/image/processing.py index 7224a8fd..7ddf3f35 100644 --- a/mediagoblin/media_types/image/processing.py +++ b/mediagoblin/media_types/image/processing.py @@ -431,11 +431,6 @@ class ImageProcessingManager(ProcessingManager): self.add_processor(Resizer) self.add_processor(MetadataProcessing) - def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): - ProcessMedia().apply_async( - [entry.id, feed_url, reprocess_action, reprocess_info], {}, - task_id=entry.queued_task_id) - if __name__ == '__main__': import sys diff --git a/mediagoblin/media_types/pdf/processing.py b/mediagoblin/media_types/pdf/processing.py index e6e6e0a9..ac4bab6d 100644 --- a/mediagoblin/media_types/pdf/processing.py +++ b/mediagoblin/media_types/pdf/processing.py @@ -470,8 +470,3 @@ class PdfProcessingManager(ProcessingManager): super(PdfProcessingManager, self).__init__() self.add_processor(InitialProcessor) self.add_processor(Resizer) - - def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): - ProcessMedia().apply_async( - [entry.id, feed_url, reprocess_action, reprocess_info], {}, - task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/raw_image/processing.py b/mediagoblin/media_types/raw_image/processing.py index 4bfd9f3a..740ba2dd 100644 --- a/mediagoblin/media_types/raw_image/processing.py +++ b/mediagoblin/media_types/raw_image/processing.py @@ -80,8 +80,3 @@ class RawImageProcessingManager(ProcessingManager): super(RawImageProcessingManager, self).__init__() self.add_processor(InitialRawProcessor) self.add_processor(Resizer) - - def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): - ProcessMedia().apply_async( - [entry.id, feed_url, reprocess_action, reprocess_info], {}, - task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/stl/processing.py b/mediagoblin/media_types/stl/processing.py index cd3ffd8c..55764aeb 100644 --- a/mediagoblin/media_types/stl/processing.py +++ b/mediagoblin/media_types/stl/processing.py @@ -368,8 +368,3 @@ class StlProcessingManager(ProcessingManager): super(StlProcessingManager, self).__init__() self.add_processor(InitialProcessor) self.add_processor(Resizer) - - def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): - ProcessMedia().apply_async( - [entry.id, feed_url, reprocess_action, reprocess_info], {}, - task_id=entry.queued_task_id) diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index 586a0c85..e881636c 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -30,7 +30,6 @@ from mediagoblin.processing import ( ProcessingManager, request_from_args, get_process_filename, store_public, copy_original, get_entry_and_processing_manager) -from mediagoblin.processing.task import ProcessMedia from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ from mediagoblin.media_types import MissingComponents @@ -543,7 +542,7 @@ class VideoProcessingManager(ProcessingManager): self.add_processor(Resizer) self.add_processor(Transcoder) - def workflow(self, entry_id, feed_url, reprocess_action, reprocess_info=None): + def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): reprocess_info = reprocess_info or {} if 'vp8_quality' not in reprocess_info: @@ -556,25 +555,21 @@ class VideoProcessingManager(ProcessingManager): reprocess_info['thumb_size'] = None transcoding_tasks = group([ - main_task.signature(args=(entry_id, '480p', ACCEPTED_RESOLUTIONS['480p']), + main_task.signature(args=(entry.id, '480p', ACCEPTED_RESOLUTIONS['480p']), kwargs=reprocess_info, queue='default', priority=5, immutable=True), - complimentary_task.signature(args=(entry_id, '360p', ACCEPTED_RESOLUTIONS['360p']), + complimentary_task.signature(args=(entry.id, '360p', ACCEPTED_RESOLUTIONS['360p']), kwargs=reprocess_info, queue='default', priority=4, immutable=True), - complimentary_task.signature(args=(entry_id, '720p', ACCEPTED_RESOLUTIONS['720p']), + complimentary_task.signature(args=(entry.id, '720p', ACCEPTED_RESOLUTIONS['720p']), kwargs=reprocess_info, queue='default', priority=3, immutable=True), ]) - cleanup_task = processing_cleanup.signature(args=(entry_id,), + cleanup_task = processing_cleanup.signature(args=(entry.id,), queue='default', immutable=True) - """ - main_task.apply_async(args=(entry_id, '480p', ACCEPTED_RESOLUTIONS['480p']), - kwargs=reprocess_info, queue='default', - priority=5, immutable=True) - processing_cleanup.apply_async(args=(entry_id,), queue='default', immutable=True) - """ - chord(transcoding_tasks)(cleanup_task) + + # Not sure what to return since we are scheduling the task here itself + return 1 -- cgit v1.2.3 From 336508bb17bc9ce51b001c5b415da116fb137f73 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Sat, 17 Jun 2017 02:34:58 +0530 Subject: Render webm_480 as default if webm_video is absent Currently, webm_video is still the higher priority in media_fetch_oder as older versions should not be broken. So, webm_480p (main_task) will be rendered by default for newer versions. All tests pass at this point. --- mediagoblin/media_types/video/__init__.py | 2 +- mediagoblin/media_types/video/processing.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/video/__init__.py b/mediagoblin/media_types/video/__init__.py index 1862ffe1..08e92080 100644 --- a/mediagoblin/media_types/video/__init__.py +++ b/mediagoblin/media_types/video/__init__.py @@ -31,7 +31,7 @@ class VideoMediaManager(MediaManagerBase): type_icon = "images/type_icons/video.png" # Used by the media_entry.get_display_media method - media_fetch_order = [u'webm_video', u'original'] + media_fetch_order = [u'webm_video', u'webm_480p', u'original'] default_webm_type = 'video/webm; codecs="vp8, vorbis"' diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index e881636c..4da1ad23 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -166,6 +166,7 @@ def store_metadata(media_entry, metadata): @celery.task() def main_task(entry_id, resolution, medium_size, **process_info): + print "\nEntry processing\n" entry, manager = get_entry_and_processing_manager(entry_id) print "\nEntered main_task\n" with CommonVideoProcessor(manager, entry) as processor: @@ -175,6 +176,10 @@ def main_task(entry_id, resolution, medium_size, **process_info): processor.generate_thumb(thumb_size=process_info['thumb_size']) processor.store_orig_metadata() print "\nExited main_task\n" + # Make state of entry as processed + entry.state = u'processed' + entry.save() + print "\nEntry processed\n" @celery.task() @@ -544,6 +549,9 @@ class VideoProcessingManager(ProcessingManager): def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + entry.state = u'processing' + entry.save() + reprocess_info = reprocess_info or {} if 'vp8_quality' not in reprocess_info: reprocess_info['vp8_quality'] = None -- cgit v1.2.3 From 5161533a6fb66e24f6906d6476069277d594e738 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Tue, 20 Jun 2017 17:28:34 +0530 Subject: Add choice of video resolutions for instance owner The instance owner can choose from the list of available resolutions. ['144p', '240p', '360p', '480p', '720p', '1080p'] Also, the default resolution is now set to 480p and the instance owner can choose the default resolution from that list as well. --- mediagoblin/media_types/video/config_spec.ini | 8 +++++++ mediagoblin/media_types/video/processing.py | 32 +++++++++++++++++---------- 2 files changed, 28 insertions(+), 12 deletions(-) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/video/config_spec.ini b/mediagoblin/media_types/video/config_spec.ini index 98714f56..e899eff5 100644 --- a/mediagoblin/media_types/video/config_spec.ini +++ b/mediagoblin/media_types/video/config_spec.ini @@ -12,6 +12,14 @@ vorbis_quality = float(default=0.3) # Autoplay the video when page is loaded? auto_play = boolean(default=False) +# List of resolutions that the video should be transcoded to +# Choose among ['144p', '240p', '360p', '480p', '720p', '1080p'], +# preferrably in the order of transcoding. +available_resolutions = string_list(default=list('480p', '360p', '720p')) + +# Default resolution of video +default_resolution = string(default='480p') + [[skip_transcode]] mime_types = string_list(default=list("video/webm")) container_formats = string_list(default=list("Matroska")) diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index 4da1ad23..2a6a716f 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -549,6 +549,10 @@ class VideoProcessingManager(ProcessingManager): def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): + video_config = mgg.global_config['plugins'][MEDIA_TYPE] + def_res = video_config['default_resolution'] + priority_num = len(video_config['available_resolutions']) + 1 + entry.state = u'processing' entry.save() @@ -562,18 +566,22 @@ class VideoProcessingManager(ProcessingManager): if 'thumb_size' not in reprocess_info: reprocess_info['thumb_size'] = None - transcoding_tasks = group([ - main_task.signature(args=(entry.id, '480p', ACCEPTED_RESOLUTIONS['480p']), - kwargs=reprocess_info, queue='default', - priority=5, immutable=True), - complimentary_task.signature(args=(entry.id, '360p', ACCEPTED_RESOLUTIONS['360p']), - kwargs=reprocess_info, queue='default', - priority=4, immutable=True), - complimentary_task.signature(args=(entry.id, '720p', ACCEPTED_RESOLUTIONS['720p']), - kwargs=reprocess_info, queue='default', - priority=3, immutable=True), - ]) - + tasks_list = [main_task.signature(args=(entry.id, def_res, + ACCEPTED_RESOLUTIONS[def_res]), + kwargs=reprocess_info, queue='default', + priority=priority_num, immutable=True)] + + for comp_res in video_config['available_resolutions']: + if comp_res != def_res: + priority_num += -1 + tasks_list.append( + complimentary_task.signature(args=(entry.id, comp_res, + ACCEPTED_RESOLUTIONS[comp_res]), + kwargs=reprocess_info, queue='default', + priority=priority_num, immutable=True) + ) + + transcoding_tasks = group(tasks_list) cleanup_task = processing_cleanup.signature(args=(entry.id,), queue='default', immutable=True) -- cgit v1.2.3 From 602cfcb78916ac54bb415d9c2023124b6f3ad5b6 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Wed, 21 Jun 2017 10:00:56 +0530 Subject: Use tuple as default return type of workflow() If None, then schedule the old Celery call, else the format is group and a single task in the tuple. Hence, Celery chord would be ideal to use. Closes #4 --- mediagoblin/media_types/video/processing.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index 2a6a716f..ee2aa443 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -22,7 +22,7 @@ import celery import six -from celery import group, chord +from celery import group from mediagoblin import mg_globals as mgg from mediagoblin.processing import ( FilenameBuilder, BaseProcessingFail, @@ -585,7 +585,4 @@ class VideoProcessingManager(ProcessingManager): cleanup_task = processing_cleanup.signature(args=(entry.id,), queue='default', immutable=True) - chord(transcoding_tasks)(cleanup_task) - - # Not sure what to return since we are scheduling the task here itself - return 1 + return (transcoding_tasks, cleanup_task) -- cgit v1.2.3 From bd0b36be7916f756a701f29bb6b0aff8cb624f9c Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Sun, 25 Jun 2017 20:23:37 +0530 Subject: Implement copy_original and _keep_best _keep_best is changed to keep_best now, and is added in processing_cleanup. webm is added to ACCEPTED_RESOLUTIONS. --- mediagoblin/media_types/video/processing.py | 35 +++++++++++++++++++++++++---- mediagoblin/media_types/video/util.py | 13 ++++++----- 2 files changed, 38 insertions(+), 10 deletions(-) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index ee2aa443..f2fba38e 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -198,6 +198,10 @@ def processing_cleanup(entry_id): print "\nEntered processing_cleanup()\n" entry, manager = get_entry_and_processing_manager(entry_id) with CommonVideoProcessor(manager, entry) as processor: + # no need to specify a resolution here + processor.common_setup() + processor.copy_original() + processor.keep_best() processor.delete_queue_file() print "\nDeleted queue_file\n" @@ -235,13 +239,36 @@ class CommonVideoProcessor(MediaProcessor): def copy_original(self): # If we didn't transcode, then we need to keep the original - raise NotImplementedError - - def _keep_best(self): + self.did_transcode = False + for each_res in self.video_config['available_resolutions']: + if ('webm_' + str(each_res)) in self.entry.media_files: + print "here == 1.1" + self.did_transcode = True + break + if not self.did_transcode or \ + (self.video_config['keep_original'] and self.did_transcode): + copy_original( + self.entry, self.process_filename, + self.name_builder.fill('{basename}{ext}')) + + def keep_best(self): """ If there is no original, keep the best file that we have """ - raise NotImplementedError + best_file = None + best_file_dim = (0, 0) + for each_res in self.video_config['available_resolutions']: + curr_dim = ACCEPTED_RESOLUTIONS[each_res] + if curr_dim[0] >= best_file_dim[0] and curr_dim[1] >= best_file_dim[1]: + best_file = each_res + best_file_dim = curr_dim + if not self.entry.media_files.get('best_quality'): + # Save the best quality file if no original? + if not self.entry.media_files.get('original') and \ + self.entry.media_files.get(str(best_file)): + self.entry.media_files['best_quality'] = self.entry \ + .media_files[str(best_file)] + def _skip_processing(self, keyname, **kwargs): file_metadata = self.entry.get_file_metadata(keyname) diff --git a/mediagoblin/media_types/video/util.py b/mediagoblin/media_types/video/util.py index 1f5e907d..cf8dc72d 100644 --- a/mediagoblin/media_types/video/util.py +++ b/mediagoblin/media_types/video/util.py @@ -19,12 +19,13 @@ import logging from mediagoblin import mg_globals as mgg ACCEPTED_RESOLUTIONS = { - '144p' : (256, 144), - '240p' : (352, 240), - '360p' : (480, 360), - '480p' : (858, 480), - '720p' : (1280, 720), - '1080p' : (1920, 1080), + '144p': (256, 144), + '240p': (352, 240), + '360p': (480, 360), + '480p': (858, 480), + '720p': (1280, 720), + '1080p': (1920, 1080), + 'webm': (640, 640), } _log = logging.getLogger(__name__) -- cgit v1.2.3 From ae7ebecdbba7c7ae66e547920502c77a65c4dc0f Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Sat, 1 Jul 2017 04:06:02 +0530 Subject: Add property to media_fetch_order Due to configurable resolution settings, this change is necessary to set the default resolution and the others. --- mediagoblin/media_types/video/__init__.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/video/__init__.py b/mediagoblin/media_types/video/__init__.py index 08e92080..ea7bc021 100644 --- a/mediagoblin/media_types/video/__init__.py +++ b/mediagoblin/media_types/video/__init__.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from mediagoblin import mg_globals as mgg from mediagoblin.media_types import MediaManagerBase from mediagoblin.media_types.video.processing import (VideoProcessingManager, sniff_handler, sniffer) @@ -31,9 +32,17 @@ class VideoMediaManager(MediaManagerBase): type_icon = "images/type_icons/video.png" # Used by the media_entry.get_display_media method - media_fetch_order = [u'webm_video', u'webm_480p', u'original'] default_webm_type = 'video/webm; codecs="vp8, vorbis"' + @property + def media_fetch_order(self): + video_config = mgg.global_config['plugins'][MEDIA_TYPE] + video_res = video_config['available_resolutions'] + video_res.remove(video_config['default_resolution']) + video_res.insert(0, video_config['default_resolution']) + video_res = map((lambda x: unicode('webm_' + str(x), 'utf-8')), video_res) + return ([u'webm_video'] + video_res + [u'original']) + def get_media_type_and_manager(ext): if ext in ACCEPTED_EXTENSIONS: -- cgit v1.2.3 From 36c1e7dc2f3c63021458c8d36fe3c291be0131bf Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Sat, 5 Aug 2017 17:23:17 +0530 Subject: Replace print statements to logging Closes #5 --- mediagoblin/media_types/video/processing.py | 43 ++++++++++++----------------- 1 file changed, 18 insertions(+), 25 deletions(-) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index f2fba38e..58a53e0f 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -161,41 +161,45 @@ def store_metadata(media_entry, metadata): if len(stored_metadata): media_entry.media_data_init(orig_metadata=stored_metadata) -# ===================== - @celery.task() def main_task(entry_id, resolution, medium_size, **process_info): - print "\nEntry processing\n" + """ + Main celery task to transcode the video to the default resolution + and store original video metadata. + """ + _log.debug('MediaEntry processing') entry, manager = get_entry_and_processing_manager(entry_id) - print "\nEntered main_task\n" with CommonVideoProcessor(manager, entry) as processor: processor.common_setup(resolution) processor.transcode(medium_size=tuple(medium_size), vp8_quality=process_info['vp8_quality'], vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) processor.generate_thumb(thumb_size=process_info['thumb_size']) processor.store_orig_metadata() - print "\nExited main_task\n" # Make state of entry as processed entry.state = u'processed' entry.save() - print "\nEntry processed\n" + _log.info(str(entry.title) + ' media entry is processed (transcoded to ' + 'default resolution: ' + str(medium_size) + ').') + _log.debug('MediaEntry processed') @celery.task() def complimentary_task(entry_id, resolution, medium_size, **process_info): + """ + Side celery task to transcode the video to other resolutions + """ entry, manager = get_entry_and_processing_manager(entry_id) - print "\nEntered complimentary_task\n" with CommonVideoProcessor(manager, entry) as processor: processor.common_setup(resolution) processor.transcode(medium_size=tuple(medium_size), vp8_quality=process_info['vp8_quality'], vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) - print "\nExited complimentary_task\n" + _log.info(str(entry.title) + ' media entry is transcoded to ' + str(medium_size)) @celery.task() def processing_cleanup(entry_id): - print "\nEntered processing_cleanup()\n" + _log.debug('Entered processing_cleanup') entry, manager = get_entry_and_processing_manager(entry_id) with CommonVideoProcessor(manager, entry) as processor: # no need to specify a resolution here @@ -203,9 +207,7 @@ def processing_cleanup(entry_id): processor.copy_original() processor.keep_best() processor.delete_queue_file() - print "\nDeleted queue_file\n" - -# ===================== + _log.debug('Deleted queue_file') class CommonVideoProcessor(MediaProcessor): @@ -235,14 +237,12 @@ class CommonVideoProcessor(MediaProcessor): self.curr_file = 'webm_video' self.part_filename = self.name_builder.fill('{basename}.medium.webm') - print self.curr_file, ": Done common_setup()" def copy_original(self): # If we didn't transcode, then we need to keep the original self.did_transcode = False for each_res in self.video_config['available_resolutions']: if ('webm_' + str(each_res)) in self.entry.media_files: - print "here == 1.1" self.did_transcode = True break if not self.did_transcode or \ @@ -251,6 +251,7 @@ class CommonVideoProcessor(MediaProcessor): self.entry, self.process_filename, self.name_builder.fill('{basename}{ext}')) + def keep_best(self): """ If there is no original, keep the best file that we have @@ -296,7 +297,6 @@ class CommonVideoProcessor(MediaProcessor): def transcode(self, medium_size=None, vp8_quality=None, vp8_threads=None, vorbis_quality=None): - print self.curr_file, ": Enter transcode" progress_callback = ProgressCallback(self.entry) tmp_dst = os.path.join(self.workbench.dir, self.part_filename) @@ -335,30 +335,25 @@ class CommonVideoProcessor(MediaProcessor): self.entry.media_files[self.curr_file].delete() else: - print self.curr_file, ": ->1" + _log.debug('Entered transcoder') self.transcoder.transcode(self.process_filename, tmp_dst, vp8_quality=vp8_quality, vp8_threads=vp8_threads, vorbis_quality=vorbis_quality, progress_callback=progress_callback, dimensions=tuple(medium_size)) - print self.curr_file, ": ->2" if self.transcoder.dst_data: - print self.curr_file, ": ->3" # Push transcoded video to public storage _log.debug('Saving medium...') store_public(self.entry, self.curr_file, tmp_dst, self.part_filename) _log.debug('Saved medium') - print self.curr_file, ": ->4" - # Is this the file_metadata that paroneayea was talking about? self.entry.set_file_metadata(self.curr_file, **file_metadata) self.did_transcode = True - print self.curr_file, ": Done transcode()" def generate_thumb(self, thumb_size=None): - print self.curr_file, ": Enter generate_thumb()" + _log.debug("Enter generate_thumb()") # Temporary file for the video thumbnail (cleaned up with workbench) tmp_thumb = os.path.join(self.workbench.dir, self.name_builder.fill( @@ -387,10 +382,8 @@ class CommonVideoProcessor(MediaProcessor): self.name_builder.fill('{basename}.thumbnail.jpg')) self.entry.set_file_metadata('thumb', thumb_size=thumb_size) - print self.curr_file, ": Done generate_thumb()" def store_orig_metadata(self): - print self.curr_file, ": Enter store_orig_metadata()" # Extract metadata and keep a record of it metadata = transcoders.discover(self.process_filename) @@ -398,7 +391,7 @@ class CommonVideoProcessor(MediaProcessor): # it gets split into DiscovererAudioInfo and DiscovererVideoInfo; # metadata itself has container-related data in tags, like video-codec store_metadata(self.entry, metadata) - print self.curr_file, ": Done store_orig_metadata()" + _log.debug("Stored original video metadata") class InitialProcessor(CommonVideoProcessor): -- cgit v1.2.3 From a0fa4b4bb2d23df05052ab15a58fef3c3bebce90 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Tue, 8 Aug 2017 22:09:08 +0530 Subject: Add unicode to accomodate title of MediaEntry Related to #5 --- mediagoblin/media_types/video/processing.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index 58a53e0f..66363476 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -179,8 +179,8 @@ def main_task(entry_id, resolution, medium_size, **process_info): # Make state of entry as processed entry.state = u'processed' entry.save() - _log.info(str(entry.title) + ' media entry is processed (transcoded to ' - 'default resolution: ' + str(medium_size) + ').') + _log.info(u'{0} media entry is processed (transcoded to default resolution' + '): {1}'.format(unicode(entry.title), unicode(medium_size))) _log.debug('MediaEntry processed') @@ -194,7 +194,7 @@ def complimentary_task(entry_id, resolution, medium_size, **process_info): processor.common_setup(resolution) processor.transcode(medium_size=tuple(medium_size), vp8_quality=process_info['vp8_quality'], vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) - _log.info(str(entry.title) + ' media entry is transcoded to ' + str(medium_size)) + _log.info(u'{0} media entry is transcoded to {1}'.format(unicode(entry.title), unicode(medium_size))) @celery.task() -- cgit v1.2.3 From 3456abe90196d5894fc5f8aa0383d02b894b1f2d Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Thu, 10 Aug 2017 19:32:29 +0530 Subject: Fix percentage reporting Report transcoding_progress as the overall percent of all resolutions. Modify Logging. Closes #6 --- mediagoblin/media_types/video/processing.py | 3 ++- mediagoblin/media_types/video/transcoders.py | 20 ++++++++++++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index 66363476..dc7264fc 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -341,7 +341,8 @@ class CommonVideoProcessor(MediaProcessor): vp8_threads=vp8_threads, vorbis_quality=vorbis_quality, progress_callback=progress_callback, - dimensions=tuple(medium_size)) + dimensions=tuple(medium_size), + media_entry=self.entry) if self.transcoder.dst_data: # Push transcoded video to public storage _log.debug('Saving medium...') diff --git a/mediagoblin/media_types/video/transcoders.py b/mediagoblin/media_types/video/transcoders.py index 2d3392f2..d94bf59f 100644 --- a/mediagoblin/media_types/video/transcoders.py +++ b/mediagoblin/media_types/video/transcoders.py @@ -21,8 +21,10 @@ import sys import logging import multiprocessing +from mediagoblin import mg_globals as mgg from mediagoblin.media_types.tools import discover from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ +from .util import ACCEPTED_RESOLUTIONS #os.environ['GST_DEBUG'] = '4,python:4' @@ -163,6 +165,9 @@ class VideoTranscoder(object): self.source_path = src self.destination_path = dst + # Get media entry + self.entry = kwargs.get('media_entry') or None + # vp8enc options self.destination_dimensions = kwargs.get('dimensions', (640, 640)) self.vp8_quality = kwargs.get('vp8_quality', 8) @@ -184,6 +189,11 @@ class VideoTranscoder(object): self._progress_callback = kwargs.get('progress_callback') or None + # Get number of resolutions available for the video + video_config = mgg.global_config['plugins']['mediagoblin.media_types.video'] + self.num_of_resolutions = len(video_config['available_resolutions']) + self.progress_percentage = 0 + if not type(self.destination_dimensions) == tuple: raise Exception('dimensions must be tuple: (width, height)') @@ -354,10 +364,16 @@ class VideoTranscoder(object): # Update progress state if it has changed (success, percent) = structure.get_int('percent') if self.progress_percentage != percent and success: + if self.progress_percentage > percent and percent == 0: + percent = 100 + percent_increment = percent - self.progress_percentage self.progress_percentage = percent if self._progress_callback: - self._progress_callback(percent) - _log.info('{percent}% done...'.format(percent=percent)) + self._progress_callback(percent_increment/self.num_of_resolutions) + _log.info('{percent}% of {dest} resolution done..' + '.'.format(percent=percent, dest=self.destination_dimensions)) + _log.info('{0:.2f}% of all resolutions done' + '...'.format(self.entry.transcoding_progress)) elif message.type == Gst.MessageType.ERROR: _log.error('Got error: {0}'.format(message.parse_error())) self.dst_data = None -- cgit v1.2.3 From 40f7e8d191080bc82d361c4fbdb8af651b77bfe1 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Thu, 10 Aug 2017 19:39:33 +0530 Subject: Change entry.title to entry.id in logging Related to #5 --- mediagoblin/media_types/video/processing.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index dc7264fc..e343930b 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -179,8 +179,8 @@ def main_task(entry_id, resolution, medium_size, **process_info): # Make state of entry as processed entry.state = u'processed' entry.save() - _log.info(u'{0} media entry is processed (transcoded to default resolution' - '): {1}'.format(unicode(entry.title), unicode(medium_size))) + _log.info(u'MediaEntry ID {0} is processed (transcoded to default resolution' + '): {1}'.format(unicode(entry.id), unicode(medium_size))) _log.debug('MediaEntry processed') @@ -194,7 +194,7 @@ def complimentary_task(entry_id, resolution, medium_size, **process_info): processor.common_setup(resolution) processor.transcode(medium_size=tuple(medium_size), vp8_quality=process_info['vp8_quality'], vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) - _log.info(u'{0} media entry is transcoded to {1}'.format(unicode(entry.title), unicode(medium_size))) + _log.info(u'MediaEntry ID {0} is transcoded to {1}'.format(unicode(entry.id), unicode(medium_size))) @celery.task() -- cgit v1.2.3 From 91a7b1226cb95a8d61f7cc19be56ae472289639c Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Fri, 11 Aug 2017 19:40:41 +0530 Subject: Small changes to fixing transcode percentage Related to #6 --- mediagoblin/media_types/video/processing.py | 5 ++--- mediagoblin/media_types/video/transcoders.py | 13 ++++++------- 2 files changed, 8 insertions(+), 10 deletions(-) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index e343930b..114c5057 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -336,13 +336,12 @@ class CommonVideoProcessor(MediaProcessor): else: _log.debug('Entered transcoder') - self.transcoder.transcode(self.process_filename, tmp_dst, + self.transcoder.transcode(self.process_filename, tmp_dst, self.entry, vp8_quality=vp8_quality, vp8_threads=vp8_threads, vorbis_quality=vorbis_quality, progress_callback=progress_callback, - dimensions=tuple(medium_size), - media_entry=self.entry) + dimensions=tuple(medium_size)) if self.transcoder.dst_data: # Push transcoded video to public storage _log.debug('Saving medium...') diff --git a/mediagoblin/media_types/video/transcoders.py b/mediagoblin/media_types/video/transcoders.py index d94bf59f..a2c8f431 100644 --- a/mediagoblin/media_types/video/transcoders.py +++ b/mediagoblin/media_types/video/transcoders.py @@ -155,18 +155,16 @@ class VideoTranscoder(object): ''' def __init__(self): _log.info('Initializing VideoTranscoder...') - self.progress_percentage = None + self.progress_percentage = 0 self.loop = GLib.MainLoop() - def transcode(self, src, dst, **kwargs): + def transcode(self, src, dst, entry, **kwargs): ''' Transcode a video file into a 'medium'-sized version. ''' self.source_path = src self.destination_path = dst - - # Get media entry - self.entry = kwargs.get('media_entry') or None + self.entry = entry # vp8enc options self.destination_dimensions = kwargs.get('dimensions', (640, 640)) @@ -190,9 +188,8 @@ class VideoTranscoder(object): self._progress_callback = kwargs.get('progress_callback') or None # Get number of resolutions available for the video - video_config = mgg.global_config['plugins']['mediagoblin.media_types.video'] + video_config = mgg.global_config['plugins']['mediagoblin.media_types.video'] self.num_of_resolutions = len(video_config['available_resolutions']) - self.progress_percentage = 0 if not type(self.destination_dimensions) == tuple: raise Exception('dimensions must be tuple: (width, height)') @@ -364,6 +361,8 @@ class VideoTranscoder(object): # Update progress state if it has changed (success, percent) = structure.get_int('percent') if self.progress_percentage != percent and success: + # FIXME: the code below is a workaround for structure.get_int('percent') + # returning 0 when the transcoding gets over (100%) if self.progress_percentage > percent and percent == 0: percent = 100 percent_increment = percent - self.progress_percentage -- cgit v1.2.3 From b89e12a23e55e0a1a4750e3107b114f27a0f0382 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Tue, 22 Aug 2017 13:30:13 +0530 Subject: Add main_transcoding_progress to ProgressCallback --- mediagoblin/media_types/video/transcoders.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/video/transcoders.py b/mediagoblin/media_types/video/transcoders.py index a2c8f431..dbe56d9e 100644 --- a/mediagoblin/media_types/video/transcoders.py +++ b/mediagoblin/media_types/video/transcoders.py @@ -190,6 +190,7 @@ class VideoTranscoder(object): # Get number of resolutions available for the video video_config = mgg.global_config['plugins']['mediagoblin.media_types.video'] self.num_of_resolutions = len(video_config['available_resolutions']) + self.default_resolution = video_config['default_resolution'] if not type(self.destination_dimensions) == tuple: raise Exception('dimensions must be tuple: (width, height)') @@ -368,7 +369,10 @@ class VideoTranscoder(object): percent_increment = percent - self.progress_percentage self.progress_percentage = percent if self._progress_callback: - self._progress_callback(percent_increment/self.num_of_resolutions) + if ACCEPTED_RESOLUTIONS[self.default_resolution] == self.destination_dimensions: + self._progress_callback(percent_increment/self.num_of_resolutions, percent) + else: + self._progress_callback(percent_increment/self.num_of_resolutions) _log.info('{percent}% of {dest} resolution done..' '.'.format(percent=percent, dest=self.destination_dimensions)) _log.info('{0:.2f}% of all resolutions done' -- cgit v1.2.3 From 7fb50b7a21a95c191e9e76bd0695300514f48eb2 Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Sun, 27 Aug 2017 23:12:46 +0530 Subject: Remove self.entry in VideoTranscoder --- mediagoblin/media_types/video/processing.py | 2 +- mediagoblin/media_types/video/transcoders.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index 114c5057..c07aa303 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -336,7 +336,7 @@ class CommonVideoProcessor(MediaProcessor): else: _log.debug('Entered transcoder') - self.transcoder.transcode(self.process_filename, tmp_dst, self.entry, + self.transcoder.transcode(self.process_filename, tmp_dst, vp8_quality=vp8_quality, vp8_threads=vp8_threads, vorbis_quality=vorbis_quality, diff --git a/mediagoblin/media_types/video/transcoders.py b/mediagoblin/media_types/video/transcoders.py index a2c8f431..437b0bf8 100644 --- a/mediagoblin/media_types/video/transcoders.py +++ b/mediagoblin/media_types/video/transcoders.py @@ -158,13 +158,12 @@ class VideoTranscoder(object): self.progress_percentage = 0 self.loop = GLib.MainLoop() - def transcode(self, src, dst, entry, **kwargs): + def transcode(self, src, dst, **kwargs): ''' Transcode a video file into a 'medium'-sized version. ''' self.source_path = src self.destination_path = dst - self.entry = entry # vp8enc options self.destination_dimensions = kwargs.get('dimensions', (640, 640)) @@ -371,8 +370,6 @@ class VideoTranscoder(object): self._progress_callback(percent_increment/self.num_of_resolutions) _log.info('{percent}% of {dest} resolution done..' '.'.format(percent=percent, dest=self.destination_dimensions)) - _log.info('{0:.2f}% of all resolutions done' - '...'.format(self.entry.transcoding_progress)) elif message.type == Gst.MessageType.ERROR: _log.error('Got error: {0}'.format(message.parse_error())) self.dst_data = None -- cgit v1.2.3 From f1cdf7420223e39a8bcf67bf163a585ae98b98fa Mon Sep 17 00:00:00 2001 From: vijeth-aradhya Date: Tue, 29 Aug 2017 11:26:12 +0530 Subject: Change complimentary_task to complementary_tas Closes #11 --- mediagoblin/media_types/video/processing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index c07aa303..c72c12f2 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -185,7 +185,7 @@ def main_task(entry_id, resolution, medium_size, **process_info): @celery.task() -def complimentary_task(entry_id, resolution, medium_size, **process_info): +def complementary_task(entry_id, resolution, medium_size, **process_info): """ Side celery task to transcode the video to other resolutions """ @@ -595,7 +595,7 @@ class VideoProcessingManager(ProcessingManager): if comp_res != def_res: priority_num += -1 tasks_list.append( - complimentary_task.signature(args=(entry.id, comp_res, + complementary_task.signature(args=(entry.id, comp_res, ACCEPTED_RESOLUTIONS[comp_res]), kwargs=reprocess_info, queue='default', priority=priority_num, immutable=True) -- cgit v1.2.3 From 4551a808beb8aa0dab3f1a331d022bfa84ee933c Mon Sep 17 00:00:00 2001 From: Boris Bobrov Date: Thu, 12 Jul 2018 18:27:44 +0200 Subject: Fixes for small bugs There were some blocking issues in the code. They are small, but break tests and/or production. Fixed some of them. --- mediagoblin/media_types/video/__init__.py | 2 +- mediagoblin/media_types/video/processing.py | 32 ++++++++++++++++++---------- mediagoblin/media_types/video/transcoders.py | 7 +++--- 3 files changed, 25 insertions(+), 16 deletions(-) (limited to 'mediagoblin/media_types') diff --git a/mediagoblin/media_types/video/__init__.py b/mediagoblin/media_types/video/__init__.py index ea7bc021..0c822f69 100644 --- a/mediagoblin/media_types/video/__init__.py +++ b/mediagoblin/media_types/video/__init__.py @@ -40,7 +40,7 @@ class VideoMediaManager(MediaManagerBase): video_res = video_config['available_resolutions'] video_res.remove(video_config['default_resolution']) video_res.insert(0, video_config['default_resolution']) - video_res = map((lambda x: unicode('webm_' + str(x), 'utf-8')), video_res) + video_res = ['webm_{}'.format(x) for x in video_res] return ([u'webm_video'] + video_res + [u'original']) diff --git a/mediagoblin/media_types/video/processing.py b/mediagoblin/media_types/video/processing.py index c72c12f2..c377d100 100644 --- a/mediagoblin/media_types/video/processing.py +++ b/mediagoblin/media_types/video/processing.py @@ -172,29 +172,34 @@ def main_task(entry_id, resolution, medium_size, **process_info): entry, manager = get_entry_and_processing_manager(entry_id) with CommonVideoProcessor(manager, entry) as processor: processor.common_setup(resolution) - processor.transcode(medium_size=tuple(medium_size), vp8_quality=process_info['vp8_quality'], - vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) + processor.transcode(medium_size=tuple(medium_size), + vp8_quality=process_info['vp8_quality'], + vp8_threads=process_info['vp8_threads'], + vorbis_quality=process_info['vorbis_quality']) processor.generate_thumb(thumb_size=process_info['thumb_size']) processor.store_orig_metadata() # Make state of entry as processed entry.state = u'processed' entry.save() - _log.info(u'MediaEntry ID {0} is processed (transcoded to default resolution' - '): {1}'.format(unicode(entry.id), unicode(medium_size))) + _log.info(u'MediaEntry ID {0} is processed (transcoded to default' + ' resolution): {1}'.format(entry.id, medium_size)) _log.debug('MediaEntry processed') @celery.task() -def complementary_task(entry_id, resolution, medium_size, **process_info): +def complementary_task(entry_id, resolution, medium_size, **process_info): """ Side celery task to transcode the video to other resolutions """ entry, manager = get_entry_and_processing_manager(entry_id) with CommonVideoProcessor(manager, entry) as processor: processor.common_setup(resolution) - processor.transcode(medium_size=tuple(medium_size), vp8_quality=process_info['vp8_quality'], - vp8_threads=process_info['vp8_threads'], vorbis_quality=process_info['vorbis_quality']) - _log.info(u'MediaEntry ID {0} is transcoded to {1}'.format(unicode(entry.id), unicode(medium_size))) + processor.transcode(medium_size=tuple(medium_size), + vp8_quality=process_info['vp8_quality'], + vp8_threads=process_info['vp8_threads'], + vorbis_quality=process_info['vorbis_quality']) + _log.info(u'MediaEntry ID {0} is transcoded to {1}'.format( + entry.id, medium_size)) @celery.task() @@ -215,7 +220,7 @@ class CommonVideoProcessor(MediaProcessor): Provides a base for various video processing steps """ acceptable_files = ['original, best_quality', 'webm_144p', 'webm_360p', - 'webm_480p', 'webm_720p', 'webm_1080p', 'webm_video'] + 'webm_480p', 'webm_720p', 'webm_1080p', 'webm_video'] def common_setup(self, resolution=None): self.video_config = mgg \ @@ -225,7 +230,7 @@ class CommonVideoProcessor(MediaProcessor): self.process_filename = get_process_filename( self.entry, self.workbench, self.acceptable_files) self.name_builder = FilenameBuilder(self.process_filename) - + self.transcoder = transcoders.VideoTranscoder() self.did_transcode = False @@ -336,7 +341,12 @@ class CommonVideoProcessor(MediaProcessor): else: _log.debug('Entered transcoder') + video_config = (mgg.global_config['plugins'] + ['mediagoblin.media_types.video']) + num_res = len(video_config['available_resolutions']) + default_res = video_config['default_resolution'] self.transcoder.transcode(self.process_filename, tmp_dst, + default_res, num_res, vp8_quality=vp8_quality, vp8_threads=vp8_threads, vorbis_quality=vorbis_quality, @@ -569,7 +579,7 @@ class VideoProcessingManager(ProcessingManager): def workflow(self, entry, feed_url, reprocess_action, reprocess_info=None): - video_config = mgg.global_config['plugins'][MEDIA_TYPE] + video_config = mgg.global_config['plugins'][MEDIA_TYPE] def_res = video_config['default_resolution'] priority_num = len(video_config['available_resolutions']) + 1 diff --git a/mediagoblin/media_types/video/transcoders.py b/mediagoblin/media_types/video/transcoders.py index 75f5ef86..d6aae7cf 100644 --- a/mediagoblin/media_types/video/transcoders.py +++ b/mediagoblin/media_types/video/transcoders.py @@ -158,7 +158,7 @@ class VideoTranscoder(object): self.progress_percentage = 0 self.loop = GLib.MainLoop() - def transcode(self, src, dst, **kwargs): + def transcode(self, src, dst, default_res, num_res, **kwargs): ''' Transcode a video file into a 'medium'-sized version. ''' @@ -187,9 +187,8 @@ class VideoTranscoder(object): self._progress_callback = kwargs.get('progress_callback') or None # Get number of resolutions available for the video - video_config = mgg.global_config['plugins']['mediagoblin.media_types.video'] - self.num_of_resolutions = len(video_config['available_resolutions']) - self.default_resolution = video_config['default_resolution'] + self.num_of_resolutions = num_res + self.default_resolution = default_res if not type(self.destination_dimensions) == tuple: raise Exception('dimensions must be tuple: (width, height)') -- cgit v1.2.3