From c30714805b1936497ea846ab96e9104f5e1176ef Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 1 Aug 2013 12:32:49 -0700 Subject: Beginnings of a reprocess command --- mediagoblin/gmg_commands/reprocess.py | 41 +++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 mediagoblin/gmg_commands/reprocess.py (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py new file mode 100644 index 00000000..1cc9f71a --- /dev/null +++ b/mediagoblin/gmg_commands/reprocess.py @@ -0,0 +1,41 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + + +def reprocess_parser_setup(subparser): + subparser.add_argument( + '--available', '-a', + action="store_true", + help="List available actions for a given media entry") + subparser.add_argument( + '--all', '-A', + action="store_true", + help="Reprocess all media entries") + subparser.add_argument( + '--state', '-s', + help="Reprocess media entries in this state" + " such as 'failed' or 'processed'") + subparser.add_argument( + '--type', '-t', + help="The type of media to be reprocessed such as 'video' or 'image'") + subparser.add_argument( + 'media_id', + nargs='*', + help="The media_entry id(s) you wish to reprocess.") + + +def reprocess(args): + pass -- cgit v1.2.3 From 99b34c4ce68b636583657bccec137d04875a5bf1 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 1 Aug 2013 12:35:49 -0700 Subject: Added a set_media_type function that has checks to only reprocess one media_type at a time --- mediagoblin/gmg_commands/reprocess.py | 48 ++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 1cc9f71a..9dbadefb 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -13,6 +13,9 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from mediagoblin.db.models import MediaEntry +from mediagoblin.gmg_commands import util as commands_util +from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ def reprocess_parser_setup(subparser): @@ -37,5 +40,48 @@ def reprocess_parser_setup(subparser): help="The media_entry id(s) you wish to reprocess.") -def reprocess(args): +class MismatchingMediaTypes(Exception): + """ + Error that should be raised if the media_types are not the same + """ pass + + +def _set_media_type(args): + if len(args[0].media_id) == 1: + media_type = MediaEntry.query.filter_by(id=args[0].media_id[0])\ + .first().media_type.split('.')[-1] + + if not args[0].type: + args[0].type = media_type + elif args[0].type != media_type: + raise MismatchingMediaTypes(_('The type that you set does not' + ' match the type of the given' + ' media_id.')) + elif len(args[0].media_id) > 1: + media_types = [] + + for id in args[0].media_id: + media_types.append(MediaEntry.query.filter_by(id=id).first()\ + .media_type.split('.')[-1]) + for type in media_types: + if media_types[0] != type: + raise MismatchingMediaTypes((u'You cannot reprocess different' + ' media_types at the same time.')) + + if not args[0].type: + args[0].type = media_types[0] + elif args[0].type != media_types[0]: + raise MismatchingMediaTypes(_('The type that you set does not' + ' match the type of the given' + ' media_ids.')) + + elif not args[0].type: + raise MismatchingMediaTypes(_('You must provide either a media_id or' + ' set the --type flag')) + + +def reprocess(args): + commands_util.setup_app(args[0]) + + _set_media_type(args) -- cgit v1.2.3 From 6fc8aaf65f483ab523cab74489dd3421314e2b7e Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 1 Aug 2013 13:23:40 -0700 Subject: add reprocess_all function. still need to add code to reprocess all failed entries --- mediagoblin/gmg_commands/reprocess.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 9dbadefb..f458cd1d 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -16,6 +16,7 @@ from mediagoblin.db.models import MediaEntry from mediagoblin.gmg_commands import util as commands_util from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ +from mediagoblin.tools.pluginapi import hook_handle def reprocess_parser_setup(subparser): @@ -81,7 +82,34 @@ def _set_media_type(args): ' set the --type flag')) +def _reprocess_all(args): + if not args[0].type: + if args[0].state == 'failed': + if args[0].available: + print '\n Available reprocess actions for all failed' \ + ' media_entries: \n \t --initial_processing' + return + else: + #TODO reprocess all failed entries + pass + else: + raise Exception(_('You must set --type when trying to reprocess' + ' all media_entries, unless you set --state' + ' to "failed".')) + + if args[0].available: + return hook_handle(('reprocess_action', args[0].type), args) + else: + return hook_handle(('media_reprocess', args[0].type), args) + + def reprocess(args): commands_util.setup_app(args[0]) + if not args[0].state: + args[0].state = 'processed' + + if args[0].all: + return _reprocess_all(args) + _set_media_type(args) -- cgit v1.2.3 From 7c1f6a6aeea47c27a56e3f39e0a6cf33d9dd2486 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 1 Aug 2013 13:47:44 -0700 Subject: added a _run_reprocessing function which handles the hook calls --- mediagoblin/gmg_commands/reprocess.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index f458cd1d..50434bd2 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -97,6 +97,10 @@ def _reprocess_all(args): ' all media_entries, unless you set --state' ' to "failed".')) + _run_reprocessing(args) + + +def _run_reprocessing(args): if args[0].available: return hook_handle(('reprocess_action', args[0].type), args) else: @@ -113,3 +117,5 @@ def reprocess(args): return _reprocess_all(args) _set_media_type(args) + + return _run_reprocessing(args) -- cgit v1.2.3 From 81d880b16adb2d6c872e0ad37ffe34bf7bfaba6c Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 1 Aug 2013 13:51:36 -0700 Subject: Just raise standard exception. Pass print statement to gettext --- mediagoblin/gmg_commands/reprocess.py | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 50434bd2..2158d36e 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -41,13 +41,6 @@ def reprocess_parser_setup(subparser): help="The media_entry id(s) you wish to reprocess.") -class MismatchingMediaTypes(Exception): - """ - Error that should be raised if the media_types are not the same - """ - pass - - def _set_media_type(args): if len(args[0].media_id) == 1: media_type = MediaEntry.query.filter_by(id=args[0].media_id[0])\ @@ -56,9 +49,8 @@ def _set_media_type(args): if not args[0].type: args[0].type = media_type elif args[0].type != media_type: - raise MismatchingMediaTypes(_('The type that you set does not' - ' match the type of the given' - ' media_id.')) + raise Exception(_('The type that you set does not match the type' + ' of the given media_id.')) elif len(args[0].media_id) > 1: media_types = [] @@ -67,27 +59,26 @@ def _set_media_type(args): .media_type.split('.')[-1]) for type in media_types: if media_types[0] != type: - raise MismatchingMediaTypes((u'You cannot reprocess different' - ' media_types at the same time.')) + raise Exception((u'You cannot reprocess different media_types' + ' at the same time.')) if not args[0].type: args[0].type = media_types[0] elif args[0].type != media_types[0]: - raise MismatchingMediaTypes(_('The type that you set does not' - ' match the type of the given' - ' media_ids.')) + raise Exception(_('The type that you set does not match the type' + ' of the given media_ids.')) elif not args[0].type: - raise MismatchingMediaTypes(_('You must provide either a media_id or' - ' set the --type flag')) + raise Exception(_('You must provide either a media_id or set the' + ' --type flag')) def _reprocess_all(args): if not args[0].type: if args[0].state == 'failed': if args[0].available: - print '\n Available reprocess actions for all failed' \ - ' media_entries: \n \t --initial_processing' + print _('\n Available reprocess actions for all failed' \ + ' media_entries: \n \t --initial_processing') return else: #TODO reprocess all failed entries -- cgit v1.2.3 From 4a36407d39a18f295e5fc09125fac8e2a7252f55 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 1 Aug 2013 13:53:20 -0700 Subject: Pep 8 --- mediagoblin/gmg_commands/reprocess.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 2158d36e..9390861f 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -44,7 +44,7 @@ def reprocess_parser_setup(subparser): def _set_media_type(args): if len(args[0].media_id) == 1: media_type = MediaEntry.query.filter_by(id=args[0].media_id[0])\ - .first().media_type.split('.')[-1] + .first().media_type.split('.')[-1] if not args[0].type: args[0].type = media_type @@ -55,7 +55,7 @@ def _set_media_type(args): media_types = [] for id in args[0].media_id: - media_types.append(MediaEntry.query.filter_by(id=id).first()\ + media_types.append(MediaEntry.query.filter_by(id=id).first() .media_type.split('.')[-1]) for type in media_types: if media_types[0] != type: @@ -77,8 +77,8 @@ def _reprocess_all(args): if not args[0].type: if args[0].state == 'failed': if args[0].available: - print _('\n Available reprocess actions for all failed' \ - ' media_entries: \n \t --initial_processing') + print _('\n Available reprocess actions for all failed' + ' media_entries: \n \t --initial_processing') return else: #TODO reprocess all failed entries -- cgit v1.2.3 From 243756e0205b2c8c009b6ac3e96eca8923508c38 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 1 Aug 2013 14:47:24 -0700 Subject: added a set_media_state function. removed the --all flag (just don't enter any media_ids to process all media). slight refactor --- mediagoblin/gmg_commands/reprocess.py | 42 ++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 15 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 9390861f..cad75c45 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -24,10 +24,6 @@ def reprocess_parser_setup(subparser): '--available', '-a', action="store_true", help="List available actions for a given media entry") - subparser.add_argument( - '--all', '-A', - action="store_true", - help="Reprocess all media entries") subparser.add_argument( '--state', '-s', help="Reprocess media entries in this state" @@ -49,7 +45,7 @@ def _set_media_type(args): if not args[0].type: args[0].type = media_type elif args[0].type != media_type: - raise Exception(_('The type that you set does not match the type' + raise Exception(_('The --type that you set does not match the type' ' of the given media_id.')) elif len(args[0].media_id) > 1: media_types = [] @@ -65,13 +61,9 @@ def _set_media_type(args): if not args[0].type: args[0].type = media_types[0] elif args[0].type != media_types[0]: - raise Exception(_('The type that you set does not match the type' + raise Exception(_('The --type that you set does not match the type' ' of the given media_ids.')) - elif not args[0].type: - raise Exception(_('You must provide either a media_id or set the' - ' --type flag')) - def _reprocess_all(args): if not args[0].type: @@ -98,15 +90,35 @@ def _run_reprocessing(args): return hook_handle(('media_reprocess', args[0].type), args) -def reprocess(args): - commands_util.setup_app(args[0]) +def _set_media_state(args): + if len(args[0].media_id) == 1: + args[0].state = MediaEntry.query.filter_by(id=args[0].media_id[0])\ + .first().state + + elif len(args[0].media_id) > 1: + media_states = [] - if not args[0].state: + for id in args[0].media_id: + media_states.append(MediaEntry.query.filter_by(id=id).first() + .state) + for state in media_states: + if state != media_states[0]: + raise Exception(_('You can only reprocess media that is in the' + ' same state.')) + + args[0].state = media_states[0] + + elif not args[0].state: args[0].state = 'processed' - if args[0].all: - return _reprocess_all(args) +def reprocess(args): + commands_util.setup_app(args[0]) + + _set_media_state(args) _set_media_type(args) + if not args[0].media_id: + return _reprocess_all(args) + return _run_reprocessing(args) -- cgit v1.2.3 From 11a99d787f899808b10410e271491cbdfb69e55f Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 1 Aug 2013 15:24:34 -0700 Subject: refactored _run_reprocessing --- mediagoblin/gmg_commands/reprocess.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index cad75c45..60df697f 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -85,7 +85,15 @@ def _reprocess_all(args): def _run_reprocessing(args): if args[0].available: - return hook_handle(('reprocess_action', args[0].type), args) + if args[0].state == 'failed': + print _('\n Available reprocess actions for all failed' + ' media_entries: \n \t --initial_processing') + else: + result = hook_handle(('reprocess_action', args[0].type), args) + if not result: + print _('Sorry there is no available reprocessing for {}' + ' entries in the {} state'.format(args[0].type, + args[0].state)) else: return hook_handle(('media_reprocess', args[0].type), args) -- cgit v1.2.3 From 842ba30529040fb47ac9905df6373d7c1f0286ed Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 1 Aug 2013 15:40:26 -0700 Subject: make media_id an optional argument --- mediagoblin/gmg_commands/reprocess.py | 82 ++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 40 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 60df697f..f3445ea6 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -32,37 +32,38 @@ def reprocess_parser_setup(subparser): '--type', '-t', help="The type of media to be reprocessed such as 'video' or 'image'") subparser.add_argument( - 'media_id', + '--media_id', nargs='*', help="The media_entry id(s) you wish to reprocess.") def _set_media_type(args): - if len(args[0].media_id) == 1: - media_type = MediaEntry.query.filter_by(id=args[0].media_id[0])\ - .first().media_type.split('.')[-1] - - if not args[0].type: - args[0].type = media_type - elif args[0].type != media_type: - raise Exception(_('The --type that you set does not match the type' - ' of the given media_id.')) - elif len(args[0].media_id) > 1: - media_types = [] - - for id in args[0].media_id: - media_types.append(MediaEntry.query.filter_by(id=id).first() - .media_type.split('.')[-1]) - for type in media_types: - if media_types[0] != type: - raise Exception((u'You cannot reprocess different media_types' - ' at the same time.')) - - if not args[0].type: - args[0].type = media_types[0] - elif args[0].type != media_types[0]: - raise Exception(_('The --type that you set does not match the type' - ' of the given media_ids.')) + if args[0].media_id: + if len(args[0].media_id) == 1: + media_type = MediaEntry.query.filter_by(id=args[0].media_id[0])\ + .first().media_type.split('.')[-1] + + if not args[0].type: + args[0].type = media_type + elif args[0].type != media_type: + raise Exception(_('The --type that you set does not match the type' + ' of the given media_id.')) + elif len(args[0].media_id) > 1: + media_types = [] + + for id in args[0].media_id: + media_types.append(MediaEntry.query.filter_by(id=id).first() + .media_type.split('.')[-1]) + for type in media_types: + if media_types[0] != type: + raise Exception((u'You cannot reprocess different media_types' + ' at the same time.')) + + if not args[0].type: + args[0].type = media_types[0] + elif args[0].type != media_types[0]: + raise Exception(_('The --type that you set does not match the type' + ' of the given media_ids.')) def _reprocess_all(args): @@ -99,24 +100,25 @@ def _run_reprocessing(args): def _set_media_state(args): - if len(args[0].media_id) == 1: - args[0].state = MediaEntry.query.filter_by(id=args[0].media_id[0])\ - .first().state + if args[0].media_id: + if len(args[0].media_id) == 1: + args[0].state = MediaEntry.query.filter_by(id=args[0].media_id[0])\ + .first().state - elif len(args[0].media_id) > 1: - media_states = [] + elif len(args[0].media_id) > 1: + media_states = [] - for id in args[0].media_id: - media_states.append(MediaEntry.query.filter_by(id=id).first() - .state) - for state in media_states: - if state != media_states[0]: - raise Exception(_('You can only reprocess media that is in the' - ' same state.')) + for id in args[0].media_id: + media_states.append(MediaEntry.query.filter_by(id=id).first() + .state) + for state in media_states: + if state != media_states[0]: + raise Exception(_('You can only reprocess media that is in the' + ' same state.')) - args[0].state = media_states[0] + args[0].state = media_states[0] - elif not args[0].state: + if not args[0].state: args[0].state = 'processed' -- cgit v1.2.3 From 065db04730936286bbe213133a3175d48950fd32 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 1 Aug 2013 15:55:39 -0700 Subject: add command option for regenerating all thumbnails --- mediagoblin/gmg_commands/reprocess.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index f3445ea6..b45543e4 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -35,6 +35,10 @@ def reprocess_parser_setup(subparser): '--media_id', nargs='*', help="The media_entry id(s) you wish to reprocess.") + subparser.add_argument( + '--thumbnails', + action="store_true", + help="Regenerate thumbnails for all processed media") def _set_media_type(args): @@ -68,20 +72,31 @@ def _set_media_type(args): def _reprocess_all(args): if not args[0].type: - if args[0].state == 'failed': + if args[0].thumbnails: + if args[0].available: + print _('Available options for regenerating all processed' + ' media thumbnails: \n' + '\t --size: max_width max_height' + ' (defaults to config specs)') + else: + #TODO regenerate all thumbnails + pass + + elif args[0].state == 'failed': if args[0].available: print _('\n Available reprocess actions for all failed' ' media_entries: \n \t --initial_processing') - return else: #TODO reprocess all failed entries pass + else: raise Exception(_('You must set --type when trying to reprocess' ' all media_entries, unless you set --state' ' to "failed".')) - _run_reprocessing(args) + else: + _run_reprocessing(args) def _run_reprocessing(args): -- cgit v1.2.3 From bf909ab048d2741b91542ad2b00789f30e51c2cf Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Thu, 1 Aug 2013 15:58:25 -0700 Subject: pep 8 --- mediagoblin/gmg_commands/reprocess.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index b45543e4..0390c48d 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -50,24 +50,24 @@ def _set_media_type(args): if not args[0].type: args[0].type = media_type elif args[0].type != media_type: - raise Exception(_('The --type that you set does not match the type' - ' of the given media_id.')) + raise Exception(_('The --type that you set does not match the' + 'type of the given media_id.')) elif len(args[0].media_id) > 1: media_types = [] for id in args[0].media_id: media_types.append(MediaEntry.query.filter_by(id=id).first() - .media_type.split('.')[-1]) + .media_type.split('.')[-1]) for type in media_types: if media_types[0] != type: - raise Exception((u'You cannot reprocess different media_types' - ' at the same time.')) + raise Exception((u'You cannot reprocess different' + ' media_types at the same time.')) if not args[0].type: args[0].type = media_types[0] elif args[0].type != media_types[0]: - raise Exception(_('The --type that you set does not match the type' - ' of the given media_ids.')) + raise Exception(_('The --type that you set does not match the' + ' type of the given media_ids.')) def _reprocess_all(args): @@ -128,8 +128,8 @@ def _set_media_state(args): .state) for state in media_states: if state != media_states[0]: - raise Exception(_('You can only reprocess media that is in the' - ' same state.')) + raise Exception(_('You can only reprocess media that is in' + ' the same state.')) args[0].state = media_states[0] -- cgit v1.2.3 From 9a2c66ca9ef763fa68dc09a483c02fe2ee02d78f Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Fri, 2 Aug 2013 11:40:41 -0700 Subject: added image reprocessing --- mediagoblin/gmg_commands/reprocess.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 0390c48d..f6b9e653 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -13,6 +13,7 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from mediagoblin import mg_globals from mediagoblin.db.models import MediaEntry from mediagoblin.gmg_commands import util as commands_util from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ @@ -143,6 +144,8 @@ def reprocess(args): _set_media_state(args) _set_media_type(args) + import ipdb + ipdb.set_trace() if not args[0].media_id: return _reprocess_all(args) -- cgit v1.2.3 From f30fbfe60c5cd84cdf5df653d2c281a0bb05bae8 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Fri, 2 Aug 2013 12:06:50 -0700 Subject: add option to not run eagerly --- mediagoblin/gmg_commands/reprocess.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index f6b9e653..9d8ede24 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -13,6 +13,8 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import os + from mediagoblin import mg_globals from mediagoblin.db.models import MediaEntry from mediagoblin.gmg_commands import util as commands_util @@ -40,6 +42,10 @@ def reprocess_parser_setup(subparser): '--thumbnails', action="store_true", help="Regenerate thumbnails for all processed media") + subparser.add_argument( + '--celery', + action='store_true', + help="Don't process eagerly, pass off to celery") def _set_media_type(args): @@ -139,13 +145,13 @@ def _set_media_state(args): def reprocess(args): + if not args[0].celery: + os.environ['CELERY_ALWAYS_EAGER'] = 'true' commands_util.setup_app(args[0]) _set_media_state(args) _set_media_type(args) - import ipdb - ipdb.set_trace() if not args[0].media_id: return _reprocess_all(args) -- cgit v1.2.3 From 3e9faf85da1ee2971e9ff2fde12b192ea470d806 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Fri, 2 Aug 2013 15:12:07 -0700 Subject: added comments and did a little refactoring. not sure if it is actually any clearer though --- mediagoblin/gmg_commands/reprocess.py | 45 ++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 11 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 9d8ede24..4df0d581 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -49,16 +49,18 @@ def reprocess_parser_setup(subparser): def _set_media_type(args): + """ + This will verify that all media id's are of the same media_type. If the + --type flag is set, it will be replaced by the given media id's type. + + If they are trying to process different media types, an Exception will be + raised. + """ if args[0].media_id: if len(args[0].media_id) == 1: - media_type = MediaEntry.query.filter_by(id=args[0].media_id[0])\ + args[0].type = MediaEntry.query.filter_by(id=args[0].media_id[0])\ .first().media_type.split('.')[-1] - if not args[0].type: - args[0].type = media_type - elif args[0].type != media_type: - raise Exception(_('The --type that you set does not match the' - 'type of the given media_id.')) elif len(args[0].media_id) > 1: media_types = [] @@ -70,15 +72,17 @@ def _set_media_type(args): raise Exception((u'You cannot reprocess different' ' media_types at the same time.')) - if not args[0].type: - args[0].type = media_types[0] - elif args[0].type != media_types[0]: - raise Exception(_('The --type that you set does not match the' - ' type of the given media_ids.')) + args[0].type = media_types[0] def _reprocess_all(args): + """ + This handles reprocessing if no media_id's are given. + """ if not args[0].type: + # If no media type is given, we can either regenerate all thumbnails, + # or try to reprocess all failed media + if args[0].thumbnails: if args[0].available: print _('Available options for regenerating all processed' @@ -89,6 +93,7 @@ def _reprocess_all(args): #TODO regenerate all thumbnails pass + # Reprocess all failed media elif args[0].state == 'failed': if args[0].available: print _('\n Available reprocess actions for all failed' @@ -97,6 +102,8 @@ def _reprocess_all(args): #TODO reprocess all failed entries pass + # If here, they didn't set the --type flag and were trying to do + # something other the generating thumbnails or initial_processing else: raise Exception(_('You must set --type when trying to reprocess' ' all media_entries, unless you set --state' @@ -107,6 +114,8 @@ def _reprocess_all(args): def _run_reprocessing(args): + # Are they just asking for the available reprocessing options for the given + # media? if args[0].available: if args[0].state == 'failed': print _('\n Available reprocess actions for all failed' @@ -118,11 +127,20 @@ def _run_reprocessing(args): ' entries in the {} state'.format(args[0].type, args[0].state)) else: + # Run media reprocessing return hook_handle(('media_reprocess', args[0].type), args) def _set_media_state(args): + """ + This will verify that all media id's are in the same state. If the + --state flag is set, it will be replaced by the given media id's state. + + If they are trying to process different media states, an Exception will be + raised. + """ if args[0].media_id: + # Only check if we are given media_ids if len(args[0].media_id) == 1: args[0].state = MediaEntry.query.filter_by(id=args[0].media_id[0])\ .first().state @@ -133,6 +151,8 @@ def _set_media_state(args): for id in args[0].media_id: media_states.append(MediaEntry.query.filter_by(id=id).first() .state) + + # Make sure that all media are in the same state for state in media_states: if state != media_states[0]: raise Exception(_('You can only reprocess media that is in' @@ -140,11 +160,13 @@ def _set_media_state(args): args[0].state = media_states[0] + # If no state was set, then we will default to the processed state if not args[0].state: args[0].state = 'processed' def reprocess(args): + # Run eagerly unless explicetly set not to if not args[0].celery: os.environ['CELERY_ALWAYS_EAGER'] = 'true' commands_util.setup_app(args[0]) @@ -152,6 +174,7 @@ def reprocess(args): _set_media_state(args) _set_media_type(args) + # If no media_ids were given, then try to reprocess all entries if not args[0].media_id: return _reprocess_all(args) -- cgit v1.2.3 From 58bacb33aca6505673f90460d31811ed487bcb4c Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Fri, 9 Aug 2013 11:20:21 -0500 Subject: More progress towards the new reprocessing infrastructure: args updating This commit sponsored by Elizabeth Webber. Thanks, sis! --- mediagoblin/gmg_commands/reprocess.py | 82 ++++++++++++++++++++++++++++------- 1 file changed, 67 insertions(+), 15 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 4df0d581..30575033 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -13,6 +13,7 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import argparse import os from mediagoblin import mg_globals @@ -23,30 +24,63 @@ from mediagoblin.tools.pluginapi import hook_handle def reprocess_parser_setup(subparser): - subparser.add_argument( - '--available', '-a', - action="store_true", - help="List available actions for a given media entry") - subparser.add_argument( + subparsers = subparser.add_subparsers(dest="reprocess_subcommand") + + ################### + # available command + ################### + available_parser = subparsers.add_parser( + "available", + help="Find out what actions are available for this media") + + available_parser.add_argument( + "id_or_type", + help="Media id or media type to check") + + + ############################################ + # run command (TODO: and bulk_run command??) + ############################################ + + run_parser = subparsers.add_parser( + "run", + help="Run a reprocessing on one or more media") + + run_parser.add_argument( '--state', '-s', help="Reprocess media entries in this state" " such as 'failed' or 'processed'") - subparser.add_argument( + run_parser.add_argument( '--type', '-t', help="The type of media to be reprocessed such as 'video' or 'image'") - subparser.add_argument( - '--media_id', - nargs='*', - help="The media_entry id(s) you wish to reprocess.") - subparser.add_argument( + run_parser.add_argument( '--thumbnails', action="store_true", help="Regenerate thumbnails for all processed media") - subparser.add_argument( + run_parser.add_argument( '--celery', action='store_true', help="Don't process eagerly, pass off to celery") + run_parser.add_argument( + 'media_id', + help="The media_entry id(s) you wish to reprocess.") + + run_parser.add_argument( + 'reprocess_command', + help="The reprocess command you intend to run") + + run_parser.add_argument( + 'reprocess_args', + nargs=argparse.REMAINDER, + help="rest of arguments to the reprocessing tool") + + + ############### + # help command? + ############### + + def _set_media_type(args): """ @@ -165,11 +199,22 @@ def _set_media_state(args): args[0].state = 'processed' -def reprocess(args): +def available(args): + # Get the media type, either by looking up media id, or by specific type + + ### TODO: look up by id + + # + pass + + +def run(args): + ### OLD CODE, review + # Run eagerly unless explicetly set not to - if not args[0].celery: + if not args.celery: os.environ['CELERY_ALWAYS_EAGER'] = 'true' - commands_util.setup_app(args[0]) + commands_util.setup_app(args) _set_media_state(args) _set_media_type(args) @@ -179,3 +224,10 @@ def reprocess(args): return _reprocess_all(args) return _run_reprocessing(args) + + +def reprocess(args): + if args.reprocess_subcommand == "run": + run(args) + elif args.reprocess_subcommand == "available": + available(args) -- cgit v1.2.3 From 85ead8ac3cf59aeee12ddd3b33ecfeb03c3aa946 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Fri, 9 Aug 2013 12:13:53 -0500 Subject: "initial" reprocessing subcommand now works! MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We are on our way now to a working reprocessing system under this redesign! This commit sponsored by Bjarni Rúnar Einarsson. Thank you! --- mediagoblin/gmg_commands/reprocess.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 30575033..d6ac99ac 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -199,13 +199,35 @@ def _set_media_state(args): args[0].state = 'processed' +def extract_entry_and_type(media_id): + raise NotImplementedError + + def available(args): # Get the media type, either by looking up media id, or by specific type + try: + media_id = int(args.id_or_type) + media_type, media_entry = extract_entry_and_type(media_id) + except ValueError: + media_type = args.id_or_type + media_entry = None + + manager_class = hook_handle(('reprocess_manager', media_type)) + manager = manager_class() - ### TODO: look up by id + if media_entry is None: + processors = manager.list_all_processors() + else: + processors = manager.list_eligible_processors(media_entry) + + print "Available processors:" + print "---------------------" - # - pass + for processor in processors: + if processor.description: + print " - %s: %s" % (processor.name, processor.description) + else: + print " - %s" % processor.name def run(args): @@ -214,7 +236,6 @@ def run(args): # Run eagerly unless explicetly set not to if not args.celery: os.environ['CELERY_ALWAYS_EAGER'] = 'true' - commands_util.setup_app(args) _set_media_state(args) _set_media_type(args) @@ -227,6 +248,8 @@ def run(args): def reprocess(args): + commands_util.setup_app(args) + if args.reprocess_subcommand == "run": run(args) elif args.reprocess_subcommand == "available": -- cgit v1.2.3 From 55a10fef0ae97cb33c8393a7a25487c2666b4cf1 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Fri, 9 Aug 2013 13:56:23 -0500 Subject: `gmg reprocess available --action-help` now tells you processor arguments! Every reprocessing action possible can inform you of its command line argument stuff! Is that awesome or what? --- mediagoblin/gmg_commands/reprocess.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index d6ac99ac..70163928 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -37,6 +37,11 @@ def reprocess_parser_setup(subparser): "id_or_type", help="Media id or media type to check") + available_parser.add_argument( + "--action-help", + action="store_true", + help="List argument help for each action available") + ############################################ # run command (TODO: and bulk_run command??) @@ -221,13 +226,22 @@ def available(args): processors = manager.list_eligible_processors(media_entry) print "Available processors:" - print "---------------------" + print "=====================" - for processor in processors: - if processor.description: - print " - %s: %s" % (processor.name, processor.description) - else: - print " - %s" % processor.name + if args.action_help: + for processor in processors: + print processor.name + print "-" * len(processor.name) + + parser = processor.generate_parser() + parser.print_help() + + else: + for processor in processors: + if processor.description: + print " - %s: %s" % (processor.name, processor.description) + else: + print " - %s" % processor.name def run(args): -- cgit v1.2.3 From 19ed12b2790ec32d993e6e73c101ea820fe3ed29 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Fri, 9 Aug 2013 14:10:52 -0500 Subject: Whitespacin' it up. --- mediagoblin/gmg_commands/reprocess.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 70163928..62a6d428 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -227,6 +227,7 @@ def available(args): print "Available processors:" print "=====================" + print "" if args.action_help: for processor in processors: @@ -235,6 +236,7 @@ def available(args): parser = processor.generate_parser() parser.print_help() + print "" else: for processor in processors: @@ -266,5 +268,6 @@ def reprocess(args): if args.reprocess_subcommand == "run": run(args) + elif args.reprocess_subcommand == "available": available(args) -- cgit v1.2.3 From 7a414c8d42d63685655c0142345dcae6c66726af Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Fri, 9 Aug 2013 14:15:18 -0500 Subject: Moving celery setup to the right place This commit sponsored by Jose Manuel Zueco Lazaro. Thank you! --- mediagoblin/gmg_commands/reprocess.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 62a6d428..2af88847 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -249,10 +249,6 @@ def available(args): def run(args): ### OLD CODE, review - # Run eagerly unless explicetly set not to - if not args.celery: - os.environ['CELERY_ALWAYS_EAGER'] = 'true' - _set_media_state(args) _set_media_type(args) @@ -264,6 +260,10 @@ def run(args): def reprocess(args): + # Run eagerly unless explicetly set not to + if not args.celery: + os.environ['CELERY_ALWAYS_EAGER'] = 'true' + commands_util.setup_app(args) if args.reprocess_subcommand == "run": -- cgit v1.2.3 From 4ba5bdd96ef3703d8da216ca3dd92f080214f164 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Fri, 9 Aug 2013 16:12:06 -0500 Subject: Steps toward working "run" reprocessing command. This commit sponsored by Philippe Casteleyn. Thank you! --- mediagoblin/gmg_commands/reprocess.py | 41 +++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 9 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 2af88847..0d8db858 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -21,6 +21,7 @@ from mediagoblin.db.models import MediaEntry from mediagoblin.gmg_commands import util as commands_util from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ from mediagoblin.tools.pluginapi import hook_handle +from mediagoblin.processing import ProcessorDoesNotExist, ProcessorNotEligible def reprocess_parser_setup(subparser): @@ -204,8 +205,17 @@ def _set_media_state(args): args[0].state = 'processed' +class MediaEntryNotFound(Exception): pass + def extract_entry_and_type(media_id): - raise NotImplementedError + """ + Fetch a media entry, as well as its media type + """ + entry = MediaEntry.query.filter_by(id=media_id).first() + if entry is None: + raise MediaEntryNotFound("Can't find media with id '%s'" % media_id) + + return entry.media_type, entry def available(args): @@ -247,16 +257,29 @@ def available(args): def run(args): - ### OLD CODE, review + media_type, media_entry = extract_entry_and_type(args.media_id) - _set_media_state(args) - _set_media_type(args) - - # If no media_ids were given, then try to reprocess all entries - if not args[0].media_id: - return _reprocess_all(args) + manager_class = hook_handle(('reprocess_manager', media_type)) + manager = manager_class() - return _run_reprocessing(args) + # TOOD: Specify in error + try: + processor_class = manager.get_processor( + args.reprocess_command, media_entry) + except ProcessorDoesNotExist: + print 'No such processor "%s" for media with id "%s"' % ( + args.reprocess_command, media_entry.id) + return + except ProcessorNotEligible: + print 'Processor "%s" exists but media "%s" is not eligible' % ( + args.reprocess_command, media_entry.id) + return + + reprocess_parser = processor_class.generate_parser() + reprocess_args = reprocess_parser.parse_args(args.reprocess_args) + + import pdb + pdb.set_trace() def reprocess(args): -- cgit v1.2.3 From d1e9913b71a6f3b7bb41f5eee1051093b92fcd8a Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Fri, 9 Aug 2013 17:30:52 -0500 Subject: Should be enough to get to the point where you can actually initialize a processing command now. However, it doesn't celery task-ify it... This commit sponsored by Catalin Cosovanu. Thank you! --- mediagoblin/gmg_commands/reprocess.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 0d8db858..6d04427e 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -262,7 +262,8 @@ def run(args): manager_class = hook_handle(('reprocess_manager', media_type)) manager = manager_class() - # TOOD: Specify in error + # TODO: (maybe?) This could probably be handled entirely by the + # processor class... try: processor_class = manager.get_processor( args.reprocess_command, media_entry) @@ -277,9 +278,9 @@ def run(args): reprocess_parser = processor_class.generate_parser() reprocess_args = reprocess_parser.parse_args(args.reprocess_args) - - import pdb - pdb.set_trace() + reprocess_request = processor_class.args_to_request(reprocess_args) + processor = processor_class(manager, media_entry) + processor.process(**reprocess_request) def reprocess(args): -- cgit v1.2.3 From 77ea4c9bd1e8372fb7206596ca5125738033ced5 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sun, 11 Aug 2013 14:34:45 -0500 Subject: Updating to the point where we can allllmost run with the new reprocessing code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit sponsored by Odin Hørthe Omdal. Thank you! --- mediagoblin/gmg_commands/reprocess.py | 38 ++++++++++++----------------------- 1 file changed, 13 insertions(+), 25 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 6d04427e..24fcde37 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -19,9 +19,12 @@ import os from mediagoblin import mg_globals from mediagoblin.db.models import MediaEntry from mediagoblin.gmg_commands import util as commands_util +from mediagoblin.submit.lib import run_process_media from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ from mediagoblin.tools.pluginapi import hook_handle -from mediagoblin.processing import ProcessorDoesNotExist, ProcessorNotEligible +from mediagoblin.processing import ( + ProcessorDoesNotExist, ProcessorNotEligible, + get_entry_and_manager, get_manager_for_type) def reprocess_parser_setup(subparser): @@ -205,31 +208,16 @@ def _set_media_state(args): args[0].state = 'processed' -class MediaEntryNotFound(Exception): pass - -def extract_entry_and_type(media_id): - """ - Fetch a media entry, as well as its media type - """ - entry = MediaEntry.query.filter_by(id=media_id).first() - if entry is None: - raise MediaEntryNotFound("Can't find media with id '%s'" % media_id) - - return entry.media_type, entry - - def available(args): # Get the media type, either by looking up media id, or by specific type try: - media_id = int(args.id_or_type) - media_type, media_entry = extract_entry_and_type(media_id) + media_entry, manager = get_entry_and_manager(args.id_or_type) + media_type = media_entry.type except ValueError: media_type = args.id_or_type media_entry = None + manager = get_manager_for_type(media_type) - manager_class = hook_handle(('reprocess_manager', media_type)) - manager = manager_class() - if media_entry is None: processors = manager.list_all_processors() else: @@ -257,10 +245,7 @@ def available(args): def run(args): - media_type, media_entry = extract_entry_and_type(args.media_id) - - manager_class = hook_handle(('reprocess_manager', media_type)) - manager = manager_class() + media_entry, manager = get_entry_and_manager(args.media_id) # TODO: (maybe?) This could probably be handled entirely by the # processor class... @@ -279,8 +264,11 @@ def run(args): reprocess_parser = processor_class.generate_parser() reprocess_args = reprocess_parser.parse_args(args.reprocess_args) reprocess_request = processor_class.args_to_request(reprocess_args) - processor = processor_class(manager, media_entry) - processor.process(**reprocess_request) + run_process_media( + media_entry, + reprocess_action=args.reprocess_command, + reprocess_info=reprocess_request) + manager.process(media_entry, args.reprocess_command, **reprocess_request) def reprocess(args): -- cgit v1.2.3 From a59f92f3eca79b8070c2142a2be0ee5a48fe4611 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sun, 11 Aug 2013 14:48:21 -0500 Subject: That manager.process() line no longer made sense --- mediagoblin/gmg_commands/reprocess.py | 1 - 1 file changed, 1 deletion(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 24fcde37..9a9196bb 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -268,7 +268,6 @@ def run(args): media_entry, reprocess_action=args.reprocess_command, reprocess_info=reprocess_request) - manager.process(media_entry, args.reprocess_command, **reprocess_request) def reprocess(args): -- cgit v1.2.3 From 55cfa3406390732173195bb920bf3f86bd1ce9f4 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Sun, 11 Aug 2013 15:22:43 -0500 Subject: Renaming the processing manager stuff to be less ambiguous. BONUS COMMIT to Ben Finney and the Free Software Melbourne crew. :) IRONY: Initially I committed this as "media manager". --- mediagoblin/gmg_commands/reprocess.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 9a9196bb..10ab50ab 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -24,7 +24,7 @@ from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ from mediagoblin.tools.pluginapi import hook_handle from mediagoblin.processing import ( ProcessorDoesNotExist, ProcessorNotEligible, - get_entry_and_manager, get_manager_for_type) + get_entry_and_processing_manager, get_processing_manager_for_type) def reprocess_parser_setup(subparser): @@ -211,12 +211,12 @@ def _set_media_state(args): def available(args): # Get the media type, either by looking up media id, or by specific type try: - media_entry, manager = get_entry_and_manager(args.id_or_type) + media_entry, manager = get_entry_and_processing_manager(args.id_or_type) media_type = media_entry.type except ValueError: media_type = args.id_or_type media_entry = None - manager = get_manager_for_type(media_type) + manager = get_processing_manager_for_type(media_type) if media_entry is None: processors = manager.list_all_processors() @@ -245,7 +245,7 @@ def available(args): def run(args): - media_entry, manager = get_entry_and_manager(args.media_id) + media_entry, manager = get_entry_and_processing_manager(args.media_id) # TODO: (maybe?) This could probably be handled entirely by the # processor class... -- cgit v1.2.3 From de332ab9f5c56589349494d1ccbbf2cfc65424ca Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Mon, 12 Aug 2013 11:19:20 -0500 Subject: Trying to fix the bug that's happening to rodney757 but not to me ;) --- mediagoblin/gmg_commands/reprocess.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 10ab50ab..4678ca6f 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -28,6 +28,11 @@ from mediagoblin.processing import ( def reprocess_parser_setup(subparser): + subparser.add_argument( + '--celery', + action='store_true', + help="Don't process eagerly, pass off to celery") + subparsers = subparser.add_subparsers(dest="reprocess_subcommand") ################### @@ -66,10 +71,6 @@ def reprocess_parser_setup(subparser): '--thumbnails', action="store_true", help="Regenerate thumbnails for all processed media") - run_parser.add_argument( - '--celery', - action='store_true', - help="Don't process eagerly, pass off to celery") run_parser.add_argument( 'media_id', -- cgit v1.2.3 From c100b8b284456e5102de4b05d77a66f829a34939 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Mon, 12 Aug 2013 11:27:43 -0500 Subject: Fixing ./bin/gmg reprocess available, which I broke :) --- mediagoblin/gmg_commands/reprocess.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 4678ca6f..bd8039b5 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -212,8 +212,9 @@ def _set_media_state(args): def available(args): # Get the media type, either by looking up media id, or by specific type try: - media_entry, manager = get_entry_and_processing_manager(args.id_or_type) - media_type = media_entry.type + media_id = int(args.id_or_type) + media_entry, manager = get_entry_and_processing_manager(media_id) + media_type = media_entry.media_type except ValueError: media_type = args.id_or_type media_entry = None -- cgit v1.2.3 From 7584080bf7d7b2d74087d31ca781e1111c2024da Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Mon, 12 Aug 2013 14:28:03 -0700 Subject: add bulk_run, thumbs, and initial sub_commands --- mediagoblin/gmg_commands/reprocess.py | 96 ++++++++++++++++++++++++++++------- 1 file changed, 77 insertions(+), 19 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index bd8039b5..34311f6d 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -41,7 +41,7 @@ def reprocess_parser_setup(subparser): available_parser = subparsers.add_parser( "available", help="Find out what actions are available for this media") - + available_parser.add_argument( "id_or_type", help="Media id or media type to check") @@ -51,27 +51,19 @@ def reprocess_parser_setup(subparser): action="store_true", help="List argument help for each action available") - - ############################################ - # run command (TODO: and bulk_run command??) - ############################################ - + available_parser.add_argument( + "--state", + help="The state of media you would like to reprocess") + + + ############# + # run command + ############# + run_parser = subparsers.add_parser( "run", help="Run a reprocessing on one or more media") - run_parser.add_argument( - '--state', '-s', - help="Reprocess media entries in this state" - " such as 'failed' or 'processed'") - run_parser.add_argument( - '--type', '-t', - help="The type of media to be reprocessed such as 'video' or 'image'") - run_parser.add_argument( - '--thumbnails', - action="store_true", - help="Regenerate thumbnails for all processed media") - run_parser.add_argument( 'media_id', help="The media_entry id(s) you wish to reprocess.") @@ -86,6 +78,48 @@ def reprocess_parser_setup(subparser): help="rest of arguments to the reprocessing tool") + ################ + # thumbs command + ################ + thumbs = subparsers.add_parser( + 'thumbs', + help='Regenerate thumbs for all processed media') + + thumbs.add_argument( + '--size', + nargs=2, + type=int, + metavar=('max_width', 'max_height')) + + ################# + # initial command + ################# + subparsers.add_parser( + 'initial', + help='Reprocess all failed media') + + ################## + # bulk_run command + ################## + bulk_run_parser = subparsers.add_parser( + 'bulk_run', + help='Run reprocessing on a given media type or state') + + bulk_run_parser.add_argument( + 'type', + help='The type of media you would like to process') + + bulk_run_parser.add_argument( + 'state', + default='processed', + help='The state of the media you would like to process. Defaults to' \ + " 'processed'") + + bulk_run_parser.add_argument( + 'reprocess_args', + nargs=argparse.REMAINDER, + help='The rest of the arguments to the reprocessing tool') + ############### # help command? ############### @@ -220,7 +254,9 @@ def available(args): media_entry = None manager = get_processing_manager_for_type(media_type) - if media_entry is None: + if args.state: + processors = manager.list_all_processors_by_state(args.state) + elif media_entry is None: processors = manager.list_all_processors() else: processors = manager.list_eligible_processors(media_entry) @@ -271,6 +307,19 @@ def run(args): reprocess_action=args.reprocess_command, reprocess_info=reprocess_request) +def bulk_run(args): + pass + + +def thumbs(args): + #TODO regenerate thumbs for all processed media + pass + + +def initial(args): + #TODO initial processing on all failed media + pass + def reprocess(args): # Run eagerly unless explicetly set not to @@ -284,3 +333,12 @@ def reprocess(args): elif args.reprocess_subcommand == "available": available(args) + + elif args.reprocess_subcommand == "bulk_run": + bulk_run(args) + + elif args.reprocess_subcommand == "thumbs": + thumbs(args) + + elif args.reprocess_subcommand == "initial": + initial(args) -- cgit v1.2.3 From 4e6013689beded08121c0d139565ffccbf3c0000 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Mon, 12 Aug 2013 14:54:02 -0700 Subject: run initial processing on all failed entries --- mediagoblin/gmg_commands/reprocess.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 34311f6d..a3c732b9 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -24,7 +24,8 @@ from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ from mediagoblin.tools.pluginapi import hook_handle from mediagoblin.processing import ( ProcessorDoesNotExist, ProcessorNotEligible, - get_entry_and_processing_manager, get_processing_manager_for_type) + get_entry_and_processing_manager, get_processing_manager_for_type, + ProcessingManagerDoesNotExist) def reprocess_parser_setup(subparser): @@ -307,6 +308,7 @@ def run(args): reprocess_action=args.reprocess_command, reprocess_info=reprocess_request) + def bulk_run(args): pass @@ -317,8 +319,19 @@ def thumbs(args): def initial(args): - #TODO initial processing on all failed media - pass + """ + Reprocess all failed media + """ + query = MediaEntry.query.filter_by(state='failed') + + for entry in query: + try: + media_entry, manager = get_entry_and_processing_manager(entry.id) + run_process_media( + media_entry, + reprocess_action='initial') + except ProcessingManagerDoesNotExist: + print 'No such processing manager for {0}'.format(entry.media_type) def reprocess(args): -- cgit v1.2.3 From 441ed10de0cbc50ef2f40bae8dc25ea8493e8f8f Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Mon, 12 Aug 2013 14:57:45 -0700 Subject: wrap get_entry_and_processing_manager in try, except block --- mediagoblin/gmg_commands/reprocess.py | 52 ++++++++++++++++++++--------------- 1 file changed, 30 insertions(+), 22 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index a3c732b9..579ba478 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -254,6 +254,9 @@ def available(args): media_type = args.id_or_type media_entry = None manager = get_processing_manager_for_type(media_type) + except ProcessingManagerDoesNotExist: + entry = MediaEntry.query.filter_by(id=args.id_or_type).first() + print 'No such processing manager for {0}'.format(entry.media_type) if args.state: processors = manager.list_all_processors_by_state(args.state) @@ -284,29 +287,34 @@ def available(args): def run(args): - media_entry, manager = get_entry_and_processing_manager(args.media_id) - - # TODO: (maybe?) This could probably be handled entirely by the - # processor class... try: - processor_class = manager.get_processor( - args.reprocess_command, media_entry) - except ProcessorDoesNotExist: - print 'No such processor "%s" for media with id "%s"' % ( - args.reprocess_command, media_entry.id) - return - except ProcessorNotEligible: - print 'Processor "%s" exists but media "%s" is not eligible' % ( - args.reprocess_command, media_entry.id) - return - - reprocess_parser = processor_class.generate_parser() - reprocess_args = reprocess_parser.parse_args(args.reprocess_args) - reprocess_request = processor_class.args_to_request(reprocess_args) - run_process_media( - media_entry, - reprocess_action=args.reprocess_command, - reprocess_info=reprocess_request) + media_entry, manager = get_entry_and_processing_manager(args.media_id) + + # TODO: (maybe?) This could probably be handled entirely by the + # processor class... + try: + processor_class = manager.get_processor( + args.reprocess_command, media_entry) + except ProcessorDoesNotExist: + print 'No such processor "%s" for media with id "%s"' % ( + args.reprocess_command, media_entry.id) + return + except ProcessorNotEligible: + print 'Processor "%s" exists but media "%s" is not eligible' % ( + args.reprocess_command, media_entry.id) + return + + reprocess_parser = processor_class.generate_parser() + reprocess_args = reprocess_parser.parse_args(args.reprocess_args) + reprocess_request = processor_class.args_to_request(reprocess_args) + run_process_media( + media_entry, + reprocess_action=args.reprocess_command, + reprocess_info=reprocess_request) + + except ProcessingManagerDoesNotExist: + entry = MediaEntry.query.filter_by(id=args.media_id).first() + print 'No such processing manager for {0}'.format(entry.media_type) def bulk_run(args): -- cgit v1.2.3 From 0c4b68a8049acdef5f75e5fe7e95be2360d524eb Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Mon, 12 Aug 2013 15:25:52 -0700 Subject: Resize all processed thumbs --- mediagoblin/gmg_commands/reprocess.py | 42 +++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 579ba478..55aa6cc9 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -322,8 +322,46 @@ def bulk_run(args): def thumbs(args): - #TODO regenerate thumbs for all processed media - pass + """ + Regenerate thumbs for all processed media + """ + query = MediaEntry.query.filter_by(state='processed') + + for entry in query: + try: + media_entry, manager = get_entry_and_processing_manager(entry.id) + + # TODO: (maybe?) This could probably be handled entirely by the + # processor class... + try: + processor_class = manager.get_processor( + 'resize', media_entry) + except ProcessorDoesNotExist: + print 'No such processor "%s" for media with id "%s"' % ( + 'resize', media_entry.id) + return + except ProcessorNotEligible: + print 'Processor "%s" exists but media "%s" is not eligible' % ( + 'resize', media_entry.id) + return + + reprocess_parser = processor_class.generate_parser() + + # prepare filetype and size to be passed into reprocess_parser + if args.size: + extra_args = 'thumb --size {0} {1}'.format(args.size[0], args.size[1]) + else: + extra_args = 'thumb' + + reprocess_args = reprocess_parser.parse_args(extra_args.split()) + reprocess_request = processor_class.args_to_request(reprocess_args) + run_process_media( + media_entry, + reprocess_action='resize', + reprocess_info=reprocess_request) + + except ProcessingManagerDoesNotExist: + print 'No such processing manager for {0}'.format(entry.media_type) def initial(args): -- cgit v1.2.3 From a7f426368dfce15f86fa0ef56200ec1228f6e4af Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Mon, 12 Aug 2013 15:45:09 -0700 Subject: bulk_run reprocessing complete --- mediagoblin/gmg_commands/reprocess.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 55aa6cc9..3ba5d92c 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -111,11 +111,16 @@ def reprocess_parser_setup(subparser): help='The type of media you would like to process') bulk_run_parser.add_argument( - 'state', + '--state', default='processed', + nargs='?', help='The state of the media you would like to process. Defaults to' \ " 'processed'") + bulk_run_parser.add_argument( + 'reprocess_command', + help='The reprocess command you intend to run') + bulk_run_parser.add_argument( 'reprocess_args', nargs=argparse.REMAINDER, @@ -286,9 +291,11 @@ def available(args): print " - %s" % processor.name -def run(args): +def run(args, media_id=None): + if not media_id: + media_id = args.media_id try: - media_entry, manager = get_entry_and_processing_manager(args.media_id) + media_entry, manager = get_entry_and_processing_manager(media_id) # TODO: (maybe?) This could probably be handled entirely by the # processor class... @@ -313,12 +320,19 @@ def run(args): reprocess_info=reprocess_request) except ProcessingManagerDoesNotExist: - entry = MediaEntry.query.filter_by(id=args.media_id).first() + entry = MediaEntry.query.filter_by(id=media_id).first() print 'No such processing manager for {0}'.format(entry.media_type) def bulk_run(args): - pass + """ + Bulk reprocessing of a given media_type + """ + query = MediaEntry.query.filter_by(media_type=args.type, + state=args.state) + + for entry in query: + run(args, entry.id) def thumbs(args): -- cgit v1.2.3 From 36c17b85c12546076067e039e5f55c7780dc54f3 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Mon, 12 Aug 2013 15:48:40 -0700 Subject: remove old code --- mediagoblin/gmg_commands/reprocess.py | 118 ---------------------------------- 1 file changed, 118 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 3ba5d92c..5285942e 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -131,124 +131,6 @@ def reprocess_parser_setup(subparser): ############### - -def _set_media_type(args): - """ - This will verify that all media id's are of the same media_type. If the - --type flag is set, it will be replaced by the given media id's type. - - If they are trying to process different media types, an Exception will be - raised. - """ - if args[0].media_id: - if len(args[0].media_id) == 1: - args[0].type = MediaEntry.query.filter_by(id=args[0].media_id[0])\ - .first().media_type.split('.')[-1] - - elif len(args[0].media_id) > 1: - media_types = [] - - for id in args[0].media_id: - media_types.append(MediaEntry.query.filter_by(id=id).first() - .media_type.split('.')[-1]) - for type in media_types: - if media_types[0] != type: - raise Exception((u'You cannot reprocess different' - ' media_types at the same time.')) - - args[0].type = media_types[0] - - -def _reprocess_all(args): - """ - This handles reprocessing if no media_id's are given. - """ - if not args[0].type: - # If no media type is given, we can either regenerate all thumbnails, - # or try to reprocess all failed media - - if args[0].thumbnails: - if args[0].available: - print _('Available options for regenerating all processed' - ' media thumbnails: \n' - '\t --size: max_width max_height' - ' (defaults to config specs)') - else: - #TODO regenerate all thumbnails - pass - - # Reprocess all failed media - elif args[0].state == 'failed': - if args[0].available: - print _('\n Available reprocess actions for all failed' - ' media_entries: \n \t --initial_processing') - else: - #TODO reprocess all failed entries - pass - - # If here, they didn't set the --type flag and were trying to do - # something other the generating thumbnails or initial_processing - else: - raise Exception(_('You must set --type when trying to reprocess' - ' all media_entries, unless you set --state' - ' to "failed".')) - - else: - _run_reprocessing(args) - - -def _run_reprocessing(args): - # Are they just asking for the available reprocessing options for the given - # media? - if args[0].available: - if args[0].state == 'failed': - print _('\n Available reprocess actions for all failed' - ' media_entries: \n \t --initial_processing') - else: - result = hook_handle(('reprocess_action', args[0].type), args) - if not result: - print _('Sorry there is no available reprocessing for {}' - ' entries in the {} state'.format(args[0].type, - args[0].state)) - else: - # Run media reprocessing - return hook_handle(('media_reprocess', args[0].type), args) - - -def _set_media_state(args): - """ - This will verify that all media id's are in the same state. If the - --state flag is set, it will be replaced by the given media id's state. - - If they are trying to process different media states, an Exception will be - raised. - """ - if args[0].media_id: - # Only check if we are given media_ids - if len(args[0].media_id) == 1: - args[0].state = MediaEntry.query.filter_by(id=args[0].media_id[0])\ - .first().state - - elif len(args[0].media_id) > 1: - media_states = [] - - for id in args[0].media_id: - media_states.append(MediaEntry.query.filter_by(id=id).first() - .state) - - # Make sure that all media are in the same state - for state in media_states: - if state != media_states[0]: - raise Exception(_('You can only reprocess media that is in' - ' the same state.')) - - args[0].state = media_states[0] - - # If no state was set, then we will default to the processed state - if not args[0].state: - args[0].state = 'processed' - - def available(args): # Get the media type, either by looking up media id, or by specific type try: -- cgit v1.2.3 From 5ac1fe806483de28656a056f10314ecc6a10aed4 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Tue, 13 Aug 2013 09:57:35 -0700 Subject: Audio Initial Processor --- mediagoblin/gmg_commands/reprocess.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 5285942e..375d9ff2 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -245,7 +245,8 @@ def thumbs(args): # prepare filetype and size to be passed into reprocess_parser if args.size: - extra_args = 'thumb --size {0} {1}'.format(args.size[0], args.size[1]) + extra_args = 'thumb --size {0} {1}'.format(args.size[0], + args.size[1]) else: extra_args = 'thumb' -- cgit v1.2.3 From 3225008f04e263c618186da881e3e01439578900 Mon Sep 17 00:00:00 2001 From: Rodney Ewing Date: Wed, 14 Aug 2013 16:25:44 -0700 Subject: include a thumb_size string with each Resizer to run gmg reprocess thumbs --- mediagoblin/gmg_commands/reprocess.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'mediagoblin/gmg_commands/reprocess.py') diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index 375d9ff2..e2f19ea3 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -245,8 +245,10 @@ def thumbs(args): # prepare filetype and size to be passed into reprocess_parser if args.size: - extra_args = 'thumb --size {0} {1}'.format(args.size[0], - args.size[1]) + extra_args = 'thumb --{0} {1} {2}'.format( + processor_class.thumb_size, + args.size[0], + args.size[1]) else: extra_args = 'thumb' -- cgit v1.2.3