diff options
| -rw-r--r-- | docs/source/devel/codebase.rst | 140 | ||||
| -rw-r--r-- | mediagoblin/media_types/video/transcoders.py | 348 | ||||
| -rw-r--r-- | mediagoblin/user_pages/views.py | 8 | ||||
| -rwxr-xr-x | runtests.sh | 14 | ||||
| -rw-r--r-- | setup.py | 2 |
5 files changed, 126 insertions, 386 deletions
diff --git a/docs/source/devel/codebase.rst b/docs/source/devel/codebase.rst index 98e0c26e..cd46242c 100644 --- a/docs/source/devel/codebase.rst +++ b/docs/source/devel/codebase.rst @@ -41,6 +41,76 @@ which explains generally how to get going with running an instance for development. +What's where +============ + +After you've run checked out mediagoblin and followed the virtualenv +instantiation instructions, you're faced with the following directory +tree:: + + mediagoblin/ + |- mediagoblin/ # source code + | |- db/ # database setup + | |- tools/ # various utilities + | |- init/ # "initialization" tools (arguably should be in tools/) + | |- tests/ # unit tests + | |- templates/ # templates for this application + | |- media_types/ # code for processing, displaying different media + | |- storage/ # different storage backends + | |- gmg_commands/ # command line tools (./bin/gmg) + | |- themes/ # pre-bundled themes + | | + | | # ... some submodules here as well for different sections + | | # of the application... here's just a few + | |- auth/ # authentication (login/registration) code + | |- user_dev/ # user pages (under /u/), including media pages + | \- submit/ # submitting media for processing + | + |- docs/ # documentation + |- devtools/ # some scripts for developer convenience + | + |- user_dev/ # local instance sessions, media, etc + | + | # the below directories are installed into your virtualenv checkout + | + |- bin/ # scripts + |- develop-eggs/ + |- lib/ # python libraries installed into your virtualenv + |- include/ + |- mediagoblin.egg-info/ + \- parts/ + + +As you can see, all the code for GNU MediaGoblin is in the +``mediagoblin`` directory. + +Here are some interesting files and what they do: + +:routing.py: maps url paths to views +:views.py: views handle http requests +:forms.py: wtforms stuff for this submodule + +You'll notice that there are several sub-directories: tests, +templates, auth, submit, ... + +``tests`` holds the unit test code. + +``templates`` holds all the templates for the output. + +``auth`` and ``submit`` are modules that enacpsulate authentication +and media item submission. If you look in these directories, you'll +see they have their own ``routing.py``, ``view.py``, and forms.py in +addition to some other code. + +You'll also notice that mediagoblin/db/ contains quite a few things, +including the following: + +:models.py: This is where the database is set up +:mixin.py: Certain functions appended to models from here +:migrations.py: When creating a new migration (a change to the + database structure), we put it here + + Software Stack ============== @@ -111,73 +181,3 @@ Software Stack * `JQuery <http://jquery.com/>`_: for groovy JavaScript things - -What's where -============ - -After you've run checked out mediagoblin and followed the virtualenv -instantiation instructions, you're faced with the following directory -tree:: - - mediagoblin/ - |- mediagoblin/ # source code - | |- db/ # database setup - | |- tools/ # various utilities - | |- init/ # "initialization" tools (arguably should be in tools/) - | |- tests/ # unit tests - | |- templates/ # templates for this application - | |- media_types/ # code for processing, displaying different media - | |- storage/ # different storage backends - | |- gmg_commands/ # command line tools (./bin/gmg) - | |- themes/ # pre-bundled themes - | | - | | # ... some submodules here as well for different sections - | | # of the application... here's just a few - | |- auth/ # authentication (login/registration) code - | |- user_dev/ # user pages (under /u/), including media pages - | \- submit/ # submitting media for processing - | - |- docs/ # documentation - |- devtools/ # some scripts for developer convenience - | - |- user_dev/ # local instance sessions, media, etc - | - | # the below directories are installed into your virtualenv checkout - | - |- bin/ # scripts - |- develop-eggs/ - |- lib/ # python libraries installed into your virtualenv - |- include/ - |- mediagoblin.egg-info/ - \- parts/ - - -As you can see, all the code for GNU MediaGoblin is in the -``mediagoblin`` directory. - -Here are some interesting files and what they do: - -:routing.py: maps url paths to views -:views.py: views handle http requests -:forms.py: wtforms stuff for this submodule - -You'll notice that there are several sub-directories: tests, -templates, auth, submit, ... - -``tests`` holds the unit test code. - -``templates`` holds all the templates for the output. - -``auth`` and ``submit`` are modules that enacpsulate authentication -and media item submission. If you look in these directories, you'll -see they have their own ``routing.py``, ``view.py``, and forms.py in -addition to some other code. - -You'll also notice that mediagoblin/db/ contains quite a few things, -including the following: - -:models.py: This is where the database is set up -:mixin.py: Certain functions appended to models from here -:migrations.py: When creating a new migration (a change to the - database structure), we put it here - diff --git a/mediagoblin/media_types/video/transcoders.py b/mediagoblin/media_types/video/transcoders.py index d8290d41..58b2c0d4 100644 --- a/mediagoblin/media_types/video/transcoders.py +++ b/mediagoblin/media_types/video/transcoders.py @@ -53,283 +53,6 @@ def pixbuf_to_pilbuf(buf): return data -class VideoThumbnailer: - # Declaration of thumbnailer states - STATE_NULL = 0 - STATE_HALTING = 1 - STATE_PROCESSING = 2 - - # The current thumbnailer state - - def __init__(self, source_path, dest_path): - ''' - Set up playbin pipeline in order to get video properties. - - Initializes and runs the gobject.MainLoop() - - Abstract - - Set up a playbin with a fake audio sink and video sink. Load the video - into the playbin - - Initialize - ''' - # This will contain the thumbnailing pipeline - self.state = self.STATE_NULL - self.thumbnail_pipeline = None - self.buffer_probes = {} - self.errors = [] - - self.source_path = source_path - self.dest_path = dest_path - - self.loop = gobject.MainLoop() - - # Set up the playbin. It will be used to discover certain - # properties of the input file - self.playbin = gst.element_factory_make('playbin') - - self.videosink = gst.element_factory_make('fakesink', 'videosink') - self.playbin.set_property('video-sink', self.videosink) - - self.audiosink = gst.element_factory_make('fakesink', 'audiosink') - self.playbin.set_property('audio-sink', self.audiosink) - - self.bus = self.playbin.get_bus() - self.bus.add_signal_watch() - self.watch_id = self.bus.connect('message', self._on_bus_message) - - self.playbin.set_property('uri', 'file:{0}'.format( - urllib.pathname2url(self.source_path))) - - self.playbin.set_state(gst.STATE_PAUSED) - - self.run() - - def run(self): - self.loop.run() - - def _on_bus_message(self, bus, message): - _log.debug(' thumbnail playbin: {0}'.format(message)) - - if message.type == gst.MESSAGE_ERROR: - _log.error('thumbnail playbin: {0}'.format(message)) - gobject.idle_add(self._on_bus_error) - - elif message.type == gst.MESSAGE_STATE_CHANGED: - # The pipeline state has changed - # Parse state changing data - _prev, state, _pending = message.parse_state_changed() - - _log.debug('State changed: {0}'.format(state)) - - if state == gst.STATE_PAUSED: - if message.src == self.playbin: - gobject.idle_add(self._on_bus_paused) - - def _on_bus_paused(self): - ''' - Set up thumbnailing pipeline - ''' - current_video = self.playbin.get_property('current-video') - - if current_video == 0: - _log.debug('Found current video from playbin') - else: - _log.error('Could not get any current video from playbin!') - - self.duration = self._get_duration(self.playbin) - _log.info('Video length: {0}'.format(self.duration / gst.SECOND)) - - _log.info('Setting up thumbnailing pipeline') - self.thumbnail_pipeline = gst.parse_launch( - 'filesrc location="{0}" ! decodebin ! ' - 'ffmpegcolorspace ! videoscale ! ' - 'video/x-raw-rgb,depth=24,bpp=24,pixel-aspect-ratio=1/1,width=180 ! ' - 'fakesink signal-handoffs=True'.format(self.source_path)) - - self.thumbnail_bus = self.thumbnail_pipeline.get_bus() - self.thumbnail_bus.add_signal_watch() - self.thumbnail_watch_id = self.thumbnail_bus.connect( - 'message', self._on_thumbnail_bus_message) - - self.thumbnail_pipeline.set_state(gst.STATE_PAUSED) - - #gobject.timeout_add(3000, self._on_timeout) - - return False - - def _on_thumbnail_bus_message(self, bus, message): - _log.debug('thumbnail: {0}'.format(message)) - - if message.type == gst.MESSAGE_ERROR: - _log.error(message) - gobject.idle_add(self._on_bus_error) - - if message.type == gst.MESSAGE_STATE_CHANGED: - _log.debug('State changed') - _prev, state, _pending = message.parse_state_changed() - - if (state == gst.STATE_PAUSED and - not self.state == self.STATE_PROCESSING and - message.src == self.thumbnail_pipeline): - _log.info('Pipeline paused, processing') - self.state = self.STATE_PROCESSING - - for sink in self.thumbnail_pipeline.sinks(): - name = sink.get_name() - factoryname = sink.get_factory().get_name() - - if factoryname == 'fakesink': - sinkpad = sink.get_pad('sink') - - self.buffer_probes[name] = sinkpad.add_buffer_probe( - self.buffer_probe_handler, name) - - _log.info('Added buffer probe') - - break - - # Apply the wadsworth constant, fallback to 1 second - # TODO: Will break if video is shorter than 1 sec - seek_amount = max(self.duration / 100 * 30, 1 * gst.SECOND) - - _log.debug('seek amount: {0}'.format(seek_amount)) - - seek_result = self.thumbnail_pipeline.seek( - 1.0, - gst.FORMAT_TIME, - gst.SEEK_FLAG_FLUSH | gst.SEEK_FLAG_ACCURATE, - gst.SEEK_TYPE_SET, - seek_amount, - gst.SEEK_TYPE_NONE, - 0) - - if not seek_result: - self.errors.append('COULD_NOT_SEEK') - _log.error('Couldn\'t seek! result: {0}'.format( - seek_result)) - _log.info(message) - self.shutdown() - else: - _log.debug('Seek successful') - self.thumbnail_pipeline.set_state(gst.STATE_PAUSED) - else: - _log.debug('Won\'t seek: \t{0}\n\t{1}'.format( - self.state, - message.src)) - - def buffer_probe_handler_real(self, pad, buff, name): - ''' - Capture buffers as gdk_pixbufs when told to. - ''' - _log.info('Capturing frame') - try: - caps = buff.caps - if caps is None: - _log.error('No caps passed to buffer probe handler!') - self.shutdown() - return False - - _log.debug('caps: {0}'.format(caps)) - - filters = caps[0] - width = filters["width"] - height = filters["height"] - - im = Image.new('RGB', (width, height)) - - data = pixbuf_to_pilbuf(buff.data) - - im.putdata(data) - - im.save(self.dest_path) - - _log.info('Saved thumbnail') - - self.shutdown() - - except gst.QueryError as e: - _log.error('QueryError: {0}'.format(e)) - - return False - - def buffer_probe_handler(self, pad, buff, name): - ''' - Proxy function for buffer_probe_handler_real - ''' - _log.debug('Attaching real buffer handler to gobject idle event') - gobject.idle_add( - lambda: self.buffer_probe_handler_real(pad, buff, name)) - - return True - - def _get_duration(self, pipeline, retries=0): - ''' - Get the duration of a pipeline. - - Retries 5 times. - ''' - if retries == 5: - return 0 - - try: - return pipeline.query_duration(gst.FORMAT_TIME)[0] - except gst.QueryError: - return self._get_duration(pipeline, retries + 1) - - def _on_timeout(self): - _log.error('Timeout in thumbnailer!') - self.shutdown() - - def _on_bus_error(self, *args): - _log.error('AHAHAHA! Error! args: {0}'.format(args)) - - def shutdown(self): - ''' - Tell gobject to call __halt when the mainloop is idle. - ''' - _log.info('Shutting down') - self.__halt() - - def __halt(self): - ''' - Halt all pipelines and shut down the main loop - ''' - _log.info('Halting...') - self.state = self.STATE_HALTING - - self.__disconnect() - - gobject.idle_add(self.__halt_final) - - def __disconnect(self): - _log.debug('Disconnecting...') - if not self.playbin is None: - self.playbin.set_state(gst.STATE_NULL) - for sink in self.playbin.sinks(): - name = sink.get_name() - factoryname = sink.get_factory().get_name() - - _log.debug('Disconnecting {0}'.format(name)) - - if factoryname == "fakesink": - pad = sink.get_pad("sink") - pad.remove_buffer_probe(self.buffer_probes[name]) - del self.buffer_probes[name] - - self.playbin = None - - if self.bus is not None: - self.bus.disconnect(self.watch_id) - self.bus = None - - def __halt_final(self): - _log.info('Done') - if self.errors: - _log.error(','.join(self.errors)) - - self.loop.quit() - - class VideoThumbnailerMarkII(object): ''' Creates a thumbnail from a video file. Rewrite of VideoThumbnailer. @@ -398,8 +121,8 @@ class VideoThumbnailerMarkII(object): self.run() except Exception as exc: _log.critical( - 'Exception "{0}" caught, disconnecting and re-raising'\ - .format(exc)) + 'Exception "{0}" caught, shutting down mainloop and re-raising'\ + .format(exc)) self.disconnect() raise @@ -410,7 +133,8 @@ class VideoThumbnailerMarkII(object): self.mainloop.run() def on_playbin_message(self, message_bus, message): - _log.debug('playbin message: {0}'.format(message)) + # Silenced to prevent clobbering of output + #_log.debug('playbin message: {0}'.format(message)) if message.type == gst.MESSAGE_ERROR: _log.error('playbin error: {0}'.format(message)) @@ -433,17 +157,20 @@ pending: {2}'.format( def on_playbin_paused(self): if self.has_reached_playbin_pause: - _log.warn('Has already reached logic for playbin pause. Aborting \ + _log.warn('Has already reached on_playbin_paused. Aborting \ without doing anything this time.') return False self.has_reached_playbin_pause = True + # XXX: Why is this even needed at this point? current_video = self.playbin.get_property('current-video') if not current_video: - _log.critical('thumbnail could not get any video data \ + _log.critical('Could not get any video data \ from playbin') + else: + _log.info('Got video data from playbin') self.duration = self.get_duration(self.playbin) self.permission_to_take_picture = True @@ -474,7 +201,8 @@ from playbin') return False def on_thumbnail_message(self, message_bus, message): - _log.debug('thumbnail message: {0}'.format(message)) + # This is silenced to prevent clobbering of the terminal window + #_log.debug('thumbnail message: {0}'.format(message)) if message.type == gst.MESSAGE_ERROR: _log.error('thumbnail error: {0}'.format(message.parse_error())) @@ -490,29 +218,10 @@ pending: {2}'.format( cur_state, pending_state)) - if cur_state == gst.STATE_PAUSED and\ - not self.state == self.STATE_PROCESSING_THUMBNAIL: - self.state = self.STATE_PROCESSING_THUMBNAIL - + if cur_state == gst.STATE_PAUSED and \ + not self.state == self.STATE_PROCESSING_THUMBNAIL: # Find the fakesink sink pad and attach the on_buffer_probe # handler to it. - for sink in self.thumbnail_pipeline.sinks(): - sink_name = sink.get_name() - sink_factory_name = sink.get_factory().get_name() - - if sink_factory_name == 'fakesink': - sink_pad = sink.get_pad('sink') - - self.buffer_probes[sink_name] = sink_pad\ - .add_buffer_probe( - self.on_pad_buffer_probe, - sink_name) - - _log.info('Attached buffer probes: {0}'.format( - self.buffer_probes)) - - break - seek_amount = self.position_callback(self.duration, gst) seek_result = self.thumbnail_pipeline.seek( @@ -525,10 +234,30 @@ pending: {2}'.format( 0) if not seek_result: - _log.critical('Could not seek.') + _log.info('Could not seek.') + else: + _log.info('Seek successful, attaching buffer probe') + self.state = self.STATE_PROCESSING_THUMBNAIL + for sink in self.thumbnail_pipeline.sinks(): + sink_name = sink.get_name() + sink_factory_name = sink.get_factory().get_name() + + if sink_factory_name == 'fakesink': + sink_pad = sink.get_pad('sink') + + self.buffer_probes[sink_name] = sink_pad\ + .add_buffer_probe( + self.on_pad_buffer_probe, + sink_name) + + _log.info('Attached buffer probes: {0}'.format( + self.buffer_probes)) + + break + elif self.state == self.STATE_PROCESSING_THUMBNAIL: - _log.debug('Already processing thumbnail') + _log.info('Already processing thumbnail') def on_pad_buffer_probe(self, *args): _log.debug('buffer probe handler: {0}'.format(args)) @@ -649,7 +378,7 @@ pending: {2}'.format( return self.get_duration(pipeline, attempt + 1) -class VideoTranscoder: +class VideoTranscoder(object): ''' Video transcoder @@ -1011,6 +740,10 @@ if __name__ == '__main__': action='store_true', help='Dear program, please be quiet unless *error*') + parser.add_option('-w', '--width', + type=int, + default=180) + (options, args) = parser.parse_args() if options.verbose: @@ -1030,6 +763,7 @@ if __name__ == '__main__': transcoder = VideoTranscoder() if options.action == 'thumbnail': + args.append(options.width) VideoThumbnailerMarkII(*args) elif options.action == 'video': def cb(data): diff --git a/mediagoblin/user_pages/views.py b/mediagoblin/user_pages/views.py index c611daa1..09ec7df9 100644 --- a/mediagoblin/user_pages/views.py +++ b/mediagoblin/user_pages/views.py @@ -208,7 +208,7 @@ def media_collect(request, media): # Make sure this user isn't duplicating an existing collection existing_collection = Collection.query.filter_by( creator=request.user.id, - title=request.form['collection_title']).first() + title=form.collection_title.data).first() if existing_collection: messages.add_message(request, messages.ERROR, _('You already have a collection called "%s"!') @@ -218,8 +218,8 @@ def media_collect(request, media): media=media.slug_or_id) collection = Collection() - collection.title = request.form['collection_title'] - collection.description = request.form.get('collection_description') + collection.title = form.collection_title.data + collection.description = form.collection_description.data collection.creator = request.user.id collection.generate_slug() collection.save() @@ -251,7 +251,7 @@ def media_collect(request, media): collection_item = request.db.CollectionItem() collection_item.collection = collection.id collection_item.media_entry = media.id - collection_item.note = request.form['note'] + collection_item.note = form.note.data collection_item.save() collection.items = collection.items + 1 diff --git a/runtests.sh b/runtests.sh index a4ceec2e..fd19caf8 100755 --- a/runtests.sh +++ b/runtests.sh @@ -38,6 +38,16 @@ else exit 1 fi + +CELERY_CONFIG_MODULE=mediagoblin.init.celery.from_tests +export CELERY_CONFIG_MODULE +echo "+ CELERY_CONFIG_MODULE=$CELERY_CONFIG_MODULE" + +# Look to see if the user has specified a specific directory/file to +# run tests out of. If not we'll need to pass along +# mediagoblin/tests/ later very specifically. Otherwise nosetests +# will try to read all directories, and this turns into a mess! + need_arg=1 for i in "$@" do @@ -47,10 +57,6 @@ do esac done -CELERY_CONFIG_MODULE=mediagoblin.init.celery.from_tests -export CELERY_CONFIG_MODULE -echo "+ CELERY_CONFIG_MODULE=$CELERY_CONFIG_MODULE" - if [ "$need_arg" = 1 ] then testdir="$basedir/mediagoblin/tests" @@ -54,7 +54,7 @@ setup( 'sphinx', 'Babel', 'argparse', - 'webtest', + 'webtest<2', 'ConfigObj', 'Markdown', 'sqlalchemy>=0.7.0', |