diff options
| -rw-r--r-- | docs/source/siteadmin/commandline-upload.rst | 51 | ||||
| -rw-r--r-- | mediagoblin/edit/forms.py | 9 | ||||
| -rw-r--r-- | mediagoblin/edit/views.py | 37 | ||||
| -rw-r--r-- | mediagoblin/gmg_commands/batchaddmedia.py | 137 | ||||
| -rw-r--r-- | mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html | 6 | ||||
| -rw-r--r-- | mediagoblin/static/css/base.css | 36 | ||||
| -rw-r--r-- | mediagoblin/templates/mediagoblin/edit/metadata.html | 97 | ||||
| -rw-r--r-- | mediagoblin/templates/mediagoblin/utils/wtforms.html | 61 | ||||
| -rw-r--r-- | mediagoblin/user_pages/routing.py | 4 |
9 files changed, 346 insertions, 92 deletions
diff --git a/docs/source/siteadmin/commandline-upload.rst b/docs/source/siteadmin/commandline-upload.rst index be19df58..d67c19dd 100644 --- a/docs/source/siteadmin/commandline-upload.rst +++ b/docs/source/siteadmin/commandline-upload.rst @@ -39,3 +39,54 @@ You can also pass in the `--celery` option if you would prefer that your media be passed over to celery to be processed rather than be processed immediately. +============================ +Command-line batch uploading +============================ + +There's another way to submit media, and it can be much more powerful, although +it is a bit more complex. + + ./bin/gmg batchaddmedia admin /path/to/your/metadata.csv + +This is an example of what a script may look like. The important part here is +that you have to create the 'metadata.csv' file.:: + + media:location,dcterms:title,dcterms:creator,dcterms:type + "http://www.example.net/path/to/nap.png","Goblin taking a nap",,"Image" + "http://www.example.net/path/to/snore.ogg","Goblin Snoring","Me","Audio" + +The above is an example of a very simple metadata.csv file. The batchaddmedia +script would read this and attempt to upload only two pieces of media, and would +be able to automatically name them appropriately. + +The csv file +============ +The media:location column +------------------------- +The media:location column is the one column that is absolutely necessary for +uploading your media. This gives a path to each piece of media you upload. This +can either a path to a local file or a direct link to remote media (with the +link in http format). As you can see in the example above the (fake) media was +stored remotely on "www.example.net". + +Other columns +------------- +Other columns can be used to provide detailed metadata about each media entry. +Our metadata system accepts any information provided for in the +`RDFa Core Initial Context`_, and the batchupload script recognizes all of the +resources provided within it. + +.. _RDFa Core Initial Context: http://www.w3.org/2011/rdfa-context/rdfa-1.1 + +The uploader may include the metadata for each piece of media, or +leave them blank if they want to. A few columns from `Dublin Core`_ are +notable because the batchaddmedia script uses them to set the default +information of uploaded media entries. + +.. _Dublin Core: http://wiki.dublincore.org/index.php/User_Guide + +- **dc:title** sets a title for your media entry. If this is left blank, the media entry will be named according to the filename of the file being uploaded. +- **dc:description** sets a description of your media entry. If this is left blank the media entry's description will not be filled in. +- **dc:rights** will set a license for your media entry `if` the data provided is a valid URI. If this is left blank 'All Rights Reserved' will be selected. + +You can of course, change these values later. diff --git a/mediagoblin/edit/forms.py b/mediagoblin/edit/forms.py index 2c9b5e99..c2355980 100644 --- a/mediagoblin/edit/forms.py +++ b/mediagoblin/edit/forms.py @@ -122,3 +122,12 @@ class ChangeEmailForm(wtforms.Form): [wtforms.validators.Required()], description=_( "Enter your password to prove you own this account.")) + +class MetaDataForm(wtforms.Form): + identifier = wtforms.TextField(_(u'Identifier')) + value = wtforms.TextField(_(u'Value')) + +class EditMetaDataForm(wtforms.Form): + media_metadata = wtforms.FieldList( + wtforms.FormField(MetaDataForm, label="") + ) diff --git a/mediagoblin/edit/views.py b/mediagoblin/edit/views.py index 80590875..34021257 100644 --- a/mediagoblin/edit/views.py +++ b/mediagoblin/edit/views.py @@ -17,6 +17,7 @@ from datetime import datetime from itsdangerous import BadSignature +from pyld import jsonld from werkzeug.exceptions import Forbidden from werkzeug.utils import secure_filename @@ -29,8 +30,10 @@ from mediagoblin.edit import forms from mediagoblin.edit.lib import may_edit_media from mediagoblin.decorators import (require_active_login, active_user_from_url, get_media_entry_by_id, user_may_alter_collection, - get_user_collection) + get_user_collection, user_has_privilege, + user_not_banned) from mediagoblin.tools.crypto import get_timed_signer_url +from mediagoblin.tools.metadata import compact_and_validate from mediagoblin.tools.mail import email_debug_message from mediagoblin.tools.response import (render_to_response, redirect, redirect_obj, render_404) @@ -432,3 +435,35 @@ def change_email(request): 'mediagoblin/edit/change_email.html', {'form': form, 'user': user}) + +@user_has_privilege(u'admin') +@require_active_login +@get_media_entry_by_id +def edit_metadata(request, media): + form = forms.EditMetaDataForm(request.form) + if request.method == "POST" and form.validate(): + metadata_dict = dict([(row['identifier'],row['value']) + for row in form.media_metadata.data]) + json_ld_metadata = compact_and_validate(metadata_dict) + media.media_metadata = json_ld_metadata + media.save() + return redirect_obj(request, media) + + if media.media_metadata: + for identifier, value in media.media_metadata.iteritems(): + if identifier == "@context": continue + form.media_metadata.append_entry({ + 'identifier':identifier, + 'value':value}) + else: + form.media_metadata.append_entry({ + 'identifier':"", + 'value':""}) + form.media_metadata.append_entry({ + 'identifier':"", + 'value':""}) + return render_to_response( + request, + 'mediagoblin/edit/metadata.html', + {'form':form, + 'media':media}) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index e540e88c..75e7b7c5 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -25,32 +25,28 @@ from mediagoblin.submit.lib import ( submit_media, get_upload_file_limits, FileUploadLimit, UserUploadLimit, UserPastUploadLimit) from mediagoblin.tools.metadata import compact_and_validate - +from mediagoblin.tools.translate import pass_to_ugettext as _ from jsonschema.exceptions import ValidationError def parser_setup(subparser): subparser.description = """\ This command allows the administrator to upload many media files at once.""" + subparser.epilog = _(u"""For more information about how to properly run this +script (and how to format the metadata csv file), read the MediaGoblin +documentation page on command line uploading +<http://docs.mediagoblin.org/siteadmin/commandline-upload.html>""") subparser.add_argument( 'username', - help="Name of user these media entries belong to") + help=_(u"Name of user these media entries belong to")) subparser.add_argument( - 'target_path', - help=("""\ -Path to a local archive or directory containing a "location.csv" and a -"metadata.csv" file. These are csv (comma seperated value) files with the -locations and metadata of the files to be uploaded. The location must be listed -with either the URL of the remote media file or the filesystem path of a local -file. The metadata should be provided with one column for each of the 15 Dublin -Core properties (http://dublincore.org/documents/dces/). Both "location.csv" and -"metadata.csv" must begin with a row demonstrating the order of the columns. We -have provided an example of these files at <url to be added> -""")) + 'metadata_path', + help=_( +u"""Path to the csv file containing metadata information.""")) subparser.add_argument( '--celery', action='store_true', - help="Don't process eagerly, pass off to celery") + help=_(u"Don't process eagerly, pass off to celery")) def batchaddmedia(args): @@ -65,48 +61,24 @@ def batchaddmedia(args): # get the user user = app.db.User.query.filter_by(username=args.username.lower()).first() if user is None: - print "Sorry, no user by username '%s' exists" % args.username + print _(u"Sorry, no user by username '{username}' exists".format( + username=args.username)) return upload_limit, max_file_size = get_upload_file_limits(user) temp_files = [] - if os.path.isdir(args.target_path): - dir_path = args.target_path - - elif tarfile.is_tarfile(args.target_path): - dir_path = tempfile.mkdtemp() - temp_files.append(dir_path) - tar = tarfile.open(args.target_path) - tar.extractall(path=dir_path) - - elif zipfile.is_zipfile(args.target_path): - dir_path = tempfile.mkdtemp() - temp_files.append(dir_path) - zipped_file = zipfile.ZipFile(args.target_path) - zipped_file.extractall(path=dir_path) + if os.path.isfile(args.metadata_path): + metadata_path = args.metadata_path else: - print "Couldn't recognize the file. This script only accepts tar files,\ -zip files and directories" - if dir_path.endswith('/'): - dir_path = dir_path[:-1] - - location_file_path = os.path.join(dir_path,"location.csv") - metadata_file_path = os.path.join(dir_path, "metadata.csv") - - # check for the location file, if it exists... - abs_location_filename = os.path.abspath(location_file_path) - if not os.path.exists(abs_location_filename): - print "Can't find a file with filename '%s'" % location_file_path - return - - # check for the metadata file, if it exists... - abs_metadata_filename = os.path.abspath(metadata_file_path) - if not os.path.exists(abs_metadata_filename): - print "Can't find a file with filename '%s'" % metadata_file_path + error = _(u'File at {path} not found, use -h flag for help'.format( + path=args.metadata_path)) + print error return + abs_metadata_filename = os.path.abspath(metadata_path) + abs_metadata_dir = os.path.dirname(abs_metadata_filename) upload_limit, max_file_size = get_upload_file_limits(user) def maybe_unicodeify(some_string): @@ -116,30 +88,38 @@ zip files and directories" else: return unicode(some_string) - with file(abs_location_filename, 'r') as all_locations: - contents = all_locations.read() - media_locations = parse_csv_file(contents) - with file(abs_metadata_filename, 'r') as all_metadata: contents = all_metadata.read() media_metadata = parse_csv_file(contents) - for media_id in media_locations.keys(): + for media_id, file_metadata in media_metadata.iteritems(): files_attempted += 1 - - file_metadata = media_metadata[media_id] + # In case the metadata was not uploaded initialize an empty dictionary. + json_ld_metadata = compact_and_validate({}) + + # Get all metadata entries starting with 'media' as variables and then + # delete them because those are for internal use only. + original_location = file_metadata['media:location'] + file_metadata = dict([(key, value) + for key, value in file_metadata.iteritems() if + key.split(":")[0] != 'media']) try: json_ld_metadata = compact_and_validate(file_metadata) except ValidationError, exc: - print "Error with '%s' value '%s': %s" % ( - media_id, exc.path[0], exc.message) + error = _(u"""Error with media '{media_id}' value '{error_path}': {error_msg} +Metadata was not uploaded.""".format( + media_id=media_id, + error_path=exc.path[0], + error_msg=exc.message)) + print error continue - original_location = media_locations[media_id]['media:original'] url = urlparse(original_location) - title = json_ld_metadata.get('dcterms:title') - description = json_ld_metadata.get('dcterms:description') + ### Pull the important media information for mediagoblin from the + ### metadata, if it is provided. + title = json_ld_metadata.get('dc:title') + description = json_ld_metadata.get('dc:description') license = json_ld_metadata.get('license') filename = url.path.split()[-1] @@ -153,14 +133,14 @@ zip files and directories" if os.path.isabs(path): file_abs_path = os.path.abspath(path) else: - file_path = os.path.join(dir_path, path) + file_path = os.path.join(abs_metadata_dir, path) file_abs_path = os.path.abspath(file_path) try: media_file = file(file_abs_path, 'r') except IOError: - print "\ -FAIL: Local file {filename} could not be accessed.".format(filename=filename) - print "Skipping it." + print _(u"""\ +FAIL: Local file {filename} could not be accessed. +{filename} will not be uploaded.""".format(filename=filename)) continue try: submit_media( @@ -174,29 +154,36 @@ FAIL: Local file {filename} could not be accessed.".format(filename=filename) metadata=json_ld_metadata, tags_string=u"", upload_limit=upload_limit, max_file_size=max_file_size) - print "Successfully uploading {filename}!".format(filename=filename) - print "" + print _(u"""Successfully submitted {filename}! +Be sure to look at the Media Processing Panel on your website to be sure it +uploaded successfully.""".format(filename=filename)) files_uploaded += 1 except FileUploadLimit: - print "FAIL: This file is larger than the upload limits for this site." + print _( +u"FAIL: This file is larger than the upload limits for this site.") except UserUploadLimit: - print "FAIL: This file will put this user past their upload limits." + print _( +"FAIL: This file will put this user past their upload limits.") except UserPastUploadLimit: - print "FAIL: This user is already past their upload limits." - print "\ -{files_uploaded} out of {files_attempted} files successfully uploaded".format( + print _("FAIL: This user is already past their upload limits.") + print _( +"{files_uploaded} out of {files_attempted} files successfully submitted".format( files_uploaded=files_uploaded, - files_attempted=files_attempted) - teardown(temp_files) + files_attempted=files_attempted)) def parse_csv_file(file_contents): + """ + The helper function which converts the csv file into a dictionary where each + item's key is the provided value 'media:id' and each item's value is another + dictionary. + """ list_of_contents = file_contents.split('\n') key, lines = (list_of_contents[0].split(','), list_of_contents[1:]) objects_dict = {} - # Build a dictionaryfrom mediagoblin.tools.translate import lazy_pass_to_ugettext as _ + # Build a dictionary for line in lines: if line.isspace() or line == '': continue values = csv_reader([line]).next() @@ -207,7 +194,3 @@ def parse_csv_file(file_contents): return objects_dict - -def teardown(temp_files): - for temp_file in temp_files: - subprocess.call(['rm','-r',temp_file]) diff --git a/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html b/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html index db12f149..73b5ec52 100644 --- a/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html +++ b/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html @@ -32,3 +32,9 @@ {%- endfor %} </table> {% endif %} +{% if request.user and request.user.has_privilege('admin') %} + <a href="{{ request.urlgen('mediagoblin.edit.metadata', + user=media_entry.get_uploader.username, + media_id=media_entry.id) }}"> + {% trans %}Edit Metadata{% endtrans %}</a> +{% endif %} diff --git a/mediagoblin/static/css/base.css b/mediagoblin/static/css/base.css index 32c6c6cb..df0e850b 100644 --- a/mediagoblin/static/css/base.css +++ b/mediagoblin/static/css/base.css @@ -610,6 +610,24 @@ a img.media_image { cursor: zoom-in; } +table.metadata_info { + font-size:85%; + margin-left:10px; +} + +table.metadata_info tr.highlight { + color:#f7f7f7; +} + +table.metadata_info th { + font-weight: bold; + border-spacing: 10px; + text-align: left; +} +table.metadata_info td { + padding: 4px 8px; +} + /* icons */ img.media_icon { @@ -938,3 +956,21 @@ p.verifier { none repeat scroll 0% 0% rgb(221, 221, 221); padding: 1em 0px; } + +/* for the media metadata editing table */ +table.metadata_editor { + + margin: 10px auto; + width: 1000px; +} + +table.metadata_editor tr th { + width:100px; +} + +table.metadata_editor tr td { + width:300px; +} +table.metadata_editor tr td.form_field_input input { + width:300px; +} diff --git a/mediagoblin/templates/mediagoblin/edit/metadata.html b/mediagoblin/templates/mediagoblin/edit/metadata.html new file mode 100644 index 00000000..b5a52e5f --- /dev/null +++ b/mediagoblin/templates/mediagoblin/edit/metadata.html @@ -0,0 +1,97 @@ +{# +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +#} +{%- extends "mediagoblin/base.html" %} +{% import "/mediagoblin/utils/wtforms.html" as wtforms_util %} +{% block mediagoblin_head %} + <script> + function add_new_row(table_id, row_number, input_prefix) { + new_row = $('<tr>').append( + $('<td>').attr( + 'class','form_field_input').append( + $('<input>').attr('name', + input_prefix + row_number + "-identifier").attr('id', + input_prefix + row_number + "-identifier") + ) + ).append( + $('<td>').attr( + 'class','form_field_input').append( + $('<input>').attr('name', + input_prefix + row_number + "-value").attr('id', + input_prefix + row_number + "-value") + ) + ); + $(table_id).append(new_row); + } + function clear_empty_rows(list_id) { + $('table'+list_id+' tr').each(function(row){ + id_input = $(this).find('td').find('input'); + value_input = $(this).find('td').next().find('input'); + if ((value_input.attr('value') == "") && + (id_input.attr('value') == "")) { + $(this).remove(); + } + }) + } + + $(document).ready(function(){ + var metadata_lines = {{ form.media_metadata | length }}; + $("#add_new_metadata_row").click(function(){ + add_new_row("#metadata_list", + metadata_lines, + 'media_metadata-'); + metadata_lines += 1; + }) + }) + $("#clear_empty_rows").click(function(){ + clear_empty_rows("#metadata_list"); + }) + }) + </script> +{% endblock %} +{% block mediagoblin_content %} + <h2>{% trans media_name=media.title -%} + Metadata for "{{ media_name }}"{% endtrans %}</h2> + <form action="" method="POST" id="metadata_form"> + +<!-- This table holds all the information about the media entry's metadata --> + <h3>{% trans %}Data{% endtrans %}</h3> + <table class="metadata_editor" id="metadata_list" > + {{ wtforms_util.render_fieldlist_as_table_rows(form.media_metadata) }} + </table> + +<!-- These are the buttons you use to control the form --> + <table class="metadata_editor" id="buttons_bottom"> + <tr> + <th></th> + <td><input type=button value="{% trans %}Add new Row{% endtrans %}" + class="button_action" id="add_new_metadata_row" /> + </td> + <th></th> + <td><input type=submit value="{% trans %}Update Metadata{% endtrans %}" + class="button_action_highlight" /></td> + </tr> + <tr> + <th></th> + <td><input type=button value="{% trans %}Clear empty Rows{% endtrans %}" + class="button_action" id="clear_empty_rows" /></td> + </tr> + </table> + {{ csrf_token }} + </form> + +{% endblock mediagoblin_content %} diff --git a/mediagoblin/templates/mediagoblin/utils/wtforms.html b/mediagoblin/templates/mediagoblin/utils/wtforms.html index e079274e..c83d53f1 100644 --- a/mediagoblin/templates/mediagoblin/utils/wtforms.html +++ b/mediagoblin/templates/mediagoblin/utils/wtforms.html @@ -70,23 +70,56 @@ {# Auto-render a form as a table #} {% macro render_table(form) -%} {% for field in form %} - <tr> - <th>{{ field.label.text }}</th> - <td> - {{field}} - {% if field.errors %} - <br /> - <ul class="errors"> - {% for error in field.errors %} - <li>{{error}}</li> - {% endfor %} - </ul> - {% endif %} - </td> - </tr> + render_field_as_table_row(field) {% endfor %} {%- endmacro %} +{% macro render_form_as_table_row(form) %} + <tr> + {%- for field in form %} + <th>{{ render_label_p(field) }}</th> + <td class="form_field_input"> + {{field}} + {%- if field.errors -%} + <br /> + <ul class="errors"> + {% for error in field.errors %} + <li>{{error}}</li> + {%- endfor %} + </ul> + {%- endif -%} + </td> + {%- endfor %} + </tr> +{%- endmacro %} + +{% macro render_field_as_table_row(field) %} + <tr> + <th>{{ field.label.text }}</th> + <td> + {{field}} + {% if field.errors %} + <br /> + <ul class="errors"> + {% for error in field.errors %} + <li>{{error}}</li> + {% endfor %} + </ul> + {% endif %} + </td> + </tr> +{% endmacro %} + +{% macro render_fieldlist_as_table_rows(fieldlist) %} + {% for field in fieldlist -%} + {%- if field.type == 'FormField' %} + {{ render_form_as_table_row(field) }} + {%- else %} + {{ render_field_as_table_row(field) }} + {%- endif %} + {% endfor -%} +{% endmacro %} + {# Render a boolean field #} {% macro render_bool(field) %} <div class="boolean"> diff --git a/mediagoblin/user_pages/routing.py b/mediagoblin/user_pages/routing.py index f0f4d8b7..8eb51c8d 100644 --- a/mediagoblin/user_pages/routing.py +++ b/mediagoblin/user_pages/routing.py @@ -101,3 +101,7 @@ add_route('mediagoblin.edit.edit_media', add_route('mediagoblin.edit.attachments', '/u/<string:user>/m/<int:media_id>/attachments/', 'mediagoblin.edit.views:edit_attachments') + +add_route('mediagoblin.edit.metadata', + '/u/<string:user>/m/<int:media_id>/metadata/', + 'mediagoblin.edit.views:edit_metadata') |