diff options
23 files changed, 939 insertions, 21 deletions
diff --git a/docs/source/siteadmin/commandline-upload.rst b/docs/source/siteadmin/commandline-upload.rst index be19df58..742c0cb2 100644 --- a/docs/source/siteadmin/commandline-upload.rst +++ b/docs/source/siteadmin/commandline-upload.rst @@ -39,3 +39,54 @@ You can also pass in the `--celery` option if you would prefer that your media be passed over to celery to be processed rather than be processed immediately. +============================ +Command-line batch uploading +============================ + +There's another way to submit media, and it can be much more powerful, although +it is a bit more complex. + + ./bin/gmg batchaddmedia admin /path/to/your/metadata.csv + +This is an example of what a script may look like. The important part here is +that you have to create the 'metadata.csv' file.:: + + media:location,dcterms:title,dcterms:creator,dcterms:type + "http://www.example.net/path/to/nap.png","Goblin taking a nap",,"Image" + "http://www.example.net/path/to/snore.ogg","Goblin Snoring","Me","Audio" + +The above is an example of a very simple metadata.csv file. The batchaddmedia +script would read this and attempt to upload only two pieces of media, and would +be able to automatically name them appropriately. + +The csv file +============ +The media:location column +------------------------- +The media:location column is the one column that is absolutely necessary for +uploading your media. This gives a path to each piece of media you upload. This +can either a path to a local file or a direct link to remote media (with the +link in http format). As you can see in the example above the (fake) media was +stored remotely on "www.example.net". + +Other columns +------------- +Other columns can be used to provide detailed metadata about each media entry. +Our metadata system accepts any information provided for in the +`RDFa Core Initial Context`_, and the batchupload script recognizes all of the +resources provided within it. + +.. _RDFa Core Initial Context: http://www.w3.org/2011/rdfa-context/rdfa-1.1 + +The uploader may include the metadata for each piece of media, or +leave them blank if they want to. A few columns from `Dublin Core`_ are +notable because the batchaddmedia script uses them to set the default +information of uploaded media entries. + +.. _Dublin Core: http://wiki.dublincore.org/index.php/User_Guide + +- **dc:title** sets a title for your media entry. If this is left blank, the media entry will be named according to the filename of the file being uploaded. +- **dc:description** sets a description of your media entry. If this is left blank the media entry's description will not be filled in. +- **dc:rights** will set a license for your media entry `if` the data provided is a valid URI. If this is left blank 'All Rights Reserved' will be selected. + +You can of course, change these values later. diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py index 426080a2..8dac3214 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/migrations.py @@ -31,6 +31,7 @@ from mediagoblin.db.migration_tools import ( RegisterMigration, inspect_table, replace_table_hack) from mediagoblin.db.models import (MediaEntry, Collection, MediaComment, User, Privilege) +from mediagoblin.db.extratypes import JSONEncoded, MutationDict MIGRATIONS = {} @@ -720,3 +721,14 @@ def drop_MediaEntry_collected(db): media_collected.drop() db.commit() + +@RegisterMigration(20, MIGRATIONS) +def add_metadata_column(db): + metadata = MetaData(bind=db.bind) + + media_entry = inspect_table(metadata, 'core__media_entries') + + col = Column('media_metadata', MutationDict.as_mutable(JSONEncoded)) + col.create(media_entry) + + db.commit() diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index b750375d..defa0849 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -264,6 +264,7 @@ class MediaEntry(Base, MediaEntryMixin): cascade="all, delete-orphan" ) collections = association_proxy("collections_helper", "in_collection") + media_metadata = Column(MutationDict.as_mutable(JSONEncoded)) ## TODO # fail_error diff --git a/mediagoblin/edit/forms.py b/mediagoblin/edit/forms.py index 2c9b5e99..c2355980 100644 --- a/mediagoblin/edit/forms.py +++ b/mediagoblin/edit/forms.py @@ -122,3 +122,12 @@ class ChangeEmailForm(wtforms.Form): [wtforms.validators.Required()], description=_( "Enter your password to prove you own this account.")) + +class MetaDataForm(wtforms.Form): + identifier = wtforms.TextField(_(u'Identifier')) + value = wtforms.TextField(_(u'Value')) + +class EditMetaDataForm(wtforms.Form): + media_metadata = wtforms.FieldList( + wtforms.FormField(MetaDataForm, label="") + ) diff --git a/mediagoblin/edit/views.py b/mediagoblin/edit/views.py index 80590875..34021257 100644 --- a/mediagoblin/edit/views.py +++ b/mediagoblin/edit/views.py @@ -17,6 +17,7 @@ from datetime import datetime from itsdangerous import BadSignature +from pyld import jsonld from werkzeug.exceptions import Forbidden from werkzeug.utils import secure_filename @@ -29,8 +30,10 @@ from mediagoblin.edit import forms from mediagoblin.edit.lib import may_edit_media from mediagoblin.decorators import (require_active_login, active_user_from_url, get_media_entry_by_id, user_may_alter_collection, - get_user_collection) + get_user_collection, user_has_privilege, + user_not_banned) from mediagoblin.tools.crypto import get_timed_signer_url +from mediagoblin.tools.metadata import compact_and_validate from mediagoblin.tools.mail import email_debug_message from mediagoblin.tools.response import (render_to_response, redirect, redirect_obj, render_404) @@ -432,3 +435,35 @@ def change_email(request): 'mediagoblin/edit/change_email.html', {'form': form, 'user': user}) + +@user_has_privilege(u'admin') +@require_active_login +@get_media_entry_by_id +def edit_metadata(request, media): + form = forms.EditMetaDataForm(request.form) + if request.method == "POST" and form.validate(): + metadata_dict = dict([(row['identifier'],row['value']) + for row in form.media_metadata.data]) + json_ld_metadata = compact_and_validate(metadata_dict) + media.media_metadata = json_ld_metadata + media.save() + return redirect_obj(request, media) + + if media.media_metadata: + for identifier, value in media.media_metadata.iteritems(): + if identifier == "@context": continue + form.media_metadata.append_entry({ + 'identifier':identifier, + 'value':value}) + else: + form.media_metadata.append_entry({ + 'identifier':"", + 'value':""}) + form.media_metadata.append_entry({ + 'identifier':"", + 'value':""}) + return render_to_response( + request, + 'mediagoblin/edit/metadata.html', + {'form':form, + 'media':media}) diff --git a/mediagoblin/gmg_commands/__init__.py b/mediagoblin/gmg_commands/__init__.py index 9de4130e..fd546aac 100644 --- a/mediagoblin/gmg_commands/__init__.py +++ b/mediagoblin/gmg_commands/__init__.py @@ -57,6 +57,10 @@ SUBCOMMAND_MAP = { 'setup': 'mediagoblin.gmg_commands.deletemedia:parser_setup', 'func': 'mediagoblin.gmg_commands.deletemedia:deletemedia', 'help': 'Delete media entries'}, + 'batchaddmedia': { + 'setup': 'mediagoblin.gmg_commands.batchaddmedia:parser_setup', + 'func': 'mediagoblin.gmg_commands.batchaddmedia:batchaddmedia', + 'help': 'Add many media entries at once'}, # 'theme': { # 'setup': 'mediagoblin.gmg_commands.theme:theme_parser_setup', # 'func': 'mediagoblin.gmg_commands.theme:theme', diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py new file mode 100644 index 00000000..75e7b7c5 --- /dev/null +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -0,0 +1,196 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import tempfile, tarfile, zipfile, subprocess, requests +from csv import reader as csv_reader +from urlparse import urlparse +from pyld import jsonld + +from mediagoblin.gmg_commands import util as commands_util +from mediagoblin.submit.lib import ( + submit_media, get_upload_file_limits, + FileUploadLimit, UserUploadLimit, UserPastUploadLimit) +from mediagoblin.tools.metadata import compact_and_validate +from mediagoblin.tools.translate import pass_to_ugettext as _ +from jsonschema.exceptions import ValidationError + + +def parser_setup(subparser): + subparser.description = """\ +This command allows the administrator to upload many media files at once.""" + subparser.epilog = _(u"""For more information about how to properly run this +script (and how to format the metadata csv file), read the MediaGoblin +documentation page on command line uploading +<http://docs.mediagoblin.org/siteadmin/commandline-upload.html>""") + subparser.add_argument( + 'username', + help=_(u"Name of user these media entries belong to")) + subparser.add_argument( + 'metadata_path', + help=_( +u"""Path to the csv file containing metadata information.""")) + subparser.add_argument( + '--celery', + action='store_true', + help=_(u"Don't process eagerly, pass off to celery")) + + +def batchaddmedia(args): + # Run eagerly unless explicetly set not to + if not args.celery: + os.environ['CELERY_ALWAYS_EAGER'] = 'true' + + app = commands_util.setup_app(args) + + files_uploaded, files_attempted = 0, 0 + + # get the user + user = app.db.User.query.filter_by(username=args.username.lower()).first() + if user is None: + print _(u"Sorry, no user by username '{username}' exists".format( + username=args.username)) + return + + upload_limit, max_file_size = get_upload_file_limits(user) + temp_files = [] + + if os.path.isfile(args.metadata_path): + metadata_path = args.metadata_path + + else: + error = _(u'File at {path} not found, use -h flag for help'.format( + path=args.metadata_path)) + print error + return + + abs_metadata_filename = os.path.abspath(metadata_path) + abs_metadata_dir = os.path.dirname(abs_metadata_filename) + upload_limit, max_file_size = get_upload_file_limits(user) + + def maybe_unicodeify(some_string): + # this is kinda terrible + if some_string is None: + return None + else: + return unicode(some_string) + + with file(abs_metadata_filename, 'r') as all_metadata: + contents = all_metadata.read() + media_metadata = parse_csv_file(contents) + + for media_id, file_metadata in media_metadata.iteritems(): + files_attempted += 1 + # In case the metadata was not uploaded initialize an empty dictionary. + json_ld_metadata = compact_and_validate({}) + + # Get all metadata entries starting with 'media' as variables and then + # delete them because those are for internal use only. + original_location = file_metadata['media:location'] + file_metadata = dict([(key, value) + for key, value in file_metadata.iteritems() if + key.split(":")[0] != 'media']) + try: + json_ld_metadata = compact_and_validate(file_metadata) + except ValidationError, exc: + error = _(u"""Error with media '{media_id}' value '{error_path}': {error_msg} +Metadata was not uploaded.""".format( + media_id=media_id, + error_path=exc.path[0], + error_msg=exc.message)) + print error + continue + + url = urlparse(original_location) + + ### Pull the important media information for mediagoblin from the + ### metadata, if it is provided. + title = json_ld_metadata.get('dc:title') + description = json_ld_metadata.get('dc:description') + + license = json_ld_metadata.get('license') + filename = url.path.split()[-1] + + if url.scheme == 'http': + res = requests.get(url.geturl(), stream=True) + media_file = res.raw + + elif url.scheme == '': + path = url.path + if os.path.isabs(path): + file_abs_path = os.path.abspath(path) + else: + file_path = os.path.join(abs_metadata_dir, path) + file_abs_path = os.path.abspath(file_path) + try: + media_file = file(file_abs_path, 'r') + except IOError: + print _(u"""\ +FAIL: Local file {filename} could not be accessed. +{filename} will not be uploaded.""".format(filename=filename)) + continue + try: + submit_media( + mg_app=app, + user=user, + submitted_file=media_file, + filename=filename, + title=maybe_unicodeify(title), + description=maybe_unicodeify(description), + license=maybe_unicodeify(license), + metadata=json_ld_metadata, + tags_string=u"", + upload_limit=upload_limit, max_file_size=max_file_size) + print _(u"""Successfully submitted {filename}! +Be sure to look at the Media Processing Panel on your website to be sure it +uploaded successfully.""".format(filename=filename)) + files_uploaded += 1 + except FileUploadLimit: + print _( +u"FAIL: This file is larger than the upload limits for this site.") + except UserUploadLimit: + print _( +"FAIL: This file will put this user past their upload limits.") + except UserPastUploadLimit: + print _("FAIL: This user is already past their upload limits.") + print _( +"{files_uploaded} out of {files_attempted} files successfully submitted".format( + files_uploaded=files_uploaded, + files_attempted=files_attempted)) + + +def parse_csv_file(file_contents): + """ + The helper function which converts the csv file into a dictionary where each + item's key is the provided value 'media:id' and each item's value is another + dictionary. + """ + list_of_contents = file_contents.split('\n') + key, lines = (list_of_contents[0].split(','), + list_of_contents[1:]) + objects_dict = {} + + # Build a dictionary + for line in lines: + if line.isspace() or line == '': continue + values = csv_reader([line]).next() + line_dict = dict([(key[i], val) + for i, val in enumerate(values)]) + media_id = line_dict['media:id'] + objects_dict[media_id] = (line_dict) + + return objects_dict + diff --git a/mediagoblin/plugins/metadata_display/__init__.py b/mediagoblin/plugins/metadata_display/__init__.py new file mode 100644 index 00000000..4a4c898f --- /dev/null +++ b/mediagoblin/plugins/metadata_display/__init__.py @@ -0,0 +1,41 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +import os +from pkg_resources import resource_filename + +from mediagoblin.plugins.metadata_display.lib import add_rdfa_to_readable_to_media_home +from mediagoblin.tools import pluginapi +from mediagoblin.tools.staticdirect import PluginStatic + +PLUGIN_DIR = os.path.dirname(__file__) + +def setup_plugin(): + # Register the template path. + pluginapi.register_template_path(os.path.join(PLUGIN_DIR, 'templates')) + + pluginapi.register_template_hooks( + {"media_sideinfo": "mediagoblin/plugins/metadata_display/metadata_table.html", + "head": "mediagoblin/plugins/metadata_display/bits/metadata_extra_head.html"}) + + +hooks = { + 'setup': setup_plugin, + 'static_setup': lambda: PluginStatic( + 'metadata_display', + resource_filename('mediagoblin.plugins.metadata_display', 'static') + ), + 'media_home_context':add_rdfa_to_readable_to_media_home + } diff --git a/mediagoblin/plugins/metadata_display/lib.py b/mediagoblin/plugins/metadata_display/lib.py new file mode 100644 index 00000000..0985208c --- /dev/null +++ b/mediagoblin/plugins/metadata_display/lib.py @@ -0,0 +1,31 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +def rdfa_to_readable(rdfa_predicate): + """ + A simple script to convert rdfa resource descriptors into a form more + accessible for humans. + """ + readable = rdfa_predicate.split(u":")[1].capitalize() + return readable + +def add_rdfa_to_readable_to_media_home(context): + """ + A context hook which adds the 'rdfa_to_readable' filter to + the media home page. + """ + context['rdfa_to_readable'] = rdfa_to_readable + return context diff --git a/mediagoblin/plugins/metadata_display/static/css/metadata_display.css b/mediagoblin/plugins/metadata_display/static/css/metadata_display.css new file mode 100644 index 00000000..e4612b02 --- /dev/null +++ b/mediagoblin/plugins/metadata_display/static/css/metadata_display.css @@ -0,0 +1,14 @@ +table.metadata_info { + font-size:85%; + margin-left:10px; +} + +table.metadata_info th { + font-weight: bold; + border-spacing: 10px; + text-align: left; +} +table.metadata_info td { + padding: 4px 8px; +} + diff --git a/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/bits/metadata_extra_head.html b/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/bits/metadata_extra_head.html new file mode 100644 index 00000000..4a380299 --- /dev/null +++ b/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/bits/metadata_extra_head.html @@ -0,0 +1,3 @@ + <link rel="stylesheet" type="text/css" + href="{{ request.staticdirect('css/metadata_display.css', + 'metadata_display') }}"/> diff --git a/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html b/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html new file mode 100644 index 00000000..3a9d872c --- /dev/null +++ b/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html @@ -0,0 +1,42 @@ +{# +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +#} +{% block metadata_information_table %} +{%- set metadata=media.media_metadata %} +{%- set metadata_context=metadata['@context'] %} +{%- if metadata %} + <h3>{% trans %}Metadata{% endtrans %}</h3> + {#- NOTE: In some smart future where the context is more extensible, + we will need to add to the prefix here-#} + <table class="metadata_info"> + {%- for key, value in metadata.iteritems() if not key=='@context' %} + {% if value -%} + <tr> + <th>{{ rdfa_to_readable(key) }}</th> + <td property="{{ key }}">{{ value }}</td> + </tr> + {%- endif -%} + {%- endfor %} + </table> +{% endif %} +{% if request.user and request.user.has_privilege('admin') %} + <a href="{{ request.urlgen('mediagoblin.edit.metadata', + user=media.get_uploader.username, + media_id=media.id) }}"> + {% trans %}Edit Metadata{% endtrans %}</a> +{% endif %} +{% endblock %} diff --git a/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html.orig b/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html.orig new file mode 100644 index 00000000..2bd1a14c --- /dev/null +++ b/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html.orig @@ -0,0 +1,60 @@ +{# +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +#} + +<<<<<<< HEAD:mediagoblin/templates/mediagoblin/utils/metadata_table.html +{%- macro render_table(request, media_entry, format_predicate) %} + {%- set metadata=media_entry.media_metadata %} + {%- set metadata_context=metadata['@context'] %} + {%- if metadata %} + <h3>{% trans %}Metadata Information{% endtrans %}</h3> + <table class="metadata_info"> + {%- for key, value in metadata.iteritems() if ( + not key=='@context' and value) %} + <tr {% if loop.index%2 == 1 %}class="highlight"{% endif %}> + <th>{{ format_predicate(key) }}</th> + <td property="{{ key }}"> + {{ value }}</td> + </tr> + {%- endfor %} + </table> + {% endif %} + {% if request.user and request.user.has_privilege('admin') %} + <a href="{{ request.urlgen('mediagoblin.edit.metadata', + user=media_entry.get_uploader.username, + media_id=media_entry.id) }}"> + {% trans %}Edit Metadata{% endtrans %}</a> + {% endif %} +{%- endmacro %} +======= +{%- set metadata=media.media_metadata %} +{%- set metadata_context=metadata['@context'] %} +{%- if metadata %} + {#- NOTE: In some smart future where the context is more extensible, + we will need to add to the prefix here-#} + <table> + {%- for key, value in metadata.iteritems() if not key=='@context' %} + {% if value -%} + <tr> + <td>{{ rdfa_to_readable(key) }}</td> + <td property="{{ key }}">{{ value }}</td> + </tr> + {%- endif -%} + {%- endfor %} + </table> +{% endif %} +>>>>>>> acfcaf6366bd4695c1c37c7aa8ff5a176b412e2a:mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html diff --git a/mediagoblin/routing.py b/mediagoblin/routing.py index 1393f01c..9f2584d3 100644 --- a/mediagoblin/routing.py +++ b/mediagoblin/routing.py @@ -28,7 +28,7 @@ _log = logging.getLogger(__name__) def get_url_map(): add_route('index', '/', 'mediagoblin.views:root_view') add_route('terms_of_service','/terms_of_service', - 'mediagoblin.views:terms_of_service') + 'mediagoblin.views:terms_of_service'), mount('/auth', auth_routes) mount('/mod', moderation_routes) diff --git a/mediagoblin/static/css/base.css b/mediagoblin/static/css/base.css index 32c6c6cb..a3b564ea 100644 --- a/mediagoblin/static/css/base.css +++ b/mediagoblin/static/css/base.css @@ -609,7 +609,6 @@ a img.media_image { cursor: -moz-zoom-in; cursor: zoom-in; } - /* icons */ img.media_icon { @@ -938,3 +937,21 @@ p.verifier { none repeat scroll 0% 0% rgb(221, 221, 221); padding: 1em 0px; } + +/* for the media metadata editing table */ +table.metadata_editor { + + margin: 10px auto; + width: 1000px; +} + +table.metadata_editor tr th { + width:100px; +} + +table.metadata_editor tr td { + width:300px; +} +table.metadata_editor tr td.form_field_input input { + width:300px; +} diff --git a/mediagoblin/static/metadata/rdfa11.jsonld b/mediagoblin/static/metadata/rdfa11.jsonld new file mode 100644 index 00000000..b2557233 --- /dev/null +++ b/mediagoblin/static/metadata/rdfa11.jsonld @@ -0,0 +1,48 @@ +{ + "@context": { + "cat": "http://www.w3.org/ns/dcat#", + "qb": "http://purl.org/linked-data/cube#", + "grddl": "http://www.w3.org/2003/g/data-view#", + "ma": "http://www.w3.org/ns/ma-ont#", + "owl": "http://www.w3.org/2002/07/owl#", + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "rdfa": "http://www.w3.org/ns/rdfa#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "rif": "http://www.w3.org/2007/rif#", + "rr": "http://www.w3.org/ns/r2rml#", + "skos": "http://www.w3.org/2004/02/skos/core#", + "skosxl": "http://www.w3.org/2008/05/skos-xl#", + "wdr": "http://www.w3.org/2007/05/powder#", + "void": "http://rdfs.org/ns/void#", + "wdrs": "http://www.w3.org/2007/05/powder-s#", + "xhv": "http://www.w3.org/1999/xhtml/vocab#", + "xml": "http://www.w3.org/XML/1998/namespace", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "prov": "http://www.w3.org/ns/prov#", + "sd": "http://www.w3.org/ns/sparql-service-description#", + "org": "http://www.w3.org/ns/org#", + "gldp": "http://www.w3.org/ns/people#", + "cnt": "http://www.w3.org/2008/content#", + "dcat": "http://www.w3.org/ns/dcat#", + "earl": "http://www.w3.org/ns/earl#", + "ht": "http://www.w3.org/2006/http#", + "ptr": "http://www.w3.org/2009/pointers#", + "cc": "http://creativecommons.org/ns#", + "ctag": "http://commontag.org/ns#", + "dc": "http://purl.org/dc/terms/", + "dc11": "http://purl.org/dc/elements/1.1/", + "dcterms": "http://purl.org/dc/terms/", + "foaf": "http://xmlns.com/foaf/0.1/", + "gr": "http://purl.org/goodrelations/v1#", + "ical": "http://www.w3.org/2002/12/cal/icaltzd#", + "og": "http://ogp.me/ns#", + "rev": "http://purl.org/stuff/rev#", + "sioc": "http://rdfs.org/sioc/ns#", + "v": "http://rdf.data-vocabulary.org/#", + "vcard": "http://www.w3.org/2006/vcard/ns#", + "schema": "http://schema.org/", + "describedby": "http://www.w3.org/2007/05/powder-s#describedby", + "license": "http://www.w3.org/1999/xhtml/vocab#license", + "role": "http://www.w3.org/1999/xhtml/vocab#role" + } +}
\ No newline at end of file diff --git a/mediagoblin/submit/lib.py b/mediagoblin/submit/lib.py index c70e2731..df3f7b62 100644 --- a/mediagoblin/submit/lib.py +++ b/mediagoblin/submit/lib.py @@ -98,7 +98,7 @@ class UserPastUploadLimit(UploadLimitError): def submit_media(mg_app, user, submitted_file, filename, title=None, description=None, - license=None, tags_string=u"", + license=None, metadata=None, tags_string=u"", upload_limit=None, max_file_size=None, callback_url=None, # If provided we'll do the feed_url update, otherwise ignore @@ -142,6 +142,8 @@ def submit_media(mg_app, user, submitted_file, filename, entry.license = license or None + entry.media_metadata = metadata or u"" + # Process the user's folksonomy "tags" entry.tags = convert_to_tag_list_of_dicts(tags_string) diff --git a/mediagoblin/templates/mediagoblin/edit/metadata.html b/mediagoblin/templates/mediagoblin/edit/metadata.html new file mode 100644 index 00000000..b5a52e5f --- /dev/null +++ b/mediagoblin/templates/mediagoblin/edit/metadata.html @@ -0,0 +1,97 @@ +{# +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +#} +{%- extends "mediagoblin/base.html" %} +{% import "/mediagoblin/utils/wtforms.html" as wtforms_util %} +{% block mediagoblin_head %} + <script> + function add_new_row(table_id, row_number, input_prefix) { + new_row = $('<tr>').append( + $('<td>').attr( + 'class','form_field_input').append( + $('<input>').attr('name', + input_prefix + row_number + "-identifier").attr('id', + input_prefix + row_number + "-identifier") + ) + ).append( + $('<td>').attr( + 'class','form_field_input').append( + $('<input>').attr('name', + input_prefix + row_number + "-value").attr('id', + input_prefix + row_number + "-value") + ) + ); + $(table_id).append(new_row); + } + function clear_empty_rows(list_id) { + $('table'+list_id+' tr').each(function(row){ + id_input = $(this).find('td').find('input'); + value_input = $(this).find('td').next().find('input'); + if ((value_input.attr('value') == "") && + (id_input.attr('value') == "")) { + $(this).remove(); + } + }) + } + + $(document).ready(function(){ + var metadata_lines = {{ form.media_metadata | length }}; + $("#add_new_metadata_row").click(function(){ + add_new_row("#metadata_list", + metadata_lines, + 'media_metadata-'); + metadata_lines += 1; + }) + }) + $("#clear_empty_rows").click(function(){ + clear_empty_rows("#metadata_list"); + }) + }) + </script> +{% endblock %} +{% block mediagoblin_content %} + <h2>{% trans media_name=media.title -%} + Metadata for "{{ media_name }}"{% endtrans %}</h2> + <form action="" method="POST" id="metadata_form"> + +<!-- This table holds all the information about the media entry's metadata --> + <h3>{% trans %}Data{% endtrans %}</h3> + <table class="metadata_editor" id="metadata_list" > + {{ wtforms_util.render_fieldlist_as_table_rows(form.media_metadata) }} + </table> + +<!-- These are the buttons you use to control the form --> + <table class="metadata_editor" id="buttons_bottom"> + <tr> + <th></th> + <td><input type=button value="{% trans %}Add new Row{% endtrans %}" + class="button_action" id="add_new_metadata_row" /> + </td> + <th></th> + <td><input type=submit value="{% trans %}Update Metadata{% endtrans %}" + class="button_action_highlight" /></td> + </tr> + <tr> + <th></th> + <td><input type=button value="{% trans %}Clear empty Rows{% endtrans %}" + class="button_action" id="clear_empty_rows" /></td> + </tr> + </table> + {{ csrf_token }} + </form> + +{% endblock mediagoblin_content %} diff --git a/mediagoblin/templates/mediagoblin/user_pages/media.html b/mediagoblin/templates/mediagoblin/user_pages/media.html index e01cce5c..949cbcde 100644 --- a/mediagoblin/templates/mediagoblin/user_pages/media.html +++ b/mediagoblin/templates/mediagoblin/user_pages/media.html @@ -230,9 +230,6 @@ {% template_hook("media_sideinfo") %} - {% block mediagoblin_sidebar %} - {% endblock %} - </div><!--end media_sidebar--> <div class="clear"></div> diff --git a/mediagoblin/templates/mediagoblin/utils/wtforms.html b/mediagoblin/templates/mediagoblin/utils/wtforms.html index e079274e..c83d53f1 100644 --- a/mediagoblin/templates/mediagoblin/utils/wtforms.html +++ b/mediagoblin/templates/mediagoblin/utils/wtforms.html @@ -70,23 +70,56 @@ {# Auto-render a form as a table #} {% macro render_table(form) -%} {% for field in form %} - <tr> - <th>{{ field.label.text }}</th> - <td> - {{field}} - {% if field.errors %} - <br /> - <ul class="errors"> - {% for error in field.errors %} - <li>{{error}}</li> - {% endfor %} - </ul> - {% endif %} - </td> - </tr> + render_field_as_table_row(field) {% endfor %} {%- endmacro %} +{% macro render_form_as_table_row(form) %} + <tr> + {%- for field in form %} + <th>{{ render_label_p(field) }}</th> + <td class="form_field_input"> + {{field}} + {%- if field.errors -%} + <br /> + <ul class="errors"> + {% for error in field.errors %} + <li>{{error}}</li> + {%- endfor %} + </ul> + {%- endif -%} + </td> + {%- endfor %} + </tr> +{%- endmacro %} + +{% macro render_field_as_table_row(field) %} + <tr> + <th>{{ field.label.text }}</th> + <td> + {{field}} + {% if field.errors %} + <br /> + <ul class="errors"> + {% for error in field.errors %} + <li>{{error}}</li> + {% endfor %} + </ul> + {% endif %} + </td> + </tr> +{% endmacro %} + +{% macro render_fieldlist_as_table_rows(fieldlist) %} + {% for field in fieldlist -%} + {%- if field.type == 'FormField' %} + {{ render_form_as_table_row(field) }} + {%- else %} + {{ render_field_as_table_row(field) }} + {%- endif %} + {% endfor -%} +{% endmacro %} + {# Render a boolean field #} {% macro render_bool(field) %} <div class="boolean"> diff --git a/mediagoblin/tools/metadata.py b/mediagoblin/tools/metadata.py new file mode 100644 index 00000000..b0cad9da --- /dev/null +++ b/mediagoblin/tools/metadata.py @@ -0,0 +1,218 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +import os +import copy +import json +import re +from pkg_resources import resource_filename + +import dateutil.parser +from pyld import jsonld +from jsonschema import validate, FormatChecker, draft4_format_checker +from jsonschema.compat import str_types + +from mediagoblin.tools.pluginapi import hook_handle + + + +######################################################## +## Set up the MediaGoblin format checker for json-schema +######################################################## + +URL_REGEX = re.compile( + r'^[a-z]+://([^/:]+|([0-9]{1,3}\.){3}[0-9]{1,3})(:[0-9]+)?(\/.*)?$', + re.IGNORECASE) + +def is_uri(instance): + """ + jsonschema uri validator + """ + if not isinstance(instance, str_types): + return True + + return URL_REGEX.match(instance) + +def is_datetime(instance): + """ + Is a date or datetime readable string. + """ + if not isinstance(instance, str_types): + return True + + return dateutil.parser.parse(instance) + + +class DefaultChecker(FormatChecker): + """ + Default MediaGoblin format checker... extended to include a few extra things + """ + checkers = copy.deepcopy(draft4_format_checker.checkers) + + +DefaultChecker.checkers[u"uri"] = (is_uri, ()) +DefaultChecker.checkers[u"date-time"] = (is_datetime, (ValueError, TypeError)) +DEFAULT_CHECKER = DefaultChecker() + +# Crappy default schema, checks for things we deem important + +DEFAULT_SCHEMA = { + "$schema": "http://json-schema.org/schema#", + + "type": "object", + "properties": { + "license": { + "format": "uri", + "type": "string", + }, + "dcterms:created": { + "format": "date-time", + "type": "string", + } + }, +} + + +def load_resource(package, resource_path): + """ + Load a resource, return it as a string. + + Args: + - package: package or module name. Eg "mediagoblin.media_types.audio" + - resource_path: path to get to this resource, a list of + directories and finally a filename. Will be joined with + os.path.sep. + """ + filename = resource_filename(package, os.path.sep.join(resource_path)) + return file(filename).read() + +def load_resource_json(package, resource_path): + """ + Load a resource json file, return a dictionary. + + Args: + - package: package or module name. Eg "mediagoblin.media_types.audio" + - resource_path: path to get to this resource, a list of + directories and finally a filename. Will be joined with + os.path.sep. + """ + return json.loads(load_resource(package, resource_path)) + + +################################## +## Load the MediaGoblin core files +################################## + + +BUILTIN_CONTEXTS = { + "http://www.w3.org/2013/json-ld-context/rdfa11": load_resource( + "mediagoblin", ["static", "metadata", "rdfa11.jsonld"])} + + +_CONTEXT_CACHE = {} + +def load_context(url): + """ + A self-aware document loader. For those contexts MediaGoblin + stores internally, load them from disk. + """ + if url in _CONTEXT_CACHE: + return _CONTEXT_CACHE[url] + + # See if it's one of our basic ones + document = BUILTIN_CONTEXTS.get(url, None) + + # No? See if we have an internal schema for this + if document is None: + document = hook_handle(("context_url_data", url)) + + # Okay, if we've gotten a document by now... let's package it up + if document is not None: + document = {'contextUrl': None, + 'documentUrl': url, + 'document': document} + + # Otherwise, use jsonld.load_document + else: + document = jsonld.load_document(url) + + # cache + _CONTEXT_CACHE[url] = document + return document + + +DEFAULT_CONTEXT = "http://www.w3.org/2013/json-ld-context/rdfa11" + +def compact_json(metadata, context=DEFAULT_CONTEXT): + """ + Compact json with supplied context. + + Note: Free floating" nodes are removed (eg a key just named + "bazzzzzz" which isn't specified in the context... something like + bazzzzzz:blerp will stay though. This is jsonld.compact behavior. + """ + compacted = jsonld.compact( + metadata, context, + options={ + "documentLoader": load_context, + # This allows for things like "license" and etc to be preserved + "expandContext": context, + "keepFreeFloatingNodes": False}) + + return compacted + + +def compact_and_validate(metadata, context=DEFAULT_CONTEXT, + schema=DEFAULT_SCHEMA): + """ + compact json with supplied context, check against schema for errors + + raises an exception (jsonschema.exceptions.ValidationError) if + there's an error. + + Note: Free floating" nodes are removed (eg a key just named + "bazzzzzz" which isn't specified in the context... something like + bazzzzzz:blerp will stay though. This is jsonld.compact behavior. + + You may wish to do this validation yourself... this is just for convenience. + """ + compacted = compact_json(metadata, context) + validate(metadata, schema, format_checker=DEFAULT_CHECKER) + + return compacted + + +def expand_json(metadata, context=DEFAULT_CONTEXT): + """ + Expand json, but be sure to use our documentLoader. + + By default this expands with DEFAULT_CONTEXT, but if you do not need this, + you can safely set this to None. + + # @@: Is the above a good idea? Maybe it should be set to None by + # default. + """ + options = { + "documentLoader": load_context} + if context is not None: + options["expandContext"] = context + return jsonld.expand(metadata, options=options) + + +def rdfa_to_readable(rdfa_predicate): + readable = rdfa_predicate.split(u":")[1].capitalize() + return readable diff --git a/mediagoblin/user_pages/routing.py b/mediagoblin/user_pages/routing.py index f0f4d8b7..8eb51c8d 100644 --- a/mediagoblin/user_pages/routing.py +++ b/mediagoblin/user_pages/routing.py @@ -101,3 +101,7 @@ add_route('mediagoblin.edit.edit_media', add_route('mediagoblin.edit.attachments', '/u/<string:user>/m/<int:media_id>/attachments/', 'mediagoblin.edit.views:edit_attachments') + +add_route('mediagoblin.edit.metadata', + '/u/<string:user>/m/<int:media_id>/metadata/', + 'mediagoblin.edit.views:edit_metadata') @@ -66,8 +66,11 @@ try: 'mock', 'itsdangerous', 'pytz', + 'six>=1.4.1', 'oauthlib==0.5.0', 'unidecode', + 'jsonschema', + 'requests', 'ExifRead', # PLEASE change this when we can; a dependency is forcing us to set this |