From 8aa015978c970d26992b3b405d3e58401b4b81e2 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 6 Feb 2014 15:17:06 -0500 Subject: This branch will create a commandline bulk-upload script. So far, I have written the code to read csv files into a usable dictionary. --- mediagoblin/gmg_commands/__init__.py | 4 ++ mediagoblin/gmg_commands/batchaddmedia.py | 110 ++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100644 mediagoblin/gmg_commands/batchaddmedia.py (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/__init__.py b/mediagoblin/gmg_commands/__init__.py index a1eb599d..1460733f 100644 --- a/mediagoblin/gmg_commands/__init__.py +++ b/mediagoblin/gmg_commands/__init__.py @@ -53,6 +53,10 @@ SUBCOMMAND_MAP = { 'setup': 'mediagoblin.gmg_commands.addmedia:parser_setup', 'func': 'mediagoblin.gmg_commands.addmedia:addmedia', 'help': 'Reprocess media entries'}, + 'batchaddmedia': { + 'setup': 'mediagoblin.gmg_commands.batchaddmedia:parser_setup', + 'func': 'mediagoblin.gmg_commands.batchaddmedia:batchaddmedia', + 'help': 'Reprocess many media entries'} # 'theme': { # 'setup': 'mediagoblin.gmg_commands.theme:theme_parser_setup', # 'func': 'mediagoblin.gmg_commands.theme:theme', diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py new file mode 100644 index 00000000..1c0f6784 --- /dev/null +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -0,0 +1,110 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import os + +from mediagoblin.gmg_commands import util as commands_util +from mediagoblin.submit.lib import ( + submit_media, get_upload_file_limits, + FileUploadLimit, UserUploadLimit, UserPastUploadLimit) + +from mediagoblin import mg_globals +import json, csv + +def parser_setup(subparser): + subparser.add_argument( + 'username', + help="Name of user this media entry belongs to") + subparser.add_argument( + 'locationfile', + help=( +"Local file on filesystem with the address of all the files to be uploaded")) + subparser.add_argument( + 'metadatafile', + help=( +"Local file on filesystem with metadata of all the files to be uploaded")) + subparser.add_argument( + "-l", "--license", + help=( + "License these media entry will be released under, if all the same" + "Should be a URL.")) + subparser.add_argument( + '--celery', + action='store_true', + help="Don't process eagerly, pass off to celery") + + +def batchaddmedia(args): + # Run eagerly unless explicetly set not to + if not args.celery: + os.environ['CELERY_ALWAYS_EAGER'] = 'true' + + app = commands_util.setup_app(args) + + # get the user + user = app.db.User.query.filter_by(username=args.username.lower()).first() + if user is None: + print "Sorry, no user by username '%s'" % args.username + return + + # check for the location file, if it exists... + location_filename = os.path.split(args.locationfile)[-1] + abs_location_filename = os.path.abspath(args.locationfile) + if not os.path.exists(abs_location_filename): + print "Can't find a file with filename '%s'" % args.locationfile + return + + # check for the location file, if it exists... + metadata_filename = os.path.split(args.metadatafile)[-1] + abs_metadata_filename = os.path.abspath(args.metadatafile) + if not os.path.exists(abs_metadata_filename): + print "Can't find a file with filename '%s'" % args.metadatafile + return + + upload_limit, max_file_size = get_upload_file_limits(user) + + def maybe_unicodeify(some_string): + # this is kinda terrible + if some_string is None: + return None + else: + return unicode(some_string) + + with file(abs_location_filename, 'r') as all_locations: + contents = all_locations.read() + media_locations = parse_csv_file(contents) + + with file(abs_metadata_filename, 'r') as all_metadata: + contents = all_metadata.read() + media_metadata = parse_csv_file(contents) + +def parse_csv_file(file_contents): + list_of_contents = file_contents.split('\n') + key, lines = (list_of_contents[0].split(','), + list_of_contents[1:]) + list_of_objects = [] + + # Build a dictionary + for line in lines: + if line.isspace() or line == '': continue + values = csv.reader([line]).next() + new_dict = dict([(key[i], val) + for i, val in enumerate(values)]) + list_of_objects.append(new_dict) + + return list_of_objects + + -- cgit v1.2.3 From 268f243074fd4cd97017062e4a9e9afd0a860b32 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Wed, 12 Feb 2014 14:37:00 -0500 Subject: The script now officially works! It works in many different situations, whether the media is to be uploaded is stored locally or on the web. Still have to clean up the code and look for errors. I may also refactor some of this into a functi- on to be used with a GUI frontend in another project. Lastly, I need to merge this with the metadata branch I've been working on, and convert the metadata.csv information into the proper format for the new metadata column. --- mediagoblin/gmg_commands/batchaddmedia.py | 130 +++++++++++++++++++++++++----- 1 file changed, 111 insertions(+), 19 deletions(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 1c0f6784..7d7a2d4f 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -15,6 +15,10 @@ # along with this program. If not, see . import os +import json, tempfile, urllib, tarfile, subprocess +from csv import reader as csv_reader +from urlparse import urlparse +from pyld import jsonld from mediagoblin.gmg_commands import util as commands_util from mediagoblin.submit.lib import ( @@ -22,20 +26,26 @@ from mediagoblin.submit.lib import ( FileUploadLimit, UserUploadLimit, UserPastUploadLimit) from mediagoblin import mg_globals -import json, csv def parser_setup(subparser): subparser.add_argument( 'username', help="Name of user this media entry belongs to") - subparser.add_argument( - 'locationfile', + target_type = subparser.add_mutually_exclusive_group() + target_type.add_argument('-d', + '--directory', action='store_const', + const='directory', dest='target_type', + default='directory', help=( +"Target is a directory")) + target_type.add_argument('-a', + '--archive', action='store_const', + const='archive', dest='target_type', help=( -"Local file on filesystem with the address of all the files to be uploaded")) +"Target is an archive.")) subparser.add_argument( - 'metadatafile', + 'target_path', help=( -"Local file on filesystem with metadata of all the files to be uploaded")) +"Path to a local archive or directory containing a location.csv and metadata.csv file")) subparser.add_argument( "-l", "--license", help=( @@ -59,19 +69,36 @@ def batchaddmedia(args): if user is None: print "Sorry, no user by username '%s'" % args.username return + + upload_limit, max_file_size = get_upload_file_limits(user) + temp_files = [] + + if args.target_type == 'archive': + dir_path = tempfile.mkdtemp() + temp_files.append(dir_path) + tar = tarfile.open(args.target_path) + tar.extractall(path=dir_path) + + elif args.target_type == 'directory': + dir_path = args.target_path + + location_file_path = "{dir_path}/location.csv".format( + dir_path=dir_path) + metadata_file_path = "{dir_path}/metadata.csv".format( + dir_path=dir_path) # check for the location file, if it exists... - location_filename = os.path.split(args.locationfile)[-1] - abs_location_filename = os.path.abspath(args.locationfile) + location_filename = os.path.split(location_file_path)[-1] + abs_location_filename = os.path.abspath(location_file_path) if not os.path.exists(abs_location_filename): - print "Can't find a file with filename '%s'" % args.locationfile + print "Can't find a file with filename '%s'" % location_file_path return - # check for the location file, if it exists... - metadata_filename = os.path.split(args.metadatafile)[-1] - abs_metadata_filename = os.path.abspath(args.metadatafile) + # check for the metadata file, if it exists... + metadata_filename = os.path.split(metadata_file_path)[-1] + abs_metadata_filename = os.path.abspath(metadata_file_path) if not os.path.exists(abs_metadata_filename): - print "Can't find a file with filename '%s'" % args.metadatafile + print "Can't find a file with filename '%s'" % metadata_file_path return upload_limit, max_file_size = get_upload_file_limits(user) @@ -91,20 +118,85 @@ def batchaddmedia(args): contents = all_metadata.read() media_metadata = parse_csv_file(contents) + dcterms_context = { 'dcterms':'http://purl.org/dc/terms/' } + + for media_id in media_locations.keys(): + file_metadata = media_metadata[media_id] + json_ld_metadata = jsonld.compact(file_metadata, dcterms_context) + original_location = media_locations[media_id]['media:original'] + url = urlparse(original_location) + + title = file_metadata.get('dcterms:title') + description = file_metadata.get('dcterms:description') + license = file_metadata.get('dcterms:license') + filename = url.path.split()[-1] + print "Working with {filename}".format(filename=filename) + + if url.scheme == 'http': + print "Downloading {filename}...".format( + filename=filename) + media_file = tempfile.TemporaryFile() + res = urllib.urlopen(url.geturl()) + media_file.write(res.read()) + media_file.seek(0) + + elif url.scheme == '': + path = url.path + if os.path.isabs(path): + file_abs_path = os.path.abspath(path) + else: + file_path = "{dir_path}/{local_path}".format( + dir_path=dir_path, + local_path=path) + file_abs_path = os.path.abspath(file_path) + try: + media_file = file(file_abs_path, 'r') + except IOError: + print "Local file {filename} could not be accessed.".format( + filename=filename) + print "Skipping it." + continue + print "Submitting {filename}...".format(filename=filename) + try: + submit_media( + mg_app=app, + user=user, + submitted_file=media_file, + filename=filename, + title=maybe_unicodeify(title), + description=maybe_unicodeify(description), + license=maybe_unicodeify(license), + tags_string=u"", + upload_limit=upload_limit, max_file_size=max_file_size) + print "Successfully uploading {filename}!".format(filename=filename) + print "" + except FileUploadLimit: + print "This file is larger than the upload limits for this site." + except UserUploadLimit: + print "This file will put this user past their upload limits." + except UserPastUploadLimit: + print "This user is already past their upload limits." + teardown(temp_files) + + + def parse_csv_file(file_contents): list_of_contents = file_contents.split('\n') key, lines = (list_of_contents[0].split(','), list_of_contents[1:]) - list_of_objects = [] + objects_dict = {} # Build a dictionary for line in lines: if line.isspace() or line == '': continue - values = csv.reader([line]).next() - new_dict = dict([(key[i], val) + values = csv_reader([line]).next() + line_dict = dict([(key[i], val) for i, val in enumerate(values)]) - list_of_objects.append(new_dict) + media_id = line_dict['media:id'] + objects_dict[media_id] = (line_dict) - return list_of_objects + return objects_dict - +def teardown(temp_files): + for temp_file in temp_files: + subprocess.call(['rm','-r',temp_file]) -- cgit v1.2.3 From 6c37aeaa33aa21d6937e392185df4d0f6bffef7a Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 13 Feb 2014 13:57:10 -0500 Subject: Minor change in the wording of argparsing. --- mediagoblin/gmg_commands/batchaddmedia.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 7d7a2d4f..2fd36dfb 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -36,12 +36,12 @@ def parser_setup(subparser): '--directory', action='store_const', const='directory', dest='target_type', default='directory', help=( -"Target is a directory")) +"Choose this option is the target is a directory.")) target_type.add_argument('-a', '--archive', action='store_const', const='archive', dest='target_type', help=( -"Target is an archive.")) +"Choose this option if the target is an archive.")) subparser.add_argument( 'target_path', help=( -- cgit v1.2.3 From 28ecc53a5aed89b5d567e0245e06aa5f34bd371d Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Wed, 19 Feb 2014 14:27:14 -0500 Subject: I made it so the command no longer requires the "Target type" to be provided, it now recognizes whether the target is a directory or an archive on its own. I added in a help message, which is still incomplete, but should make it easier for admins to know how to use this new command. I believe we should also provi- -de an example of the location.csv and metadata.csv files, so there is no conf- -usion. Also, I made it possible for the command to recognize zip files as a valid archive. I also made some minor changes to the commands description w/i the larger gmg command help menu. --- mediagoblin/gmg_commands/__init__.py | 2 +- mediagoblin/gmg_commands/batchaddmedia.py | 55 ++++++++++++++++++------------- 2 files changed, 34 insertions(+), 23 deletions(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/__init__.py b/mediagoblin/gmg_commands/__init__.py index 1460733f..55e85116 100644 --- a/mediagoblin/gmg_commands/__init__.py +++ b/mediagoblin/gmg_commands/__init__.py @@ -56,7 +56,7 @@ SUBCOMMAND_MAP = { 'batchaddmedia': { 'setup': 'mediagoblin.gmg_commands.batchaddmedia:parser_setup', 'func': 'mediagoblin.gmg_commands.batchaddmedia:batchaddmedia', - 'help': 'Reprocess many media entries'} + 'help': 'Add many media entries at once'} # 'theme': { # 'setup': 'mediagoblin.gmg_commands.theme:theme_parser_setup', # 'func': 'mediagoblin.gmg_commands.theme:theme', diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 2fd36dfb..d3ab7733 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -15,7 +15,7 @@ # along with this program. If not, see . import os -import json, tempfile, urllib, tarfile, subprocess +import json, tempfile, urllib, tarfile, zipfile, subprocess from csv import reader as csv_reader from urlparse import urlparse from pyld import jsonld @@ -28,29 +28,28 @@ from mediagoblin.submit.lib import ( from mediagoblin import mg_globals def parser_setup(subparser): + subparser.description = """\ +This command allows the administrator to upload many media files at once.""" subparser.add_argument( 'username', - help="Name of user this media entry belongs to") - target_type = subparser.add_mutually_exclusive_group() - target_type.add_argument('-d', - '--directory', action='store_const', - const='directory', dest='target_type', - default='directory', help=( -"Choose this option is the target is a directory.")) - target_type.add_argument('-a', - '--archive', action='store_const', - const='archive', dest='target_type', - help=( -"Choose this option if the target is an archive.")) + help="Name of user these media entries belong to") subparser.add_argument( 'target_path', - help=( -"Path to a local archive or directory containing a location.csv and metadata.csv file")) + help=("""\ +Path to a local archive or directory containing a "location.csv" and a +"metadata.csv" file. These are csv (comma seperated value) files with the +locations and metadata of the files to be uploaded. The location must be listed +with either the URL of the remote media file or the filesystem path of a local +file. The metadata should be provided with one column for each of the 15 Dublin +Core properties (http://dublincore.org/documents/dces/). Both "location.csv" and +"metadata.csv" must begin with a row demonstrating the order of the columns. We +have provided an example of these files at +""")) subparser.add_argument( "-l", "--license", help=( - "License these media entry will be released under, if all the same" - "Should be a URL.")) + "License these media entry will be released under, if all the same. " + "Should be a URL.")) subparser.add_argument( '--celery', action='store_true', @@ -67,26 +66,38 @@ def batchaddmedia(args): # get the user user = app.db.User.query.filter_by(username=args.username.lower()).first() if user is None: - print "Sorry, no user by username '%s'" % args.username + print "Sorry, no user by username '%s' exists" % args.username return upload_limit, max_file_size = get_upload_file_limits(user) temp_files = [] - if args.target_type == 'archive': + if tarfile.is_tarfile(args.target_path): dir_path = tempfile.mkdtemp() temp_files.append(dir_path) tar = tarfile.open(args.target_path) tar.extractall(path=dir_path) - elif args.target_type == 'directory': + elif zipfile.is_zipfile(args.target_path): + dir_path = tempfile.mkdtemp() + temp_files.append(dir_path) + zipped_file = zipfile.ZipFile(args.target_path) + zipped_file.extractall(path=dir_path) + + elif os.path.isdir(args.target_path): dir_path = args.target_path + else: + print "Couldn't recognize the file. This script only accepts tar files,\ +zip files and directories" + if dir_path.endswith('/'): + dir_path = dir_path[:-1] + location_file_path = "{dir_path}/location.csv".format( dir_path=dir_path) metadata_file_path = "{dir_path}/metadata.csv".format( dir_path=dir_path) - + # check for the location file, if it exists... location_filename = os.path.split(location_file_path)[-1] abs_location_filename = os.path.abspath(location_file_path) @@ -178,7 +189,7 @@ def batchaddmedia(args): print "This user is already past their upload limits." teardown(temp_files) - + def parse_csv_file(file_contents): list_of_contents = file_contents.split('\n') -- cgit v1.2.3 From 5c14f62d19abefb99f392cf1a92b07ec0de26bc4 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Fri, 21 Feb 2014 12:38:02 -0500 Subject: Changed some of the print messages as well as tweaked the order of the commands attempts to figure out what type of file the target file is. --- mediagoblin/gmg_commands/batchaddmedia.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index d3ab7733..678c8ab4 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -63,6 +63,8 @@ def batchaddmedia(args): app = commands_util.setup_app(args) + files_uploaded, files_attempted = 0, 0 + # get the user user = app.db.User.query.filter_by(username=args.username.lower()).first() if user is None: @@ -72,7 +74,10 @@ def batchaddmedia(args): upload_limit, max_file_size = get_upload_file_limits(user) temp_files = [] - if tarfile.is_tarfile(args.target_path): + if os.path.isdir(args.target_path): + dir_path = args.target_path + + elif tarfile.is_tarfile(args.target_path): dir_path = tempfile.mkdtemp() temp_files.append(dir_path) tar = tarfile.open(args.target_path) @@ -84,9 +89,6 @@ def batchaddmedia(args): zipped_file = zipfile.ZipFile(args.target_path) zipped_file.extractall(path=dir_path) - elif os.path.isdir(args.target_path): - dir_path = args.target_path - else: print "Couldn't recognize the file. This script only accepts tar files,\ zip files and directories" @@ -141,11 +143,9 @@ zip files and directories" description = file_metadata.get('dcterms:description') license = file_metadata.get('dcterms:license') filename = url.path.split()[-1] - print "Working with {filename}".format(filename=filename) + files_attempted += 1 if url.scheme == 'http': - print "Downloading {filename}...".format( - filename=filename) media_file = tempfile.TemporaryFile() res = urllib.urlopen(url.geturl()) media_file.write(res.read()) @@ -163,11 +163,10 @@ zip files and directories" try: media_file = file(file_abs_path, 'r') except IOError: - print "Local file {filename} could not be accessed.".format( - filename=filename) + print "\ +FAIL: Local file {filename} could not be accessed.".format(filename=filename) print "Skipping it." continue - print "Submitting {filename}...".format(filename=filename) try: submit_media( mg_app=app, @@ -181,12 +180,17 @@ zip files and directories" upload_limit=upload_limit, max_file_size=max_file_size) print "Successfully uploading {filename}!".format(filename=filename) print "" + files_uploaded += 1 except FileUploadLimit: - print "This file is larger than the upload limits for this site." + print "FAIL: This file is larger than the upload limits for this site." except UserUploadLimit: - print "This file will put this user past their upload limits." + print "FAIL: This file will put this user past their upload limits." except UserPastUploadLimit: - print "This user is already past their upload limits." + print "FAIL: This user is already past their upload limits." + print "\ +{files_uploaded} out of {files_attempted} files successfully uploaded".format( + files_uploaded=files_uploaded, + files_attempted=files_attempted) teardown(temp_files) -- cgit v1.2.3 From 3e76b2bc7712e012bdb2f971961adf96741e0df3 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 27 Mar 2014 13:31:04 -0400 Subject: Began work on metadata validation --- mediagoblin/gmg_commands/batchaddmedia.py | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 678c8ab4..83aea7b7 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -26,6 +26,7 @@ from mediagoblin.submit.lib import ( FileUploadLimit, UserUploadLimit, UserPastUploadLimit) from mediagoblin import mg_globals +from jsonschema import validate def parser_setup(subparser): subparser.description = """\ @@ -215,3 +216,35 @@ def parse_csv_file(file_contents): def teardown(temp_files): for temp_file in temp_files: subprocess.call(['rm','-r',temp_file]) + +def check_metadata_format(metadata_dict): + schema = json.loads(""" +{ + "$schema":"http://json-schema.org/schema#", + "properties":{ + "@context":{}, + "contributor":{}, + "coverage":{}, + "created":{}, + "creator":{}, + "date":{}, + "description":{}, + "format":{}, + "identifier":{}, + "language":{}, + "publisher":{}, + "relation":{}, + "rights" : { + "format":"uri", + "type":"string" + }, + "source":{}, + "subject":{}, + "title":{}, + "type":{} + }, + "additionalProperties": false, + "required":["title","@context"] +}""") + try: + validate(metadata_dict, schema) -- cgit v1.2.3 From 26b3d6cf27d8653bfcbd8caf9bec4abfb5c16c5a Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 27 Mar 2014 13:55:15 -0400 Subject: Added exception handling into the metadata format checking function. --- mediagoblin/gmg_commands/batchaddmedia.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 83aea7b7..f06bc2e8 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -24,9 +24,11 @@ from mediagoblin.gmg_commands import util as commands_util from mediagoblin.submit.lib import ( submit_media, get_upload_file_limits, FileUploadLimit, UserUploadLimit, UserPastUploadLimit) +from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ from mediagoblin import mg_globals -from jsonschema import validate +from jsonschema import validate +from jsonschema.exceptions import ValidationError def parser_setup(subparser): subparser.description = """\ @@ -135,7 +137,10 @@ zip files and directories" dcterms_context = { 'dcterms':'http://purl.org/dc/terms/' } for media_id in media_locations.keys(): - file_metadata = media_metadata[media_id] + file_metadata = media_metadata[media_id] + santized_metadata = check_metadata_format(file_metadata) + if sanitized_metadata == {}: continue + json_ld_metadata = jsonld.compact(file_metadata, dcterms_context) original_location = media_locations[media_id]['media:original'] url = urlparse(original_location) @@ -248,3 +253,14 @@ def check_metadata_format(metadata_dict): }""") try: validate(metadata_dict, schema) + output_dict = metadata_dict + except ValidationError, exc: + title = metadata_dict.get('title') or metadata_dict.get('media:id') or \ + _(u'UNKNOWN FILE') + print _( +u"""WARN: Could not find appropriate metadata for file {title}. File will be +skipped""".format(title=title)) + output_dict = {} + except: + raise + return output_dict -- cgit v1.2.3 From 8c7cccf6cc6d2a4f58d10a116a535854d6ae9e63 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 27 Mar 2014 14:11:12 -0400 Subject: Fixed up some fatal errors. Is still not ready. --- mediagoblin/gmg_commands/batchaddmedia.py | 46 ++++++++++++++++--------------- 1 file changed, 24 insertions(+), 22 deletions(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index f06bc2e8..414e969c 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -138,7 +138,7 @@ zip files and directories" for media_id in media_locations.keys(): file_metadata = media_metadata[media_id] - santized_metadata = check_metadata_format(file_metadata) + sanitized_metadata = check_metadata_format(file_metadata) if sanitized_metadata == {}: continue json_ld_metadata = jsonld.compact(file_metadata, dcterms_context) @@ -207,7 +207,7 @@ def parse_csv_file(file_contents): list_of_contents[1:]) objects_dict = {} - # Build a dictionary + # Build a dictionaryfrom mediagoblin.tools.translate import lazy_pass_to_ugettext as _ for line in lines: if line.isspace() or line == '': continue values = csv_reader([line]).next() @@ -228,38 +228,40 @@ def check_metadata_format(metadata_dict): "$schema":"http://json-schema.org/schema#", "properties":{ "@context":{}, - "contributor":{}, - "coverage":{}, - "created":{}, - "creator":{}, - "date":{}, - "description":{}, - "format":{}, - "identifier":{}, - "language":{}, - "publisher":{}, - "relation":{}, - "rights" : { + "dcterms:contributor":{}, + "dcterms:coverage":{}, + "dcterms:created":{}, + "dcterms:creator":{}, + "dcterms:date":{}, + "dcterms:description":{}, + "dcterms:format":{}, + "dcterms:identifier":{}, + "dcterms:language":{}, + "dcterms:publisher":{}, + "dcterms:relation":{}, + "dcterms:rights" : { "format":"uri", "type":"string" }, - "source":{}, - "subject":{}, - "title":{}, - "type":{} + "dcterms:source":{}, + "dcterms:subject":{}, + "dcterms:title":{}, + "dcterms:type":{}, + "media:id":{} }, "additionalProperties": false, - "required":["title","@context"] + "required":["dcterms:title","@context","media:id"] }""") + metadata_dict["@context"] = u"http://127.0.0.1:6543/metadata_context/v1/" try: validate(metadata_dict, schema) output_dict = metadata_dict except ValidationError, exc: - title = metadata_dict.get('title') or metadata_dict.get('media:id') or \ + title = metadata_dict.get('dcterms:title') or metadata_dict.get('media:id') or \ _(u'UNKNOWN FILE') print _( -u"""WARN: Could not find appropriate metadata for file {title}. File will be -skipped""".format(title=title)) +u"""WARN: Could not find appropriate metadata for file {title}. +File will be skipped""".format(title=title)) output_dict = {} except: raise -- cgit v1.2.3 From e46760d3155873803fe3ee0e1d10cd0142eacae1 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 27 Mar 2014 17:10:31 -0400 Subject: Fixed a minor error in the batch upload script and modified the json-ld context. --- mediagoblin/gmg_commands/batchaddmedia.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 414e969c..012a5ee4 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -137,6 +137,8 @@ zip files and directories" dcterms_context = { 'dcterms':'http://purl.org/dc/terms/' } for media_id in media_locations.keys(): + files_attempted += 1 + file_metadata = media_metadata[media_id] sanitized_metadata = check_metadata_format(file_metadata) if sanitized_metadata == {}: continue @@ -149,7 +151,6 @@ zip files and directories" description = file_metadata.get('dcterms:description') license = file_metadata.get('dcterms:license') filename = url.path.split()[-1] - files_attempted += 1 if url.scheme == 'http': media_file = tempfile.TemporaryFile() @@ -228,6 +229,7 @@ def check_metadata_format(metadata_dict): "$schema":"http://json-schema.org/schema#", "properties":{ "@context":{}, + "dcterms:contributor":{}, "dcterms:coverage":{}, "dcterms:created":{}, @@ -246,8 +248,7 @@ def check_metadata_format(metadata_dict): "dcterms:source":{}, "dcterms:subject":{}, "dcterms:title":{}, - "dcterms:type":{}, - "media:id":{} + "dcterms:type":{} }, "additionalProperties": false, "required":["dcterms:title","@context","media:id"] @@ -260,7 +261,7 @@ def check_metadata_format(metadata_dict): title = metadata_dict.get('dcterms:title') or metadata_dict.get('media:id') or \ _(u'UNKNOWN FILE') print _( -u"""WARN: Could not find appropriate metadata for file {title}. +u"""WARN: Could not find appropriate metadata for file "{title}". File will be skipped""".format(title=title)) output_dict = {} except: -- cgit v1.2.3 From 907d9626e356cfcbc6b3e47a92771650a8eee4e1 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 27 Mar 2014 17:29:34 -0400 Subject: Wrote more comprehensive error messages. --- mediagoblin/gmg_commands/batchaddmedia.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 012a5ee4..fe345d5f 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -229,7 +229,7 @@ def check_metadata_format(metadata_dict): "$schema":"http://json-schema.org/schema#", "properties":{ "@context":{}, - + "media:id":{}, "dcterms:contributor":{}, "dcterms:coverage":{}, "dcterms:created":{}, @@ -251,18 +251,32 @@ def check_metadata_format(metadata_dict): "dcterms:type":{} }, "additionalProperties": false, - "required":["dcterms:title","@context","media:id"] + "required":["dcterms:title","@context","media:id","bell"] }""") metadata_dict["@context"] = u"http://127.0.0.1:6543/metadata_context/v1/" try: validate(metadata_dict, schema) output_dict = metadata_dict except ValidationError, exc: - title = metadata_dict.get('dcterms:title') or metadata_dict.get('media:id') or \ - _(u'UNKNOWN FILE') - print _( -u"""WARN: Could not find appropriate metadata for file "{title}". -File will be skipped""".format(title=title)) + title = (metadata_dict.get('dcterms:title') or + metadata_dict.get('media:id') or _(u'UNKNOWN FILE')) + + if exc.validator == "additionalProperties": + message = _(u'Invalid metadata provided for file "{title}". This \ +script only accepts the Dublin Core metadata terms.'.format(title=title)) + + elif exc.validator == "required": + message = _( +u'All necessary metadata was not provided for file "{title}", you must include \ +a "dcterms:title" column for each media file'.format(title=title)) + + else: + message = _(u'Could not find appropriate metadata for file \ +"{title}".'.format(title=title)) + + print _(u"""WARN: {message} \nSkipping File...\n""".format( + message=message)) + output_dict = {} except: raise -- cgit v1.2.3 From 77d51d4f3378de8f3bbf54dd4c99e62399688800 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Thu, 3 Apr 2014 12:18:17 -0400 Subject: Fixed a bad get of 'dcterms:rights' and am throwing away the idea of an external context file for the json-ld because it feels unnecessary seeing as we are just using the dc core terms --- mediagoblin/gmg_commands/batchaddmedia.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index fe345d5f..68993aa2 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -15,7 +15,7 @@ # along with this program. If not, see . import os -import json, tempfile, urllib, tarfile, zipfile, subprocess +import tempfile, urllib, tarfile, zipfile, subprocess from csv import reader as csv_reader from urlparse import urlparse from pyld import jsonld @@ -149,7 +149,7 @@ zip files and directories" title = file_metadata.get('dcterms:title') description = file_metadata.get('dcterms:description') - license = file_metadata.get('dcterms:license') + license = file_metadata.get('dcterms:rights') filename = url.path.split()[-1] if url.scheme == 'http': @@ -201,7 +201,6 @@ FAIL: Local file {filename} could not be accessed.".format(filename=filename) teardown(temp_files) - def parse_csv_file(file_contents): list_of_contents = file_contents.split('\n') key, lines = (list_of_contents[0].split(','), @@ -219,16 +218,16 @@ def parse_csv_file(file_contents): return objects_dict + def teardown(temp_files): for temp_file in temp_files: subprocess.call(['rm','-r',temp_file]) + def check_metadata_format(metadata_dict): - schema = json.loads(""" -{ + schema = { "$schema":"http://json-schema.org/schema#", "properties":{ - "@context":{}, "media:id":{}, "dcterms:contributor":{}, "dcterms:coverage":{}, @@ -250,13 +249,14 @@ def check_metadata_format(metadata_dict): "dcterms:title":{}, "dcterms:type":{} }, - "additionalProperties": false, - "required":["dcterms:title","@context","media:id","bell"] -}""") - metadata_dict["@context"] = u"http://127.0.0.1:6543/metadata_context/v1/" + "additionalProperties": False, + "required":["dcterms:title","media:id"] +} try: validate(metadata_dict, schema) output_dict = metadata_dict + del output_dict['media:id'] + except ValidationError, exc: title = (metadata_dict.get('dcterms:title') or metadata_dict.get('media:id') or _(u'UNKNOWN FILE')) @@ -280,4 +280,5 @@ a "dcterms:title" column for each media file'.format(title=title)) output_dict = {} except: raise + return output_dict -- cgit v1.2.3 From 18a9c50db6db680070948fd42043ea0a89deaa6f Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Fri, 11 Apr 2014 13:06:09 -0400 Subject: Fixed incorrectly coded references to filesystem paths --- mediagoblin/gmg_commands/batchaddmedia.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 68993aa2..b058a47e 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -98,10 +98,8 @@ zip files and directories" if dir_path.endswith('/'): dir_path = dir_path[:-1] - location_file_path = "{dir_path}/location.csv".format( - dir_path=dir_path) - metadata_file_path = "{dir_path}/metadata.csv".format( - dir_path=dir_path) + location_file_path = os.path.join(dir_path,"location.csv") + metadata_file_path = os.path.join(dir_path, "metadata.csv") # check for the location file, if it exists... location_filename = os.path.split(location_file_path)[-1] @@ -163,9 +161,7 @@ zip files and directories" if os.path.isabs(path): file_abs_path = os.path.abspath(path) else: - file_path = "{dir_path}/{local_path}".format( - dir_path=dir_path, - local_path=path) + file_path = os.path.join(dir_path, path) file_abs_path = os.path.abspath(file_path) try: media_file = file(file_abs_path, 'r') -- cgit v1.2.3 From ecea4847e8259125dec4617c6f11c7d7e3962925 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 15 Apr 2014 13:35:22 -0400 Subject: Added the 'requests' library as a dependency and switched over to using it to fetch remote pieces of media in the batchupload script --- mediagoblin/gmg_commands/batchaddmedia.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index b058a47e..deb6c5bd 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -15,9 +15,10 @@ # along with this program. If not, see . import os -import tempfile, urllib, tarfile, zipfile, subprocess +import tempfile, tarfile, zipfile, subprocess, requests from csv import reader as csv_reader from urlparse import urlparse +import requests from pyld import jsonld from mediagoblin.gmg_commands import util as commands_util @@ -151,10 +152,8 @@ zip files and directories" filename = url.path.split()[-1] if url.scheme == 'http': - media_file = tempfile.TemporaryFile() - res = urllib.urlopen(url.geturl()) - media_file.write(res.read()) - media_file.seek(0) + res = requests.get(url.geturl()) + media_file = res.raw elif url.scheme == '': path = url.path -- cgit v1.2.3 From 45f426ddee9900439c086a8bb3d1cfaedf3eca6f Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Mon, 21 Apr 2014 12:07:33 -0400 Subject: Made it possible to submit media with the metadata provided --- mediagoblin/gmg_commands/batchaddmedia.py | 1 + 1 file changed, 1 insertion(+) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index deb6c5bd..b6fd2763 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -178,6 +178,7 @@ FAIL: Local file {filename} could not be accessed.".format(filename=filename) title=maybe_unicodeify(title), description=maybe_unicodeify(description), license=maybe_unicodeify(license), + metadata=json_ld_metadata, tags_string=u"", upload_limit=upload_limit, max_file_size=max_file_size) print "Successfully uploading {filename}!".format(filename=filename) -- cgit v1.2.3 From e00ce53ef63abb20524399518e260c9262cc041b Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Mon, 21 Apr 2014 12:18:29 -0400 Subject: I had imported requests twice --- mediagoblin/gmg_commands/batchaddmedia.py | 1 - 1 file changed, 1 deletion(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index b6fd2763..f50425f3 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -18,7 +18,6 @@ import os import tempfile, tarfile, zipfile, subprocess, requests from csv import reader as csv_reader from urlparse import urlparse -import requests from pyld import jsonld from mediagoblin.gmg_commands import util as commands_util -- cgit v1.2.3 From 7ff99dabfbb3e854afe2aba17a79a0aee9062e44 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Mon, 21 Apr 2014 12:29:00 -0400 Subject: Fixed a problem that was causing errors in batch uploading remote files. --- mediagoblin/gmg_commands/batchaddmedia.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index f50425f3..61a068d2 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -151,7 +151,7 @@ zip files and directories" filename = url.path.split()[-1] if url.scheme == 'http': - res = requests.get(url.geturl()) + res = requests.get(url.geturl(), stream=True) media_file = res.raw elif url.scheme == '': -- cgit v1.2.3 From 9f5d388ec01c195ffbacc4a1fd876fb507a6f62d Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Mon, 21 Apr 2014 19:07:28 -0400 Subject: In the middle of some major changes --- mediagoblin/gmg_commands/batchaddmedia.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 61a068d2..43c24f6d 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -132,7 +132,8 @@ zip files and directories" contents = all_metadata.read() media_metadata = parse_csv_file(contents) - dcterms_context = { 'dcterms':'http://purl.org/dc/terms/' } + metadata_context = { 'dcterms':'http://purl.org/dc/terms/', + 'xsd': 'http://www.w3.org/2001/XMLSchema#'} for media_id in media_locations.keys(): files_attempted += 1 @@ -141,13 +142,14 @@ zip files and directories" sanitized_metadata = check_metadata_format(file_metadata) if sanitized_metadata == {}: continue - json_ld_metadata = jsonld.compact(file_metadata, dcterms_context) + json_ld_metadata = jsonld.compact(build_json_ld_metadata(file_metadata), + metadata_context) original_location = media_locations[media_id]['media:original'] url = urlparse(original_location) - title = file_metadata.get('dcterms:title') - description = file_metadata.get('dcterms:description') - license = file_metadata.get('dcterms:rights') + title = sanitized_metadata.get('dcterms:title') + description = sanitized_metadata.get('dcterms:description') + license = sanitized_metadata.get('dcterms:rights') filename = url.path.split()[-1] if url.scheme == 'http': @@ -218,6 +220,19 @@ def teardown(temp_files): for temp_file in temp_files: subprocess.call(['rm','-r',temp_file]) +def build_json_ld_metadata(metadata_dict): + output_dict = {} + for p in metadata_dict.keys(): + if p in ["dcterms:rights", "dcterms:relation"]: + m_type = "xsd:uri" + elif p in ["dcterms:date", "dcterms:created"]: + m_type = "xsd:date" + else: + m_type = "xsd:string" + description = {"@value": metadata_dict[p], + "@type" : m_type} + output_dict[p] = description + return output_dict def check_metadata_format(metadata_dict): schema = { @@ -250,6 +265,7 @@ def check_metadata_format(metadata_dict): try: validate(metadata_dict, schema) output_dict = metadata_dict + # "media:id" is only for internal use, so we delete it for the output del output_dict['media:id'] except ValidationError, exc: -- cgit v1.2.3 From af3a9107a9aef453b62f8fd83e03e9a1bbe416b8 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Wed, 7 May 2014 13:36:52 -0500 Subject: The URL format checker now works correctly ...though it isn't checking the right thing --- mediagoblin/gmg_commands/batchaddmedia.py | 77 ++++++++++++++++++------------- 1 file changed, 44 insertions(+), 33 deletions(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 43c24f6d..41fb86c9 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -15,7 +15,7 @@ # along with this program. If not, see . import os -import tempfile, tarfile, zipfile, subprocess, requests +import copy, tempfile, tarfile, zipfile, subprocess, re, requests from csv import reader as csv_reader from urlparse import urlparse from pyld import jsonld @@ -27,8 +27,10 @@ from mediagoblin.submit.lib import ( from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ from mediagoblin import mg_globals -from jsonschema import validate +from jsonschema import validate, FormatChecker, draft4_format_checker from jsonschema.exceptions import ValidationError +from jsonschema.compat import str_types + def parser_setup(subparser): subparser.description = """\ @@ -48,11 +50,6 @@ Core properties (http://dublincore.org/documents/dces/). Both "location.csv" and "metadata.csv" must begin with a row demonstrating the order of the columns. We have provided an example of these files at """)) - subparser.add_argument( - "-l", "--license", - help=( - "License these media entry will be released under, if all the same. " - "Should be a URL.")) subparser.add_argument( '--celery', action='store_true', @@ -149,6 +146,8 @@ zip files and directories" title = sanitized_metadata.get('dcterms:title') description = sanitized_metadata.get('dcterms:description') + + # TODO: this isn't the same thing license = sanitized_metadata.get('dcterms:rights') filename = url.path.split()[-1] @@ -234,36 +233,48 @@ def build_json_ld_metadata(metadata_dict): output_dict[p] = description return output_dict + +## Set up the MediaGoblin checker +# + +URL_REGEX = re.compile( + r'^[a-z]+://([^/:]+|([0-9]{1,3}\.){3}[0-9]{1,3})(:[0-9]+)?(\/.*)?$', + re.IGNORECASE) + +def is_uri(instance): + if not isinstance(instance, str_types): + return True + + return URL_REGEX.match(instance) + + +class DefaultChecker(FormatChecker): + checkers = copy.deepcopy(draft4_format_checker.checkers) + +DefaultChecker.checkers[u"uri"] = (is_uri, ()) + +DEFAULT_CHECKER = DefaultChecker() + def check_metadata_format(metadata_dict): schema = { - "$schema":"http://json-schema.org/schema#", - "properties":{ - "media:id":{}, - "dcterms:contributor":{}, - "dcterms:coverage":{}, - "dcterms:created":{}, - "dcterms:creator":{}, - "dcterms:date":{}, - "dcterms:description":{}, - "dcterms:format":{}, - "dcterms:identifier":{}, - "dcterms:language":{}, - "dcterms:publisher":{}, - "dcterms:relation":{}, - "dcterms:rights" : { - "format":"uri", - "type":"string" + "$schema": "http://json-schema.org/schema#", + + "type": "object", + "properties": { + "dcterms:rights": { + "format": "uri", + "type": "string", + }, + "dcterms:created": { + + } }, - "dcterms:source":{}, - "dcterms:subject":{}, - "dcterms:title":{}, - "dcterms:type":{} - }, - "additionalProperties": False, - "required":["dcterms:title","media:id"] -} + # "required": ["dcterms:title", "media:id"], + } + try: - validate(metadata_dict, schema) + validate(metadata_dict, schema, + format_checker=DEFAULT_CHECKER) output_dict = metadata_dict # "media:id" is only for internal use, so we delete it for the output del output_dict['media:id'] -- cgit v1.2.3 From 2a2c534e51f47cbf7c537c9b659648f23146a64b Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Wed, 7 May 2014 15:21:10 -0500 Subject: Removing build_json_ld_metadata --- mediagoblin/gmg_commands/batchaddmedia.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 41fb86c9..cf362d83 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -139,8 +139,7 @@ zip files and directories" sanitized_metadata = check_metadata_format(file_metadata) if sanitized_metadata == {}: continue - json_ld_metadata = jsonld.compact(build_json_ld_metadata(file_metadata), - metadata_context) + json_ld_metadata = jsonld.compact(file_metadata, metadata_context) original_location = media_locations[media_id]['media:original'] url = urlparse(original_location) @@ -219,20 +218,6 @@ def teardown(temp_files): for temp_file in temp_files: subprocess.call(['rm','-r',temp_file]) -def build_json_ld_metadata(metadata_dict): - output_dict = {} - for p in metadata_dict.keys(): - if p in ["dcterms:rights", "dcterms:relation"]: - m_type = "xsd:uri" - elif p in ["dcterms:date", "dcterms:created"]: - m_type = "xsd:date" - else: - m_type = "xsd:string" - description = {"@value": metadata_dict[p], - "@type" : m_type} - output_dict[p] = description - return output_dict - ## Set up the MediaGoblin checker # -- cgit v1.2.3 From a4486286363cca8d0ef9d1026883b13e7f84d8e0 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Wed, 7 May 2014 15:21:58 -0500 Subject: Removing unused variables --- mediagoblin/gmg_commands/batchaddmedia.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index cf362d83..07c0b3fc 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -26,7 +26,6 @@ from mediagoblin.submit.lib import ( FileUploadLimit, UserUploadLimit, UserPastUploadLimit) from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ -from mediagoblin import mg_globals from jsonschema import validate, FormatChecker, draft4_format_checker from jsonschema.exceptions import ValidationError from jsonschema.compat import str_types @@ -99,14 +98,12 @@ zip files and directories" metadata_file_path = os.path.join(dir_path, "metadata.csv") # check for the location file, if it exists... - location_filename = os.path.split(location_file_path)[-1] abs_location_filename = os.path.abspath(location_file_path) if not os.path.exists(abs_location_filename): print "Can't find a file with filename '%s'" % location_file_path return # check for the metadata file, if it exists... - metadata_filename = os.path.split(metadata_file_path)[-1] abs_metadata_filename = os.path.abspath(metadata_file_path) if not os.path.exists(abs_metadata_filename): print "Can't find a file with filename '%s'" % metadata_file_path -- cgit v1.2.3 From 6fab7734d6b6817b310865409f260ab87907eaa0 Mon Sep 17 00:00:00 2001 From: Christopher Allan Webber Date: Wed, 7 May 2014 18:50:48 -0500 Subject: Updating batchaddmedia to use new metadata tools --- mediagoblin/gmg_commands/batchaddmedia.py | 99 ++++--------------------------- 1 file changed, 12 insertions(+), 87 deletions(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 07c0b3fc..e540e88c 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -15,7 +15,7 @@ # along with this program. If not, see . import os -import copy, tempfile, tarfile, zipfile, subprocess, re, requests +import tempfile, tarfile, zipfile, subprocess, requests from csv import reader as csv_reader from urlparse import urlparse from pyld import jsonld @@ -24,11 +24,9 @@ from mediagoblin.gmg_commands import util as commands_util from mediagoblin.submit.lib import ( submit_media, get_upload_file_limits, FileUploadLimit, UserUploadLimit, UserPastUploadLimit) -from mediagoblin.tools.translate import lazy_pass_to_ugettext as _ +from mediagoblin.tools.metadata import compact_and_validate -from jsonschema import validate, FormatChecker, draft4_format_checker from jsonschema.exceptions import ValidationError -from jsonschema.compat import str_types def parser_setup(subparser): @@ -126,25 +124,24 @@ zip files and directories" contents = all_metadata.read() media_metadata = parse_csv_file(contents) - metadata_context = { 'dcterms':'http://purl.org/dc/terms/', - 'xsd': 'http://www.w3.org/2001/XMLSchema#'} - for media_id in media_locations.keys(): files_attempted += 1 - file_metadata = media_metadata[media_id] - sanitized_metadata = check_metadata_format(file_metadata) - if sanitized_metadata == {}: continue + file_metadata = media_metadata[media_id] + try: + json_ld_metadata = compact_and_validate(file_metadata) + except ValidationError, exc: + print "Error with '%s' value '%s': %s" % ( + media_id, exc.path[0], exc.message) + continue - json_ld_metadata = jsonld.compact(file_metadata, metadata_context) original_location = media_locations[media_id]['media:original'] url = urlparse(original_location) - title = sanitized_metadata.get('dcterms:title') - description = sanitized_metadata.get('dcterms:description') + title = json_ld_metadata.get('dcterms:title') + description = json_ld_metadata.get('dcterms:description') - # TODO: this isn't the same thing - license = sanitized_metadata.get('dcterms:rights') + license = json_ld_metadata.get('license') filename = url.path.split()[-1] if url.scheme == 'http': @@ -214,75 +211,3 @@ def parse_csv_file(file_contents): def teardown(temp_files): for temp_file in temp_files: subprocess.call(['rm','-r',temp_file]) - - -## Set up the MediaGoblin checker -# - -URL_REGEX = re.compile( - r'^[a-z]+://([^/:]+|([0-9]{1,3}\.){3}[0-9]{1,3})(:[0-9]+)?(\/.*)?$', - re.IGNORECASE) - -def is_uri(instance): - if not isinstance(instance, str_types): - return True - - return URL_REGEX.match(instance) - - -class DefaultChecker(FormatChecker): - checkers = copy.deepcopy(draft4_format_checker.checkers) - -DefaultChecker.checkers[u"uri"] = (is_uri, ()) - -DEFAULT_CHECKER = DefaultChecker() - -def check_metadata_format(metadata_dict): - schema = { - "$schema": "http://json-schema.org/schema#", - - "type": "object", - "properties": { - "dcterms:rights": { - "format": "uri", - "type": "string", - }, - "dcterms:created": { - - } - }, - # "required": ["dcterms:title", "media:id"], - } - - try: - validate(metadata_dict, schema, - format_checker=DEFAULT_CHECKER) - output_dict = metadata_dict - # "media:id" is only for internal use, so we delete it for the output - del output_dict['media:id'] - - except ValidationError, exc: - title = (metadata_dict.get('dcterms:title') or - metadata_dict.get('media:id') or _(u'UNKNOWN FILE')) - - if exc.validator == "additionalProperties": - message = _(u'Invalid metadata provided for file "{title}". This \ -script only accepts the Dublin Core metadata terms.'.format(title=title)) - - elif exc.validator == "required": - message = _( -u'All necessary metadata was not provided for file "{title}", you must include \ -a "dcterms:title" column for each media file'.format(title=title)) - - else: - message = _(u'Could not find appropriate metadata for file \ -"{title}".'.format(title=title)) - - print _(u"""WARN: {message} \nSkipping File...\n""".format( - message=message)) - - output_dict = {} - except: - raise - - return output_dict -- cgit v1.2.3 From e8d64d453b5b712a82677759c8c9acab706a98d9 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Mon, 12 May 2014 13:19:03 -0400 Subject: Cleaned up the 'batchaddmedia' command a bit --- mediagoblin/gmg_commands/batchaddmedia.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index e540e88c..a9364daa 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -38,7 +38,7 @@ This command allows the administrator to upload many media files at once.""" subparser.add_argument( 'target_path', help=("""\ -Path to a local archive or directory containing a "location.csv" and a +Path to a local archive or directory containing a "location.csv" and a "metadata.csv" file. These are csv (comma seperated value) files with the locations and metadata of the files to be uploaded. The location must be listed with either the URL of the remote media file or the filesystem path of a local @@ -128,6 +128,12 @@ zip files and directories" files_attempted += 1 file_metadata = media_metadata[media_id] + + ### Remove all metadata entries starting with 'media' because we are ### + ### only using those for internal use. ### + file_metadata = dict([(key, value) + for key, value in file_metadata.iteritems() if + key.split(":")[0] != 'media']) try: json_ld_metadata = compact_and_validate(file_metadata) except ValidationError, exc: @@ -138,8 +144,10 @@ zip files and directories" original_location = media_locations[media_id]['media:original'] url = urlparse(original_location) - title = json_ld_metadata.get('dcterms:title') - description = json_ld_metadata.get('dcterms:description') + ### Pull the important media information for mediagoblin from the ### + ### metadata, if it is provided. ### + title = json_ld_metadata.get('dc:title') + description = json_ld_metadata.get('dc:description') license = json_ld_metadata.get('license') filename = url.path.split()[-1] -- cgit v1.2.3 From 7d52eb770501240e6872d3007d8e9ef203632eb0 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Mon, 12 May 2014 13:59:28 -0400 Subject: Modified the batchaddmedia script so that it only looks for one csv file instead of the previous method which looked for two files. --- mediagoblin/gmg_commands/batchaddmedia.py | 123 +++++++++++------------------- 1 file changed, 46 insertions(+), 77 deletions(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index a9364daa..7ba8db1e 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -25,7 +25,7 @@ from mediagoblin.submit.lib import ( submit_media, get_upload_file_limits, FileUploadLimit, UserUploadLimit, UserPastUploadLimit) from mediagoblin.tools.metadata import compact_and_validate - +from mediagoblin.tools.translate import pass_to_ugettext as _ from jsonschema.exceptions import ValidationError @@ -36,17 +36,7 @@ This command allows the administrator to upload many media files at once.""" 'username', help="Name of user these media entries belong to") subparser.add_argument( - 'target_path', - help=("""\ -Path to a local archive or directory containing a "location.csv" and a -"metadata.csv" file. These are csv (comma seperated value) files with the -locations and metadata of the files to be uploaded. The location must be listed -with either the URL of the remote media file or the filesystem path of a local -file. The metadata should be provided with one column for each of the 15 Dublin -Core properties (http://dublincore.org/documents/dces/). Both "location.csv" and -"metadata.csv" must begin with a row demonstrating the order of the columns. We -have provided an example of these files at -""")) + 'metadata_path') subparser.add_argument( '--celery', action='store_true', @@ -65,48 +55,24 @@ def batchaddmedia(args): # get the user user = app.db.User.query.filter_by(username=args.username.lower()).first() if user is None: - print "Sorry, no user by username '%s' exists" % args.username + print _(u"Sorry, no user by username '{username}' exists".format( + username=args.username)) return upload_limit, max_file_size = get_upload_file_limits(user) temp_files = [] - if os.path.isdir(args.target_path): - dir_path = args.target_path - - elif tarfile.is_tarfile(args.target_path): - dir_path = tempfile.mkdtemp() - temp_files.append(dir_path) - tar = tarfile.open(args.target_path) - tar.extractall(path=dir_path) - - elif zipfile.is_zipfile(args.target_path): - dir_path = tempfile.mkdtemp() - temp_files.append(dir_path) - zipped_file = zipfile.ZipFile(args.target_path) - zipped_file.extractall(path=dir_path) + if os.path.isfile(args.metadata_path): + metadata_path = args.metadata_path else: - print "Couldn't recognize the file. This script only accepts tar files,\ -zip files and directories" - if dir_path.endswith('/'): - dir_path = dir_path[:-1] - - location_file_path = os.path.join(dir_path,"location.csv") - metadata_file_path = os.path.join(dir_path, "metadata.csv") - - # check for the location file, if it exists... - abs_location_filename = os.path.abspath(location_file_path) - if not os.path.exists(abs_location_filename): - print "Can't find a file with filename '%s'" % location_file_path - return - - # check for the metadata file, if it exists... - abs_metadata_filename = os.path.abspath(metadata_file_path) - if not os.path.exists(abs_metadata_filename): - print "Can't find a file with filename '%s'" % metadata_file_path + error = _(u'File at {path} not found, use -h flag for help'.format( + path=args.metadata_path)) + print error return + abs_metadata_filename = os.path.abspath(metadata_path) + abs_metadata_dir = os.path.dirname(abs_metadata_filename) upload_limit, max_file_size = get_upload_file_limits(user) def maybe_unicodeify(some_string): @@ -116,36 +82,36 @@ zip files and directories" else: return unicode(some_string) - with file(abs_location_filename, 'r') as all_locations: - contents = all_locations.read() - media_locations = parse_csv_file(contents) - with file(abs_metadata_filename, 'r') as all_metadata: contents = all_metadata.read() media_metadata = parse_csv_file(contents) - for media_id in media_locations.keys(): + for media_id, file_metadata in media_metadata.iteritems(): files_attempted += 1 + # In case the metadata was not uploaded initialize an empty dictionary. + json_ld_metadata = compact_and_validate({}) - file_metadata = media_metadata[media_id] - - ### Remove all metadata entries starting with 'media' because we are ### - ### only using those for internal use. ### + # Get all metadata entries starting with 'media' as variables and then + # delete them because those are for internal use only. + original_location = file_metadata['media:location'] file_metadata = dict([(key, value) for key, value in file_metadata.iteritems() if key.split(":")[0] != 'media']) try: json_ld_metadata = compact_and_validate(file_metadata) except ValidationError, exc: - print "Error with '%s' value '%s': %s" % ( - media_id, exc.path[0], exc.message) + error = _(u"""Error with media '{media_id}' value '{error_path}': {error_msg} +Metadata was not uploaded.""".format( + media_id=media_id, + error_path=exc.path[0], + error_msg=exc.message)) + print error continue - original_location = media_locations[media_id]['media:original'] url = urlparse(original_location) - ### Pull the important media information for mediagoblin from the ### - ### metadata, if it is provided. ### + ### Pull the important media information for mediagoblin from the + ### metadata, if it is provided. title = json_ld_metadata.get('dc:title') description = json_ld_metadata.get('dc:description') @@ -161,14 +127,14 @@ zip files and directories" if os.path.isabs(path): file_abs_path = os.path.abspath(path) else: - file_path = os.path.join(dir_path, path) + file_path = os.path.join(abs_metadata_dir, path) file_abs_path = os.path.abspath(file_path) try: media_file = file(file_abs_path, 'r') except IOError: - print "\ -FAIL: Local file {filename} could not be accessed.".format(filename=filename) - print "Skipping it." + print _(u"""\ +FAIL: Local file {filename} could not be accessed. +{filename} will not be uploaded.""".format(filename=filename)) continue try: submit_media( @@ -182,29 +148,36 @@ FAIL: Local file {filename} could not be accessed.".format(filename=filename) metadata=json_ld_metadata, tags_string=u"", upload_limit=upload_limit, max_file_size=max_file_size) - print "Successfully uploading {filename}!".format(filename=filename) - print "" + print _(u"""Successfully submitted {filename}! +Be sure to look at the Media Processing Panel on your website to be sure it +uploaded successfully.""".format(filename=filename)) files_uploaded += 1 except FileUploadLimit: - print "FAIL: This file is larger than the upload limits for this site." + print _( +u"FAIL: This file is larger than the upload limits for this site.") except UserUploadLimit: - print "FAIL: This file will put this user past their upload limits." + print _( +"FAIL: This file will put this user past their upload limits.") except UserPastUploadLimit: - print "FAIL: This user is already past their upload limits." - print "\ -{files_uploaded} out of {files_attempted} files successfully uploaded".format( + print _("FAIL: This user is already past their upload limits.") + print _( +"{files_uploaded} out of {files_attempted} files successfully submitted".format( files_uploaded=files_uploaded, - files_attempted=files_attempted) - teardown(temp_files) + files_attempted=files_attempted)) def parse_csv_file(file_contents): + """ + The helper function which converts the csv file into a dictionary where each + item's key is the provided value 'media:id' and each item's value is another + dictionary. + """ list_of_contents = file_contents.split('\n') key, lines = (list_of_contents[0].split(','), list_of_contents[1:]) objects_dict = {} - # Build a dictionaryfrom mediagoblin.tools.translate import lazy_pass_to_ugettext as _ + # Build a dictionary for line in lines: if line.isspace() or line == '': continue values = csv_reader([line]).next() @@ -215,7 +188,3 @@ def parse_csv_file(file_contents): return objects_dict - -def teardown(temp_files): - for temp_file in temp_files: - subprocess.call(['rm','-r',temp_file]) -- cgit v1.2.3 From 8524a6bdc575312b35df6e6aa3118ed139303d72 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Mon, 12 May 2014 17:02:12 -0400 Subject: Added documentation for the batchaddmedia gmg tool to the mediagoblin docs. --- mediagoblin/gmg_commands/batchaddmedia.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 7ba8db1e..75e7b7c5 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -32,15 +32,21 @@ from jsonschema.exceptions import ValidationError def parser_setup(subparser): subparser.description = """\ This command allows the administrator to upload many media files at once.""" + subparser.epilog = _(u"""For more information about how to properly run this +script (and how to format the metadata csv file), read the MediaGoblin +documentation page on command line uploading +""") subparser.add_argument( 'username', - help="Name of user these media entries belong to") + help=_(u"Name of user these media entries belong to")) subparser.add_argument( - 'metadata_path') + 'metadata_path', + help=_( +u"""Path to the csv file containing metadata information.""")) subparser.add_argument( '--celery', action='store_true', - help="Don't process eagerly, pass off to celery") + help=_(u"Don't process eagerly, pass off to celery")) def batchaddmedia(args): -- cgit v1.2.3 From c0ea2bad04c7c0ce28659a73bd63ca409c847519 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 13 May 2014 16:53:28 -0400 Subject: Prepared for input without an 'id' column and made all of the internal nodes into free floating nodes so that compact_and_validate will remove them. --- mediagoblin/gmg_commands/batchaddmedia.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 75e7b7c5..58ca7e74 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -99,10 +99,7 @@ def batchaddmedia(args): # Get all metadata entries starting with 'media' as variables and then # delete them because those are for internal use only. - original_location = file_metadata['media:location'] - file_metadata = dict([(key, value) - for key, value in file_metadata.iteritems() if - key.split(":")[0] != 'media']) + original_location = file_metadata['location'] try: json_ld_metadata = compact_and_validate(file_metadata) except ValidationError, exc: @@ -175,7 +172,7 @@ u"FAIL: This file is larger than the upload limits for this site.") def parse_csv_file(file_contents): """ The helper function which converts the csv file into a dictionary where each - item's key is the provided value 'media:id' and each item's value is another + item's key is the provided value 'id' and each item's value is another dictionary. """ list_of_contents = file_contents.split('\n') @@ -184,12 +181,12 @@ def parse_csv_file(file_contents): objects_dict = {} # Build a dictionary - for line in lines: + for index, line in enumerate(lines): if line.isspace() or line == '': continue values = csv_reader([line]).next() line_dict = dict([(key[i], val) for i, val in enumerate(values)]) - media_id = line_dict['media:id'] + media_id = line_dict.get('id') or index objects_dict[media_id] = (line_dict) return objects_dict -- cgit v1.2.3 From 80fefb851485a73d1ff9d526bdcc6ebc6052af55 Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 13 May 2014 16:59:02 -0400 Subject: Removed unused imports. --- mediagoblin/gmg_commands/batchaddmedia.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 58ca7e74..d83774ee 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -15,10 +15,9 @@ # along with this program. If not, see . import os -import tempfile, tarfile, zipfile, subprocess, requests +import requests from csv import reader as csv_reader from urlparse import urlparse -from pyld import jsonld from mediagoblin.gmg_commands import util as commands_util from mediagoblin.submit.lib import ( -- cgit v1.2.3 From 65f5714118f5b59bc7f51c67ffc6ef23f2c603cc Mon Sep 17 00:00:00 2001 From: tilly-Q Date: Tue, 13 May 2014 18:15:28 -0400 Subject: Adjusted batchaddmedia to make use of more internal nodes. Added to the docs. --- mediagoblin/gmg_commands/batchaddmedia.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'mediagoblin/gmg_commands') diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index d83774ee..b7f2569c 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -33,7 +33,7 @@ def parser_setup(subparser): This command allows the administrator to upload many media files at once.""" subparser.epilog = _(u"""For more information about how to properly run this script (and how to format the metadata csv file), read the MediaGoblin -documentation page on command line uploading +documentation page on command line uploading """) subparser.add_argument( 'username', @@ -99,6 +99,14 @@ def batchaddmedia(args): # Get all metadata entries starting with 'media' as variables and then # delete them because those are for internal use only. original_location = file_metadata['location'] + + ### Pull the important media information for mediagoblin from the + ### metadata, if it is provided. + title = file_metadata.get('title') or file_metadata.get('dc:title') + description = (file_metadata.get('description') or + file_metadata.get('dc:description')) + + license = file_metadata.get('license') try: json_ld_metadata = compact_and_validate(file_metadata) except ValidationError, exc: @@ -111,13 +119,6 @@ Metadata was not uploaded.""".format( continue url = urlparse(original_location) - - ### Pull the important media information for mediagoblin from the - ### metadata, if it is provided. - title = json_ld_metadata.get('dc:title') - description = json_ld_metadata.get('dc:description') - - license = json_ld_metadata.get('license') filename = url.path.split()[-1] if url.scheme == 'http': -- cgit v1.2.3