diff options
| -rw-r--r-- | .dockerignore | 3 | ||||
| -rw-r--r-- | Dockerfile-python2 | 102 | ||||
| -rw-r--r-- | Dockerfile-python3 | 100 | ||||
| -rw-r--r-- | docker-compose.yml | 33 | ||||
| -rw-r--r-- | docs/source/siteadmin/commandline-upload.rst | 2 | ||||
| -rw-r--r-- | docs/source/siteadmin/media-types.rst | 5 | ||||
| -rw-r--r-- | extlib/freesound/audioprocessing.py | 25 | ||||
| -rw-r--r-- | guix-env.scm | 104 | ||||
| -rw-r--r-- | mediagoblin/gmg_commands/batchaddmedia.py | 136 | ||||
| -rw-r--r-- | mediagoblin/init/config.py | 9 | ||||
| -rw-r--r-- | mediagoblin/plugins/metadata_display/static/css/metadata_display.css | 4 | ||||
| -rw-r--r-- | mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html | 2 | ||||
| -rw-r--r-- | mediagoblin/static/css/base.css | 4 |
13 files changed, 404 insertions, 125 deletions
diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..12ef3383 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +# This helps to reduce the size of the Docker build context. +user_dev +node_modules diff --git a/Dockerfile-python2 b/Dockerfile-python2 new file mode 100644 index 00000000..849ca8ae --- /dev/null +++ b/Dockerfile-python2 @@ -0,0 +1,102 @@ +# A Dockerfile for MediaGoblin hacking. + +# docker build -t mediagoblin-python2 -f Dockerfile-python2 . +# docker build -t mediagoblin +# docker run -it -p 6543:6543 -v ~/ws/mediagoblin/mediagoblin:/opt/mediagoblin/mediagoblin -v ~/ws/mediagoblin/extlib:/opt/mediagoblin/extlib mediagoblin-python2 +# docker stop [container-name/id] +# docker start [container-name/id] +# docker kill [container-name/id] + +FROM debian:buster + +# Install bootstrap and configure dependencies. Currently requires virtualenv +# rather than the more modern python3-venv (should be fixed). +RUN apt-get update && apt-get install -y \ +automake \ +git \ +nodejs \ +npm \ +python-dev \ +virtualenv + +# Install make and runtime dependencies. +RUN apt-get install -y \ +python-alembic \ +python-celery \ +python-jsonschema \ +python-kombu \ +python-lxml \ +python-migrate \ +python-mock \ +python-py \ +python-pytest \ +python-pytest-xdist \ +python-six \ +python-sphinx \ +python-webtest + +# Install audio dependencies. +RUN apt-get install -y \ +gstreamer1.0-libav \ +gstreamer1.0-plugins-bad \ +gstreamer1.0-plugins-base \ +gstreamer1.0-plugins-good \ +gstreamer1.0-plugins-ugly \ +libsndfile1-dev \ +python-gst-1.0 \ +python-numpy \ +python-scipy + +# Install video dependencies. +RUN apt-get install -y \ +gir1.2-gst-plugins-base-1.0 \ +gir1.2-gstreamer-1.0 \ +gstreamer1.0-tools \ +python-gi + +# Create working directory. +RUN mkdir /opt/mediagoblin +RUN chown -R www-data:www-data /opt/mediagoblin +WORKDIR /opt/mediagoblin + +# Create /var/www because Bower writes some cache files into /var/www during +# make, failing if it doesn't exist. +RUN mkdir /var/www +RUN chown root:www-data /var/www +RUN chmod g+w /var/www + +USER www-data + +# Clone MediaGoblin for use during the install. Could alternately copy across +# just the files needed to run bootstrap/configure/make. +RUN git clone git://git.savannah.gnu.org/mediagoblin.git -b master . +RUN git submodule init && git submodule update + +RUN ./bootstrap.sh +RUN VIRTUALENV_FLAGS='--system-site-packages' ./configure +RUN make + +# Re-run installation of Python dependencies - seems to install more things that +# didn't get installed with make. That shouldn't happen. +RUN ./bin/python setup.py develop --upgrade + +# Only supported on Python 2. +RUN ./bin/pip install scikits.audiolab + +# Patch to fix the config defaults that are failing at runtime. Needed here +# since we're running `dbupdate` during the Docker build. +COPY mediagoblin/init/config.py /opt/mediagoblin/mediagoblin/init/config.py + +RUN echo '[[mediagoblin.media_types.audio]]' >> mediagoblin.ini +RUN echo '[[mediagoblin.media_types.video]]' >> mediagoblin.ini + +RUN cat mediagoblin.ini + +# Using default sqlite database for now. +RUN ./bin/gmg dbupdate + +RUN ./bin/gmg adduser --username admin --password a --email admin@example.com +RUN ./bin/gmg makeadmin admin + +# You can change this to /bin/bash if you'd prefer a shell. +CMD ["./lazyserver.sh", "--server-name=broadcast"] diff --git a/Dockerfile-python3 b/Dockerfile-python3 new file mode 100644 index 00000000..c617e205 --- /dev/null +++ b/Dockerfile-python3 @@ -0,0 +1,100 @@ +# A Dockerfile for MediaGoblin hacking. + +# docker build -t mediagoblin-python3 -f Dockerfile-python3 . +# docker run -it -p 6543:6543 -v ~/ws/mediagoblin/mediagoblin:/opt/mediagoblin/mediagoblin -v ~/ws/mediagoblin/extlib:/opt/mediagoblin/extlib mediagoblin-python3 +# docker stop [container-name/id] +# docker start [container-name/id] +# docker kill [container-name/id] + +FROM debian:buster + +# Install bootstrap and configure dependencies. Currently requires virtualenv +# rather than the more modern python3-venv (should be fixed). +RUN apt-get update && apt-get install -y \ +automake \ +git \ +nodejs \ +npm \ +python3-dev \ +virtualenv + +# Install make and runtime dependencies. +RUN apt-get install -y \ +python3-alembic \ +python3-celery \ +python3-jsonschema \ +python3-kombu \ +python3-lxml \ +python3-migrate \ +python3-py \ +python3-pytest \ +python3-pytest-xdist \ +python3-six \ +python3-sphinx \ +python3-webtest + +# Install audio dependencies. +RUN apt-get install -y \ +gstreamer1.0-libav \ +gstreamer1.0-plugins-bad \ +gstreamer1.0-plugins-base \ +gstreamer1.0-plugins-good \ +gstreamer1.0-plugins-ugly \ +libsndfile1-dev \ +python3-gst-1.0 \ +python3-numpy \ +python3-scipy + +# Install video dependencies. +RUN apt-get install -y \ +gir1.2-gst-plugins-base-1.0 \ +gir1.2-gstreamer-1.0 \ +gstreamer1.0-tools \ +python3-gi + +# Create working directory. +RUN mkdir /opt/mediagoblin +RUN chown -R www-data:www-data /opt/mediagoblin +WORKDIR /opt/mediagoblin + +# Create /var/www because Bower writes some cache files into /var/www during +# make, failing if it doesn't exist. +RUN mkdir /var/www +RUN chown root:www-data /var/www +RUN chmod g+w /var/www + +USER www-data + +# Clone MediaGoblin for use during the install. Could alternately copy across +# just the files needed to run bootstrap/configure/make. +RUN git clone git://git.savannah.gnu.org/mediagoblin.git -b master . +RUN git submodule init && git submodule update + +RUN ./bootstrap.sh +RUN VIRTUALENV_FLAGS='--system-site-packages' ./configure --with-python3 +RUN make + +# Re-run installation of Python dependencies - seems to install more things that +# didn't get installed with make. That shouldn't happen. +RUN ./bin/python setup.py develop --upgrade + +# Only supported on Python 2. +# RUN ./bin/pip install scikits.audiolab + +# Patch to fix the config defaults that are failing at runtime. Needed here +# since we're running `dbupdate` during the Docker build. +COPY mediagoblin/init/config.py /opt/mediagoblin/mediagoblin/init/config.py + +RUN echo '[[mediagoblin.media_types.audio]]' >> mediagoblin.ini +RUN echo '[[mediagoblin.media_types.video]]' >> mediagoblin.ini + +RUN cat mediagoblin.ini + +# Using default sqlite database for now. +RUN ./bin/gmg dbupdate + +RUN ./bin/gmg adduser --username admin --password a --email admin@example.com +RUN ./bin/gmg makeadmin admin + +# You can change this to /bin/bash if you'd prefer a shell. +CMD ["./lazyserver.sh", "--server-name=broadcast"] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..ac11257b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,33 @@ +# A docker-compose recipe for MediaGoblin hacking. +# +# Tested on Trisquel 8. Currently runs Python 3 and works for photos and video. +# Audio raises an exception "NameError: name 'audiolab' is not defined". + +# docker-compose up --build +# docker-compose run --rm web bin/python +# docker-compose start [service] +# docker-compose stop [service] +# docker-compose down + +version: '2' + +services: + web: + build: + context: . + dockerfile: Dockerfile-python3 + # Is user required here, or does it just pick up from the last USER in Dockerfile? + user: www-data + # Consider running dbupdate here (at runtime), rather than in Dockerfile. + command: ./lazyserver.sh --server-name=broadcast + volumes: + # Mount your local copy of the source for hecking on MediaGoblin. + - ./mediagoblin:/opt/mediagoblin/mediagoblin + + # Mount your local media/secrets. Requires some initial setup: + # + # $ mkdir user_dev/media user_dev/crypto + # $ chmod 777 user_dev/media user_dev_crypto + - ./user_dev:/opt/mediagoblin/user_dev + ports: + - "6543:6543" diff --git a/docs/source/siteadmin/commandline-upload.rst b/docs/source/siteadmin/commandline-upload.rst index 756f5fa8..af4fd1bd 100644 --- a/docs/source/siteadmin/commandline-upload.rst +++ b/docs/source/siteadmin/commandline-upload.rst @@ -58,7 +58,7 @@ it is a bit more complex. This is an example of what a script may look like. The important part here is that you have to create the 'metadata.csv' file.:: - location,dcterms:title,dcterms:creator,dcterms:type + location,dc:title,dc:creator,dc:type "http://www.example.net/path/to/nap.png","Goblin taking a nap",,"Image" "http://www.example.net/path/to/snore.ogg","Goblin Snoring","Me","Audio" diff --git a/docs/source/siteadmin/media-types.rst b/docs/source/siteadmin/media-types.rst index 8f9239be..e06739ec 100644 --- a/docs/source/siteadmin/media-types.rst +++ b/docs/source/siteadmin/media-types.rst @@ -131,10 +131,13 @@ To install these on Debianoid systems, run:: not compile it with alsa support. Alsa support is not necessary for the GNU MediaGoblin application. -Then install ``scikits.audiolab`` for the spectrograms:: +If you're running Python 2, install ``scikits.audiolab`` for the spectrograms:: ./bin/pip install scikits.audiolab +Audio spectrograms are currently not available on Python 3, since scikits.audiolab +does not provide Python 3 support. + Add ``[[mediagoblin.media_types.audio]]`` under the ``[plugins]`` section in your ``mediagoblin.ini`` and restart MediaGoblin. diff --git a/extlib/freesound/audioprocessing.py b/extlib/freesound/audioprocessing.py index 7ef8d5d4..b9a96a97 100644 --- a/extlib/freesound/audioprocessing.py +++ b/extlib/freesound/audioprocessing.py @@ -44,6 +44,31 @@ try: import scikits.audiolab as audiolab except ImportError: print("WARNING: audiolab is not installed so wav2png will not work") + + # Hack to prevent errors when uploading audio files. The issue is that + # scikits.audiolab does not support Python 3. By replacing it with a mock + # implementation here, we can accept audio files, but we won't get the nice + # waveform image. + import six + if six.PY3: + class MockSndfile(object): + def __init__(self, *args, **kwargs): + self.nframes = 0 + self.channels = 1 + self.samplerate = 44100 + + def read_frames(self, *args): + return [] + + def seek(self, *args): + return + + def close(self): + return + import unittest.mock as mock + audiolab = mock.Mock() + audiolab.Sndfile = MockSndfile + import subprocess class AudioProcessingException(Exception): diff --git a/guix-env.scm b/guix-env.scm index d56f8539..acff8886 100644 --- a/guix-env.scm +++ b/guix-env.scm @@ -1,6 +1,7 @@ ;;; GNU MediaGoblin -- federated, autonomous media hosting ;;; Copyright © 2015, 2016 David Thompson <davet@gnu.org> ;;; Copyright © 2016 Christopher Allan Webber <cwebber@dustycloud.org> +;;; Copyright © 2019 Ben Sturmfels <ben@sturm.com.au> ;;; ;;; This program is free software: you can redistribute it and/or modify ;;; it under the terms of the GNU General Public License as published by @@ -19,7 +20,7 @@ ;;; also borrows some code directly from Guix. ;;; ;;; ======================================== -;;; +;;; ;;; With `guix environment' you can use guix as kind of a universal ;;; virtualenv, except a universal virtualenv with magical time traveling ;;; properties and also, not just for Python. @@ -28,11 +29,19 @@ ;;; Then do: ;;; guix environment -l guix-env.scm --pure ;;; -;;; And the first time you use it: +;;; You'll need to run the above command every time you close your terminal or +;;; restart your system, so a handy way to save having to remember is to install +;;; "direnv" an then create a ".envrc" file in your current directory containing +;;; the following and then run "direnv allow" when prompted: +;;; use guix -l guix-env.scm --pure +;;; +;;; To set things up for the first time, you'll also need to run: +;;; git submodule init +;;; git submodule update ;;; ./bootstrap.sh ;;; ./configure --with-python3 --without-virtualenv ;;; make -;;; virtualenv . && ./bin/python setup.py develop --no-deps +;;; python3 -m venv --system-site-packages . && bin/python setup.py develop --no-deps ;;; ;;; ... wait whaaat, what's that last line! I thought you said this ;;; was a reasonable virtualenv replacement! Well it is and it will @@ -41,6 +50,14 @@ ;;; for certain things to run, so we have a virtualenv with nothing ;;; in it but this project itself. ;;; +;;; The devtools/update_extlib.sh script won't run on Guix due to missing +;;; "/usr/bin/env", so then run: +;;; node node_modules/.bin/bower install +;;; ./devtools/update_extlib.sh +;;; bin/gmg dbupdate +;;; bin/gmg adduser --username admin --password a --email admin@example.com +;;; ./lazyserver.sh +;;; ;;; So anyway, now you can do: ;;; PYTHONPATH="${PYTHONPATH}:$(pwd)" ./runtests.sh ;;; @@ -49,6 +66,9 @@ ;;; the virtualenv and path-hacking stuff unnecessary. ;;; ;;; Have fun! +;;; +;;; Known issues: +;;; - currently fails to upload h264 source video: "GStreamer: missing H.264 decoder" (use-modules (ice-9 match) (srfi srfi-1) @@ -61,11 +81,19 @@ (gnu packages) (gnu packages autotools) (gnu packages base) + (gnu packages certs) + (gnu packages check) + (gnu packages databases) (gnu packages python) + (gnu packages python-crypto) + (gnu packages python-web) + (gnu packages python-xyz) + (gnu packages sphinx) (gnu packages gstreamer) (gnu packages glib) (gnu packages rsync) (gnu packages ssh) + (gnu packages time) (gnu packages version-control) ((guix licenses) #:select (expat zlib) #:prefix license:)) @@ -75,43 +103,28 @@ ;; ourselves to... ;; ================================================================= -(define python-sqlalchemy-0.9.10 +(define python-pytest-forked (package - (inherit python-sqlalchemy) - (version "0.9.10") - (source - (origin - (method url-fetch) - (uri (string-append "https://pypi.python.org/packages/source/S/" - "SQLAlchemy/SQLAlchemy-" version ".tar.gz")) - (sha256 - (base32 - "0fqnssf7pxvc7dvd5l83vnqz2wfvpq7y01kcl1537f9nbqnvlp24")))) - - ;; Temporarily skipping tests. It's the stuff that got fixed in - ;; the recent sqlalchemy release we struggled with on-list. The - ;; patch would have to be backported here to 0.9.10. - (arguments - '(#:tests? #f)))) - -(define python-alembic-0.6.6 - (package - (inherit python-alembic) - (version "0.6.6") - (source - (origin - (method url-fetch) - (uri (pypi-uri "alembic" version)) - (sha256 - (base32 - "0i3nic56blq079vj1iskkmllwjp980vnvvx898d3bm5qa416crcn")))) - (native-inputs - `(("python-nose" ,python-nose) - ,@(package-native-inputs python-alembic))) - (propagated-inputs - `(("python-sqlalchemy" ,python-sqlalchemy-0.9.10) - ("python-mako" ,python-mako) - ("python-editor" ,python-editor))))) + (name "python-pytest-forked") + (version "1.0.2") + (source + (origin + (method url-fetch) + (uri (pypi-uri "pytest-forked" version)) + (sha256 + (base32 + "0f4y1jhcg70xhm220pdb8r24n01knhn749aqlr14vmgbsb7allnk")))) + (build-system python-build-system) + (propagated-inputs + `(("python-pytest" ,python-pytest) + ("python-setuptools-scm" ,python-setuptools-scm))) + (home-page + "https://github.com/pytest-dev/pytest-forked") + (synopsis + "run tests in isolated forked subprocesses") + (description + "run tests in isolated forked subprocesses") + (license license:expat))) ;; ================================================================= @@ -127,11 +140,16 @@ (base32 "0p2gj4z351166d1zqmmd8wc9bzb69w0fjm8qq1fs8dw2yhcg2wwv")))) (build-system python-build-system) + (arguments + ;; Complains about missing gunicorn. Not sure where that comes from. + '(#:tests? #f)) (native-inputs - `(("python-pytest" ,python-pytest))) + `(("python-pytest" ,python-pytest) + ("nss-certs" ,nss-certs))) (propagated-inputs `(("python-alembic" ,python-alembic) ("python-pytest-xdist" ,python-pytest-xdist) + ("python-pytest-forked" ,python-pytest-forked) ("python-celery" ,python-celery) ("python-kombu" ,python-kombu) ("python-webtest" ,python-webtest) @@ -141,7 +159,7 @@ ("python-translitcodec" ,python-translitcodec) ("python-babel" ,python-babel) ("python-configobj" ,python-configobj) - ("python-dateutil-2" ,python-dateutil-2) + ("python-dateutil" ,python-dateutil) ("python-itsdangerous" ,python-itsdangerous) ("python-jinja2" ,python-jinja2) ("python-jsonschema" ,python-jsonschema) @@ -175,8 +193,10 @@ media.") (inputs `(;;; audio/video stuff ("gstreamer" ,gstreamer) + ("gst-libav" ,gst-plugins-base) ("gst-plugins-base" ,gst-plugins-base) ("gst-plugins-good" ,gst-plugins-good) + ("gst-plugins-bad" ,gst-plugins-bad) ("gst-plugins-ugly" ,gst-plugins-ugly) ("gobject-introspection" ,gobject-introspection) ;; useful to have! @@ -185,7 +205,7 @@ media.") ("which" ,which) ("git" ,git) ("automake" ,automake) - ("autoconf" ,(autoconf-wrapper)) + ("autoconf" ,autoconf) ,@(package-inputs mediagoblin))) (propagated-inputs `(("python" ,python) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 55ed865b..88fa3e5a 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -14,19 +14,18 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -from __future__ import print_function +from __future__ import print_function, unicode_literals -import codecs import csv import os -import sys +import shutil +import tempfile import requests import six - from six.moves.urllib.parse import urlparse -from mediagoblin.db.models import LocalUser +from mediagoblin.db.models import LocalUser, MediaEntry from mediagoblin.gmg_commands import util as commands_util from mediagoblin.submit.lib import ( submit_media, FileUploadLimit, UserUploadLimit, UserPastUploadLimit) @@ -38,21 +37,21 @@ from jsonschema.exceptions import ValidationError def parser_setup(subparser): subparser.description = """\ This command allows the administrator to upload many media files at once.""" - subparser.epilog = _(u"""For more information about how to properly run this + subparser.epilog = _("""For more information about how to properly run this script (and how to format the metadata csv file), read the MediaGoblin documentation page on command line uploading <http://docs.mediagoblin.org/siteadmin/commandline-upload.html>""") subparser.add_argument( 'username', - help=_(u"Name of user these media entries belong to")) + help=_("Name of user these media entries belong to")) subparser.add_argument( 'metadata_path', help=_( -u"""Path to the csv file containing metadata information.""")) +"""Path to the csv file containing metadata information.""")) subparser.add_argument( '--celery', action='store_true', - help=_(u"Don't process eagerly, pass off to celery")) + help=_("Don't process eagerly, pass off to celery")) def batchaddmedia(args): @@ -69,7 +68,7 @@ def batchaddmedia(args): LocalUser.username==args.username.lower() ).first() if user is None: - print(_(u"Sorry, no user by username '{username}' exists".format( + print(_("Sorry, no user by username '{username}' exists".format( username=args.username))) return @@ -77,7 +76,7 @@ def batchaddmedia(args): metadata_path = args.metadata_path else: - error = _(u'File at {path} not found, use -h flag for help'.format( + error = _('File at {path} not found, use -h flag for help'.format( path=args.metadata_path)) print(error) return @@ -85,19 +84,12 @@ def batchaddmedia(args): abs_metadata_filename = os.path.abspath(metadata_path) abs_metadata_dir = os.path.dirname(abs_metadata_filename) - def maybe_unicodeify(some_string): - # this is kinda terrible - if some_string is None: - return None - else: - return six.text_type(some_string) - - with codecs.open( - abs_metadata_filename, 'r', encoding='utf-8') as all_metadata: - contents = all_metadata.read() - media_metadata = parse_csv_file(contents) + all_metadata = open(abs_metadata_filename, 'r') + media_metadata = csv.DictReader(all_metadata) + for index, file_metadata in enumerate(media_metadata): + if six.PY2: + file_metadata = {k.decode('utf-8'): v.decode('utf-8') for k, v in file_metadata.items()} - for media_id, file_metadata in media_metadata.items(): files_attempted += 1 # In case the metadata was not uploaded initialize an empty dictionary. json_ld_metadata = compact_and_validate({}) @@ -108,6 +100,7 @@ def batchaddmedia(args): ### Pull the important media information for mediagoblin from the ### metadata, if it is provided. + slug = file_metadata.get('slug') title = file_metadata.get('title') or file_metadata.get('dc:title') description = (file_metadata.get('description') or file_metadata.get('dc:description')) @@ -117,7 +110,8 @@ def batchaddmedia(args): try: json_ld_metadata = compact_and_validate(file_metadata) except ValidationError as exc: - error = _(u"""Error with media '{media_id}' value '{error_path}': {error_msg} + media_id = file_metadata.get('id') or index + error = _("""Error with media '{media_id}' value '{error_path}': {error_msg} Metadata was not uploaded.""".format( media_id=media_id, error_path=exc.path[0], @@ -125,12 +119,36 @@ Metadata was not uploaded.""".format( print(error) continue + if slug and MediaEntry.query.filter_by(actor=user.id, slug=slug).count(): + # Avoid re-importing media from a previous batch run. Note that this + # check isn't quite robust enough, since it requires that a slug is + # specified. Probably needs to be based on "location" since this is + # the only required field. + error = '{}: {}'.format( + slug, _('An entry with that slug already exists for this user.')) + print(error) + continue + url = urlparse(original_location) filename = url.path.split()[-1] - if url.scheme == 'http': + if url.scheme.startswith('http'): res = requests.get(url.geturl(), stream=True) - media_file = res.raw + if res.headers.get('content-encoding'): + # The requests library's "raw" method does not deal with content + # encoding. Alternative could be to use iter_content(), and + # write chunks to the temporary file. + raise NotImplementedError('URL-based media with content-encoding (eg. gzip) are not currently supported.') + + # To avoid loading the media into memory all at once, we write it to + # a file before importing. This currently requires free space up to + # twice the size of the media file. Memory use can be tested by + # running something like `ulimit -Sv 200000` before running + # `batchaddmedia` to upload a file larger than 200MB. + media_file = tempfile.TemporaryFile() + shutil.copyfileobj(res.raw, media_file) + if six.PY2: + media_file.seek(0) elif url.scheme == '': path = url.path @@ -142,76 +160,42 @@ Metadata was not uploaded.""".format( try: media_file = open(file_abs_path, 'rb') except IOError: - print(_(u"""\ + print(_("""\ FAIL: Local file {filename} could not be accessed. {filename} will not be uploaded.""".format(filename=filename))) continue try: - submit_media( + entry = submit_media( mg_app=app, user=user, submitted_file=media_file, filename=filename, - title=maybe_unicodeify(title), - description=maybe_unicodeify(description), - collection_slug=maybe_unicodeify(collection_slug), - license=maybe_unicodeify(license), + title=title, + description=description, + collection_slug=collection_slug, + license=license, metadata=json_ld_metadata, - tags_string=u"") - print(_(u"""Successfully submitted {filename}! + tags_string="") + if slug: + # Slug is automatically set by submit_media, so overwrite it + # with the desired slug. + entry.slug = slug + entry.save() + print(_("""Successfully submitted {filename}! Be sure to look at the Media Processing Panel on your website to be sure it uploaded successfully.""".format(filename=filename))) files_uploaded += 1 except FileUploadLimit: print(_( -u"FAIL: This file is larger than the upload limits for this site.")) +"FAIL: This file is larger than the upload limits for this site.")) except UserUploadLimit: print(_( "FAIL: This file will put this user past their upload limits.")) except UserPastUploadLimit: print(_("FAIL: This user is already past their upload limits.")) + finally: + media_file.close() print(_( "{files_uploaded} out of {files_attempted} files successfully submitted".format( files_uploaded=files_uploaded, files_attempted=files_attempted))) - - -def unicode_csv_reader(unicode_csv_data, dialect=csv.excel, **kwargs): - # csv.py doesn't do Unicode; encode temporarily as UTF-8: - # TODO: this probably won't be necessary in Python 3 - csv_reader = csv.reader(utf_8_encoder(unicode_csv_data), - dialect=dialect, **kwargs) - for row in csv_reader: - # decode UTF-8 back to Unicode, cell by cell: - yield [six.text_type(cell, 'utf-8') for cell in row] - -def utf_8_encoder(unicode_csv_data): - for line in unicode_csv_data: - yield line.encode('utf-8') - -def parse_csv_file(file_contents): - """ - The helper function which converts the csv file into a dictionary where each - item's key is the provided value 'id' and each item's value is another - dictionary. - """ - list_of_contents = file_contents.split('\n') - key, lines = (list_of_contents[0].split(','), - list_of_contents[1:]) - objects_dict = {} - - # Build a dictionary - for index, line in enumerate(lines): - if line.isspace() or line == u'': continue - if (sys.version_info[0] == 3): - # Python 3's csv.py supports Unicode out of the box. - reader = csv.reader([line]) - else: - reader = unicode_csv_reader([line]) - values = next(reader) - line_dict = dict([(key[i], val) - for i, val in enumerate(values)]) - media_id = line_dict.get('id') or index - objects_dict[media_id] = (line_dict) - - return objects_dict diff --git a/mediagoblin/init/config.py b/mediagoblin/init/config.py index fe469156..2e22083a 100644 --- a/mediagoblin/init/config.py +++ b/mediagoblin/init/config.py @@ -84,6 +84,15 @@ def read_mediagoblin_config(config_path, config_spec_path=CONFIG_SPEC_PATH): config_spec_path, encoding="UTF8", list_values=False, _inspec=True) + # HACK to get MediaGoblin running under Docker/Python 3. Without this line, + # `./bin/gmg dbupdate` fails as the configuration under 'DEFAULT' in + # config_spec still had %(here)s markers in it, when these should have been + # replaced with actual paths, resulting in + # "configobj.MissingInterpolationOption: missing option "here" in + # interpolation". This issue doesn't seem to appear when running on Guix, + # but adding this line also doesn't appear to cause problems on Guix. + _setup_defaults(config_spec, config_path) + # Set up extra defaults that will be pushed into the rest of the # configs. This is a combined extrapolation of defaults based on mainconfig_defaults = copy.copy(config_spec.get("DEFAULT", {})) diff --git a/mediagoblin/plugins/metadata_display/static/css/metadata_display.css b/mediagoblin/plugins/metadata_display/static/css/metadata_display.css index e4612b02..dd787e94 100644 --- a/mediagoblin/plugins/metadata_display/static/css/metadata_display.css +++ b/mediagoblin/plugins/metadata_display/static/css/metadata_display.css @@ -1,6 +1,6 @@ table.metadata_info { font-size:85%; - margin-left:10px; + margin: 8px 0 16px 8px; } table.metadata_info th { @@ -8,7 +8,7 @@ table.metadata_info th { border-spacing: 10px; text-align: left; } + table.metadata_info td { padding: 4px 8px; } - diff --git a/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html b/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html index 15ea1536..6fc46212 100644 --- a/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html +++ b/mediagoblin/plugins/metadata_display/templates/mediagoblin/plugins/metadata_display/metadata_table.html @@ -23,7 +23,7 @@ {#- NOTE: In some smart future where the context is more extensible, we will need to add to the prefix here-#} <table class="metadata_info"> - {%- for key, value in metadata.iteritems() if not key=='@context' %} + {%- for key, value in metadata.items() if key != '@context' %} {% if value -%} <tr> <th>{{ rdfa_to_readable(key) }}</th> diff --git a/mediagoblin/static/css/base.css b/mediagoblin/static/css/base.css index 6da19f94..11558fe5 100644 --- a/mediagoblin/static/css/base.css +++ b/mediagoblin/static/css/base.css @@ -142,7 +142,7 @@ header { .header_right { width: 47%; - margin: 8px 8px 4px 0; + margin: 8px 8px 8px 0; display: inline-block; float: right; text-align: right; @@ -195,7 +195,7 @@ a.logo { .logo img { vertical-align: middle; - margin: 6px 8px 6px 0; + margin: 8px 8px 6px 0; } .welcomeimage { |