diff options
Diffstat (limited to 'mediagoblin/tools')
26 files changed, 2518 insertions, 0 deletions
diff --git a/mediagoblin/tools/__init__.py b/mediagoblin/tools/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/mediagoblin/tools/__init__.py diff --git a/mediagoblin/tools/common.py b/mediagoblin/tools/common.py new file mode 100644 index 00000000..34586611 --- /dev/null +++ b/mediagoblin/tools/common.py @@ -0,0 +1,73 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import sys + + +global TESTS_ENABLED +TESTS_ENABLED = False + + +def import_component(import_string): + """ + Import a module component defined by STRING. Probably a method, + class, or global variable. + + Args: + - import_string: a string that defines what to import. Written + in the format of "module1.module2:component" + """ + module_name, func_name = import_string.split(':', 1) + __import__(module_name) + module = sys.modules[module_name] + func = getattr(module, func_name) + return func + + +def simple_printer(string): + """ + Prints a string, but without an auto \n at the end. + + Useful for places where we want to dependency inject for printing. + """ + sys.stdout.write(string) + sys.stdout.flush() + + +class CollectingPrinter(object): + """ + Another printer object, this one useful for capturing output for + examination during testing or otherwise. + + Use this like: + + >>> printer = CollectingPrinter() + >>> printer("herp derp\n") + >>> printer("lollerskates\n") + >>> printer.combined_string + "herp derp\nlollerskates\n" + """ + def __init__(self): + self.collection = [] + + def __call__(self, string): + self.collection.append(string) + + @property + def combined_string(self): + return u''.join(self.collection) + + diff --git a/mediagoblin/tools/crypto.py b/mediagoblin/tools/crypto.py new file mode 100644 index 00000000..1379d21b --- /dev/null +++ b/mediagoblin/tools/crypto.py @@ -0,0 +1,113 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2013 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import errno +import itsdangerous +import logging +import os.path +import random +import tempfile +from mediagoblin import mg_globals + +_log = logging.getLogger(__name__) + + +# Use the system (hardware-based) random number generator if it exists. +# -- this optimization is lifted from Django +try: + getrandbits = random.SystemRandom().getrandbits +except AttributeError: + getrandbits = random.getrandbits + + +__itsda_secret = None + + +def load_key(filename): + global __itsda_secret + key_file = open(filename) + try: + __itsda_secret = key_file.read() + finally: + key_file.close() + + +def create_key(key_dir, key_filepath): + global __itsda_secret + old_umask = os.umask(077) + key_file = None + try: + if not os.path.isdir(key_dir): + os.makedirs(key_dir) + _log.info("Created %s", key_dir) + key = str(getrandbits(192)) + key_file = tempfile.NamedTemporaryFile(dir=key_dir, suffix='.bin', + delete=False) + key_file.write(key) + key_file.flush() + os.rename(key_file.name, key_filepath) + key_file.close() + finally: + os.umask(old_umask) + if (key_file is not None) and (not key_file.closed): + key_file.close() + os.unlink(key_file.name) + __itsda_secret = key + _log.info("Saved new key for It's Dangerous") + + +def setup_crypto(): + global __itsda_secret + key_dir = mg_globals.app_config["crypto_path"] + key_filepath = os.path.join(key_dir, 'itsdangeroussecret.bin') + try: + load_key(key_filepath) + except IOError, error: + if error.errno != errno.ENOENT: + raise + create_key(key_dir, key_filepath) + + +def get_timed_signer_url(namespace): + """ + This gives a basic signing/verifying object. + + The namespace makes sure signed tokens can't be used in + a different area. Like using a forgot-password-token as + a session cookie. + + Basic usage: + + .. code-block:: python + + _signer = None + TOKEN_VALID_DAYS = 10 + def setup(): + global _signer + _signer = get_timed_signer_url("session cookie") + def create_token(obj): + return _signer.dumps(obj) + def parse_token(token): + # This might raise an exception in case + # of an invalid token, or an expired token. + return _signer.loads(token, max_age=TOKEN_VALID_DAYS*24*3600) + + For more details see + http://pythonhosted.org/itsdangerous/#itsdangerous.URLSafeTimedSerializer + """ + assert __itsda_secret is not None + return itsdangerous.URLSafeTimedSerializer(__itsda_secret, + salt=namespace) diff --git a/mediagoblin/tools/exif.py b/mediagoblin/tools/exif.py new file mode 100644 index 00000000..6b3639e8 --- /dev/null +++ b/mediagoblin/tools/exif.py @@ -0,0 +1,187 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +try: + from EXIF import process_file, Ratio +except ImportError: + from mediagoblin.tools.extlib.EXIF import process_file, Ratio + +from mediagoblin.processing import BadMediaFail +from mediagoblin.tools.translate import pass_to_ugettext as _ + +# A list of tags that should be stored for faster access +USEFUL_TAGS = [ + 'Image Make', + 'Image Model', + 'EXIF FNumber', + 'EXIF Flash', + 'EXIF FocalLength', + 'EXIF ExposureTime', + 'EXIF ApertureValue', + 'EXIF ExposureMode', + 'EXIF ISOSpeedRatings', + 'EXIF UserComment', + ] + + +def exif_image_needs_rotation(exif_tags): + """ + Returns True if EXIF orientation requires rotation + """ + return 'Image Orientation' in exif_tags \ + and exif_tags['Image Orientation'].values[0] != 1 + + +def exif_fix_image_orientation(im, exif_tags): + """ + Translate any EXIF orientation to raw orientation + + Cons: + - Well, it changes the image, which means we'll recompress + it... not a problem if scaling it down already anyway. We might + lose some quality in recompressing if it's at the same-size + though + + Pros: + - Prevents neck pain + """ + # Rotate image + if 'Image Orientation' in exif_tags: + rotation_map = { + 3: 180, + 6: 270, + 8: 90} + orientation = exif_tags['Image Orientation'].values[0] + if orientation in rotation_map: + im = im.rotate( + rotation_map[orientation]) + + return im + + +def extract_exif(filename): + """ + Returns EXIF tags found in file at ``filename`` + """ + try: + with file(filename) as image: + return process_file(image, details=False) + except IOError: + raise BadMediaFail(_('Could not read the image file.')) + + +def clean_exif(exif): + ''' + Clean the result from anything the database cannot handle + ''' + # Discard any JPEG thumbnail, for database compatibility + # and that I cannot see a case when we would use it. + # It takes up some space too. + disabled_tags = [ + 'Thumbnail JPEGInterchangeFormatLength', + 'JPEGThumbnail', + 'Thumbnail JPEGInterchangeFormat'] + + return dict((key, _ifd_tag_to_dict(value)) for (key, value) + in exif.iteritems() if key not in disabled_tags) + + +def _ifd_tag_to_dict(tag): + ''' + Takes an IFD tag object from the EXIF library and converts it to a dict + that can be stored as JSON in the database. + ''' + data = { + 'printable': tag.printable, + 'tag': tag.tag, + 'field_type': tag.field_type, + 'field_offset': tag.field_offset, + 'field_length': tag.field_length, + 'values': None} + + if isinstance(tag.printable, str): + # Force it to be decoded as UTF-8 so that it'll fit into the DB + data['printable'] = tag.printable.decode('utf8', 'replace') + + if type(tag.values) == list: + data['values'] = [_ratio_to_list(val) if isinstance(val, Ratio) else val + for val in tag.values] + else: + if isinstance(tag.values, str): + # Force UTF-8, so that it fits into the DB + data['values'] = tag.values.decode('utf8', 'replace') + else: + data['values'] = tag.values + + return data + + +def _ratio_to_list(ratio): + return [ratio.num, ratio.den] + + +def get_useful(tags): + return dict((key, tag) for (key, tag) in tags.iteritems()) + + +def get_gps_data(tags): + """ + Processes EXIF data returned by EXIF.py + """ + gps_data = {} + + if not 'Image GPSInfo' in tags: + return gps_data + + try: + dms_data = { + 'latitude': tags['GPS GPSLatitude'], + 'longitude': tags['GPS GPSLongitude']} + + for key, dat in dms_data.iteritems(): + gps_data[key] = ( + lambda v: + float(v[0].num) / float(v[0].den) \ + + (float(v[1].num) / float(v[1].den) / 60) \ + + (float(v[2].num) / float(v[2].den) / (60 * 60)) + )(dat.values) + + if tags['GPS GPSLatitudeRef'].values == 'S': + gps_data['latitude'] /= -1 + + if tags['GPS GPSLongitudeRef'].values == 'W': + gps_data['longitude'] /= -1 + + except KeyError: + pass + + try: + gps_data['direction'] = ( + lambda d: + float(d.num) / float(d.den) + )(tags['GPS GPSImgDirection'].values[0]) + except KeyError: + pass + + try: + gps_data['altitude'] = ( + lambda a: + float(a.num) / float(a.den) + )(tags['GPS GPSAltitude'].values[0]) + except KeyError: + pass + + return gps_data diff --git a/mediagoblin/tools/extlib/EXIF.py b/mediagoblin/tools/extlib/EXIF.py new file mode 120000 index 00000000..82a2fb30 --- /dev/null +++ b/mediagoblin/tools/extlib/EXIF.py @@ -0,0 +1 @@ +../../../extlib/exif/EXIF.py
\ No newline at end of file diff --git a/mediagoblin/tools/extlib/__init__.py b/mediagoblin/tools/extlib/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/mediagoblin/tools/extlib/__init__.py diff --git a/mediagoblin/tools/extlib/wtf_html5.py b/mediagoblin/tools/extlib/wtf_html5.py new file mode 120000 index 00000000..5028c599 --- /dev/null +++ b/mediagoblin/tools/extlib/wtf_html5.py @@ -0,0 +1 @@ +../../../extlib/flask-wtf/html5.py
\ No newline at end of file diff --git a/mediagoblin/tools/files.py b/mediagoblin/tools/files.py new file mode 100644 index 00000000..848c86f2 --- /dev/null +++ b/mediagoblin/tools/files.py @@ -0,0 +1,43 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from mediagoblin import mg_globals + + +def delete_media_files(media): + """ + Delete all files associated with a MediaEntry + + Arguments: + - media: A MediaEntry document + """ + no_such_files = [] + for listpath in media.media_files.itervalues(): + try: + mg_globals.public_store.delete_file( + listpath) + except OSError: + no_such_files.append("/".join(listpath)) + + for attachment in media.attachment_files: + try: + mg_globals.public_store.delete_file( + attachment['filepath']) + except OSError: + no_such_files.append("/".join(attachment['filepath'])) + + if no_such_files: + raise OSError(", ".join(no_such_files)) diff --git a/mediagoblin/tools/licenses.py b/mediagoblin/tools/licenses.py new file mode 100644 index 00000000..a964980e --- /dev/null +++ b/mediagoblin/tools/licenses.py @@ -0,0 +1,69 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from collections import namedtuple +# Give a License attribute names: uri, name, abbreviation +License = namedtuple("License", ["abbreviation", "name", "uri"]) + +SORTED_LICENSES = [ + License("All rights reserved", "No license specified", ""), + License("CC BY 3.0", "Creative Commons Attribution Unported 3.0", + "http://creativecommons.org/licenses/by/3.0/"), + License("CC BY-SA 3.0", + "Creative Commons Attribution-ShareAlike Unported 3.0", + "http://creativecommons.org/licenses/by-sa/3.0/"), + License("CC BY-ND 3.0", + "Creative Commons Attribution-NoDerivs 3.0 Unported", + "http://creativecommons.org/licenses/by-nd/3.0/"), + License("CC BY-NC 3.0", + "Creative Commons Attribution-NonCommercial Unported 3.0", + "http://creativecommons.org/licenses/by-nc/3.0/"), + License("CC BY-NC-SA 3.0", + "Creative Commons Attribution-NonCommercial-ShareAlike 3.0 Unported", + "http://creativecommons.org/licenses/by-nc-sa/3.0/"), + License("CC BY-NC-ND 3.0", + "Creative Commons Attribution-NonCommercial-NoDerivs 3.0 Unported", + "http://creativecommons.org/licenses/by-nc-nd/3.0/"), + License("CC0 1.0", + "Creative Commons CC0 1.0 Universal", + "http://creativecommons.org/publicdomain/zero/1.0/"), + License("Public Domain","Public Domain", + "http://creativecommons.org/publicdomain/mark/1.0/"), + ] + +# dict {uri: License,...} to enable fast license lookup by uri. Ideally, +# we'd want to use an OrderedDict (python 2.7+) here to avoid having the +# same data in two structures +SUPPORTED_LICENSES = dict(((l.uri, l) for l in SORTED_LICENSES)) + + +def get_license_by_url(url): + """Look up a license by its url and return the License object""" + try: + return SUPPORTED_LICENSES[url] + except KeyError: + # in case of an unknown License, just display the url given + # rather than exploding in the user's face. + return License(url, url, url) + + +def licenses_as_choices(): + """List of (uri, abbreviation) tuples for HTML choice field population + + The data seems to be consumed/deleted during usage, so hand over a + throwaway list, rather than just a generator. + """ + return [(lic.uri, lic.abbreviation) for lic in SORTED_LICENSES] diff --git a/mediagoblin/tools/mail.py b/mediagoblin/tools/mail.py new file mode 100644 index 00000000..6886c859 --- /dev/null +++ b/mediagoblin/tools/mail.py @@ -0,0 +1,150 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import smtplib +from email.MIMEText import MIMEText +from mediagoblin import mg_globals, messages +from mediagoblin.tools import common + +### ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +### Special email test stuff begins HERE +### ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +# We have two "test inboxes" here: +# +# EMAIL_TEST_INBOX: +# ---------------- +# If you're writing test views, you'll probably want to check this. +# It contains a list of MIMEText messages. +# +# EMAIL_TEST_MBOX_INBOX: +# ---------------------- +# This collects the messages from the FakeMhost inbox. It's reslly +# just here for testing the send_email method itself. +# +# Anyway this contains: +# - from +# - to: a list of email recipient addresses +# - message: not just the body, but the whole message, including +# headers, etc. +# +# ***IMPORTANT!*** +# ---------------- +# Before running tests that call functions which send email, you should +# always call _clear_test_inboxes() to "wipe" the inboxes clean. + +EMAIL_TEST_INBOX = [] +EMAIL_TEST_MBOX_INBOX = [] + + +class FakeMhost(object): + """ + Just a fake mail host so we can capture and test messages + from send_email + """ + def login(self, *args, **kwargs): + pass + + def sendmail(self, from_addr, to_addrs, message): + EMAIL_TEST_MBOX_INBOX.append( + {'from': from_addr, + 'to': to_addrs, + 'message': message}) + + +def _clear_test_inboxes(): + global EMAIL_TEST_INBOX + global EMAIL_TEST_MBOX_INBOX + EMAIL_TEST_INBOX = [] + EMAIL_TEST_MBOX_INBOX = [] + + +### ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +### </Special email test stuff> +### ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +def send_email(from_addr, to_addrs, subject, message_body): + """ + Simple email sending wrapper, use this so we can capture messages + for unit testing purposes. + + Args: + - from_addr: address you're sending the email from + - to_addrs: list of recipient email addresses + - subject: subject of the email + - message_body: email body text + """ + if common.TESTS_ENABLED or mg_globals.app_config['email_debug_mode']: + mhost = FakeMhost() + elif not mg_globals.app_config['email_debug_mode']: + mhost = smtplib.SMTP( + mg_globals.app_config['email_smtp_host'], + mg_globals.app_config['email_smtp_port']) + + # SMTP.__init__ Issues SMTP.connect implicitly if host + if not mg_globals.app_config['email_smtp_host']: # e.g. host = '' + mhost.connect() # We SMTP.connect explicitly + + if ((not common.TESTS_ENABLED) + and (mg_globals.app_config['email_smtp_user'] + or mg_globals.app_config['email_smtp_pass'])): + mhost.login( + mg_globals.app_config['email_smtp_user'], + mg_globals.app_config['email_smtp_pass']) + + message = MIMEText(message_body.encode('utf-8'), 'plain', 'utf-8') + message['Subject'] = subject + message['From'] = from_addr + message['To'] = ', '.join(to_addrs) + + if common.TESTS_ENABLED: + EMAIL_TEST_INBOX.append(message) + + elif mg_globals.app_config['email_debug_mode']: + print u"===== Email =====" + print u"From address: %s" % message['From'] + print u"To addresses: %s" % message['To'] + print u"Subject: %s" % message['Subject'] + print u"-- Body: --" + print message.get_payload(decode=True) + + return mhost.sendmail(from_addr, to_addrs, message.as_string()) + + +def normalize_email(email): + """return case sensitive part, lower case domain name + + :returns: None in case of broken email addresses""" + try: + em_user, em_dom = email.split('@', 1) + except ValueError: + # email contained no '@' + return None + email = "@".join((em_user, em_dom.lower())) + return email + + +def email_debug_message(request): + """ + If the server is running in email debug mode (which is + the current default), give a debug message to the user + so that they have an idea where to find their email. + """ + if mg_globals.app_config['email_debug_mode']: + # DEBUG message, no need to translate + messages.add_message(request, messages.DEBUG, + u"This instance is running in email debug mode. " + u"The email will be on the console of the server process.") diff --git a/mediagoblin/tools/pagination.py b/mediagoblin/tools/pagination.py new file mode 100644 index 00000000..d0f08c94 --- /dev/null +++ b/mediagoblin/tools/pagination.py @@ -0,0 +1,113 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import urllib +import copy +from math import ceil, floor +from itertools import izip, count + + +PAGINATION_DEFAULT_PER_PAGE = 30 + + +class Pagination(object): + """ + Pagination class for database queries. + + Initialization through __init__(self, cursor, page=1, per_page=2), + get actual data slice through __call__(). + """ + + def __init__(self, page, cursor, per_page=PAGINATION_DEFAULT_PER_PAGE, + jump_to_id=False): + """ + Initializes Pagination + + Args: + - page: requested page + - per_page: number of objects per page + - cursor: db cursor + - jump_to_id: object id, sets the page to the page containing the + object with id == jump_to_id. + """ + self.page = page + self.per_page = per_page + self.cursor = cursor + self.total_count = self.cursor.count() + self.active_id = None + + if jump_to_id: + cursor = copy.copy(self.cursor) + + for (doc, increment) in izip(cursor, count(0)): + if doc.id == jump_to_id: + self.page = 1 + int(floor(increment / self.per_page)) + + self.active_id = jump_to_id + break + + def __call__(self): + """ + Returns slice of objects for the requested page + """ + # TODO, return None for out of index so templates can + # distinguish between empty galleries and out-of-bound pages??? + return self.cursor.slice( + (self.page - 1) * self.per_page, + self.page * self.per_page) + + @property + def pages(self): + return int(ceil(self.total_count / float(self.per_page))) + + @property + def has_prev(self): + return self.page > 1 + + @property + def has_next(self): + return self.page < self.pages + + def iter_pages(self, left_edge=2, left_current=2, + right_current=5, right_edge=2): + last = 0 + for num in xrange(1, self.pages + 1): + if num <= left_edge or \ + (num > self.page - left_current - 1 and \ + num < self.page + right_current) or \ + num > self.pages - right_edge: + if last + 1 != num: + yield None + yield num + last = num + + def get_page_url_explicit(self, base_url, get_params, page_no): + """ + Get a page url by adding a page= parameter to the base url + """ + new_get_params = dict(get_params) or {} + new_get_params['page'] = page_no + return "%s?%s" % ( + base_url, urllib.urlencode(new_get_params)) + + def get_page_url(self, request, page_no): + """ + Get a new page url based of the request, and the new page number. + + This is a nice wrapper around get_page_url_explicit() + """ + return self.get_page_url_explicit( + request.full_path, request.GET, page_no) diff --git a/mediagoblin/tools/pluginapi.py b/mediagoblin/tools/pluginapi.py new file mode 100644 index 00000000..3f98aa8a --- /dev/null +++ b/mediagoblin/tools/pluginapi.py @@ -0,0 +1,367 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +""" +This module implements the plugin api bits. + +Two things about things in this module: + +1. they should be excessively well documented because we should pull + from this file for the docs + +2. they should be well tested + + +How do plugins work? +==================== + +Plugins are structured like any Python project. You create a Python package. +In that package, you define a high-level ``__init__.py`` module that has a +``hooks`` dict that maps hooks to callables that implement those hooks. + +Additionally, you want a LICENSE file that specifies the license and a +``setup.py`` that specifies the metadata for packaging your plugin. A rough +file structure could look like this:: + + myplugin/ + |- setup.py # plugin project packaging metadata + |- README # holds plugin project information + |- LICENSE # holds license information + |- myplugin/ # plugin package directory + |- __init__.py # has hooks dict and code + + +Lifecycle +========= + +1. All the modules listed as subsections of the ``plugins`` section in + the config file are imported. MediaGoblin registers any hooks in + the ``hooks`` dict of those modules. + +2. After all plugin modules are imported, the ``setup`` hook is called + allowing plugins to do any set up they need to do. + +""" + +import logging + +from functools import wraps + +from mediagoblin import mg_globals + + +_log = logging.getLogger(__name__) + + +class PluginManager(object): + """Manager for plugin things + + .. Note:: + + This is a Borg class--there is one and only one of this class. + """ + __state = { + # list of plugin classes + "plugins": [], + + # map of hook names -> list of callables for that hook + "hooks": {}, + + # list of registered template paths + "template_paths": set(), + + # list of template hooks + "template_hooks": {}, + + # list of registered routes + "routes": [], + } + + def clear(self): + """This is only useful for testing.""" + # Why lists don't have a clear is not clear. + del self.plugins[:] + del self.routes[:] + self.hooks.clear() + self.template_paths.clear() + + def __init__(self): + self.__dict__ = self.__state + + def register_plugin(self, plugin): + """Registers a plugin class""" + self.plugins.append(plugin) + + def register_hooks(self, hook_mapping): + """Takes a hook_mapping and registers all the hooks""" + for hook, callables in hook_mapping.items(): + if isinstance(callables, (list, tuple)): + self.hooks.setdefault(hook, []).extend(list(callables)) + else: + # In this case, it's actually a single callable---not a + # list of callables. + self.hooks.setdefault(hook, []).append(callables) + + def get_hook_callables(self, hook_name): + return self.hooks.get(hook_name, []) + + def register_template_path(self, path): + """Registers a template path""" + self.template_paths.add(path) + + def get_template_paths(self): + """Returns a tuple of registered template paths""" + return tuple(self.template_paths) + + def register_route(self, route): + """Registers a single route""" + _log.debug('registering route: {0}'.format(route)) + self.routes.append(route) + + def get_routes(self): + return tuple(self.routes) + + def register_template_hooks(self, template_hooks): + for hook, templates in template_hooks.items(): + if isinstance(templates, (list, tuple)): + self.template_hooks.setdefault(hook, []).extend(list(templates)) + else: + # In this case, it's actually a single callable---not a + # list of callables. + self.template_hooks.setdefault(hook, []).append(templates) + + def get_template_hooks(self, hook_name): + return self.template_hooks.get(hook_name, []) + + +def register_routes(routes): + """Registers one or more routes + + If your plugin handles requests, then you need to call this with + the routes your plugin handles. + + A "route" is a `routes.Route` object. See `the routes.Route + documentation + <http://routes.readthedocs.org/en/latest/modules/route.html>`_ for + more details. + + Example passing in a single route: + + >>> register_routes(('about-view', '/about', + ... 'mediagoblin.views:about_view_handler')) + + Example passing in a list of routes: + + >>> register_routes([ + ... ('contact-view', '/contact', 'mediagoblin.views:contact_handler'), + ... ('about-view', '/about', 'mediagoblin.views:about_handler') + ... ]) + + + .. Note:: + + Be careful when designing your route urls. If they clash with + core urls, then it could result in DISASTER! + """ + if isinstance(routes, list): + for route in routes: + PluginManager().register_route(route) + else: + PluginManager().register_route(routes) + + +def register_template_path(path): + """Registers a path for template loading + + If your plugin has templates, then you need to call this with + the absolute path of the root of templates directory. + + Example: + + >>> my_plugin_dir = os.path.dirname(__file__) + >>> template_dir = os.path.join(my_plugin_dir, 'templates') + >>> register_template_path(template_dir) + + .. Note:: + + You can only do this in `setup_plugins()`. Doing this after + that will have no effect on template loading. + + """ + PluginManager().register_template_path(path) + + +def get_config(key): + """Retrieves the configuration for a specified plugin by key + + Example: + + >>> get_config('mediagoblin.plugins.sampleplugin') + {'foo': 'bar'} + >>> get_config('myplugin') + {} + >>> get_config('flatpages') + {'directory': '/srv/mediagoblin/pages', 'nesting': 1}} + + """ + + global_config = mg_globals.global_config + plugin_section = global_config.get('plugins', {}) + return plugin_section.get(key, {}) + + +def register_template_hooks(template_hooks): + """ + Register a dict of template hooks. + + Takes template_hooks as an argument, which is a dictionary of + template hook names/keys to the templates they should provide. + (The value can either be a single template path or an iterable + of paths.) + + Example: + + .. code-block:: python + + {"media_sidebar": "/plugin/sidemess/mess_up_the_side.html", + "media_descriptionbox": ["/plugin/sidemess/even_more_mess.html", + "/plugin/sidemess/so_much_mess.html"]} + """ + PluginManager().register_template_hooks(template_hooks) + + +def get_hook_templates(hook_name): + """ + Get a list of hook templates for this hook_name. + + Note: for the most part, you access this via a template tag, not + this method directly, like so: + + .. code-block:: html+jinja + + {% template_hook "media_sidebar" %} + + ... which will include all templates for you, partly using this + method. + + However, this method is exposed to templates, and if you wish, you + can iterate over templates in a template hook manually like so: + + .. code-block:: html+jinja + + {% for template_path in get_hook_templates("media_sidebar") %} + <div class="extra_structure"> + {% include template_path %} + </div> + {% endfor %} + + Returns: + A list of strings representing template paths. + """ + return PluginManager().get_template_hooks(hook_name) + + +############################# +## Hooks: The Next Generation +############################# + + +def hook_handle(hook_name, *args, **kwargs): + """ + Run through hooks attempting to find one that handle this hook. + + All callables called with the same arguments until one handles + things and returns a non-None value. + + (If you are writing a handler and you don't have a particularly + useful value to return even though you've handled this, returning + True is a good solution.) + + Note that there is a special keyword argument: + if "default_handler" is passed in as a keyword argument, this will + be used if no handler is found. + + Some examples of using this: + - You need an interface implemented, but only one fit for it + - You need to *do* something, but only one thing needs to do it. + """ + default_handler = kwargs.pop('default_handler', None) + + callables = PluginManager().get_hook_callables(hook_name) + + result = None + + for callable in callables: + result = callable(*args, **kwargs) + + if result is not None: + break + + if result is None and default_handler is not None: + result = default_handler(*args, **kwargs) + + return result + + +def hook_runall(hook_name, *args, **kwargs): + """ + Run through all callable hooks and pass in arguments. + + All non-None results are accrued in a list and returned from this. + (Other "false-like" values like False and friends are still + accrued, however.) + + Some examples of using this: + - You have an interface call where actually multiple things can + and should implement it + - You need to get a list of things from various plugins that + handle them and do something with them + - You need to *do* something, and actually multiple plugins need + to do it separately + """ + callables = PluginManager().get_hook_callables(hook_name) + + results = [] + + for callable in callables: + result = callable(*args, **kwargs) + + if result is not None: + results.append(result) + + return results + + +def hook_transform(hook_name, arg): + """ + Run through a bunch of hook callables and transform some input. + + Note that unlike the other hook tools, this one only takes ONE + argument. This argument is passed to each function, which in turn + returns something that becomes the input of the next callable. + + Some examples of using this: + - You have an object, say a form, but you want plugins to each be + able to modify it. + """ + result = arg + + callables = PluginManager().get_hook_callables(hook_name) + + for callable in callables: + result = callable(result) + + return result diff --git a/mediagoblin/tools/processing.py b/mediagoblin/tools/processing.py new file mode 100644 index 00000000..2abe6452 --- /dev/null +++ b/mediagoblin/tools/processing.py @@ -0,0 +1,87 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import logging +import json +import traceback + +from urllib2 import urlopen, Request, HTTPError +from urllib import urlencode + +_log = logging.getLogger(__name__) + +TESTS_CALLBACKS = {} + + +def create_post_request(url, data, **kw): + ''' + Issue a HTTP POST request. + + Args: + url: The URL to which the POST request should be issued + data: The data to be send in the body of the request + **kw: + data_parser: The parser function that is used to parse the `data` + argument + ''' + data_parser = kw.get('data_parser', urlencode) + headers = kw.get('headers', {}) + + return Request(url, data_parser(data), headers=headers) + + +def json_processing_callback(entry): + ''' + Send an HTTP post to the registered callback url, if any. + ''' + if not entry.processing_metadata: + _log.debug('No processing callback for {0}'.format(entry)) + return + + url = entry.processing_metadata[0].callback_url + + _log.debug('Sending processing callback for {0} ({1})'.format( + entry, + url)) + + headers = { + 'Content-Type': 'application/json'} + + data = { + 'id': entry.id, + 'state': entry.state} + + # Trigger testing mode, no callback will be sent + if url.endswith('secrettestmediagoblinparam'): + TESTS_CALLBACKS.update({url: data}) + return True + + request = create_post_request( + url, + data, + headers=headers, + data_parser=json.dumps) + + try: + urlopen(request) + _log.debug('Processing callback for {0} sent'.format(entry)) + + return True + except HTTPError: + _log.error('Failed to send callback: {0}'.format( + traceback.format_exc())) + + return False diff --git a/mediagoblin/tools/request.py b/mediagoblin/tools/request.py new file mode 100644 index 00000000..ee342eae --- /dev/null +++ b/mediagoblin/tools/request.py @@ -0,0 +1,38 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import logging +from mediagoblin.db.models import User + +_log = logging.getLogger(__name__) + + +def setup_user_in_request(request): + """ + Examine a request and tack on a request.user parameter if that's + appropriate. + """ + if 'user_id' not in request.session: + request.user = None + return + + request.user = User.query.get(request.session['user_id']) + + if not request.user: + # Something's wrong... this user doesn't exist? Invalidate + # this session. + _log.warn("Killing session for user id %r", request.session['user_id']) + request.session.delete() diff --git a/mediagoblin/tools/response.py b/mediagoblin/tools/response.py new file mode 100644 index 00000000..aaf31d0b --- /dev/null +++ b/mediagoblin/tools/response.py @@ -0,0 +1,108 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import werkzeug.utils +from werkzeug.wrappers import Response as wz_Response +from mediagoblin.tools.template import render_template +from mediagoblin.tools.translate import (lazy_pass_to_ugettext as _, + pass_to_ugettext) + +class Response(wz_Response): + """Set default response mimetype to HTML, otherwise we get text/plain""" + default_mimetype = u'text/html' + + +def render_to_response(request, template, context, status=200): + """Much like Django's shortcut.render()""" + return Response( + render_template(request, template, context), + status=status) + + +def render_error(request, status=500, title=_('Oops!'), + err_msg=_('An error occured')): + """Render any error page with a given error code, title and text body + + Title and description are passed through as-is to allow html. Make + sure no user input is contained therein for security reasons. The + description will be wrapped in <p></p> tags. + """ + return Response(render_template(request, 'mediagoblin/error.html', + {'err_code': status, 'title': title, 'err_msg': err_msg}), + status=status) + + +def render_403(request): + """Render a standard 403 page""" + _ = pass_to_ugettext + title = _('Operation not allowed') + err_msg = _("Sorry Dave, I can't let you do that!</p><p>You have tried " + " to perform a function that you are not allowed to. Have you " + "been trying to delete all user accounts again?") + return render_error(request, 403, title, err_msg) + +def render_404(request): + """Render a standard 404 page.""" + _ = pass_to_ugettext + err_msg = _("There doesn't seem to be a page at this address. Sorry!</p>" + "<p>If you're sure the address is correct, maybe the page " + "you're looking for has been moved or deleted.") + return render_error(request, 404, err_msg=err_msg) + + +def render_http_exception(request, exc, description): + """Return Response() given a werkzeug.HTTPException + + :param exc: werkzeug.HTTPException or subclass thereof + :description: message describing the error.""" + # If we were passed the HTTPException stock description on + # exceptions where we have localized ones, use those: + stock_desc = (description == exc.__class__.description) + + if stock_desc and exc.code == 403: + return render_403(request) + elif stock_desc and exc.code == 404: + return render_404(request) + + return render_error(request, title=exc.args[0], + err_msg=description, + status=exc.code) + + +def redirect(request, *args, **kwargs): + """Redirects to an URL, using urlgen params or location string + + :param querystring: querystring to be appended to the URL + :param location: If the location keyword is given, redirect to the URL + """ + querystring = kwargs.pop('querystring', None) + + # Redirect to URL if given by "location=..." + if 'location' in kwargs: + location = kwargs.pop('location') + else: + location = request.urlgen(*args, **kwargs) + + if querystring: + location += querystring + return werkzeug.utils.redirect(location) + + +def redirect_obj(request, obj): + """Redirect to the page for the given object. + + Requires obj to have a .url_for_self method.""" + return redirect(request, location=obj.url_for_self(request.urlgen)) diff --git a/mediagoblin/tools/routing.py b/mediagoblin/tools/routing.py new file mode 100644 index 00000000..a15795fe --- /dev/null +++ b/mediagoblin/tools/routing.py @@ -0,0 +1,67 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import logging + +import six +from werkzeug.routing import Map, Rule +from mediagoblin.tools.common import import_component + + +_log = logging.getLogger(__name__) + +url_map = Map() + + +class MGRoute(Rule): + def __init__(self, endpoint, url, controller): + Rule.__init__(self, url, endpoint=endpoint) + self.gmg_controller = controller + + def empty(self): + new_rule = Rule.empty(self) + new_rule.gmg_controller = self.gmg_controller + return new_rule + + +def endpoint_to_controller(rule): + endpoint = rule.endpoint + view_func = rule.gmg_controller + + _log.debug('endpoint: {0} view_func: {1}'.format(endpoint, view_func)) + + # import the endpoint, or if it's already a callable, call that + if isinstance(view_func, six.string_types): + view_func = import_component(view_func) + rule.gmg_controller = view_func + + return view_func + + +def add_route(endpoint, url, controller): + """ + Add a route to the url mapping + """ + url_map.add(MGRoute(endpoint, url, controller)) + + +def mount(mountpoint, routes): + """ + Mount a bunch of routes to this mountpoint + """ + for endpoint, url, controller in routes: + url = "%s/%s" % (mountpoint.rstrip('/'), url.lstrip('/')) + add_route(endpoint, url, controller) diff --git a/mediagoblin/tools/session.py b/mediagoblin/tools/session.py new file mode 100644 index 00000000..fdc32523 --- /dev/null +++ b/mediagoblin/tools/session.py @@ -0,0 +1,68 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2013 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import itsdangerous +import logging + +import crypto + +_log = logging.getLogger(__name__) + +class Session(dict): + def __init__(self, *args, **kwargs): + self.send_new_cookie = False + dict.__init__(self, *args, **kwargs) + + def save(self): + self.send_new_cookie = True + + def is_updated(self): + return self.send_new_cookie + + def delete(self): + self.clear() + self.save() + + +class SessionManager(object): + def __init__(self, cookie_name='MGSession', namespace=None): + if namespace is None: + namespace = cookie_name + self.signer = crypto.get_timed_signer_url(namespace) + self.cookie_name = cookie_name + + def load_session_from_cookie(self, request): + cookie = request.cookies.get(self.cookie_name) + if not cookie: + return Session() + ### FIXME: Future cookie-blacklisting code + # m = BadCookie.query.filter_by(cookie = cookie) + # if m: + # _log.warn("Bad cookie received: %s", m.reason) + # raise BadRequest() + try: + return Session(self.signer.loads(cookie)) + except itsdangerous.BadData: + return Session() + + def save_session_to_cookie(self, session, request, response): + if not session.is_updated(): + return + elif not session: + response.delete_cookie(self.cookie_name) + else: + response.set_cookie(self.cookie_name, self.signer.dumps(session), + httponly=True) diff --git a/mediagoblin/tools/staticdirect.py b/mediagoblin/tools/staticdirect.py new file mode 100644 index 00000000..ef8b20d0 --- /dev/null +++ b/mediagoblin/tools/staticdirect.py @@ -0,0 +1,101 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +#################################### +# Staticdirect infrastructure. +# Borrowed largely from cc.engine +# by Chris Webber & Creative Commons +# +# This needs documentation! +#################################### + +import logging + +_log = logging.getLogger(__name__) + + +class StaticDirect(object): + """ + Direct to a static resource. + + This StaticDirect class can take a series of "domains" to + staticdirect to. In general, you should supply a None domain, as + that's the "default" domain. + + Things work like this: + >>> staticdirect = StaticDirect( + ... {None: "/static/", + ... "theme": "http://example.org/themestatic/"}) + >>> staticdirect("css/monkeys.css") + "/static/css/monkeys.css" + >>> staticdirect("images/lollerskate.png", "theme") + "http://example.org/themestatic/images/lollerskate.png" + """ + def __init__(self, domains): + self.domains = dict( + [(key, value.rstrip('/')) + for key, value in domains.iteritems()]) + self.cache = {} + + def __call__(self, filepath, domain=None): + if domain in self.cache and filepath in self.cache[domain]: + return self.cache[domain][filepath] + + static_direction = self.cache.setdefault( + domain, {})[filepath] = self.get(filepath, domain) + return static_direction + + def get(self, filepath, domain=None): + return '%s/%s' % ( + self.domains[domain], filepath.lstrip('/')) + + +class PluginStatic(object): + """Pass this into the ``'static_setup'`` hook to register your + plugin's static directory. + + This has two mandatory attributes that you must pass in on class + init: + - name: this name will be both used for lookup in "urlgen" for + your plugin's static resources and for the subdirectory that + it'll be "mounted" to for serving via your web browser. It + *MUST* be unique. If writing a plugin bundled with MediaGoblin + please use the pattern 'coreplugin__foo' where 'foo' is your + plugin name. All external plugins should use their modulename, + so if your plugin is 'mg_bettertags' you should also call this + name 'mg_bettertags'. + - file_path: the directory your plugin's static resources are + located in. It's recommended that you use + pkg_resources.resource_filename() for this. + + An example of using this:: + + from pkg_resources import resource_filename + from mediagoblin.tools.staticdirect import PluginStatic + + hooks = { + 'static_setup': lambda: PluginStatic( + 'mg_bettertags', + resource_filename('mg_bettertags', 'static')) + } + + """ + def __init__(self, name, file_path): + self.name = name + self.file_path = file_path + + def __call__(self): + return self diff --git a/mediagoblin/tools/template.py b/mediagoblin/tools/template.py new file mode 100644 index 00000000..3d651a6e --- /dev/null +++ b/mediagoblin/tools/template.py @@ -0,0 +1,160 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +import jinja2 +from jinja2.ext import Extension +from jinja2.nodes import Include, Const + +from babel.localedata import exists +from werkzeug.urls import url_quote_plus + +from mediagoblin import mg_globals +from mediagoblin import messages +from mediagoblin import _version +from mediagoblin.tools import common +from mediagoblin.tools.translate import set_thread_locale +from mediagoblin.tools.pluginapi import get_hook_templates, hook_transform +from mediagoblin.tools.timesince import timesince +from mediagoblin.meddleware.csrf import render_csrf_form_token + + + +SETUP_JINJA_ENVS = {} + + +def get_jinja_env(template_loader, locale): + """ + Set up the Jinja environment, + + (In the future we may have another system for providing theming; + for now this is good enough.) + """ + set_thread_locale(locale) + + # If we have a jinja environment set up with this locale, just + # return that one. + if locale in SETUP_JINJA_ENVS: + return SETUP_JINJA_ENVS[locale] + + # jinja2.StrictUndefined will give exceptions on references + # to undefined/unknown variables in templates. + template_env = jinja2.Environment( + loader=template_loader, autoescape=True, + undefined=jinja2.StrictUndefined, + extensions=[ + 'jinja2.ext.i18n', 'jinja2.ext.autoescape', + TemplateHookExtension]) + + template_env.install_gettext_callables( + mg_globals.thread_scope.translations.ugettext, + mg_globals.thread_scope.translations.ungettext) + + # All templates will know how to ... + # ... fetch all waiting messages and remove them from the queue + # ... construct a grid of thumbnails or other media + # ... have access to the global and app config + template_env.globals['fetch_messages'] = messages.fetch_messages + template_env.globals['app_config'] = mg_globals.app_config + template_env.globals['global_config'] = mg_globals.global_config + template_env.globals['version'] = _version.__version__ + + template_env.filters['urlencode'] = url_quote_plus + + # add human readable fuzzy date time + template_env.globals['timesince'] = timesince + + # allow for hooking up plugin templates + template_env.globals['get_hook_templates'] = get_hook_templates + + template_env.globals = hook_transform( + 'template_global_context', template_env.globals) + + if exists(locale): + SETUP_JINJA_ENVS[locale] = template_env + + return template_env + + +# We'll store context information here when doing unit tests +TEMPLATE_TEST_CONTEXT = {} + + +def render_template(request, template_path, context): + """ + Render a template with context. + + Always inserts the request into the context, so you don't have to. + Also stores the context if we're doing unit tests. Helpful! + """ + template = request.template_env.get_template( + template_path) + context['request'] = request + rendered_csrf_token = render_csrf_form_token(request) + if rendered_csrf_token is not None: + context['csrf_token'] = render_csrf_form_token(request) + + # allow plugins to do things to the context + if request.controller_name: + context = hook_transform( + (request.controller_name, template_path), + context) + + # More evil: allow plugins to possibly do something to the context + # in every request ever with access to the request and other + # variables. Note: this is slower than using + # template_global_context + context = hook_transform( + 'template_context_prerender', context) + + rendered = template.render(context) + + if common.TESTS_ENABLED: + TEMPLATE_TEST_CONTEXT[template_path] = context + + return rendered + + +def clear_test_template_context(): + global TEMPLATE_TEST_CONTEXT + TEMPLATE_TEST_CONTEXT = {} + + +class TemplateHookExtension(Extension): + """ + Easily loop through a bunch of templates from a template hook. + + Use: + {% template_hook("comment_extras") %} + + ... will include all templates hooked into the comment_extras section. + """ + + tags = set(["template_hook"]) + + def parse(self, parser): + includes = [] + expr = parser.parse_expression() + lineno = expr.lineno + hook_name = expr.args[0].value + + for template_name in get_hook_templates(hook_name): + includes.append( + parser.parse_import_context( + Include(Const(template_name), True, False, lineno=lineno), + True)) + + return includes diff --git a/mediagoblin/tools/testing.py b/mediagoblin/tools/testing.py new file mode 100644 index 00000000..7f2bcbfb --- /dev/null +++ b/mediagoblin/tools/testing.py @@ -0,0 +1,45 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from mediagoblin.tools import common +from mediagoblin.tools.template import clear_test_template_context +from mediagoblin.tools.mail import EMAIL_TEST_INBOX, EMAIL_TEST_MBOX_INBOX + +def _activate_testing(): + """ + Call this to activate testing in util.py + """ + + common.TESTS_ENABLED = True + +def clear_test_buckets(): + """ + We store some things for testing purposes that should be cleared + when we want a "clean slate" of information for our next round of + tests. Call this function to wipe all that stuff clean. + + Also wipes out some other things we might redefine during testing, + like the jinja envs. + """ + global SETUP_JINJA_ENVS + SETUP_JINJA_ENVS = {} + + global EMAIL_TEST_INBOX + global EMAIL_TEST_MBOX_INBOX + EMAIL_TEST_INBOX = [] + EMAIL_TEST_MBOX_INBOX = [] + + clear_test_template_context() diff --git a/mediagoblin/tools/text.py b/mediagoblin/tools/text.py new file mode 100644 index 00000000..96df49d2 --- /dev/null +++ b/mediagoblin/tools/text.py @@ -0,0 +1,124 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import wtforms +import markdown +from lxml.html.clean import Cleaner + +from mediagoblin import mg_globals +from mediagoblin.tools import url + + +# A super strict version of the lxml.html cleaner class +HTML_CLEANER = Cleaner( + scripts=True, + javascript=True, + comments=True, + style=True, + links=True, + page_structure=True, + processing_instructions=True, + embedded=True, + frames=True, + forms=True, + annoying_tags=True, + allow_tags=[ + 'div', 'b', 'i', 'em', 'strong', 'p', 'ul', 'ol', 'li', 'a', 'br', + 'pre', 'code'], + remove_unknown_tags=False, # can't be used with allow_tags + safe_attrs_only=True, + add_nofollow=True, # for now + host_whitelist=(), + whitelist_tags=set([])) + + +def clean_html(html): + # clean_html barfs on an empty string + if not html: + return u'' + + return HTML_CLEANER.clean_html(html) + + +def convert_to_tag_list_of_dicts(tag_string): + """ + Filter input from incoming string containing user tags, + + Strips trailing, leading, and internal whitespace, and also converts + the "tags" text into an array of tags + """ + taglist = [] + if tag_string: + + # Strip out internal, trailing, and leading whitespace + stripped_tag_string = u' '.join(tag_string.strip().split()) + + # Split the tag string into a list of tags + for tag in stripped_tag_string.split(','): + tag = tag.strip() + # Ignore empty or duplicate tags + if tag and tag not in [t['name'] for t in taglist]: + taglist.append({'name': tag, + 'slug': url.slugify(tag)}) + return taglist + + +def media_tags_as_string(media_entry_tags): + """ + Generate a string from a media item's tags, stored as a list of dicts + + This is the opposite of convert_to_tag_list_of_dicts + """ + tags_string = '' + if media_entry_tags: + tags_string = u', '.join([tag['name'] for tag in media_entry_tags]) + return tags_string + + +TOO_LONG_TAG_WARNING = \ + u'Tags must be shorter than %s characters. Tags that are too long: %s' + + +def tag_length_validator(form, field): + """ + Make sure tags do not exceed the maximum tag length. + """ + tags = convert_to_tag_list_of_dicts(field.data) + too_long_tags = [ + tag['name'] for tag in tags + if len(tag['name']) > mg_globals.app_config['tags_max_length']] + + if too_long_tags: + raise wtforms.ValidationError( + TOO_LONG_TAG_WARNING % (mg_globals.app_config['tags_max_length'], + ', '.join(too_long_tags))) + + +# Don't use the safe mode, because lxml.html.clean is better and we are using +# it anyway +UNSAFE_MARKDOWN_INSTANCE = markdown.Markdown() + + +def cleaned_markdown_conversion(text): + """ + Take a block of text, run it through MarkDown, and clean its HTML. + """ + # Markdown will do nothing with and clean_html can do nothing with + # an empty string :) + if not text: + return u'' + + return clean_html(UNSAFE_MARKDOWN_INSTANCE.convert(text)) diff --git a/mediagoblin/tools/theme.py b/mediagoblin/tools/theme.py new file mode 100644 index 00000000..97b041a6 --- /dev/null +++ b/mediagoblin/tools/theme.py @@ -0,0 +1,89 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +""" +""" + + +import pkg_resources +import os + +from configobj import ConfigObj + + +BUILTIN_THEME_DIR = pkg_resources.resource_filename('mediagoblin', 'themes') + + +def themedata_for_theme_dir(name, theme_dir): + """ + Given a theme directory, extract important theme information. + """ + # open config + config = ConfigObj(os.path.join(theme_dir, 'theme.ini')).get('theme', {}) + + templates_dir = os.path.join(theme_dir, 'templates') + if not os.path.exists(templates_dir): + templates_dir = None + + assets_dir = os.path.join(theme_dir, 'assets') + if not os.path.exists(assets_dir): + assets_dir = None + + themedata = { + 'name': config.get('name', name), + 'description': config.get('description'), + 'licensing': config.get('licensing'), + 'dir': theme_dir, + 'templates_dir': templates_dir, + 'assets_dir': assets_dir, + 'config': config} + + return themedata + + +def register_themes(app_config, builtin_dir=BUILTIN_THEME_DIR): + """ + Register all themes relevant to this application. + """ + registry = {} + + def _install_themes_in_dir(directory): + for themedir in os.listdir(directory): + abs_themedir = os.path.join(directory, themedir) + if not os.path.isdir(abs_themedir): + continue + + themedata = themedata_for_theme_dir(themedir, abs_themedir) + registry[themedir] = themedata + + # Built-in themes + if os.path.exists(builtin_dir): + _install_themes_in_dir(builtin_dir) + + # Installed themes + theme_install_dir = app_config.get('theme_install_dir') + if theme_install_dir and os.path.exists(theme_install_dir): + _install_themes_in_dir(theme_install_dir) + + current_theme_name = app_config.get('theme') + if current_theme_name \ + and registry.has_key(current_theme_name): + current_theme = registry[current_theme_name] + else: + current_theme = None + + return registry, current_theme + diff --git a/mediagoblin/tools/timesince.py b/mediagoblin/tools/timesince.py new file mode 100644 index 00000000..b761c1be --- /dev/null +++ b/mediagoblin/tools/timesince.py @@ -0,0 +1,95 @@ +# Copyright (c) Django Software Foundation and individual contributors. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# 3. Neither the name of Django nor the names of its contributors may be used +# to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import unicode_literals + +import datetime +import pytz + +from mediagoblin.tools.translate import pass_to_ugettext, lazy_pass_to_ungettext as _ + +"""UTC time zone as a tzinfo instance.""" +utc = pytz.utc if pytz else UTC() + +def is_aware(value): + """ + Determines if a given datetime.datetime is aware. + + The logic is described in Python's docs: + http://docs.python.org/library/datetime.html#datetime.tzinfo + """ + return value.tzinfo is not None and value.tzinfo.utcoffset(value) is not None + +def timesince(d, now=None, reversed=False): + """ + Takes two datetime objects and returns the time between d and now + as a nicely formatted string, e.g. "10 minutes". If d occurs after now, + then "0 minutes" is returned. + + Units used are years, months, weeks, days, hours, and minutes. + Seconds and microseconds are ignored. Up to two adjacent units will be + displayed. For example, "2 weeks, 3 days" and "1 year, 3 months" are + possible outputs, but "2 weeks, 3 hours" and "1 year, 5 days" are not. + + Adapted from http://blog.natbat.co.uk/archive/2003/Jun/14/time_since + """ + chunks = ( + (60 * 60 * 24 * 365, lambda n: _('year', 'years', n)), + (60 * 60 * 24 * 30, lambda n: _('month', 'months', n)), + (60 * 60 * 24 * 7, lambda n : _('week', 'weeks', n)), + (60 * 60 * 24, lambda n : _('day', 'days', n)), + (60 * 60, lambda n: _('hour', 'hours', n)), + (60, lambda n: _('minute', 'minutes', n)) + ) + # Convert datetime.date to datetime.datetime for comparison. + if not isinstance(d, datetime.datetime): + d = datetime.datetime(d.year, d.month, d.day) + if now and not isinstance(now, datetime.datetime): + now = datetime.datetime(now.year, now.month, now.day) + + if not now: + now = datetime.datetime.now(utc if is_aware(d) else None) + + delta = (d - now) if reversed else (now - d) + # ignore microseconds + since = delta.days * 24 * 60 * 60 + delta.seconds + if since <= 0: + # d is in the future compared to now, stop processing. + return '0 ' + pass_to_ugettext('minutes') + for i, (seconds, name) in enumerate(chunks): + count = since // seconds + if count != 0: + break + s = pass_to_ugettext('%(number)d %(type)s') % {'number': count, 'type': name(count)} + if i + 1 < len(chunks): + # Now get the second item + seconds2, name2 = chunks[i + 1] + count2 = (since - (seconds * count)) // seconds2 + if count2 != 0: + s += pass_to_ugettext(', %(number)d %(type)s') % {'number': count2, 'type': name2(count2)} + return s diff --git a/mediagoblin/tools/translate.py b/mediagoblin/tools/translate.py new file mode 100644 index 00000000..b20e57d1 --- /dev/null +++ b/mediagoblin/tools/translate.py @@ -0,0 +1,211 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import gettext +import pkg_resources + + +from babel import localedata +from babel.support import LazyProxy + +from mediagoblin import mg_globals + +################### +# Translation tools +################### + +AVAILABLE_LOCALES = None +TRANSLATIONS_PATH = pkg_resources.resource_filename( + 'mediagoblin', 'i18n') + + +def set_available_locales(): + """Set available locales for which we have translations""" + global AVAILABLE_LOCALES + locales=['en', 'en_US'] # these are available without translations + for locale in localedata.list(): + if gettext.find('mediagoblin', TRANSLATIONS_PATH, [locale]): + locales.append(locale) + AVAILABLE_LOCALES = locales + + +class ReallyLazyProxy(LazyProxy): + """ + Like LazyProxy, except that it doesn't cache the value ;) + """ + @property + def value(self): + return self._func(*self._args, **self._kwargs) + + def __repr__(self): + return "<%s for %s(%r, %r)>" % ( + self.__class__.__name__, + self._func, + self._args, + self._kwargs) + + +def locale_to_lower_upper(locale): + """ + Take a locale, regardless of style, and format it like "en_US" + """ + if '-' in locale: + lang, country = locale.split('-', 1) + return '%s_%s' % (lang.lower(), country.upper()) + elif '_' in locale: + lang, country = locale.split('_', 1) + return '%s_%s' % (lang.lower(), country.upper()) + else: + return locale.lower() + + +def locale_to_lower_lower(locale): + """ + Take a locale, regardless of style, and format it like "en_us" + """ + if '_' in locale: + lang, country = locale.split('_', 1) + return '%s-%s' % (lang.lower(), country.lower()) + else: + return locale.lower() + + +def get_locale_from_request(request): + """ + Return most appropriate language based on prefs/request request + """ + request_args = (request.args, request.form)[request.method=='POST'] + + if 'lang' in request_args: + # User explicitely demanded a language, normalize lower_uppercase + target_lang = locale_to_lower_upper(request_args['lang']) + + elif 'target_lang' in request.session: + # TODO: Uh, ohh, this is never ever set anywhere? + target_lang = request.session['target_lang'] + else: + # Pull the most acceptable language based on browser preferences + # This returns one of AVAILABLE_LOCALES which is aready case-normalized. + # Note: in our tests request.accept_languages is None, so we need + # to explicitely fallback to en here. + target_lang = request.accept_languages.best_match(AVAILABLE_LOCALES) \ + or "en_US" + + return target_lang + +SETUP_GETTEXTS = {} + +def get_gettext_translation(locale): + """ + Return the gettext instance based on this locale + """ + # Later on when we have plugins we may want to enable the + # multi-translations system they have so we can handle plugin + # translations too + + # TODO: fallback nicely on translations from pt_PT to pt if not + # available, etc. + if locale in SETUP_GETTEXTS: + this_gettext = SETUP_GETTEXTS[locale] + else: + this_gettext = gettext.translation( + 'mediagoblin', TRANSLATIONS_PATH, [locale], fallback=True) + if localedata.exists(locale): + SETUP_GETTEXTS[locale] = this_gettext + return this_gettext + + +def set_thread_locale(locale): + """Set the current translation for this thread""" + mg_globals.thread_scope.translations = get_gettext_translation(locale) + + +def pass_to_ugettext(*args, **kwargs): + """ + Pass a translation on to the appropriate ugettext method. + + The reason we can't have a global ugettext method is because + mg_globals gets swapped out by the application per-request. + """ + return mg_globals.thread_scope.translations.ugettext( + *args, **kwargs) + +def pass_to_ungettext(*args, **kwargs): + """ + Pass a translation on to the appropriate ungettext method. + + The reason we can't have a global ugettext method is because + mg_globals gets swapped out by the application per-request. + """ + return mg_globals.thread_scope.translations.ungettext( + *args, **kwargs) + + +def lazy_pass_to_ugettext(*args, **kwargs): + """ + Lazily pass to ugettext. + + This is useful if you have to define a translation on a module + level but you need it to not translate until the time that it's + used as a string. For example, in: + def func(self, message=_('Hello boys and girls')) + + you would want to use the lazy version for _. + """ + return ReallyLazyProxy(pass_to_ugettext, *args, **kwargs) + + +def pass_to_ngettext(*args, **kwargs): + """ + Pass a translation on to the appropriate ngettext method. + + The reason we can't have a global ngettext method is because + mg_globals gets swapped out by the application per-request. + """ + return mg_globals.thread_scope.translations.ngettext( + *args, **kwargs) + + +def lazy_pass_to_ngettext(*args, **kwargs): + """ + Lazily pass to ngettext. + + This is useful if you have to define a translation on a module + level but you need it to not translate until the time that it's + used as a string. + """ + return ReallyLazyProxy(pass_to_ngettext, *args, **kwargs) + +def lazy_pass_to_ungettext(*args, **kwargs): + """ + Lazily pass to ungettext. + + This is useful if you have to define a translation on a module + level but you need it to not translate until the time that it's + used as a string. + """ + return ReallyLazyProxy(pass_to_ungettext, *args, **kwargs) + + +def fake_ugettext_passthrough(string): + """ + Fake a ugettext call for extraction's sake ;) + + In wtforms there's a separate way to define a method to translate + things... so we just need to mark up the text so that it can be + extracted, not so that it's actually run through gettext. + """ + return string diff --git a/mediagoblin/tools/url.py b/mediagoblin/tools/url.py new file mode 100644 index 00000000..d9179f9e --- /dev/null +++ b/mediagoblin/tools/url.py @@ -0,0 +1,44 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import re +# This import *is* used; see word.encode('tranlit/long') below. +from unicodedata import normalize + +try: + import translitcodec + USING_TRANSLITCODEC = True +except ImportError: + USING_TRANSLITCODEC = False + + +_punct_re = re.compile(r'[\t !"#:$%&\'()*\-/<=>?@\[\\\]^_`{|},.]+') + + +def slugify(text, delim=u'-'): + """ + Generates an ASCII-only slug. Taken from http://flask.pocoo.org/snippets/5/ + """ + result = [] + for word in _punct_re.split(text.lower()): + if USING_TRANSLITCODEC: + word = word.encode('translit/long') + else: + word = normalize('NFKD', word).encode('ascii', 'ignore') + + if word: + result.append(word) + return unicode(delim.join(result)) diff --git a/mediagoblin/tools/workbench.py b/mediagoblin/tools/workbench.py new file mode 100644 index 00000000..0bd4096b --- /dev/null +++ b/mediagoblin/tools/workbench.py @@ -0,0 +1,164 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import shutil +import tempfile + + +# Actual workbench stuff +# ---------------------- + +class Workbench(object): + """ + Represent the directory for the workbench + + WARNING: DO NOT create Workbench objects on your own, + let the WorkbenchManager do that for you! + """ + def __init__(self, dir): + """ + WARNING: DO NOT create Workbench objects on your own, + let the WorkbenchManager do that for you! + """ + self.dir = dir + + def __unicode__(self): + return unicode(self.dir) + + def __str__(self): + return str(self.dir) + + def __repr__(self): + try: + return str(self) + except AttributeError: + return 'None' + + def joinpath(self, *args): + return os.path.join(self.dir, *args) + + def localized_file(self, storage, filepath, + filename_if_copying=None, + keep_extension_if_copying=True): + """ + Possibly localize the file from this storage system (for read-only + purposes, modifications should be written to a new file.). + + If the file is already local, just return the absolute filename of that + local file. Otherwise, copy the file locally to the workbench, and + return the absolute path of the new file. + + If it is copying locally, we might want to require a filename like + "source.jpg" to ensure that we won't conflict with other filenames in + our workbench... if that's the case, make sure filename_if_copying is + set to something like 'source.jpg'. Relatedly, if you set + keep_extension_if_copying, you don't have to set an extension on + filename_if_copying yourself, it'll be set for you (assuming such an + extension can be extacted from the filename in the filepath). + + Returns: + localized_filename + + Examples: + >>> wb_manager.localized_file( + ... '/our/workbench/subdir', local_storage, + ... ['path', 'to', 'foobar.jpg']) + u'/local/storage/path/to/foobar.jpg' + + >>> wb_manager.localized_file( + ... '/our/workbench/subdir', remote_storage, + ... ['path', 'to', 'foobar.jpg']) + '/our/workbench/subdir/foobar.jpg' + + >>> wb_manager.localized_file( + ... '/our/workbench/subdir', remote_storage, + ... ['path', 'to', 'foobar.jpg'], 'source.jpeg', False) + '/our/workbench/subdir/foobar.jpeg' + + >>> wb_manager.localized_file( + ... '/our/workbench/subdir', remote_storage, + ... ['path', 'to', 'foobar.jpg'], 'source', True) + '/our/workbench/subdir/foobar.jpg' + """ + if storage.local_storage: + return storage.get_local_path(filepath) + else: + if filename_if_copying is None: + dest_filename = filepath[-1] + else: + orig_filename, orig_ext = os.path.splitext(filepath[-1]) + if keep_extension_if_copying and orig_ext: + dest_filename = filename_if_copying + orig_ext + else: + dest_filename = filename_if_copying + + full_dest_filename = os.path.join( + self.dir, dest_filename) + + # copy it over + storage.copy_locally( + filepath, full_dest_filename) + + return full_dest_filename + + def destroy(self): + """ + Destroy this workbench! Deletes the directory and all its contents! + + WARNING: Does no checks for a sane value in self.dir! + """ + # just in case + workbench = os.path.abspath(self.dir) + shutil.rmtree(workbench) + del self.dir + + def __enter__(self): + """Make Workbench a context manager so we can use `with Workbench() as bench:`""" + return self + + def __exit__(self, *args): + """Clean up context manager, aka ourselves, deleting the workbench""" + self.destroy() + + +class WorkbenchManager(object): + """ + A system for generating and destroying workbenches. + + Workbenches are actually just subdirectories of a (local) temporary + storage space for during the processing stage. The preferred way to + create them is to use: + + with workbenchmger.create() as workbench: + do stuff... + + This will automatically clean up all temporary directories even in + case of an exceptions. Also check the + @mediagoblin.decorators.get_workbench decorator for a convenient + wrapper. + """ + + def __init__(self, base_workbench_dir): + self.base_workbench_dir = os.path.abspath(base_workbench_dir) + if not os.path.exists(self.base_workbench_dir): + os.makedirs(self.base_workbench_dir) + + def create(self): + """ + Create and return the path to a new workbench (directory). + """ + return Workbench(tempfile.mkdtemp(dir=self.base_workbench_dir)) |