diff options
49 files changed, 1112 insertions, 526 deletions
@@ -11,4 +11,5 @@ mediagoblin.egg-info docs/_build/ user_dev/ server-log.txt -*~
\ No newline at end of file +*~ +*.swp diff --git a/docs/codebase.rst b/docs/codebase.rst index 4f5f215f..898eadfe 100644 --- a/docs/codebase.rst +++ b/docs/codebase.rst @@ -4,6 +4,10 @@ Codebase Documentation ======================== +.. contents:: Sections + :local: + + This chapter covers the libraries that GNU MediaGoblin uses as well as various recipes for getting things done. diff --git a/docs/conf.py b/docs/conf.py index fedaf33c..0e75a617 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -48,9 +48,9 @@ copyright = u'2011, Free Software Foundation, Inc and contributors' # built documents. # # The short X.Y version. -version = '0.0.1' +version = '0.0.2' # The full version, including alpha/beta/rc tags. -release = '0.0.1' +release = '0.0.2' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/docs/contributinghowto.rst b/docs/contributinghowto.rst index e980a5e0..06d2814e 100644 --- a/docs/contributinghowto.rst +++ b/docs/contributinghowto.rst @@ -4,6 +4,10 @@ Contributing HOWTO ==================== +.. contents:: Sections + :local: + + .. _join-the-community-section: Join the community! diff --git a/docs/designdecisions.rst b/docs/designdecisions.rst index 50dfe3e8..afa1e26b 100644 --- a/docs/designdecisions.rst +++ b/docs/designdecisions.rst @@ -4,6 +4,10 @@ Design Decisions ================== +.. contents:: Sections + :local: + + This chapter talks a bit about design decisions. diff --git a/docs/git.rst b/docs/git.rst index 0db1dacf..c3f7ccce 100644 --- a/docs/git.rst +++ b/docs/git.rst @@ -2,11 +2,21 @@ Git, Cloning and Patches ========================== -GNU MediaGoblin uses git for all our version control and we have -the repositories hosted on `Gitorious <http://gitorious.org/>`_. +.. contents:: Sections + :local: -We have two repositories. One is for the project and the other is for -the project website. + +GNU MediaGoblin uses git for all our version control and we have the +repositories hosted on `Gitorious <http://gitorious.org/>`_. We have +two repositories: + +* MediaGoblin software: http://gitorious.org/mediagoblin/mediagoblin +* MediaGoblin website: http://gitorious.org/mediagoblin/mediagoblin-website + +It's most likely you want to look at the software repository--not the +website one. + +The rest of this chapter talks about using the software repository. How to clone the project @@ -17,49 +27,173 @@ Do:: git clone git://gitorious.org/mediagoblin/mediagoblin.git -How to send in patches -====================== +How to contribute changes +========================= + +Tie your changes to issues in the issue tracker +----------------------------------------------- All patches should be tied to issues in the `issue tracker -<http://bugs.foocorp.net/projects/mediagoblin/issues>`_. -That makes it a lot easier for everyone to track proposed changes and -make sure your hard work doesn't get dropped on the floor! +<http://bugs.foocorp.net/projects/mediagoblin/issues>`_. That makes +it a lot easier for everyone to track proposed changes and make sure +your hard work doesn't get dropped on the floor! If there isn't an +issue for what you're working on, please create one. The better the +description of what it is you're trying to fix/implement, the better +everyone else is able to understand why you're doing what you're +doing. + + +Use bugfix branches to make changes +----------------------------------- + +The best way to isolate your changes is to create a branch based off +of the MediaGoblin repository master branch, do the changes related to +that one issue there, and then let us know how to get it. + +It's much easier on us if you isolate your changes to a branch focused +on the issue. Then we don't have to sift through things. + +It's much easier on you if you isolate your changes to a branch +focused on the issue. Then when we merge your changes in, you just +have to do a ``git fetch`` and that's it. This is especially true if +we reject some of your changes, but accept others or otherwise tweak +your changes. + +Further, if you isolate your changes to a branch, then you can work on +multiple issues at the same time and they don't conflict with one +another. + + +Properly document your changes +------------------------------ + +Include comments in the code. + +Write comprehensive commit messages. The better your commit message +is at describing what you did and why, the easier it is for us to +quickly accept your patch. + +Write comprehensive comments in the issue tracker about what you're +doing and why. + + +How to send us your changes +--------------------------- + +There are three ways to let us know how to get it: + +1. (preferred) **push changes to publicly available git clone and let + us know where to find it** + + Push your feature/bugfix/issue branch to your publicly available + git clone and add a comment to the issue with the url for your + clone and the branch to look at. + +2. **attaching the patch files to the issue** + + Run:: + + git format-patch -o patches <remote>/master + + Then tar up the newly created ``patches`` directory and attach the + directory to the issue. + + +Example workflow +================ +Here's an example workflow. + + +Contributing changes +-------------------- + +Slartibartfast from the planet Magrathea far off in the universe has +decided that he is bored with fjords and wants to fix issue 42 and +send us the changes. + +Slartibartfast has cloned the MediaGoblin repository and his clone +lives on gitorious. + +Slartibartfast works locally. The remote named ``origin`` points to +his clone on gitorious. The remote named ``gmg`` points to the +MediaGoblin repository. + +Slartibartfast does the following: + +1. Fetches the latest from the MediaGoblin repository:: + + git fetch --all -p + +2. Creates a branch from the tip of the MediaGoblin repository (the + remote is named ``gmg``) master branch called ``issue_42``:: + + git checkout -b issue_42 gmg/master + +3. Slartibartfast works hard on his changes in the ``issue_42`` + branch. When done, he wants to notify us that he has made changes + he wants us to see. + +4. Slartibartfast pushes his changes to his clone (the remote is named + ``origin``):: + + git push origin issue_42 --set-upstream + +5. Slartibartfast adds a comment to issue 42 with the url for his + repository and the name of the branch he put the code in. He also + explains what he did and why it addresses the issue. + + +Updating a contribution +----------------------- + +Slartibartfast brushes his hands off with the sense of accomplishment +that comes with the knowledge of a job well done. He stands, wanders +over to get a cup of water, then realizes that he forgot to run the +unit tests! + +He runs the unit tests and discovers there's a bug in the code! + +Then he does this: + +1. He checks out the ``issue_42`` branch:: -If there isn't an issue for what you're working on, please create -one. The better the description of what it is you're trying to -fix/implement, the better everyone else is able to understand why -you're doing what you're doing. + git checkout issue_42 -There are two ways you could send in a patch. +2. He fixes the bug and checks it into the ``issue_42`` branch. +3. He pushes his changes to his clone (the remote is named ``origin``):: -How to send in a patch from a publicly available clone ------------------------------------------------------- + git push origin issue_42 -Add a comment to the issue you're working on with the following bits -of information: +4. He adds another comment to issue 42 explaining about the mistake + and how he fixed it and that he's pushed the new change to the + ``issue_42`` branch of his publicly available clone. -* the url for your clone -* the revs you want looked at -* any details, questions, or other things that should be known +What happens next +----------------- -How to send in a patch if you don't have a publicly available clone -------------------------------------------------------------------- +Slartibartfast is once again happy with his work. He finds issue 42 +in the issue tracker and adds a comment saying he submitted a merge +request with his changes and explains what they are. -Assuming that the remote is our repository on gitorious and the branch -to compare against is master, do the following: +Later, someone checks out his code and finds a problem with it. He +adds a comment to the issue tracker specifying the problem and asks +Slartibartfast to fix it. Slartibartfst goes through the above steps +again, fixes the issue, pushes it to his ``issue_42`` branch and adds +another comment to the issue tracker about how he fixed it. -1. checkout the branch you did your work in -2. do:: +Later, someone checks out his code and is happy with it. Someone +pulls it into the master branch of the MediaGoblin repository and adds +another comment to the issue and probably closes the issue out. - git format-patch -o patches origin/master +Slartibartfast is notified of this. Slartibartfast does a:: -3. either: + git fetch --all - * tar up and attach the tarball to the issue you're working on, OR - * attach the patch files to the issue you're working on one at a - time +The changes show up in the ``master`` branch of the ``gmg`` remote. +Slartibartfast now deletes his ``issue_42`` branch because he doesn't +need it anymore. How to learn git diff --git a/docs/hackinghowto.rst b/docs/hackinghowto.rst index a9aadb62..fcab5844 100644 --- a/docs/hackinghowto.rst +++ b/docs/hackinghowto.rst @@ -4,6 +4,10 @@ Hacking HOWTO =============== +.. contents:: Sections + :local: + + So you want to hack on GNU MediaGoblin? ======================================= @@ -119,40 +123,42 @@ To do this, do:: Running the server ================== -Run:: +If you want to get things running quickly and without hassle, just +run:: + + ./lazyserver.sh + +This will start up a python server where you can begin playing with +mediagoblin. It will also run celery in "always eager" mode so you +don't have to start a separate process for it. - ./bin/paster serve mediagoblin.ini --reload +This is fine in development, but if you want to actually run celery +separately for testing (or deployment purposes), you'll want to run +the server independently:: + + ./bin/paster serve server.ini --reload Running celeryd =============== -You need to do this if you want your media to process and actually -show up. It's probably a good idea in development to have the web -server (above) running in one terminal and celeryd in another window. +If you aren't using ./lazyserver.sh or otherwise aren't running celery +in always eager mode, you'll need to do this if you want your media to +process and actually show up. It's probably a good idea in +development to have the web server (above) running in one terminal and +celeryd in another window. Run:: CELERY_CONFIG_MODULE=mediagoblin.celery_setup.from_celery ./bin/celeryd -Too much work? Don't want to run an http server and celeryd at the -same time? For development purposes there's a shortcut:: - - CELERY_ALWAYS_EAGER=true ./bin/paster serve mediagoblin.ini --reload - -This way the web server will block on processing items until they are -done, but you don't need to run celery separately (which is probably -good enough for development purposes, but something you almost -certainly shouldn't do in production). - - Running the test suite ====================== Run:: - CELERY_CONFIG_MODULE=mediagoblin.celery_setup.from_tests ./bin/nosetests + ./bin/nosetests Running a shell diff --git a/mediagoblin/celery_setup/from_tests.py b/lazyserver.sh index fe7d7314..bd93b109 100644..100755 --- a/mediagoblin/celery_setup/from_tests.py +++ b/lazyserver.sh @@ -1,3 +1,5 @@ +#!/bin/sh + # GNU MediaGoblin -- federated, autonomous media hosting # Copyright (C) 2011 Free Software Foundation, Inc # @@ -14,30 +16,15 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import os - -from mediagoblin.tests.tools import TEST_APP_CONFIG -from mediagoblin import util -from mediagoblin.celery_setup import setup_celery_from_config -from mediagoblin.globals import setup_globals - - -OUR_MODULENAME = 'mediagoblin.celery_setup.from_tests' - - -def setup_self(setup_globals_func=setup_globals): - """ - Set up celery for testing's sake, which just needs to set up - celery and celery only. - """ - mgoblin_conf = util.read_config_file(TEST_APP_CONFIG) - mgoblin_section = mgoblin_conf['app:mediagoblin'] - - setup_celery_from_config( - mgoblin_section, mgoblin_conf, - settings_module=OUR_MODULENAME, - set_environ=False) - - -if os.environ.get('CELERY_CONFIG_MODULE') == OUR_MODULENAME: - setup_self() +if [ -f ./bin/paster ]; then + echo "Using ./bin/paster"; + export PASTER="./bin/paster"; +elif which paster > /dev/null; then + echo "Using paster from \$PATH"; + export PASTER="paster"; +else + echo "No paster found, exiting! X_X"; + exit 1 +fi + +CELERY_ALWAYS_EAGER=true $PASTER serve server.ini --reload diff --git a/mediagoblin.ini b/mediagoblin.ini index b85f22ea..596107dc 100644 --- a/mediagoblin.ini +++ b/mediagoblin.ini @@ -1,42 +1,15 @@ -[DEFAULT] -debug = true - -[composite:main] -use = egg:Paste#urlmap -/ = mediagoblin -/mgoblin_media/ = publicstore_serve -/mgoblin_static/ = mediagoblin_static - -[app:mediagoblin] -use = egg:mediagoblin#app -filter-with = beaker +[mediagoblin] queuestore_base_dir = %(here)s/user_dev/media/queue publicstore_base_dir = %(here)s/user_dev/media/public publicstore_base_url = /mgoblin_media/ direct_remote_path = /mgoblin_static/ email_sender_address = "notice@mediagoblin.example.org" + # set to false to enable sending notices email_debug_mode = true + ## Uncomment this to put some user-overriding templates here #local_templates = %(here)s/user_dev/templates/ -[app:publicstore_serve] -use = egg:Paste#static -document_root = %(here)s/user_dev/media/public - -[app:mediagoblin_static] -use = egg:Paste#static -document_root = %(here)s/mediagoblin/static/ - -[filter:beaker] -use = egg:Beaker#beaker_session -cache_dir = %(here)s/user_dev/beaker -beaker.session.key = mediagoblin -# beaker.session.secret = somesupersecret -beaker.session.data_dir = %(here)s/user_dev/beaker/sessions/data -beaker.session.lock_dir = %(here)s/user_dev/beaker/sessions/lock - -[server:main] -use = egg:Paste#http -host = 127.0.0.1 -port = 6543 +[celery] +# Put celery stuff here diff --git a/mediagoblin/app.py b/mediagoblin/app.py index 5d594039..b27b5761 100644 --- a/mediagoblin/app.py +++ b/mediagoblin/app.py @@ -18,14 +18,15 @@ import os import urllib import routes -from paste.deploy.converters import asbool from webob import Request, exc from mediagoblin import routing, util, storage, staticdirect +from mediagoblin.config import ( + read_mediagoblin_config, generate_validation_report) from mediagoblin.db.open import setup_connection_and_db_from_config -from mediagoblin.globals import setup_globals +from mediagoblin.mg_globals import setup_globals from mediagoblin.celery_setup import setup_celery_from_config -from mediagoblin.workbench import WorkbenchManager, DEFAULT_WORKBENCH_DIR +from mediagoblin.workbench import WorkbenchManager class Error(Exception): pass @@ -34,42 +35,102 @@ class ImproperlyConfigured(Error): pass class MediaGoblinApp(object): """ - Really basic wsgi app using routes and WebOb. + WSGI application of MediaGoblin + + ... this is the heart of the program! """ - def __init__(self, connection, db, - public_store, queue_store, - staticdirector, - email_sender_address, email_debug_mode, - user_template_path=None, - workbench_path=DEFAULT_WORKBENCH_DIR): + def __init__(self, config_path, setup_celery=True): + """ + Initialize the application based on a configuration file. + + Arguments: + - config_path: path to the configuration file we're opening. + - setup_celery: whether or not to setup celery during init. + (Note: setting 'celery_setup_elsewhere' also disables + setting up celery.) + """ + ############## + # Setup config + ############## + + # Open and setup the config + global_config, validation_result = read_mediagoblin_config(config_path) + app_config = global_config['mediagoblin'] + # report errors if necessary + validation_report = generate_validation_report( + global_config, validation_result) + if validation_report: + raise ImproperlyConfigured(validation_report) + + ########################################## + # Setup other connections / useful objects + ########################################## + + # Set up the database + self.connection, self.db = setup_connection_and_db_from_config( + app_config) + # Get the template environment - self.template_loader = util.get_jinja_loader(user_template_path) + self.template_loader = util.get_jinja_loader( + app_config.get('user_template_path')) # Set up storage systems - self.public_store = public_store - self.queue_store = queue_store - - # Set up database - self.connection = connection - self.db = db + self.public_store = storage.storage_system_from_config( + app_config, 'publicstore') + self.queue_store = storage.storage_system_from_config( + app_config, 'queuestore') # set up routing self.routing = routing.get_mapper() # set up staticdirector tool - self.staticdirector = staticdirector - + if app_config.has_key('direct_remote_path'): + self.staticdirector = staticdirect.RemoteStaticDirect( + app_config['direct_remote_path'].strip()) + elif app_config.has_key('direct_remote_paths'): + direct_remote_path_lines = app_config[ + 'direct_remote_paths'].strip().splitlines() + self.staticdirector = staticdirect.MultiRemoteStaticDirect( + dict([line.strip().split(' ', 1) + for line in direct_remote_path_lines])) + else: + raise ImproperlyConfigured( + "One of direct_remote_path or " + "direct_remote_paths must be provided") + + # Setup celery, if appropriate + if setup_celery and not app_config.get('celery_setup_elsewhere'): + if os.environ.get('CELERY_ALWAYS_EAGER'): + setup_celery_from_config( + app_config, global_config, + force_celery_always_eager=True) + else: + setup_celery_from_config(app_config, global_config) + + ####################################################### + # Insert appropriate things into mediagoblin.mg_globals + # # certain properties need to be accessed globally eg from # validators, etc, which might not access to the request # object. + ####################################################### + setup_globals( - email_sender_address=email_sender_address, - email_debug_mode=email_debug_mode, - db_connection=connection, + app_config=app_config, + global_config=global_config, + + # TODO: No need to set these two up as globals, we could + # just read them out of mg_globals.app_config + email_sender_address=app_config['email_sender_address'], + email_debug_mode=app_config['email_debug_mode'], + + # Actual, useful to everyone objects + app=self, + db_connection=self.connection, database=self.db, public_store=self.public_store, queue_store=self.queue_store, - workbench_manager=WorkbenchManager(workbench_path)) + workbench_manager=WorkbenchManager(app_config['workbench_path'])) def __call__(self, environ, start_response): request = Request(environ) @@ -119,45 +180,6 @@ class MediaGoblinApp(object): def paste_app_factory(global_config, **app_config): - # Get the database connection - connection, db = setup_connection_and_db_from_config(app_config) - - # Set up the storage systems. - public_store = storage.storage_system_from_paste_config( - app_config, 'publicstore') - queue_store = storage.storage_system_from_paste_config( - app_config, 'queuestore') - - # Set up the staticdirect system - if app_config.has_key('direct_remote_path'): - staticdirector = staticdirect.RemoteStaticDirect( - app_config['direct_remote_path'].strip()) - elif app_config.has_key('direct_remote_paths'): - direct_remote_path_lines = app_config[ - 'direct_remote_paths'].strip().splitlines() - staticdirector = staticdirect.MultiRemoteStaticDirect( - dict([line.strip().split(' ', 1) - for line in direct_remote_path_lines])) - else: - raise ImproperlyConfigured( - "One of direct_remote_path or direct_remote_paths must be provided") - - if not asbool(app_config.get('celery_setup_elsewhere')): - if asbool(os.environ.get('CELERY_ALWAYS_EAGER')): - setup_celery_from_config( - app_config, global_config, - force_celery_always_eager=True) - else: - setup_celery_from_config(app_config, global_config) - - mgoblin_app = MediaGoblinApp( - connection, db, - public_store=public_store, queue_store=queue_store, - staticdirector=staticdirector, - email_sender_address=app_config.get( - 'email_sender_address', 'notice@mediagoblin.example.org'), - email_debug_mode=asbool(app_config.get('email_debug_mode')), - user_template_path=app_config.get('local_templates'), - workbench_path=app_config.get('workbench_path', DEFAULT_WORKBENCH_DIR)) + mgoblin_app = MediaGoblinApp(app_config['config']) return mgoblin_app diff --git a/mediagoblin/auth/lib.py b/mediagoblin/auth/lib.py index f40e560f..08bbdd16 100644 --- a/mediagoblin/auth/lib.py +++ b/mediagoblin/auth/lib.py @@ -20,7 +20,7 @@ import random import bcrypt from mediagoblin.util import send_email, render_template -from mediagoblin import globals as mgoblin_globals +from mediagoblin import mg_globals def bcrypt_check_password(raw_pass, stored_hash, extra_salt=None): @@ -112,7 +112,7 @@ def send_verification_email(user, request): # TODO: There is no error handling in place send_email( - mgoblin_globals.email_sender_address, + mg_globals.email_sender_address, [user['email']], # TODO # Due to the distributed nature of GNU MediaGoblin, we should diff --git a/mediagoblin/celery_setup/__init__.py b/mediagoblin/celery_setup/__init__.py index d4f25b07..b6e35e99 100644 --- a/mediagoblin/celery_setup/__init__.py +++ b/mediagoblin/celery_setup/__init__.py @@ -17,86 +17,35 @@ import os import sys -from paste.deploy.converters import asbool, asint, aslist - - -KNOWN_CONFIG_BOOLS = [ - 'CELERY_RESULT_PERSISTENT', - 'CELERY_CREATE_MISSING_QUEUES', - 'BROKER_USE_SSL', 'BROKER_CONNECTION_RETRY', - 'CELERY_ALWAYS_EAGER', 'CELERY_EAGER_PROPAGATES_EXCEPTIONS', - 'CELERY_IGNORE_RESULT', 'CELERY_TRACK_STARTED', - 'CELERY_DISABLE_RATE_LIMITS', 'CELERY_ACKS_LATE', - 'CELERY_STORE_ERRORS_EVEN_IF_IGNORED', - 'CELERY_SEND_TASK_ERROR_EMAILS', - 'CELERY_SEND_EVENTS', 'CELERY_SEND_TASK_SENT_EVENT', - 'CELERYD_LOG_COLOR', 'CELERY_REDIRECT_STDOUTS', - ] - -KNOWN_CONFIG_INTS = [ - 'CELERYD_CONCURRENCY', - 'CELERYD_PREFETCH_MULTIPLIER', - 'CELERY_AMQP_TASK_RESULT_EXPIRES', - 'CELERY_AMQP_TASK_RESULT_CONNECTION_MAX', - 'REDIS_PORT', 'REDIS_DB', - 'BROKER_PORT', 'BROKER_CONNECTION_TIMEOUT', - 'CELERY_BROKER_CONNECTION_MAX_RETRIES', - 'CELERY_TASK_RESULT_EXPIRES', 'CELERY_MAX_CACHED_RESULTS', - 'CELERY_DEFAULT_RATE_LIMIT', # ?? - 'CELERYD_MAX_TASKS_PER_CHILD', 'CELERYD_TASK_TIME_LIMIT', - 'CELERYD_TASK_SOFT_TIME_LIMIT', - 'MAIL_PORT', 'CELERYBEAT_MAX_LOOP_INTERVAL', - ] - -KNOWN_CONFIG_FLOATS = [ - 'CELERYD_ETA_SCHEDULER_PRECISION', - ] - -KNOWN_CONFIG_LISTS = [ - 'CELERY_ROUTES', 'CELERY_IMPORTS', - ] - - -## Needs special processing: -# ADMINS, ??? -# there are a lot more; we should list here or process specially. - - -def asfloat(obj): - try: - return float(obj) - except (TypeError, ValueError), e: - raise ValueError( - "Bad float value: %r" % obj) - MANDATORY_CELERY_IMPORTS = ['mediagoblin.process_media'] DEFAULT_SETTINGS_MODULE = 'mediagoblin.celery_setup.dummy_settings_module' + def setup_celery_from_config(app_config, global_config, settings_module=DEFAULT_SETTINGS_MODULE, force_celery_always_eager=False, set_environ=True): """ - Take a mediagoblin app config and the global config from a paste - factory and try to set up a celery settings module from this. + Take a mediagoblin app config and try to set up a celery settings + module from this. Args: - app_config: the application config section - - global_config: the entire paste config, all sections + - global_config: the entire ConfigObj loaded config, all sections - settings_module: the module to populate, as a string - - + - force_celery_always_eager: whether or not to force celery into + always eager mode; good for development and small installs - set_environ: if set, this will CELERY_CONFIG_MODULE to the settings_module """ - if asbool(app_config.get('use_celery_environment_var')) == True: + if app_config.get('celery_setup_elsewhere') == True: # Don't setup celery based on our config file. return - celery_conf_section = app_config.get('celery_section', 'celery') - if global_config.has_key(celery_conf_section): - celery_conf = global_config[celery_conf_section] + if global_config.has_key('celery'): + celery_conf = global_config['celery'] else: celery_conf = {} @@ -114,9 +63,9 @@ def setup_celery_from_config(app_config, global_config, if celery_settings['BROKER_BACKEND'] == 'mongodb': celery_settings['BROKER_HOST'] = app_config['db_host'] if app_config.has_key('db_port'): - celery_mongo_settings['port'] = asint(app_config['db_port']) + celery_mongo_settings['port'] = app_config['db_port'] if celery_settings['BROKER_BACKEND'] == 'mongodb': - celery_settings['BROKER_PORT'] = asint(app_config['db_port']) + celery_settings['BROKER_PORT'] = app_config['db_port'] celery_mongo_settings['database'] = app_config.get('db_name', 'mediagoblin') celery_settings['CELERY_MONGODB_BACKEND_SETTINGS'] = celery_mongo_settings @@ -124,14 +73,6 @@ def setup_celery_from_config(app_config, global_config, # Add anything else for key, value in celery_conf.iteritems(): key = key.upper() - if key in KNOWN_CONFIG_BOOLS: - value = asbool(value) - elif key in KNOWN_CONFIG_INTS: - value = asint(value) - elif key in KNOWN_CONFIG_FLOATS: - value = asfloat(value) - elif key in KNOWN_CONFIG_LISTS: - value = aslist(value) celery_settings[key] = value # add mandatory celery imports diff --git a/mediagoblin/celery_setup/from_celery.py b/mediagoblin/celery_setup/from_celery.py index 5fa9ba76..45e65e52 100644 --- a/mediagoblin/celery_setup/from_celery.py +++ b/mediagoblin/celery_setup/from_celery.py @@ -16,81 +16,39 @@ import os -from paste.deploy.loadwsgi import NicerConfigParser -from paste.deploy.converters import asbool - -from mediagoblin import storage -from mediagoblin.db.open import setup_connection_and_db_from_config +from mediagoblin import app, mg_globals from mediagoblin.celery_setup import setup_celery_from_config -from mediagoblin.globals import setup_globals -from mediagoblin.workbench import WorkbenchManager, DEFAULT_WORKBENCH_DIR -OUR_MODULENAME = 'mediagoblin.celery_setup.from_celery' +OUR_MODULENAME = __name__ -def setup_self(setup_globals_func=setup_globals): +def setup_self(): """ Transform this module into a celery config module by reading the mediagoblin config file. Set the environment variable - MEDIAGOBLIN_CONFIG to specify where this config file is at and - what section it uses. - - By default it defaults to 'mediagoblin.ini:app:mediagoblin'. + MEDIAGOBLIN_CONFIG to specify where this config file is. - The first colon ":" is a delimiter between the filename and the - config section, so in this case the filename is 'mediagoblin.ini' - and the section where mediagoblin is defined is 'app:mediagoblin'. + By default it defaults to 'mediagoblin.ini'. - Args: - - 'setup_globals_func': this is for testing purposes only. Don't - set this! + Note that if celery_setup_elsewhere is set in your config file, + this simply won't work. """ - mgoblin_conf_file, mgoblin_section = os.environ.get( - 'MEDIAGOBLIN_CONFIG', 'mediagoblin.ini:app:mediagoblin').split(':', 1) + mgoblin_conf_file = os.path.abspath( + os.environ.get('MEDIAGOBLIN_CONFIG', 'mediagoblin.ini')) if not os.path.exists(mgoblin_conf_file): raise IOError( "MEDIAGOBLIN_CONFIG not set or file does not exist") - parser = NicerConfigParser(mgoblin_conf_file) - parser.read(mgoblin_conf_file) - parser._defaults.setdefault( - 'here', os.path.dirname(os.path.abspath(mgoblin_conf_file))) - parser._defaults.setdefault( - '__file__', os.path.abspath(mgoblin_conf_file)) - - mgoblin_section = dict(parser.items(mgoblin_section)) - mgoblin_conf = dict( - [(section_name, dict(parser.items(section_name))) - for section_name in parser.sections()]) + # By setting the environment variable here we should ensure that + # this is the module that gets set up. + os.environ['CELERY_CONFIG_MODULE'] = OUR_MODULENAME + app.MediaGoblinApp(mgoblin_conf_file, setup_celery=False) setup_celery_from_config( - mgoblin_section, mgoblin_conf, + mg_globals.app_config, mg_globals.global_config, settings_module=OUR_MODULENAME, set_environ=False) - connection, db = setup_connection_and_db_from_config(mgoblin_section) - - # Set up the storage systems. - public_store = storage.storage_system_from_paste_config( - mgoblin_section, 'publicstore') - queue_store = storage.storage_system_from_paste_config( - mgoblin_section, 'queuestore') - - workbench_manager = WorkbenchManager( - mgoblin_section.get( - 'workbench_path', DEFAULT_WORKBENCH_DIR)) - - setup_globals_func( - db_connection=connection, - database=db, - public_store=public_store, - email_debug_mode=asbool(mgoblin_section.get('email_debug_mode')), - email_sender_address=mgoblin_section.get( - 'email_sender_address', - 'notice@mediagoblin.example.org'), - queue_store=queue_store, - workbench_manager=workbench_manager) - if os.environ['CELERY_CONFIG_MODULE'] == OUR_MODULENAME: setup_self() diff --git a/mediagoblin/config.py b/mediagoblin/config.py new file mode 100644 index 00000000..2f93d32c --- /dev/null +++ b/mediagoblin/config.py @@ -0,0 +1,122 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import pkg_resources + +from configobj import ConfigObj, flatten_errors +from validate import Validator + + +CONFIG_SPEC_PATH = pkg_resources.resource_filename( + 'mediagoblin', 'config_spec.ini') + + +def _setup_defaults(config, config_path): + """ + Setup DEFAULTS in a config object from an (absolute) config_path. + """ + config.setdefault('DEFAULT', {}) + config['DEFAULT']['here'] = os.path.dirname(config_path) + config['DEFAULT']['__file__'] = config_path + + +def read_mediagoblin_config(config_path, config_spec=CONFIG_SPEC_PATH): + """ + Read a config object from config_path. + + Does automatic value transformation based on the config_spec. + Also provides %(__file__)s and %(here)s values of this file and + its directory respectively similar to paste deploy. + + This function doesn't itself raise any exceptions if validation + fails, you'll have to do something + + Args: + - config_path: path to the config file + - config_spec: config file that provides defaults and value types + for validation / conversion. Defaults to mediagoblin/config_spec.ini + + Returns: + A tuple like: (config, validation_result) + ... where 'conf' is the parsed config object and 'validation_result' + is the information from the validation process. + """ + config_path = os.path.abspath(config_path) + + config_spec = ConfigObj( + config_spec, + encoding='UTF8', list_values=False, _inspec=True) + + _setup_defaults(config_spec, config_path) + + config = ConfigObj( + config_path, + configspec=config_spec, + interpolation='ConfigParser') + + _setup_defaults(config, config_path) + + # For now the validator just works with the default functions, + # but in the future if we want to add additional validation/configuration + # functions we'd add them to validator.functions here. + # + # See also: + # http://www.voidspace.org.uk/python/validate.html#adding-functions + validator = Validator() + validation_result = config.validate(validator, preserve_errors=True) + + return config, validation_result + + +REPORT_HEADER = u"""\ +There were validation problems loading this config file: +-------------------------------------------------------- +""" + + +def generate_validation_report(config, validation_result): + """ + Generate a report if necessary of problems while validating. + + Returns: + Either a string describing for a user the problems validating + this config or None if there are no problems. + """ + report = [] + + # Organize the report + for entry in flatten_errors(config, validation_result): + # each entry is a tuple + section_list, key, error = entry + + if key is not None: + section_list.append(key) + else: + section_list.append(u'[missing section]') + + section_string = u':'.join(section_list) + + if error == False: + # We don't care about missing values for now. + continue + + report.append(u"%s = %s" % (section_string, error)) + + if report: + return REPORT_HEADER + u"\n".join(report) + else: + return None diff --git a/mediagoblin/config_spec.ini b/mediagoblin/config_spec.ini new file mode 100644 index 00000000..aadf5c21 --- /dev/null +++ b/mediagoblin/config_spec.ini @@ -0,0 +1,76 @@ +[mediagoblin] +# database stuff +db_host = string() +db_name = string() +db_port = integer() + +# +queuestore_base_dir = string(default="%(here)s/user_dev/media/queue") +publicstore_base_dir = string(default="%(here)s/user_dev/media/public") + +# Where temporary files used in processing and etc are kept +workbench_path = string(default="%(here)s/user_dev/media/workbench") + +# +publicstore_base_url = string(default="/mgoblin_media/") + +# Where mediagoblin-builtin static assets are kept +direct_remote_path = string(default="/mgoblin_static/") + +# set to false to enable sending notices +email_debug_mode = boolean(default=True) +email_sender_address = string(default="notice@mediagoblin.example.org") + +# By default not set, but you might want something like: +# "%(here)s/user_dev/templates/" +local_templates = string() + +# Whether or not celery is set up via an environment variable or +# something else (and thus mediagoblin should not attempt to set it up +# itself) +celery_setup_elsewhere = boolean(default=False) + +[celery] +# known booleans +celery_result_persistent = boolean() +celery_create_missing_queues = boolean() +broker_use_ssl = boolean() +broker_connection_retry = boolean() +celery_always_eager = boolean() +celery_eager_propagates_exceptions = boolean() +celery_ignore_result = boolean() +celery_track_started = boolean() +celery_disable_rate_limits = boolean() +celery_acks_late = boolean() +celery_store_errors_even_if_ignored = boolean() +celery_send_task_error_emails = boolean() +celery_send_events = boolean() +celery_send_task_sent_event = boolean() +celeryd_log_color = boolean() +celery_redirect_stdouts = boolean() + +# known ints +celeryd_concurrency = integer() +celeryd_prefetch_multiplier = integer() +celery_amqp_task_result_expires = integer() +celery_amqp_task_result_connection_max = integer() +redis_port = integer() +redis_db = integer() +broker_port = integer() +broker_connection_timeout = integer() +celery_broker_connection_max_retries = integer() +celery_task_result_expires = integer() +celery_max_cached_results = integer() +celery_default_rate_limit = integer() +celeryd_max_tasks_per_child = integer() +celeryd_task_time_limit = integer() +celeryd_task_soft_time_limit = integer() +mail_port = integer() +celerybeat_max_loop_interval = integer() + +# known floats +celeryd_eta_scheduler_precision = float() + +# known lists +celery_routes = string_list() +celery_imports = string_list()
\ No newline at end of file diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py index d035b15b..f1f625b7 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/migrations.py @@ -16,8 +16,6 @@ from mongokit import DocumentMigration -from mediagoblin import globals as mediagoblin_globals - class MediaEntryMigration(DocumentMigration): def allmigration01_uploader_to_reference(self): diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 3da97a49..d77cf619 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -20,7 +20,7 @@ from mongokit import Document, Set from mediagoblin import util from mediagoblin.auth import lib as auth_lib -from mediagoblin import globals as mediagoblin_globals +from mediagoblin import mg_globals from mediagoblin.db import migrations from mediagoblin.db.util import ObjectId @@ -114,7 +114,7 @@ class MediaEntry(Document): def generate_slug(self): self['slug'] = util.slugify(self['title']) - duplicate = mediagoblin_globals.database.media_entries.find_one( + duplicate = mg_globals.database.media_entries.find_one( {'slug': self['slug']}) if duplicate: diff --git a/mediagoblin/gmg_commands/migrate.py b/mediagoblin/gmg_commands/migrate.py index e04fb343..9e01d51c 100644 --- a/mediagoblin/gmg_commands/migrate.py +++ b/mediagoblin/gmg_commands/migrate.py @@ -17,16 +17,12 @@ from mediagoblin.db import migrations from mediagoblin.gmg_commands import util as commands_util -from mediagoblin import globals as mgoblin_globals def migrate_parser_setup(subparser): subparser.add_argument( '-cf', '--conf_file', default='mediagoblin.ini', help="Config file used to set up environment") - subparser.add_argument( - '-cs', '--app_section', default='app:mediagoblin', - help="Section of the config file where the app config is stored.") def migrate(args): diff --git a/mediagoblin/gmg_commands/shell.py b/mediagoblin/gmg_commands/shell.py index 9c0259de..dc1621d1 100644 --- a/mediagoblin/gmg_commands/shell.py +++ b/mediagoblin/gmg_commands/shell.py @@ -17,7 +17,7 @@ import code -from mediagoblin import globals as mgoblin_globals +from mediagoblin import mg_globals from mediagoblin.gmg_commands import util as commands_util @@ -25,9 +25,6 @@ def shell_parser_setup(subparser): subparser.add_argument( '-cf', '--conf_file', default='mediagoblin.ini', help="Config file used to set up environment") - subparser.add_argument( - '-cs', '--app_section', default='app:mediagoblin', - help="Section of the config file where the app config is stored.") SHELL_BANNER = """\ @@ -35,7 +32,7 @@ GNU MediaGoblin shell! ---------------------- Available vars: - mgoblin_app: instantiated mediagoblin application - - mgoblin_globals: mediagoblin.globals + - mg_globals: mediagoblin.globals - db: database instance """ @@ -50,5 +47,5 @@ def shell(args): banner=SHELL_BANNER, local={ 'mgoblin_app': mgoblin_app, - 'mgoblin_globals': mgoblin_globals, - 'db': mgoblin_globals.database}) + 'mg_globals': mg_globals, + 'db': mg_globals.database}) diff --git a/mediagoblin/gmg_commands/util.py b/mediagoblin/gmg_commands/util.py index 41a21a1e..8dcac913 100644 --- a/mediagoblin/gmg_commands/util.py +++ b/mediagoblin/gmg_commands/util.py @@ -15,10 +15,6 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. -import os - -from paste.deploy.loadwsgi import NicerConfigParser - from mediagoblin import app @@ -26,20 +22,6 @@ def setup_app(args): """ Setup the application after reading the mediagoblin config files """ - # Duplicated from from_celery.py, remove when we have the generic util - parser = NicerConfigParser(args.conf_file) - parser.read(args.conf_file) - parser._defaults.setdefault( - 'here', os.path.dirname(os.path.abspath(args.conf_file))) - parser._defaults.setdefault( - '__file__', os.path.abspath(args.conf_file)) - - mgoblin_section = dict(parser.items(args.app_section)) - mgoblin_conf = dict( - [(section_name, dict(parser.items(section_name))) - for section_name in parser.sections()]) - - mgoblin_app = app.paste_app_factory( - mgoblin_conf, **mgoblin_section) + mgoblin_app = app.MediaGoblinApp(args.conf_file) return mgoblin_app diff --git a/mediagoblin/globals.py b/mediagoblin/mg_globals.py index 49a513a2..3d0ed61d 100644 --- a/mediagoblin/globals.py +++ b/mediagoblin/mg_globals.py @@ -36,7 +36,7 @@ translations = gettext.find( def setup_globals(**kwargs): - from mediagoblin import globals as mg_globals + from mediagoblin import mg_globals for key, value in kwargs.iteritems(): if not hasattr(mg_globals, key): diff --git a/mediagoblin/process_media/__init__.py b/mediagoblin/process_media/__init__.py index 531eb16d..0dce1418 100644 --- a/mediagoblin/process_media/__init__.py +++ b/mediagoblin/process_media/__init__.py @@ -18,22 +18,29 @@ import Image from mediagoblin.db.util import ObjectId from celery.task import task -from mediagoblin import globals as mg_globals +from mediagoblin import mg_globals as mgg THUMB_SIZE = 200, 200 +def create_pub_filepath(entry, filename): + return mgg.public_store.get_unique_filepath( + ['media_entries', + unicode(entry['_id']), + filename]) + + @task def process_media_initial(media_id): - workbench = mg_globals.workbench_manager.create_workbench() + workbench = mgg.workbench_manager.create_workbench() - entry = mg_globals.database.MediaEntry.one( + entry = mgg.database.MediaEntry.one( {'_id': ObjectId(media_id)}) queued_filepath = entry['queued_media_file'] - queued_filename = mg_globals.workbench_manager.localized_file( - workbench, mg_globals.queue_store, queued_filepath, + queued_filename = workbench.localized_file( + mgg.queue_store, queued_filepath, 'source') queued_file = file(queued_filename, 'r') @@ -41,13 +48,13 @@ def process_media_initial(media_id): with queued_file: thumb = Image.open(queued_file) thumb.thumbnail(THUMB_SIZE, Image.ANTIALIAS) + # ensure color mode is compatible with jpg + if thumb.mode != "RGB": + thumb = thumb.convert("RGB") - thumb_filepath = mg_globals.public_store.get_unique_filepath( - ['media_entries', - unicode(entry['_id']), - 'thumbnail.jpg']) + thumb_filepath = create_pub_filepath(entry, 'thumbnail.jpg') - thumb_file = mg_globals.public_store.get_file(thumb_filepath, 'w') + thumb_file = mgg.public_store.get_file(thumb_filepath, 'w') with thumb_file: thumb.save(thumb_file, "JPEG") @@ -56,15 +63,13 @@ def process_media_initial(media_id): queued_file = file(queued_filename, 'rb') with queued_file: - main_filepath = mg_globals.public_store.get_unique_filepath( - ['media_entries', - unicode(entry['_id']), - queued_filepath[-1]]) + main_filepath = create_pub_filepath(entry, queued_filepath[-1]) - with mg_globals.public_store.get_file(main_filepath, 'wb') as main_file: + with mgg.public_store.get_file(main_filepath, 'wb') as main_file: main_file.write(queued_file.read()) - mg_globals.queue_store.delete_file(queued_filepath) + mgg.queue_store.delete_file(queued_filepath) + entry['queued_media_file'] = [] media_files_dict = entry.setdefault('media_files', {}) media_files_dict['thumb'] = thumb_filepath media_files_dict['main'] = main_filepath @@ -72,4 +77,4 @@ def process_media_initial(media_id): entry.save() # clean up workbench - mg_globals.workbench_manager.destroy_workbench(workbench) + workbench.destroy_self() diff --git a/mediagoblin/storage.py b/mediagoblin/storage.py index ba6ac017..5d6faa4c 100644 --- a/mediagoblin/storage.py +++ b/mediagoblin/storage.py @@ -247,7 +247,7 @@ def clean_listy_filepath(listy_filepath): return cleaned_filepath -def storage_system_from_paste_config(paste_config, storage_prefix): +def storage_system_from_config(paste_config, storage_prefix): """ Utility for setting up a storage system from the paste app config. @@ -266,7 +266,7 @@ def storage_system_from_paste_config(paste_config, storage_prefix): An instantiated storage system. Example: - storage_system_from_paste_config( + storage_system_from_config( {'publicstore_base_url': '/media/', 'publicstore_base_dir': '/var/whatever/media/'}, 'publicstore') diff --git a/mediagoblin/templates/mediagoblin/base.html b/mediagoblin/templates/mediagoblin/base.html index 704e5aa7..c7313173 100644 --- a/mediagoblin/templates/mediagoblin/base.html +++ b/mediagoblin/templates/mediagoblin/base.html @@ -35,6 +35,8 @@ <div class="mediagoblin_header_right"> {% if request.user %} {{ request.user['username'] }}'s account + <a href="{{ request.urlgen('mediagoblin.user_pages.user_gallery', + user= request.user['username']) }}">gallery</a> (<a href="{{ request.urlgen('mediagoblin.auth.logout') }}">logout</a>) {% else %} <a href="{{ request.urlgen('mediagoblin.auth.login') }}"> diff --git a/mediagoblin/templates/mediagoblin/user_pages/gallery.html b/mediagoblin/templates/mediagoblin/user_pages/gallery.html new file mode 100644 index 00000000..1d8fbdaa --- /dev/null +++ b/mediagoblin/templates/mediagoblin/user_pages/gallery.html @@ -0,0 +1,42 @@ +{# +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. +#} +{% extends "mediagoblin/base.html" %} + +{% block mediagoblin_head %} + <link rel="alternate" type="application/atom+xml" + href="{{ request.urlgen( + 'mediagoblin.user_pages.atom_feed', + user=user.username) }}"> +{% endblock mediagoblin_head %} + +{% block mediagoblin_content -%} + {% if user %} + <h1>gallery for <a href="{{ request.urlgen( + 'mediagoblin.user_pages.user_home', + user=user.username) }}">{{ user.username }}</a></h1> + + {% include "mediagoblin/utils/object_gallery.html" %} + + <a href={{ request.urlgen( + 'mediagoblin.user_pages.atom_feed', + user=user.username) }}> atom feed</a> + {% else %} + {# This *should* not occur as the view makes sure we pass in a user. #} + <p>Sorry, no such user found.<p/> + {% endif %} +{% endblock %} diff --git a/mediagoblin/tests/__init__.py b/mediagoblin/tests/__init__.py index c129cbf8..1f1e23e9 100644 --- a/mediagoblin/tests/__init__.py +++ b/mediagoblin/tests/__init__.py @@ -13,3 +13,15 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + +from mediagoblin import mg_globals + + +def setup_package(): + pass + +def teardown_package(): + if mg_globals.db_connection: + print "Killing db ..." + mg_globals.db_connection.drop_database(mg_globals.database.name) + print "... done" diff --git a/mediagoblin/tests/fake_carrot_conf_bad.ini b/mediagoblin/tests/fake_carrot_conf_bad.ini new file mode 100644 index 00000000..0c79b354 --- /dev/null +++ b/mediagoblin/tests/fake_carrot_conf_bad.ini @@ -0,0 +1,14 @@ +[carrotapp] +# Whether or not our carrots are going to be turned into cake. +## These should throw errors +carrotcake = slobber +num_carrots = GROSS + +# A message encouraging our users to eat their carrots. +encouragement_phrase = 586956856856 # shouldn't throw error + +# Something extra! +blah_blah = "blah!" # shouldn't throw error either + +[celery] +eat_celery_with_carrots = pants # yeah that's def an error right there. diff --git a/mediagoblin/tests/fake_carrot_conf_empty.ini b/mediagoblin/tests/fake_carrot_conf_empty.ini new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/mediagoblin/tests/fake_carrot_conf_empty.ini diff --git a/mediagoblin/tests/fake_carrot_conf_good.ini b/mediagoblin/tests/fake_carrot_conf_good.ini new file mode 100644 index 00000000..fed14d07 --- /dev/null +++ b/mediagoblin/tests/fake_carrot_conf_good.ini @@ -0,0 +1,13 @@ +[carrotapp] +# Whether or not our carrots are going to be turned into cake. +carrotcake = true +num_carrots = 88 + +# A message encouraging our users to eat their carrots. +encouragement_phrase = "I'd love it if you eat your carrots!" + +# Something extra! +blah_blah = "blah!" + +[celery] +eat_celery_with_carrots = False diff --git a/mediagoblin/tests/fake_celery_conf.ini b/mediagoblin/tests/fake_celery_conf.ini new file mode 100644 index 00000000..3e52ac3a --- /dev/null +++ b/mediagoblin/tests/fake_celery_conf.ini @@ -0,0 +1,9 @@ +['mediagoblin'] +# I got nothin' in this file! + +['celery'] +some_variable = floop +mail_port = 2000 +celeryd_eta_scheduler_precision = 1.3 +celery_result_persistent = true +celery_imports = foo.bar.baz, this.is.an.import diff --git a/mediagoblin/tests/fake_celery_conf_mgdb.ini b/mediagoblin/tests/fake_celery_conf_mgdb.ini new file mode 100644 index 00000000..52671c14 --- /dev/null +++ b/mediagoblin/tests/fake_celery_conf_mgdb.ini @@ -0,0 +1,14 @@ +['mediagoblin'] +db_host = mongodb.example.org +db_port = 8080 +db_name = captain_lollerskates + +['something'] +or = other + +['celery'] +some_variable = poolf +mail_port = 2020 +celeryd_eta_scheduler_precision = 3.1 +celery_result_persistent = false +celery_imports = baz.bar.foo, import.is.a.this diff --git a/mediagoblin/tests/fake_config_spec.ini b/mediagoblin/tests/fake_config_spec.ini new file mode 100644 index 00000000..9421ce36 --- /dev/null +++ b/mediagoblin/tests/fake_config_spec.ini @@ -0,0 +1,10 @@ +[carrotapp] +# Whether or not our carrots are going to be turned into cake. +carrotcake = boolean(default=False) +num_carrots = integer(default=1) + +# A message encouraging our users to eat their carrots. +encouragement_phrase = string() + +[celery] +eat_celery_with_carrots = boolean(default=True)
\ No newline at end of file diff --git a/mediagoblin/tests/test_auth.py b/mediagoblin/tests/test_auth.py index cdfeccab..b8389f8d 100644 --- a/mediagoblin/tests/test_auth.py +++ b/mediagoblin/tests/test_auth.py @@ -20,7 +20,7 @@ from nose.tools import assert_equal from mediagoblin.auth import lib as auth_lib from mediagoblin.tests.tools import setup_fresh_app -from mediagoblin import globals as mgoblin_globals +from mediagoblin import mg_globals from mediagoblin import util @@ -77,7 +77,7 @@ def test_register_views(test_app): # Make sure it rendered with the appropriate template assert util.TEMPLATE_TEST_CONTEXT.has_key( 'mediagoblin/auth/register.html') - + # Try to register without providing anything, should error # -------------------------------------------------------- @@ -137,7 +137,7 @@ def test_register_views(test_app): u'Passwords must match.'] ## At this point there should be no users in the database ;) - assert not mgoblin_globals.database.User.find().count() + assert not mg_globals.database.User.find().count() # Successful register # ------------------- @@ -158,7 +158,7 @@ def test_register_views(test_app): 'mediagoblin/auth/register_success.html') ## Make sure user is in place - new_user = mgoblin_globals.database.User.find_one( + new_user = mg_globals.database.User.find_one( {'username': 'happygirl'}) assert new_user assert new_user['status'] == u'needs_email_verification' @@ -182,7 +182,7 @@ def test_register_views(test_app): unicode(new_user['_id'])] assert parsed_get_params['token'] == [ new_user['verification_key']] - + ## Try verifying with bs verification key, shouldn't work util.clear_test_template_context() test_app.get( @@ -191,7 +191,7 @@ def test_register_views(test_app): context = util.TEMPLATE_TEST_CONTEXT[ 'mediagoblin/auth/verify_email.html'] assert context['verification_successful'] == False - new_user = mgoblin_globals.database.User.find_one( + new_user = mg_globals.database.User.find_one( {'username': 'happygirl'}) assert new_user assert new_user['status'] == u'needs_email_verification' @@ -203,14 +203,12 @@ def test_register_views(test_app): context = util.TEMPLATE_TEST_CONTEXT[ 'mediagoblin/auth/verify_email.html'] assert context['verification_successful'] == True - new_user = mgoblin_globals.database.User.find_one( + new_user = mg_globals.database.User.find_one( {'username': 'happygirl'}) assert new_user assert new_user['status'] == u'active' assert new_user['email_verified'] == True - ## TODO: Try logging in - # Uniqueness checks # ----------------- ## We shouldn't be able to register with that user twice @@ -221,7 +219,7 @@ def test_register_views(test_app): 'password': 'iamsohappy2', 'confirm_password': 'iamsohappy2', 'email': 'happygrrl2@example.org'}) - + context = util.TEMPLATE_TEST_CONTEXT[ 'mediagoblin/auth/register.html'] form = context['register_form'] @@ -229,3 +227,43 @@ def test_register_views(test_app): u'Sorry, a user with that name already exists.'] ## TODO: Also check for double instances of an email address? + + +@setup_fresh_app +def test_authentication_views(test_app): + """ + Test logging in and logging out + """ + # Make a new user + test_user = mg_globals.database.User() + test_user['username'] = u'chris' + test_user['email'] = u'chris@example.com' + test_user['pw_hash'] = auth_lib.bcrypt_gen_password_hash('toast') + test_user.save() + + # Get login + test_app.get('/auth/login/') + # Make sure it rendered with the appropriate template + assert util.TEMPLATE_TEST_CONTEXT.has_key( + 'mediagoblin/auth/login.html') + + # Log in as that user + util.clear_test_template_context() + response = test_app.post( + '/auth/login/', { + 'username': u'chris', + 'password': 'toast'}) + response.follow() + assert_equal( + urlparse.urlsplit(response.location)[2], + '/') + assert util.TEMPLATE_TEST_CONTEXT.has_key( + 'mediagoblin/root.html') + + # Make sure we're in the session or something + session = util.TEMPLATE_TEST_CONTEXT['mediagoblin/root.html']['request'].session + assert session['user_id'] == unicode(test_user['_id']) + + # Log out as that user + # Make sure we're not in the session + diff --git a/mediagoblin/tests/test_celery_setup.py b/mediagoblin/tests/test_celery_setup.py index 558eb458..8bf97ae4 100644 --- a/mediagoblin/tests/test_celery_setup.py +++ b/mediagoblin/tests/test_celery_setup.py @@ -17,6 +17,13 @@ import pkg_resources from mediagoblin import celery_setup +from mediagoblin.config import read_mediagoblin_config + + +TEST_CELERY_CONF_NOSPECIALDB = pkg_resources.resource_filename( + 'mediagoblin.tests', 'fake_celery_conf.ini') +TEST_CELERY_CONF_MGSPECIALDB = pkg_resources.resource_filename( + 'mediagoblin.tests', 'fake_celery_conf_mgdb.ini') def test_setup_celery_from_config(): @@ -27,14 +34,12 @@ def test_setup_celery_from_config(): for var in vars_to_wipe: delattr(module, var) + global_config, validation_result = read_mediagoblin_config( + TEST_CELERY_CONF_NOSPECIALDB) + app_config = global_config['mediagoblin'] + celery_setup.setup_celery_from_config( - {}, - {'something': {'or': 'other'}, - 'celery': {'some_variable': 'floop', - 'mail_port': '2000', - 'CELERYD_ETA_SCHEDULER_PRECISION': '1.3', - 'celery_result_persistent': 'true', - 'celery_imports': 'foo.bar.baz this.is.an.import'}}, + app_config, global_config, 'mediagoblin.tests.fake_celery_module', set_environ=False) from mediagoblin.tests import fake_celery_module @@ -53,17 +58,12 @@ def test_setup_celery_from_config(): _wipe_testmodule_clean(fake_celery_module) + global_config, validation_result = read_mediagoblin_config( + TEST_CELERY_CONF_MGSPECIALDB) + app_config = global_config['mediagoblin'] + celery_setup.setup_celery_from_config( - {'db_host': 'mongodb.example.org', - 'db_port': '8080', - 'db_name': 'captain_lollerskates', - 'celery_section': 'vegetable'}, - {'something': {'or': 'other'}, - 'vegetable': {'some_variable': 'poolf', - 'mail_port': '2020', - 'CELERYD_ETA_SCHEDULER_PRECISION': '3.1', - 'celery_result_persistent': 'false', - 'celery_imports': 'baz.bar.foo import.is.a.this'}}, + app_config, global_config, 'mediagoblin.tests.fake_celery_module', set_environ=False) from mediagoblin.tests import fake_celery_module diff --git a/mediagoblin/tests/test_config.py b/mediagoblin/tests/test_config.py new file mode 100644 index 00000000..244f05e5 --- /dev/null +++ b/mediagoblin/tests/test_config.py @@ -0,0 +1,97 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 Free Software Foundation, Inc +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import pkg_resources + +from mediagoblin import config + + +CARROT_CONF_GOOD = pkg_resources.resource_filename( + 'mediagoblin.tests', 'fake_carrot_conf_good.ini') +CARROT_CONF_EMPTY = pkg_resources.resource_filename( + 'mediagoblin.tests', 'fake_carrot_conf_empty.ini') +CARROT_CONF_BAD = pkg_resources.resource_filename( + 'mediagoblin.tests', 'fake_carrot_conf_bad.ini') +FAKE_CONFIG_SPEC = pkg_resources.resource_filename( + 'mediagoblin.tests', 'fake_config_spec.ini') + + +def test_read_mediagoblin_config(): + # An empty file + this_conf, validation_results = config.read_mediagoblin_config( + CARROT_CONF_EMPTY, FAKE_CONFIG_SPEC) + + assert this_conf['carrotapp']['carrotcake'] == False + assert this_conf['carrotapp']['num_carrots'] == 1 + assert not this_conf['carrotapp'].has_key('encouragement_phrase') + assert this_conf['celery']['eat_celery_with_carrots'] == True + + # A good file + this_conf, validation_results = config.read_mediagoblin_config( + CARROT_CONF_GOOD, FAKE_CONFIG_SPEC) + + assert this_conf['carrotapp']['carrotcake'] == True + assert this_conf['carrotapp']['num_carrots'] == 88 + assert this_conf['carrotapp']['encouragement_phrase'] == \ + "I'd love it if you eat your carrots!" + assert this_conf['carrotapp']['blah_blah'] == "blah!" + assert this_conf['celery']['eat_celery_with_carrots'] == False + + # A bad file + this_conf, validation_results = config.read_mediagoblin_config( + CARROT_CONF_BAD, FAKE_CONFIG_SPEC) + + # These should still open but will have errors that we'll test for + # in test_generate_validation_report() + assert this_conf['carrotapp']['carrotcake'] == 'slobber' + assert this_conf['carrotapp']['num_carrots'] == 'GROSS' + assert this_conf['carrotapp']['encouragement_phrase'] == \ + "586956856856" + assert this_conf['carrotapp']['blah_blah'] == "blah!" + assert this_conf['celery']['eat_celery_with_carrots'] == "pants" + + +def test_generate_validation_report(): + # Empty + this_conf, validation_results = config.read_mediagoblin_config( + CARROT_CONF_EMPTY, FAKE_CONFIG_SPEC) + report = config.generate_validation_report(this_conf, validation_results) + assert report is None + + # Good + this_conf, validation_results = config.read_mediagoblin_config( + CARROT_CONF_GOOD, FAKE_CONFIG_SPEC) + report = config.generate_validation_report(this_conf, validation_results) + assert report is None + + # Bad + this_conf, validation_results = config.read_mediagoblin_config( + CARROT_CONF_BAD, FAKE_CONFIG_SPEC) + report = config.generate_validation_report(this_conf, validation_results) + + assert report.startswith("""\ +There were validation problems loading this config file: +--------------------------------------------------------""") + + expected_warnings = [ + 'carrotapp:carrotcake = the value "slobber" is of the wrong type.', + 'carrotapp:num_carrots = the value "GROSS" is of the wrong type.', + 'celery:eat_celery_with_carrots = the value "pants" is of the wrong type.'] + warnings = report.splitlines()[2:] + + assert len(warnings) == 3 + for warning in expected_warnings: + assert warning in warnings diff --git a/mediagoblin/tests/test_globals.py b/mediagoblin/tests/test_globals.py index b285cdf5..63578d62 100644 --- a/mediagoblin/tests/test_globals.py +++ b/mediagoblin/tests/test_globals.py @@ -16,21 +16,30 @@ from nose.tools import assert_raises -from mediagoblin import globals as mg_globals +from mediagoblin import mg_globals -def test_setup_globals(): - mg_globals.setup_globals( - db_connection='my favorite db_connection!', - database='my favorite database!', - public_store='my favorite public_store!', - queue_store='my favorite queue_store!') - - assert mg_globals.db_connection == 'my favorite db_connection!' - assert mg_globals.database == 'my favorite database!' - assert mg_globals.public_store == 'my favorite public_store!' - assert mg_globals.queue_store == 'my favorite queue_store!' +class TestGlobals(object): + def setUp(self): + self.old_connection = mg_globals.db_connection + self.old_database = mg_globals.database - assert_raises( - AssertionError, - mg_globals.setup_globals, - no_such_global_foo = "Dummy") + def tearDown(self): + mg_globals.db_connection = self.old_connection + mg_globals.database = self.old_database + + def test_setup_globals(self): + mg_globals.setup_globals( + db_connection='my favorite db_connection!', + database='my favorite database!', + public_store='my favorite public_store!', + queue_store='my favorite queue_store!') + + assert mg_globals.db_connection == 'my favorite db_connection!' + assert mg_globals.database == 'my favorite database!' + assert mg_globals.public_store == 'my favorite public_store!' + assert mg_globals.queue_store == 'my favorite queue_store!' + + assert_raises( + AssertionError, + mg_globals.setup_globals, + no_such_global_foo = "Dummy") diff --git a/mediagoblin/tests/test_mgoblin_app.ini b/mediagoblin/tests/test_mgoblin_app.ini new file mode 100644 index 00000000..94eafb5a --- /dev/null +++ b/mediagoblin/tests/test_mgoblin_app.ini @@ -0,0 +1,12 @@ +[mediagoblin] +queuestore_base_dir = %(here)s/test_user_dev/media/queue +publicstore_base_dir = %(here)s/test_user_dev/media/public +publicstore_base_url = /mgoblin_media/ +direct_remote_path = /mgoblin_static/ +email_sender_address = "notice@mediagoblin.example.org" +email_debug_mode = true +db_name = __mediagoblin_tests__ + +# Celery shouldn't be set up by the paste app factory as it's set up +# elsewhere +celery_setup_elsewhere = true diff --git a/mediagoblin/tests/mgoblin_test_app.ini b/mediagoblin/tests/test_server.ini index abed2615..929a1ccf 100644 --- a/mediagoblin/tests/mgoblin_test_app.ini +++ b/mediagoblin/tests/test_server.ini @@ -10,16 +10,7 @@ use = egg:Paste#urlmap [app:mediagoblin] use = egg:mediagoblin#app filter-with = beaker -queuestore_base_dir = %(here)s/test_user_dev/media/queue -publicstore_base_dir = %(here)s/test_user_dev/media/public -publicstore_base_url = /mgoblin_media/ -direct_remote_path = /mgoblin_static/ -email_sender_address = "notice@mediagoblin.example.org" -email_debug_mode = true -db_name = __mediagoblin_tests__ -# Celery shouldn't be set up by the paste app factory as it's set up -# elsewhere -celery_setup_elsewhere = true +config = %(here)s/test_mgoblin_app.ini [app:publicstore_serve] use = egg:Paste#static diff --git a/mediagoblin/tests/test_storage.py b/mediagoblin/tests/test_storage.py index 55b66e84..1800c29d 100644 --- a/mediagoblin/tests/test_storage.py +++ b/mediagoblin/tests/test_storage.py @@ -58,8 +58,8 @@ class FakeRemoteStorage(storage.BasicFileStorage): local_storage = False -def test_storage_system_from_paste_config(): - this_storage = storage.storage_system_from_paste_config( +def test_storage_system_from_config(): + this_storage = storage.storage_system_from_config( {'somestorage_base_url': 'http://example.org/moodia/', 'somestorage_base_dir': '/tmp/', 'somestorage_garbage_arg': 'garbage_arg', @@ -69,7 +69,7 @@ def test_storage_system_from_paste_config(): assert this_storage.base_dir == '/tmp/' assert this_storage.__class__ is storage.BasicFileStorage - this_storage = storage.storage_system_from_paste_config( + this_storage = storage.storage_system_from_config( {'somestorage_foobie': 'eiboof', 'somestorage_blech': 'hcelb', 'somestorage_garbage_arg': 'garbage_arg', diff --git a/mediagoblin/tests/test_tests.py b/mediagoblin/tests/test_tests.py index 3ecbfac7..8ac7f0a4 100644 --- a/mediagoblin/tests/test_tests.py +++ b/mediagoblin/tests/test_tests.py @@ -16,7 +16,7 @@ from mediagoblin.tests.tools import get_test_app -from mediagoblin import globals as mgoblin_globals +from mediagoblin import mg_globals def test_get_test_app_wipes_db(): @@ -24,15 +24,15 @@ def test_get_test_app_wipes_db(): Make sure we get a fresh database on every wipe :) """ get_test_app() - assert mgoblin_globals.database.User.find().count() == 0 + assert mg_globals.database.User.find().count() == 0 - new_user = mgoblin_globals.database.User() + new_user = mg_globals.database.User() new_user['username'] = u'lolcat' new_user['email'] = u'lol@cats.example.org' new_user['pw_hash'] = u'pretend_this_is_a_hash' new_user.save() - assert mgoblin_globals.database.User.find().count() == 1 + assert mg_globals.database.User.find().count() == 1 get_test_app() - assert mgoblin_globals.database.User.find().count() == 0 + assert mg_globals.database.User.find().count() == 0 diff --git a/mediagoblin/tests/test_util.py b/mediagoblin/tests/test_util.py index 7b00a074..75e28aca 100644 --- a/mediagoblin/tests/test_util.py +++ b/mediagoblin/tests/test_util.py @@ -103,3 +103,22 @@ def test_locale_to_lower_lower(): # crazy renditions. Useful? assert util.locale_to_lower_lower('en-US') == 'en-us' assert util.locale_to_lower_lower('en_us') == 'en-us' + + +def test_html_cleaner(): + # Remove images + result = util.clean_html( + '<p>Hi everybody! ' + '<img src="http://example.org/huge-purple-barney.png" /></p>\n' + '<p>:)</p>') + assert result == ( + '<div>' + '<p>Hi everybody! </p>\n' + '<p>:)</p>' + '</div>') + + # Remove evil javascript + result = util.clean_html( + '<p><a href="javascript:nasty_surprise">innocent link!</a></p>') + assert result == ( + '<p><a href="">innocent link!</a></p>') diff --git a/mediagoblin/tests/test_workbench.py b/mediagoblin/tests/test_workbench.py index 89f2ef33..953835a1 100644 --- a/mediagoblin/tests/test_workbench.py +++ b/mediagoblin/tests/test_workbench.py @@ -30,29 +30,29 @@ class TestWorkbench(object): def test_create_workbench(self): workbench = self.workbench_manager.create_workbench() - assert os.path.isdir(workbench) - assert workbench.startswith(self.workbench_manager.base_workbench_dir) + assert os.path.isdir(workbench.dir) + assert workbench.dir.startswith(self.workbench_manager.base_workbench_dir) + + def test_joinpath(self): + this_workbench = self.workbench_manager.create_workbench() + tmpname = this_workbench.joinpath('temp.txt') + assert tmpname == os.path.join(this_workbench.dir, 'temp.txt') + this_workbench.destroy_self() def test_destroy_workbench(self): # kill a workbench this_workbench = self.workbench_manager.create_workbench() - tmpfile = file(os.path.join(this_workbench, 'temp.txt'), 'w') + tmpfile_name = this_workbench.joinpath('temp.txt') + tmpfile = file(tmpfile_name, 'w') with tmpfile: tmpfile.write('lollerskates') - assert os.path.exists(os.path.join(this_workbench, 'temp.txt')) - - self.workbench_manager.destroy_workbench(this_workbench) - assert not os.path.exists(os.path.join(this_workbench, 'temp.txt')) - assert not os.path.exists(this_workbench) - - # make sure we can't kill other stuff though - dont_kill_this = tempfile.mkdtemp() + assert os.path.exists(tmpfile_name) - assert_raises( - workbench.WorkbenchOutsideScope, - self.workbench_manager.destroy_workbench, - dont_kill_this) + wb_dir = this_workbench.dir + this_workbench.destroy_self() + assert not os.path.exists(tmpfile_name) + assert not os.path.exists(wb_dir) def test_localized_file(self): tmpdir, this_storage = get_tmp_filestorage() @@ -65,8 +65,7 @@ class TestWorkbench(object): our_file.write('Our file') # with a local file storage - filename = self.workbench_manager.localized_file( - this_workbench, this_storage, filepath) + filename = this_workbench.localized_file(this_storage, filepath) assert filename == os.path.join( tmpdir, 'dir1/dir2/ourfile.txt') @@ -77,20 +76,19 @@ class TestWorkbench(object): with this_storage.get_file(filepath, 'w') as our_file: our_file.write('Our file') - filename = self.workbench_manager.localized_file( - this_workbench, this_storage, filepath) + filename = this_workbench.localized_file(this_storage, filepath) assert filename == os.path.join( - this_workbench, 'ourfile.txt') + this_workbench.dir, 'ourfile.txt') # fake remote file storage, filename_if_copying set - filename = self.workbench_manager.localized_file( - this_workbench, this_storage, filepath, 'thisfile') + filename = this_workbench.localized_file( + this_storage, filepath, 'thisfile') assert filename == os.path.join( - this_workbench, 'thisfile.txt') + this_workbench.dir, 'thisfile.txt') # fake remote file storage, filename_if_copying set, # keep_extension_if_copying set to false - filename = self.workbench_manager.localized_file( - this_workbench, this_storage, filepath, 'thisfile.text', False) + filename = this_workbench.localized_file( + this_storage, filepath, 'thisfile.text', False) assert filename == os.path.join( - this_workbench, 'thisfile.text') + this_workbench.dir, 'thisfile.text') diff --git a/mediagoblin/tests/tools.py b/mediagoblin/tests/tools.py index 342b54b7..b1fe56a0 100644 --- a/mediagoblin/tests/tools.py +++ b/mediagoblin/tests/tools.py @@ -18,20 +18,25 @@ import pkg_resources import os, shutil -from paste.deploy import appconfig, loadapp +from paste.deploy import loadapp from webtest import TestApp -from mediagoblin import util +from mediagoblin import util, mg_globals +from mediagoblin.config import read_mediagoblin_config +from mediagoblin.celery_setup import setup_celery_from_config from mediagoblin.decorators import _make_safe from mediagoblin.db.open import setup_connection_and_db_from_config MEDIAGOBLIN_TEST_DB_NAME = '__mediagoblinunittests__' +TEST_SERVER_CONFIG = pkg_resources.resource_filename( + 'mediagoblin.tests', 'test_server.ini') TEST_APP_CONFIG = pkg_resources.resource_filename( - 'mediagoblin.tests', 'mgoblin_test_app.ini') + 'mediagoblin.tests', 'test_mgoblin_app.ini') TEST_USER_DEV = pkg_resources.resource_filename( 'mediagoblin.tests', 'test_user_dev') MGOBLIN_APP = None +CELERY_SETUP = False USER_DEV_DIRECTORIES_TO_SETUP = [ 'media/public', 'media/queue', @@ -42,12 +47,13 @@ class BadCeleryEnviron(Exception): pass def get_test_app(dump_old_app=True): - if not os.environ.get('CELERY_CONFIG_MODULE') == \ - 'mediagoblin.celery_setup.from_tests': + if os.environ.get('CELERY_CONFIG_MODULE'): raise BadCeleryEnviron( - u"Sorry, you *absolutely* must run nosetests with the\n" - u"mediagoblin.celery_setup.from_tests module. Like so:\n" - u"$ CELERY_CONFIG_MODULE=mediagoblin.celery_setup.from_tests ./bin/nosetests") + u"Sorry, you *ABSOLUTELY MUST *NOT* run nosetests with the\n" + u"CELERY_CONFIG_MODULE set to anything.") + + global MGOBLIN_APP + global CELERY_SETUP # Just return the old app if that exists and it's okay to set up # and return @@ -63,16 +69,13 @@ def get_test_app(dump_old_app=True): os.makedirs(full_dir) # Get app config - config = appconfig( - 'config:' + os.path.basename(TEST_APP_CONFIG), - relative_to=os.path.dirname(TEST_APP_CONFIG), - name='mediagoblin') + global_config, validation_result = read_mediagoblin_config(TEST_APP_CONFIG) + app_config = global_config['mediagoblin'] # Wipe database # @@: For now we're dropping collections, but we could also just # collection.remove() ? - connection, db = setup_connection_and_db_from_config( - config.local_conf) + connection, db = setup_connection_and_db_from_config(app_config) collections_to_wipe = [ collection @@ -90,9 +93,19 @@ def get_test_app(dump_old_app=True): # setup app and return test_app = loadapp( - 'config:' + TEST_APP_CONFIG) + 'config:' + TEST_SERVER_CONFIG) + + app = TestApp(test_app) + MGOBLIN_APP = app + + # setup celery + if not CELERY_SETUP: + setup_celery_from_config( + mg_globals.app_config, mg_globals.global_config, + set_environ=True) + CELERY_SETUP = True - return TestApp(test_app) + return app def setup_fresh_app(func): diff --git a/mediagoblin/user_pages/routing.py b/mediagoblin/user_pages/routing.py index c5e9a984..92998726 100644 --- a/mediagoblin/user_pages/routing.py +++ b/mediagoblin/user_pages/routing.py @@ -19,6 +19,8 @@ from routes.route import Route user_routes = [ Route('mediagoblin.user_pages.user_home', "/{user}/", controller="mediagoblin.user_pages.views:user_home"), + Route('mediagoblin.user_pages.user_gallery', "/{user}/gallery/", + controller="mediagoblin.user_pages.views:user_gallery"), Route('mediagoblin.user_pages.media_home', '/{user}/m/{media}/', requirements=dict(m_id="[0-9a-fA-F]{24}"), controller="mediagoblin.user_pages.views:media_home"), diff --git a/mediagoblin/user_pages/views.py b/mediagoblin/user_pages/views.py index 323c3e54..88b5dfe5 100644 --- a/mediagoblin/user_pages/views.py +++ b/mediagoblin/user_pages/views.py @@ -49,6 +49,33 @@ def user_home(request, page): 'media_entries': media_entries, 'pagination': pagination}) +@uses_pagination +def user_gallery(request, page): + """'Gallery' of a User()""" + user = request.db.User.find_one({ + 'username': request.matchdict['user'], + 'status': 'active'}) + if not user: + return exc.HTTPNotFound() + + cursor = request.db.MediaEntry.find( + {'uploader': user['_id'], + 'state': 'processed'}).sort('created', DESCENDING) + + pagination = Pagination(page, cursor) + media_entries = pagination() + + #if no data is available, return NotFound + if media_entries == None: + return exc.HTTPNotFound() + + return render_to_response( + request, + 'mediagoblin/user_pages/gallery.html', + {'user': user, + 'media_entries': media_entries, + 'pagination': pagination}) + @get_user_media_entry def media_home(request, media): diff --git a/mediagoblin/util.py b/mediagoblin/util.py index 64e21ca9..349bc027 100644 --- a/mediagoblin/util.py +++ b/mediagoblin/util.py @@ -18,7 +18,6 @@ from email.MIMEText import MIMEText import gettext import pkg_resources import smtplib -import os import sys import re import urllib @@ -28,10 +27,10 @@ import copy from babel.localedata import exists import jinja2 import translitcodec -from paste.deploy.loadwsgi import NicerConfigParser from webob import Response, exc +from lxml.html.clean import Cleaner -from mediagoblin import globals as mgoblin_globals +from mediagoblin import mg_globals from mediagoblin.db.util import ObjectId @@ -102,8 +101,8 @@ def get_jinja_env(template_loader, locale): extensions=['jinja2.ext.i18n']) template_env.install_gettext_callables( - mgoblin_globals.translations.gettext, - mgoblin_globals.translations.ngettext) + mg_globals.translations.gettext, + mg_globals.translations.ngettext) if exists(locale): SETUP_JINJA_ENVS[locale] = template_env @@ -264,9 +263,9 @@ def send_email(from_addr, to_addrs, subject, message_body): - message_body: email body text """ # TODO: make a mock mhost if testing is enabled - if TESTS_ENABLED or mgoblin_globals.email_debug_mode: + if TESTS_ENABLED or mg_globals.email_debug_mode: mhost = FakeMhost() - elif not mgoblin_globals.email_debug_mode: + elif not mg_globals.email_debug_mode: mhost = smtplib.SMTP() mhost.connect() @@ -279,7 +278,7 @@ def send_email(from_addr, to_addrs, subject, message_body): if TESTS_ENABLED: EMAIL_TEST_INBOX.append(message) - if getattr(mgoblin_globals, 'email_debug_mode', False): + if getattr(mg_globals, 'email_debug_mode', False): print u"===== Email =====" print u"From address: %s" % message['From'] print u"To addresses: %s" % message['To'] @@ -351,26 +350,30 @@ def get_locale_from_request(request): return locale_to_lower_upper(target_lang) -def read_config_file(conf_file): - """ - Read a paste deploy style config file and process it. - """ - if not os.path.exists(conf_file): - raise IOError( - "MEDIAGOBLIN_CONFIG not set or file does not exist") - - parser = NicerConfigParser(conf_file) - parser.read(conf_file) - parser._defaults.setdefault( - 'here', os.path.dirname(os.path.abspath(conf_file))) - parser._defaults.setdefault( - '__file__', os.path.abspath(conf_file)) - - mgoblin_conf = dict( - [(section_name, dict(parser.items(section_name))) - for section_name in parser.sections()]) - - return mgoblin_conf +# A super strict version of the lxml.html cleaner class +HTML_CLEANER = Cleaner( + scripts=True, + javascript=True, + comments=True, + style=True, + links=True, + page_structure=True, + processing_instructions=True, + embedded=True, + frames=True, + forms=True, + annoying_tags=True, + allow_tags=[ + 'div', 'b', 'i', 'em', 'strong', 'p', 'ul', 'ol', 'li', 'a', 'br'], + remove_unknown_tags=False, # can't be used with allow_tags + safe_attrs_only=True, + add_nofollow=True, # for now + host_whitelist=(), + whitelist_tags=set([])) + + +def clean_html(html): + return HTML_CLEANER.clean_html(html) SETUP_GETTEXTS = {} @@ -393,7 +396,7 @@ def setup_gettext(locale): if exists(locale): SETUP_GETTEXTS[locale] = this_gettext - mgoblin_globals.setup_globals( + mg_globals.setup_globals( translations=this_gettext) diff --git a/mediagoblin/workbench.py b/mediagoblin/workbench.py index d7252623..f83c4fa0 100644 --- a/mediagoblin/workbench.py +++ b/mediagoblin/workbench.py @@ -23,54 +23,34 @@ DEFAULT_WORKBENCH_DIR = os.path.join( tempfile.gettempdir(), u'mgoblin_workbench') -# Exception(s) -# ------------ - -class WorkbenchOutsideScope(Exception): - """ - Raised when a workbench is outside a WorkbenchManager scope. - """ - pass - - # Actual workbench stuff # ---------------------- -class WorkbenchManager(object): +class Workbench(object): """ - A system for generating and destroying workbenches. + Represent the directory for the workbench - Workbenches are actually just subdirectories of a temporary storage space - for during the processing stage. + WARNING: DO NOT create Workbench objects on your own, + let the WorkbenchManager do that for you! """ - - def __init__(self, base_workbench_dir): - self.base_workbench_dir = os.path.abspath(base_workbench_dir) - if not os.path.exists(self.base_workbench_dir): - os.makedirs(self.base_workbench_dir) - - def create_workbench(self): + def __init__(self, dir): """ - Create and return the path to a new workbench (directory). + WARNING: DO NOT create Workbench objects on your own, + let the WorkbenchManager do that for you! """ - return tempfile.mkdtemp(dir=self.base_workbench_dir) + self.dir = dir - def destroy_workbench(self, workbench): - """ - Destroy this workbench! Deletes the directory and all its contents! + def __unicode__(self): + return unicode(self.dir) + def __str__(self): + return str(self.dir) + def __repr__(self): + return repr(self.dir) - Makes sure the workbench actually belongs to this manager though. - """ - # just in case - workbench = os.path.abspath(workbench) - - if not workbench.startswith(self.base_workbench_dir): - raise WorkbenchOutsideScope( - "Can't destroy workbench outside the base workbench dir") - - shutil.rmtree(workbench) + def joinpath(self, *args): + return os.path.join(self.dir, *args) - def localized_file(self, workbench, storage, filepath, + def localized_file(self, storage, filepath, filename_if_copying=None, keep_extension_if_copying=True): """ @@ -126,10 +106,43 @@ class WorkbenchManager(object): dest_filename = filename_if_copying full_dest_filename = os.path.join( - workbench, dest_filename) + self.dir, dest_filename) # copy it over storage.copy_locally( filepath, full_dest_filename) return full_dest_filename + + def destroy_self(self): + """ + Destroy this workbench! Deletes the directory and all its contents! + + WARNING: Does no checks for a sane value in self.dir! + """ + # just in case + workbench = os.path.abspath(self.dir) + + shutil.rmtree(workbench) + + del self.dir + + +class WorkbenchManager(object): + """ + A system for generating and destroying workbenches. + + Workbenches are actually just subdirectories of a temporary storage space + for during the processing stage. + """ + + def __init__(self, base_workbench_dir): + self.base_workbench_dir = os.path.abspath(base_workbench_dir) + if not os.path.exists(self.base_workbench_dir): + os.makedirs(self.base_workbench_dir) + + def create_workbench(self): + """ + Create and return the path to a new workbench (directory). + """ + return Workbench(tempfile.mkdtemp(dir=self.base_workbench_dir)) diff --git a/server.ini b/server.ini new file mode 100644 index 00000000..73fbe8e8 --- /dev/null +++ b/server.ini @@ -0,0 +1,34 @@ +[DEFAULT] +debug = true + +[composite:main] +use = egg:Paste#urlmap +/ = mediagoblin +/mgoblin_media/ = publicstore_serve +/mgoblin_static/ = mediagoblin_static + +[app:mediagoblin] +use = egg:mediagoblin#app +filter-with = beaker +config = %(here)s/mediagoblin.ini + +[app:publicstore_serve] +use = egg:Paste#static +document_root = %(here)s/user_dev/media/public/ + +[app:mediagoblin_static] +use = egg:Paste#static +document_root = %(here)s/mediagoblin/static/ + +[filter:beaker] +use = egg:Beaker#beaker_session +cache_dir = %(here)s/user_dev/beaker +beaker.session.key = mediagoblin +# beaker.session.secret = somesupersecret +beaker.session.data_dir = %(here)s/user_dev/beaker/sessions/data +beaker.session.lock_dir = %(here)s/user_dev/beaker/sessions/lock + +[server:main] +use = egg:Paste#http +host = 127.0.0.1 +port = 6543 @@ -42,6 +42,10 @@ setup( 'translitcodec', 'argparse', 'webtest', + 'ConfigObj', + ## For now we're expecting that users will install this from + ## their package managers. + # 'lxml', ], test_suite='nose.collector', |