diff options
| -rw-r--r-- | mediagoblin/config_spec.ini | 4 | ||||
| -rw-r--r-- | mediagoblin/db/models.py | 5 | ||||
| -rw-r--r-- | mediagoblin/gmg_commands/__init__.py | 14 | ||||
| -rw-r--r-- | mediagoblin/gmg_commands/import_export.py | 256 | ||||
| -rw-r--r-- | mediagoblin/processing/task.py | 14 |
5 files changed, 19 insertions, 274 deletions
diff --git a/mediagoblin/config_spec.ini b/mediagoblin/config_spec.ini index dade8420..f769e4eb 100644 --- a/mediagoblin/config_spec.ini +++ b/mediagoblin/config_spec.ini @@ -113,9 +113,7 @@ user_privilege_scheme = string(default="uploader,commenter,reporter") # Frequency garbage collection will run (setting to 0 or false to disable) # Setting units are minutes. -## NOTE: This is temporarily disabled, but we want to set it back: -## garbage_collection = integer(default=60) -garbage_collection = integer(default=0) +garbage_collection = integer(default=60) [jinja2] # Jinja2 supports more directives than the minimum required by mediagoblin. diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index 8f722cff..7afd428d 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -769,12 +769,15 @@ class MediaComment(Base, MediaCommentMixin): ) media = MediaEntry.query.filter_by(id=self.media_entry).first() author = self.get_author + published = UTC.localize(self.created) context = { "id": href, "objectType": self.object_type, "content": self.content, "inReplyTo": media.serialize(request, show_comments=False), - "author": author.serialize(request) + "author": author.serialize(request), + "published": published.isoformat(), + "updated": published.isoformat(), } if self.location: diff --git a/mediagoblin/gmg_commands/__init__.py b/mediagoblin/gmg_commands/__init__.py index d8d982b1..e5f9a395 100644 --- a/mediagoblin/gmg_commands/__init__.py +++ b/mediagoblin/gmg_commands/__init__.py @@ -81,20 +81,6 @@ SUBCOMMAND_MAP = { # 'func': 'mediagoblin.gmg_commands.theme:theme', # 'help': 'Theming commands', # } - - ## These might be useful, mayyyybe, but don't really work anymore - ## due to mongo change and the "versatility" of sql options. - ## - ## For now, commenting out. Might re-enable soonish? - # - # 'env_export': { - # 'setup': 'mediagoblin.gmg_commands.import_export:import_export_parse_setup', - # 'func': 'mediagoblin.gmg_commands.import_export:env_export', - # 'help': 'Exports the data for this MediaGoblin instance'}, - # 'env_import': { - # 'setup': 'mediagoblin.gmg_commands.import_export:import_export_parse_setup', - # 'func': 'mediagoblin.gmg_commands.import_export:env_import', - # 'help': 'Imports the data for this MediaGoblin instance'}, } diff --git a/mediagoblin/gmg_commands/import_export.py b/mediagoblin/gmg_commands/import_export.py deleted file mode 100644 index fbac09f6..00000000 --- a/mediagoblin/gmg_commands/import_export.py +++ /dev/null @@ -1,256 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -from mediagoblin import mg_globals -from mediagoblin.db.open import setup_connection_and_db_from_config -from mediagoblin.gmg_commands import util as commands_util -from mediagoblin.storage.filestorage import BasicFileStorage -from mediagoblin.init import setup_storage, setup_global_and_app_config - -import shutil -import tarfile -import tempfile -import subprocess -import os.path -import os -import sys -import logging -from contextlib import closing - -_log = logging.getLogger('gmg.import_export') -logging.basicConfig() -_log.setLevel(logging.INFO) - - -def import_export_parse_setup(subparser): - # TODO: Add default - subparser.add_argument( - 'tar_file') - subparser.add_argument( - '--mongodump_path', default='mongodump', - help='mongodump binary') - subparser.add_argument( - '--mongorestore_path', default='mongorestore', - help='mongorestore binary') - subparser.add_argument( - '--cache_path', - help='Temporary directory where files will be temporarily dumped') - - -def _import_media(db, args): - ''' - Import media files - - Must be called after _import_database() - ''' - _log.info('-> Importing media...') - - media_cache = BasicFileStorage( - args._cache_path['media']) - - # TODO: Add import of queue files - queue_cache = BasicFileStorage(args._cache_path['queue']) - - for entry in db.MediaEntry.query.filter_by(): - for name, path in entry.media_files.items(): - _log.info('Importing: {0} - {1}'.format( - entry.title.encode('ascii', 'replace'), - name)) - - media_file = mg_globals.public_store.get_file(path, mode='wb') - media_file.write( - media_cache.get_file(path, mode='rb').read()) - - _log.info('...Media imported') - - -def _import_database(db, args): - ''' - Restore mongo database from ___.bson files - ''' - _log.info('-> Importing database...') - - p = subprocess.Popen([ - args.mongorestore_path, - '-d', db.name, - os.path.join(args._cache_path['database'], db.name)]) - - p.wait() - - _log.info('...Database imported') - - -def env_import(args): - ''' - Restore mongo database and media files from a tar archive - ''' - if not args.cache_path: - args.cache_path = tempfile.mkdtemp() - - setup_global_and_app_config(args.conf_file) - - # Creates mg_globals.public_store and mg_globals.queue_store - setup_storage() - - global_config, app_config = setup_global_and_app_config(args.conf_file) - db = setup_connection_and_db_from_config( - app_config) - - tf = tarfile.open( - args.tar_file, - mode='r|gz') - - tf.extractall(args.cache_path) - - args.cache_path = os.path.join( - args.cache_path, 'mediagoblin-data') - args = _setup_paths(args) - - # Import database from extracted data - _import_database(db, args) - - _import_media(db, args) - - _clean(args) - - -def _setup_paths(args): - ''' - Populate ``args`` variable with cache subpaths - ''' - args._cache_path = dict() - PATH_MAP = { - 'media': 'media', - 'queue': 'queue', - 'database': 'database'} - - for key, val in PATH_MAP.items(): - args._cache_path[key] = os.path.join(args.cache_path, val) - - return args - - -def _create_archive(args): - ''' - Create the tar archive - ''' - _log.info('-> Compressing to archive') - - tf = tarfile.open( - args.tar_file, - mode='w|gz') - - with closing(tf): - tf.add(args.cache_path, 'mediagoblin-data/') - - _log.info('...Archiving done') - - -def _clean(args): - ''' - Remove cache directory - ''' - shutil.rmtree(args.cache_path) - - -def _export_check(args): - ''' - Run security checks for export command - ''' - if os.path.exists(args.tar_file): - overwrite = raw_input( - 'The output file already exists. ' - 'Are you **SURE** you want to overwrite it? ' - '(yes/no)> ') - if not overwrite == 'yes': - print 'Aborting.' - - return False - - return True - - -def _export_database(db, args): - _log.info('-> Exporting database...') - - p = subprocess.Popen([ - args.mongodump_path, - '-d', db.name, - '-o', args._cache_path['database']]) - - p.wait() - - _log.info('...Database exported') - - -def _export_media(db, args): - _log.info('-> Exporting media...') - - media_cache = BasicFileStorage( - args._cache_path['media']) - - # TODO: Add export of queue files - queue_cache = BasicFileStorage(args._cache_path['queue']) - - for entry in db.MediaEntry.query.filter_by(): - for name, path in entry.media_files.items(): - _log.info(u'Exporting {0} - {1}'.format( - entry.title, - name)) - try: - mc_file = media_cache.get_file(path, mode='wb') - mc_file.write( - mg_globals.public_store.get_file(path, mode='rb').read()) - except Exception as e: - _log.error('Failed: {0}'.format(e)) - - _log.info('...Media exported') - - -def env_export(args): - ''' - Export database and media files to a tar archive - ''' - commands_util.check_unrecognized_args(args) - if args.cache_path: - if os.path.exists(args.cache_path): - _log.error('The cache directory must not exist ' - 'before you run this script') - _log.error('Cache directory: {0}'.format(args.cache_path)) - - return False - else: - args.cache_path = tempfile.mkdtemp() - - args = _setup_paths(args) - - if not _export_check(args): - _log.error('Checks did not pass, exiting') - sys.exit(0) - - globa_config, app_config = setup_global_and_app_config(args.conf_file) - - setup_storage() - - db = setup_connection_and_db_from_config(app_config) - - _export_database(db, args) - - _export_media(db, args) - - _create_archive(args) - - _clean(args) diff --git a/mediagoblin/processing/task.py b/mediagoblin/processing/task.py index 1a21c6d2..0c254767 100644 --- a/mediagoblin/processing/task.py +++ b/mediagoblin/processing/task.py @@ -155,5 +155,19 @@ class ProcessMedia(celery.Task): entry = mgg.database.MediaEntry.query.filter_by(id=entry_id).first() json_processing_callback(entry) + mgg.database.reset_after_request() + + def after_return(self, *args, **kwargs): + """ + This is called after the task has returned, we should clean up. + + We need to rollback the database to prevent ProgrammingError exceptions + from being raised. + """ + # In eager mode we get DetachedInstanceError, we do rollback on_failure + # to deal with that case though when in eager mode. + if not celery.app.default_app.conf['CELERY_ALWAYS_EAGER']: + mgg.database.reset_after_request() + tasks.register(ProcessMedia) |