diff options
Diffstat (limited to 'mediagoblin/gmg_commands')
| -rw-r--r-- | mediagoblin/gmg_commands/__init__.py | 54 | ||||
| -rw-r--r-- | mediagoblin/gmg_commands/addmedia.py | 33 | ||||
| -rw-r--r-- | mediagoblin/gmg_commands/alembic_commands.py | 71 | ||||
| -rw-r--r-- | mediagoblin/gmg_commands/batchaddmedia.py | 60 | ||||
| -rw-r--r-- | mediagoblin/gmg_commands/dbupdate.py | 131 | ||||
| -rw-r--r-- | mediagoblin/gmg_commands/deletemedia.py | 9 | ||||
| -rw-r--r-- | mediagoblin/gmg_commands/import_export.py | 256 | ||||
| -rw-r--r-- | mediagoblin/gmg_commands/reprocess.py | 43 | ||||
| -rw-r--r-- | mediagoblin/gmg_commands/serve.py | 65 | ||||
| -rw-r--r-- | mediagoblin/gmg_commands/shell.py | 58 | ||||
| -rw-r--r-- | mediagoblin/gmg_commands/users.py | 63 | ||||
| -rw-r--r-- | mediagoblin/gmg_commands/util.py | 4 |
12 files changed, 446 insertions, 401 deletions
diff --git a/mediagoblin/gmg_commands/__init__.py b/mediagoblin/gmg_commands/__init__.py index 0cb239a2..98b097a6 100644 --- a/mediagoblin/gmg_commands/__init__.py +++ b/mediagoblin/gmg_commands/__init__.py @@ -16,9 +16,16 @@ import argparse import os +import shutil + +import six from mediagoblin.tools.common import import_component +import logging +_log = logging.getLogger(__name__) +logging.basicConfig() + SUBCOMMAND_MAP = { 'shell': { @@ -61,29 +68,25 @@ SUBCOMMAND_MAP = { 'setup': 'mediagoblin.gmg_commands.deletemedia:parser_setup', 'func': 'mediagoblin.gmg_commands.deletemedia:deletemedia', 'help': 'Delete media entries'}, + 'serve': { + 'setup': 'mediagoblin.gmg_commands.serve:parser_setup', + 'func': 'mediagoblin.gmg_commands.serve:serve', + 'help': 'PasteScript replacement'}, 'batchaddmedia': { 'setup': 'mediagoblin.gmg_commands.batchaddmedia:parser_setup', 'func': 'mediagoblin.gmg_commands.batchaddmedia:batchaddmedia', 'help': 'Add many media entries at once'}, + 'alembic': { + 'setup': 'mediagoblin.gmg_commands.alembic_commands:parser_setup', + 'func': 'mediagoblin.gmg_commands.alembic_commands:raw_alembic_cli', + 'help': ( + 'Run raw alembic commands with our local database. ' + '(Unless you know what you\'re doing, use dbupdate instead!)')}, # 'theme': { # 'setup': 'mediagoblin.gmg_commands.theme:theme_parser_setup', # 'func': 'mediagoblin.gmg_commands.theme:theme', # 'help': 'Theming commands', # } - - ## These might be useful, mayyyybe, but don't really work anymore - ## due to mongo change and the "versatility" of sql options. - ## - ## For now, commenting out. Might re-enable soonish? - # - # 'env_export': { - # 'setup': 'mediagoblin.gmg_commands.import_export:import_export_parse_setup', - # 'func': 'mediagoblin.gmg_commands.import_export:env_export', - # 'help': 'Exports the data for this MediaGoblin instance'}, - # 'env_import': { - # 'setup': 'mediagoblin.gmg_commands.import_export:import_export_parse_setup', - # 'func': 'mediagoblin.gmg_commands.import_export:env_import', - # 'help': 'Imports the data for this MediaGoblin instance'}, } @@ -98,7 +101,7 @@ def main_cli(): "otherwise mediagoblin.ini")) subparsers = parser.add_subparsers(help='sub-command help') - for command_name, command_struct in SUBCOMMAND_MAP.iteritems(): + for command_name, command_struct in six.iteritems(SUBCOMMAND_MAP): if 'help' in command_struct: subparser = subparsers.add_parser( command_name, help=command_struct['help']) @@ -121,6 +124,27 @@ def main_cli(): else: args.conf_file = 'mediagoblin.ini' + # This is a hopefully TEMPORARY hack for adding a mediagoblin.ini + # if none exists, to make up for a deficiency as we are migrating + # our docs to the new "no mediagoblin.ini by default" workflow. + # Normally, the docs should provide instructions for this, but + # since 0.7.1 docs say "install from master!" and yet we removed + # mediagoblin.ini, we need to cover our bases... + + parent_directory = os.path.split(os.path.abspath(args.conf_file))[0] + if os.path.split(args.conf_file)[1] == 'mediagoblin.ini' \ + and not os.path.exists(args.conf_file) \ + and os.path.exists( + os.path.join( + parent_directory, 'mediagoblin.example.ini')): + # Do the copy-over of the mediagoblin config for the user + _log.warning( + "No mediagoblin.ini found and no other path given, " + "so making one for you.") + shutil.copy( + os.path.join(parent_directory, "mediagoblin.example.ini"), + os.path.join(parent_directory, "mediagoblin.ini")) + args.func(args) diff --git a/mediagoblin/gmg_commands/addmedia.py b/mediagoblin/gmg_commands/addmedia.py index 2f095760..babc6ed7 100644 --- a/mediagoblin/gmg_commands/addmedia.py +++ b/mediagoblin/gmg_commands/addmedia.py @@ -14,8 +14,13 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +from __future__ import print_function + import os +import six + +from mediagoblin.db.models import LocalUser from mediagoblin.gmg_commands import util as commands_util from mediagoblin.submit.lib import ( submit_media, get_upload_file_limits, @@ -71,41 +76,41 @@ def addmedia(args): app = commands_util.setup_app(args) # get the user - user = app.db.User.query.filter_by(username=args.username.lower()).first() + user = app.db.LocalUser.query.filter( + LocalUser.username==args.username.lower() + ).first() if user is None: - print "Sorry, no user by username '%s'" % args.username + print("Sorry, no user by username '%s'" % args.username) return - + # check for the file, if it exists... filename = os.path.split(args.filename)[-1] abs_filename = os.path.abspath(args.filename) if not os.path.exists(abs_filename): - print "Can't find a file with filename '%s'" % args.filename + print("Can't find a file with filename '%s'" % args.filename) return - upload_limit, max_file_size = get_upload_file_limits(user) - def maybe_unicodeify(some_string): # this is kinda terrible if some_string is None: return None - else: - return unicode(some_string) + if six.PY2: + return six.text_type(some_string, 'utf-8') + return some_string try: submit_media( mg_app=app, user=user, - submitted_file=file(abs_filename, 'r'), filename=filename, + submitted_file=open(abs_filename, 'rb'), filename=filename, title=maybe_unicodeify(args.title), description=maybe_unicodeify(args.description), collection_slug=args.collection_slug, license=maybe_unicodeify(args.license), - tags_string=maybe_unicodeify(args.tags) or u"", - upload_limit=upload_limit, max_file_size=max_file_size) + tags_string=maybe_unicodeify(args.tags) or u"") except FileUploadLimit: - print "This file is larger than the upload limits for this site." + print("This file is larger than the upload limits for this site.") except UserUploadLimit: - print "This file will put this user past their upload limits." + print("This file will put this user past their upload limits.") except UserPastUploadLimit: - print "This user is already past their upload limits." + print("This user is already past their upload limits.") diff --git a/mediagoblin/gmg_commands/alembic_commands.py b/mediagoblin/gmg_commands/alembic_commands.py new file mode 100644 index 00000000..4e73ec13 --- /dev/null +++ b/mediagoblin/gmg_commands/alembic_commands.py @@ -0,0 +1,71 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import argparse + +from alembic import config +from sqlalchemy.orm import sessionmaker + +from mediagoblin.db.open import setup_connection_and_db_from_config +from mediagoblin.init import setup_global_and_app_config +from mediagoblin.db.migration_tools import build_alembic_config +from mediagoblin.tools.common import import_component + + +class FudgedCommandLine(config.CommandLine): + def main(self, args, db, global_config): + options = self.parser.parse_args(args.args_for_alembic) + + # This is useful for people running commands involving plugins, such + # as building autogenerated revisions. + if args.with_plugins: + plugins = global_config.get('plugins', {}).keys() + for plugin in plugins: + try: + import_component('{0}.models:MODELS'.format(plugin)) + except ImportError: + # It doesn't really matter if there's no models to import + # here. + pass + + # This code is inspired by a hack in Alembic, but isn't the same really. + # Regardless, Alembic is Expat licensed. + if not hasattr(options, "cmd"): + print( + "* Only use this command if you know what you are doing! *\n" + "If not, use the 'gmg dbupdate' command instead.\n\n" + "(You can also pass --with-plugins here.)" + "Alembic help:\n") + self.parser.print_help() + return + else: + Session = sessionmaker(bind=db.engine) + session = Session() + + cfg = build_alembic_config(global_config, options, session) + + self.run_cmd(cfg, options) + +def parser_setup(subparser): + subparser.add_argument( + "--with-plugins", action="store_true", + help="Import all plugins' models before running alembic commands.") + subparser.add_argument("args_for_alembic", nargs=argparse.REMAINDER) + +def raw_alembic_cli(args): + global_config, app_config = setup_global_and_app_config(args.conf_file) + db = setup_connection_and_db_from_config(app_config, migrations=False) + FudgedCommandLine().main(args, db, global_config) diff --git a/mediagoblin/gmg_commands/batchaddmedia.py b/mediagoblin/gmg_commands/batchaddmedia.py index 4931bda2..274d72bc 100644 --- a/mediagoblin/gmg_commands/batchaddmedia.py +++ b/mediagoblin/gmg_commands/batchaddmedia.py @@ -14,11 +14,18 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import os -import requests, codecs +from __future__ import print_function + +import codecs import csv -from urlparse import urlparse +import os +import requests +import six + +from six.moves.urllib.parse import urlparse + +from mediagoblin.db.models import LocalUser from mediagoblin.gmg_commands import util as commands_util from mediagoblin.submit.lib import ( submit_media, get_upload_file_limits, @@ -58,13 +65,14 @@ def batchaddmedia(args): files_uploaded, files_attempted = 0, 0 # get the user - user = app.db.User.query.filter_by(username=args.username.lower()).first() + user = app.db.LocalUser.query.filter( + LocalUser.username==args.username.lower() + ).first() if user is None: - print _(u"Sorry, no user by username '{username}' exists".format( - username=args.username)) + print(_(u"Sorry, no user by username '{username}' exists".format( + username=args.username))) return - upload_limit, max_file_size = get_upload_file_limits(user) temp_files = [] if os.path.isfile(args.metadata_path): @@ -73,19 +81,18 @@ def batchaddmedia(args): else: error = _(u'File at {path} not found, use -h flag for help'.format( path=args.metadata_path)) - print error + print(error) return abs_metadata_filename = os.path.abspath(metadata_path) abs_metadata_dir = os.path.dirname(abs_metadata_filename) - upload_limit, max_file_size = get_upload_file_limits(user) def maybe_unicodeify(some_string): # this is kinda terrible if some_string is None: return None else: - return unicode(some_string) + return six.text_type(some_string) with codecs.open( abs_metadata_filename, 'r', encoding='utf-8') as all_metadata: @@ -110,13 +117,13 @@ def batchaddmedia(args): license = file_metadata.get('license') try: json_ld_metadata = compact_and_validate(file_metadata) - except ValidationError, exc: + except ValidationError as exc: error = _(u"""Error with media '{media_id}' value '{error_path}': {error_msg} Metadata was not uploaded.""".format( media_id=media_id, error_path=exc.path[0], error_msg=exc.message)) - print error + print(error) continue url = urlparse(original_location) @@ -136,9 +143,9 @@ Metadata was not uploaded.""".format( try: media_file = file(file_abs_path, 'r') except IOError: - print _(u"""\ + print(_(u"""\ FAIL: Local file {filename} could not be accessed. -{filename} will not be uploaded.""".format(filename=filename)) +{filename} will not be uploaded.""".format(filename=filename))) continue try: submit_media( @@ -150,33 +157,33 @@ FAIL: Local file {filename} could not be accessed. description=maybe_unicodeify(description), license=maybe_unicodeify(license), metadata=json_ld_metadata, - tags_string=u"", - upload_limit=upload_limit, max_file_size=max_file_size) - print _(u"""Successfully submitted {filename}! + tags_string=u"") + print(_(u"""Successfully submitted {filename}! Be sure to look at the Media Processing Panel on your website to be sure it -uploaded successfully.""".format(filename=filename)) +uploaded successfully.""".format(filename=filename))) files_uploaded += 1 except FileUploadLimit: - print _( -u"FAIL: This file is larger than the upload limits for this site.") + print(_( +u"FAIL: This file is larger than the upload limits for this site.")) except UserUploadLimit: - print _( -"FAIL: This file will put this user past their upload limits.") + print(_( +"FAIL: This file will put this user past their upload limits.")) except UserPastUploadLimit: - print _("FAIL: This user is already past their upload limits.") - print _( + print(_("FAIL: This user is already past their upload limits.")) + print(_( "{files_uploaded} out of {files_attempted} files successfully submitted".format( files_uploaded=files_uploaded, - files_attempted=files_attempted)) + files_attempted=files_attempted))) def unicode_csv_reader(unicode_csv_data, dialect=csv.excel, **kwargs): # csv.py doesn't do Unicode; encode temporarily as UTF-8: + # TODO: this probably won't be necessary in Python 3 csv_reader = csv.reader(utf_8_encoder(unicode_csv_data), dialect=dialect, **kwargs) for row in csv_reader: # decode UTF-8 back to Unicode, cell by cell: - yield [unicode(cell, 'utf-8') for cell in row] + yield [six.text_type(cell, 'utf-8') for cell in row] def utf_8_encoder(unicode_csv_data): for line in unicode_csv_data: @@ -203,4 +210,3 @@ def parse_csv_file(file_contents): objects_dict[media_id] = (line_dict) return objects_dict - diff --git a/mediagoblin/gmg_commands/dbupdate.py b/mediagoblin/gmg_commands/dbupdate.py index 27283a20..bafe76bb 100644 --- a/mediagoblin/gmg_commands/dbupdate.py +++ b/mediagoblin/gmg_commands/dbupdate.py @@ -16,10 +16,13 @@ import logging +import six +from alembic import command from sqlalchemy.orm import sessionmaker from mediagoblin.db.open import setup_connection_and_db_from_config -from mediagoblin.db.migration_tools import MigrationManager +from mediagoblin.db.migration_tools import ( + MigrationManager, build_alembic_config, populate_table_foundations) from mediagoblin.init import setup_global_and_app_config from mediagoblin.tools.common import import_component @@ -34,19 +37,20 @@ def dbupdate_parse_setup(subparser): class DatabaseData(object): - def __init__(self, name, models, foundations, migrations): + def __init__(self, name, models, migrations): self.name = name self.models = models - self.foundations = foundations self.migrations = migrations def make_migration_manager(self, session): return MigrationManager( - self.name, self.models, self.foundations, self.migrations, session) + self.name, self.models, self.migrations, session) def gather_database_data(plugins): """ + Gather all database data relevant to the extensions installed. + Gather all database data relevant to the extensions we have installed so we can do migrations and table initialization. @@ -57,11 +61,10 @@ def gather_database_data(plugins): # Add main first from mediagoblin.db.models import MODELS as MAIN_MODELS from mediagoblin.db.migrations import MIGRATIONS as MAIN_MIGRATIONS - from mediagoblin.db.models import FOUNDATIONS as MAIN_FOUNDATIONS managed_dbdata.append( DatabaseData( - u'__main__', MAIN_MODELS, MAIN_FOUNDATIONS, MAIN_MIGRATIONS)) + u'__main__', MAIN_MODELS, MAIN_MIGRATIONS)) for plugin in plugins: try: @@ -73,8 +76,8 @@ def gather_database_data(plugins): models = [] except AttributeError as exc: - _log.warning('Could not find MODELS in {0}.models, have you \ -forgotten to add it? ({1})'.format(plugin, exc)) + _log.warning('Could not find MODELS in {0}.models, have you ' + 'forgotten to add it? ({1})'.format(plugin, exc)) models = [] try: @@ -87,23 +90,50 @@ forgotten to add it? ({1})'.format(plugin, exc)) migrations = {} except AttributeError as exc: - _log.debug('Could not find MIGRATIONS in {0}.migrations, have you \ -forgotten to add it? ({1})'.format(plugin, exc)) + _log.debug('Could not find MIGRATIONS in {0}.migrations, have you' + 'forgotten to add it? ({1})'.format(plugin, exc)) migrations = {} + if models: + managed_dbdata.append( + DatabaseData(plugin, models, migrations)) + + return managed_dbdata + + +def run_foundations(db, global_config): + """ + Gather foundations data and run it. + """ + from mediagoblin.db.models import FOUNDATIONS as MAIN_FOUNDATIONS + all_foundations = [(u"__main__", MAIN_FOUNDATIONS)] + + Session = sessionmaker(bind=db.engine) + session = Session() + + plugins = global_config.get('plugins', {}) + + for plugin in plugins: try: - foundations = import_component('{0}.models:FOUNDATIONS'.format(plugin)) + foundations = import_component( + '{0}.models:FOUNDATIONS'.format(plugin)) + all_foundations.append((plugin, foundations)) except ImportError as exc: - foundations = {} + continue except AttributeError as exc: - foundations = {} + continue - if models: - managed_dbdata.append( - DatabaseData(plugin, models, foundations, migrations)) + for name, foundations in all_foundations: + populate_table_foundations(session, foundations, name) - return managed_dbdata +def run_alembic_migrations(db, app_config, global_config): + """Initialize a database and runs all Alembic migrations.""" + Session = sessionmaker(bind=db.engine) + session = Session() + cfg = build_alembic_config(global_config, None, session) + + return command.upgrade(cfg, 'heads') def run_dbupdate(app_config, global_config): @@ -113,15 +143,36 @@ def run_dbupdate(app_config, global_config): Will also initialize or migrate all extensions (media types, and in the future, plugins) """ - # Set up the database db = setup_connection_and_db_from_config(app_config, migrations=True) - #Run the migrations - run_all_migrations(db, app_config, global_config) + + # Do we have migrations + should_run_sqam_migrations = db.engine.has_table("core__migrations") and \ + sqam_migrations_to_run(db, app_config, + global_config) + + # Looks like a fresh database! + # (We set up this variable here because doing "run_all_migrations" below + # will change things.) + fresh_database = ( + not db.engine.has_table("core__migrations") and + not db.engine.has_table("alembic_version")) + + # Run the migrations + if should_run_sqam_migrations: + run_all_migrations(db, app_config, global_config) + + run_alembic_migrations(db, app_config, global_config) + + # If this was our first time initializing the database, + # we must lay down the foundations + if fresh_database: + run_foundations(db, global_config) def run_all_migrations(db, app_config, global_config): - """ + """Initialize or migrates a database. + Initializes or migrates a database that already has a connection setup and also initializes or migrates all extensions based on the config files. @@ -131,7 +182,7 @@ def run_all_migrations(db, app_config, global_config): """ # Gather information from all media managers / projects dbdatas = gather_database_data( - global_config.get('plugins', {}).keys()) + list(global_config.get('plugins', {}).keys())) Session = sessionmaker(bind=db.engine) @@ -142,6 +193,42 @@ def run_all_migrations(db, app_config, global_config): migration_manager.init_or_migrate() +def sqam_migrations_to_run(db, app_config, global_config): + """ + Check whether any plugins have sqlalchemy-migrate migrations left to run. + + This is a kludge so we can transition away from sqlalchemy-migrate + except where necessary. + """ + # @@: This shares a lot of code with run_all_migrations, but both + # are legacy and will be removed at some point. + + # Gather information from all media managers / projects + dbdatas = gather_database_data( + list(global_config.get('plugins', {}).keys())) + + Session = sessionmaker(bind=db.engine) + + # We can bail out early if it turns out that sqlalchemy-migrate + # was never installed with any migrations + from mediagoblin.db.models import MigrationData + if Session().query(MigrationData).filter_by( + name=u"__main__").first() is None: + return False + + # Setup media managers for all dbdata, run init/migrate and print info + # For each component, create/migrate tables + for dbdata in dbdatas: + migration_manager = dbdata.make_migration_manager(Session()) + if migration_manager.migrations_to_run(): + # If *any* migration managers have migrations to run, + # we'll have to run them. + return True + + # Otherwise, scot free! + return False + + def dbupdate(args): global_config, app_config = setup_global_and_app_config(args.conf_file) run_dbupdate(app_config, global_config) diff --git a/mediagoblin/gmg_commands/deletemedia.py b/mediagoblin/gmg_commands/deletemedia.py index ab5a81f6..bf50abde 100644 --- a/mediagoblin/gmg_commands/deletemedia.py +++ b/mediagoblin/gmg_commands/deletemedia.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +from __future__ import print_function import sys from mediagoblin.gmg_commands import util as commands_util @@ -29,7 +30,7 @@ def deletemedia(args): media_ids = set([int(mid) for mid in args.media_ids.split(',') if mid.isdigit()]) if not media_ids: - print 'Can\'t find any valid media ID(s).' + print('Can\'t find any valid media ID(s).') sys.exit(1) found_medias = set() filter_ids = app.db.MediaEntry.id.in_(media_ids) @@ -37,8 +38,8 @@ def deletemedia(args): for media in medias: found_medias.add(media.id) media.delete() - print 'Media ID %d has been deleted.' % media.id + print('Media ID %d has been deleted.' % media.id) for media in media_ids - found_medias: - print 'Can\'t find a media with ID %d.' % media - print 'Done.' + print('Can\'t find a media with ID %d.' % media) + print('Done.') sys.exit(0) diff --git a/mediagoblin/gmg_commands/import_export.py b/mediagoblin/gmg_commands/import_export.py deleted file mode 100644 index fbac09f6..00000000 --- a/mediagoblin/gmg_commands/import_export.py +++ /dev/null @@ -1,256 +0,0 @@ -# GNU MediaGoblin -- federated, autonomous media hosting -# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -from mediagoblin import mg_globals -from mediagoblin.db.open import setup_connection_and_db_from_config -from mediagoblin.gmg_commands import util as commands_util -from mediagoblin.storage.filestorage import BasicFileStorage -from mediagoblin.init import setup_storage, setup_global_and_app_config - -import shutil -import tarfile -import tempfile -import subprocess -import os.path -import os -import sys -import logging -from contextlib import closing - -_log = logging.getLogger('gmg.import_export') -logging.basicConfig() -_log.setLevel(logging.INFO) - - -def import_export_parse_setup(subparser): - # TODO: Add default - subparser.add_argument( - 'tar_file') - subparser.add_argument( - '--mongodump_path', default='mongodump', - help='mongodump binary') - subparser.add_argument( - '--mongorestore_path', default='mongorestore', - help='mongorestore binary') - subparser.add_argument( - '--cache_path', - help='Temporary directory where files will be temporarily dumped') - - -def _import_media(db, args): - ''' - Import media files - - Must be called after _import_database() - ''' - _log.info('-> Importing media...') - - media_cache = BasicFileStorage( - args._cache_path['media']) - - # TODO: Add import of queue files - queue_cache = BasicFileStorage(args._cache_path['queue']) - - for entry in db.MediaEntry.query.filter_by(): - for name, path in entry.media_files.items(): - _log.info('Importing: {0} - {1}'.format( - entry.title.encode('ascii', 'replace'), - name)) - - media_file = mg_globals.public_store.get_file(path, mode='wb') - media_file.write( - media_cache.get_file(path, mode='rb').read()) - - _log.info('...Media imported') - - -def _import_database(db, args): - ''' - Restore mongo database from ___.bson files - ''' - _log.info('-> Importing database...') - - p = subprocess.Popen([ - args.mongorestore_path, - '-d', db.name, - os.path.join(args._cache_path['database'], db.name)]) - - p.wait() - - _log.info('...Database imported') - - -def env_import(args): - ''' - Restore mongo database and media files from a tar archive - ''' - if not args.cache_path: - args.cache_path = tempfile.mkdtemp() - - setup_global_and_app_config(args.conf_file) - - # Creates mg_globals.public_store and mg_globals.queue_store - setup_storage() - - global_config, app_config = setup_global_and_app_config(args.conf_file) - db = setup_connection_and_db_from_config( - app_config) - - tf = tarfile.open( - args.tar_file, - mode='r|gz') - - tf.extractall(args.cache_path) - - args.cache_path = os.path.join( - args.cache_path, 'mediagoblin-data') - args = _setup_paths(args) - - # Import database from extracted data - _import_database(db, args) - - _import_media(db, args) - - _clean(args) - - -def _setup_paths(args): - ''' - Populate ``args`` variable with cache subpaths - ''' - args._cache_path = dict() - PATH_MAP = { - 'media': 'media', - 'queue': 'queue', - 'database': 'database'} - - for key, val in PATH_MAP.items(): - args._cache_path[key] = os.path.join(args.cache_path, val) - - return args - - -def _create_archive(args): - ''' - Create the tar archive - ''' - _log.info('-> Compressing to archive') - - tf = tarfile.open( - args.tar_file, - mode='w|gz') - - with closing(tf): - tf.add(args.cache_path, 'mediagoblin-data/') - - _log.info('...Archiving done') - - -def _clean(args): - ''' - Remove cache directory - ''' - shutil.rmtree(args.cache_path) - - -def _export_check(args): - ''' - Run security checks for export command - ''' - if os.path.exists(args.tar_file): - overwrite = raw_input( - 'The output file already exists. ' - 'Are you **SURE** you want to overwrite it? ' - '(yes/no)> ') - if not overwrite == 'yes': - print 'Aborting.' - - return False - - return True - - -def _export_database(db, args): - _log.info('-> Exporting database...') - - p = subprocess.Popen([ - args.mongodump_path, - '-d', db.name, - '-o', args._cache_path['database']]) - - p.wait() - - _log.info('...Database exported') - - -def _export_media(db, args): - _log.info('-> Exporting media...') - - media_cache = BasicFileStorage( - args._cache_path['media']) - - # TODO: Add export of queue files - queue_cache = BasicFileStorage(args._cache_path['queue']) - - for entry in db.MediaEntry.query.filter_by(): - for name, path in entry.media_files.items(): - _log.info(u'Exporting {0} - {1}'.format( - entry.title, - name)) - try: - mc_file = media_cache.get_file(path, mode='wb') - mc_file.write( - mg_globals.public_store.get_file(path, mode='rb').read()) - except Exception as e: - _log.error('Failed: {0}'.format(e)) - - _log.info('...Media exported') - - -def env_export(args): - ''' - Export database and media files to a tar archive - ''' - commands_util.check_unrecognized_args(args) - if args.cache_path: - if os.path.exists(args.cache_path): - _log.error('The cache directory must not exist ' - 'before you run this script') - _log.error('Cache directory: {0}'.format(args.cache_path)) - - return False - else: - args.cache_path = tempfile.mkdtemp() - - args = _setup_paths(args) - - if not _export_check(args): - _log.error('Checks did not pass, exiting') - sys.exit(0) - - globa_config, app_config = setup_global_and_app_config(args.conf_file) - - setup_storage() - - db = setup_connection_and_db_from_config(app_config) - - _export_database(db, args) - - _export_media(db, args) - - _create_archive(args) - - _clean(args) diff --git a/mediagoblin/gmg_commands/reprocess.py b/mediagoblin/gmg_commands/reprocess.py index e2f19ea3..85cae6df 100644 --- a/mediagoblin/gmg_commands/reprocess.py +++ b/mediagoblin/gmg_commands/reprocess.py @@ -13,6 +13,9 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. + +from __future__ import print_function + import argparse import os @@ -143,7 +146,7 @@ def available(args): manager = get_processing_manager_for_type(media_type) except ProcessingManagerDoesNotExist: entry = MediaEntry.query.filter_by(id=args.id_or_type).first() - print 'No such processing manager for {0}'.format(entry.media_type) + print('No such processing manager for {0}'.format(entry.media_type)) if args.state: processors = manager.list_all_processors_by_state(args.state) @@ -152,25 +155,25 @@ def available(args): else: processors = manager.list_eligible_processors(media_entry) - print "Available processors:" - print "=====================" - print "" + print("Available processors:") + print("=====================") + print("") if args.action_help: for processor in processors: - print processor.name - print "-" * len(processor.name) + print(processor.name) + print("-" * len(processor.name)) parser = processor.generate_parser() parser.print_help() - print "" + print("") else: for processor in processors: if processor.description: - print " - %s: %s" % (processor.name, processor.description) + print(" - %s: %s" % (processor.name, processor.description)) else: - print " - %s" % processor.name + print(" - %s" % processor.name) def run(args, media_id=None): @@ -185,12 +188,12 @@ def run(args, media_id=None): processor_class = manager.get_processor( args.reprocess_command, media_entry) except ProcessorDoesNotExist: - print 'No such processor "%s" for media with id "%s"' % ( - args.reprocess_command, media_entry.id) + print('No such processor "%s" for media with id "%s"' % ( + args.reprocess_command, media_entry.id)) return except ProcessorNotEligible: - print 'Processor "%s" exists but media "%s" is not eligible' % ( - args.reprocess_command, media_entry.id) + print('Processor "%s" exists but media "%s" is not eligible' % ( + args.reprocess_command, media_entry.id)) return reprocess_parser = processor_class.generate_parser() @@ -203,7 +206,7 @@ def run(args, media_id=None): except ProcessingManagerDoesNotExist: entry = MediaEntry.query.filter_by(id=media_id).first() - print 'No such processing manager for {0}'.format(entry.media_type) + print('No such processing manager for {0}'.format(entry.media_type)) def bulk_run(args): @@ -233,12 +236,12 @@ def thumbs(args): processor_class = manager.get_processor( 'resize', media_entry) except ProcessorDoesNotExist: - print 'No such processor "%s" for media with id "%s"' % ( - 'resize', media_entry.id) + print('No such processor "%s" for media with id "%s"' % ( + 'resize', media_entry.id)) return except ProcessorNotEligible: - print 'Processor "%s" exists but media "%s" is not eligible' % ( - 'resize', media_entry.id) + print('Processor "%s" exists but media "%s" is not eligible' % ( + 'resize', media_entry.id)) return reprocess_parser = processor_class.generate_parser() @@ -260,7 +263,7 @@ def thumbs(args): reprocess_info=reprocess_request) except ProcessingManagerDoesNotExist: - print 'No such processing manager for {0}'.format(entry.media_type) + print('No such processing manager for {0}'.format(entry.media_type)) def initial(args): @@ -276,7 +279,7 @@ def initial(args): media_entry, reprocess_action='initial') except ProcessingManagerDoesNotExist: - print 'No such processing manager for {0}'.format(entry.media_type) + print('No such processing manager for {0}'.format(entry.media_type)) def reprocess(args): diff --git a/mediagoblin/gmg_commands/serve.py b/mediagoblin/gmg_commands/serve.py new file mode 100644 index 00000000..6ded1cfc --- /dev/null +++ b/mediagoblin/gmg_commands/serve.py @@ -0,0 +1,65 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011, 2012 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +from __future__ import print_function + +from paste.deploy import loadapp, loadserver + + +class ServeCommand(object): + + def loadserver(self, server_spec, name, relative_to, **kwargs): + return loadserver(server_spec, name=name, relative_to=relative_to, + **kwargs) + + def loadapp(self, app_spec, name, relative_to, **kwargs): + return loadapp(app_spec, name=name, relative_to=relative_to, **kwargs) + + def daemonize(self): + pass + + def restart_with_reloader(self): + pass + + def restart_with_monitor(self, reloader=False): + pass + + def run(self): + print('Running...') + + +def parser_setup(subparser): + subparser.add_argument('config', metavar='CONFIG_FILE') + subparser.add_argument('command', + choices=['start', 'stop', 'restart', 'status'], + nargs='?', default='start') + subparser.add_argument('-n', '--app-name', + dest='app_name', + metavar='NAME', + help="Load the named application (default main)") + subparser.add_argument('-s', '--server', + dest='server', + metavar='SERVER_TYPE', + help="Use the named server.") + subparser.add_argument('--reload', + dest='reload', + action='store_true', + help="Use auto-restart file monitor") + + +def serve(args): + serve_cmd = ServeCommand() # TODO: pass args to it + serve_cmd.run() diff --git a/mediagoblin/gmg_commands/shell.py b/mediagoblin/gmg_commands/shell.py index 4998acd7..4d3ec241 100644 --- a/mediagoblin/gmg_commands/shell.py +++ b/mediagoblin/gmg_commands/shell.py @@ -20,6 +20,8 @@ import code from mediagoblin import mg_globals from mediagoblin.gmg_commands import util as commands_util +from mediagoblin.tools.transition import DISABLE_GLOBALS + def shell_parser_setup(subparser): subparser.add_argument( @@ -27,14 +29,23 @@ def shell_parser_setup(subparser): action="store_true") -SHELL_BANNER = """\ -GNU MediaGoblin shell! ----------------------- -Available vars: - - mgoblin_app: instantiated mediagoblin application - - mg_globals: mediagoblin.globals - - db: database instance -""" +if DISABLE_GLOBALS: + SHELL_BANNER = ( + "GNU MediaGoblin shell!\n" + "----------------------\n" + "Available vars:\n" + " - app: instantiated mediagoblin application\n" + " - db: database session\n" + " - ctx: context object\n") +else: + SHELL_BANNER = ( + "GNU MediaGoblin shell!\n" + "----------------------\n" + "Available vars:\n" + " - app: instantiated mediagoblin application\n" + " - mg_globals: mediagoblin.globals\n" + " - db: database instance\n" + " - ctx: context object\n") def py_shell(**user_namespace): """ @@ -59,18 +70,27 @@ def ipython_shell(**user_namespace): user_ns=user_namespace) return True + def shell(args): """ Setup a shell for the user either a normal Python shell or an IPython one """ - user_namespace = { - 'mg_globals': mg_globals, - 'mgoblin_app': commands_util.setup_app(args), - 'db': mg_globals.database} - - if args.ipython: - ipython_shell(**user_namespace) - else: - # Try ipython_shell first and fall back if not available - if not ipython_shell(**user_namespace): - py_shell(**user_namespace) + app = commands_util.setup_app(args) + + def run_shell(db, ctx): + user_namespace = { + 'mg_globals': mg_globals, + 'app': app, + 'db': db, + "ctx": ctx} + + if args.ipython: + ipython_shell(**user_namespace) + else: + # Try ipython_shell first and fall back if not available + if not ipython_shell(**user_namespace): + py_shell(**user_namespace) + + with app.gen_context() as ctx: + db = ctx.db + run_shell(db, ctx) diff --git a/mediagoblin/gmg_commands/users.py b/mediagoblin/gmg_commands/users.py index 71149497..d1a8b72d 100644 --- a/mediagoblin/gmg_commands/users.py +++ b/mediagoblin/gmg_commands/users.py @@ -14,6 +14,13 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +from __future__ import print_function + +import sys + +import six + +from mediagoblin.db.models import LocalUser from mediagoblin.gmg_commands import util as commands_util from mediagoblin import auth from mediagoblin import mg_globals @@ -34,24 +41,25 @@ def adduser(args): #TODO: Lets trust admins this do not validate Emails :) commands_util.setup_app(args) - args.username = unicode(commands_util.prompt_if_not_set(args.username, "Username:")) + args.username = six.text_type(commands_util.prompt_if_not_set(args.username, "Username:")) args.password = commands_util.prompt_if_not_set(args.password, "Password:",True) args.email = commands_util.prompt_if_not_set(args.email, "Email:") db = mg_globals.database users_with_username = \ - db.User.query.filter_by( - username=args.username.lower() + db.LocalUser.query.filter( + LocalUser.username==args.username.lower() ).count() if users_with_username: - print u'Sorry, a user with that name already exists.' + print(u'Sorry, a user with that name already exists.') + sys.exit(1) else: # Create the user - entry = db.User() - entry.username = args.username.lower() - entry.email = unicode(args.email) + entry = db.LocalUser() + entry.username = six.text_type(args.username.lower()) + entry.email = six.text_type(args.email) entry.pw_hash = auth.gen_password_hash(args.password) default_privileges = [ db.Privilege.query.filter( @@ -66,13 +74,14 @@ def adduser(args): entry.all_privileges = default_privileges entry.save() - print "User created (and email marked as verified)" + print(u"User created (and email marked as verified).") def makeadmin_parser_setup(subparser): subparser.add_argument( 'username', - help="Username to give admin level") + help="Username to give admin level", + type=six.text_type) def makeadmin(args): @@ -80,23 +89,26 @@ def makeadmin(args): db = mg_globals.database - user = db.User.query.filter_by( - username=unicode(args.username.lower())).one() + user = db.LocalUser.query.filter( + LocalUser.username==args.username.lower() + ).first() if user: user.all_privileges.append( db.Privilege.query.filter( db.Privilege.privilege_name==u'admin').one() ) user.save() - print 'The user is now Admin' + print(u'The user %s is now an admin.' % args.username) else: - print 'The user doesn\'t exist' + print(u'The user %s doesn\'t exist.' % args.username) + sys.exit(1) def changepw_parser_setup(subparser): subparser.add_argument( 'username', - help="Username used to login") + help="Username used to login", + type=six.text_type) subparser.add_argument( 'password', help="Your NEW supersecret word to login") @@ -107,20 +119,23 @@ def changepw(args): db = mg_globals.database - user = db.User.query.filter_by( - username=unicode(args.username.lower())).one() + user = db.LocalUser.query.filter( + LocalUser.username==args.username.lower() + ).first() if user: user.pw_hash = auth.gen_password_hash(args.password) user.save() - print 'Password successfully changed' + print(u'Password successfully changed for user %s.' % args.username) else: - print 'The user doesn\'t exist' + print(u'The user %s doesn\'t exist.' % args.username) + sys.exit(1) def deleteuser_parser_setup(subparser): subparser.add_argument( 'username', - help="Username to delete") + help="Username to delete", + type=six.text_type) def deleteuser(args): @@ -128,10 +143,12 @@ def deleteuser(args): db = mg_globals.database - user = db.User.query.filter_by( - username=unicode(args.username.lower())).one() + user = db.LocalUser.query.filter( + LocalUser.username==args.username.lower() + ).first() if user: user.delete() - print 'The user %s has been deleted' % args.username + print('The user %s has been deleted.' % args.username) else: - print 'The user %s doesn\'t exist' % args.username + print('The user %s doesn\'t exist.' % args.username) + sys.exit(1) diff --git a/mediagoblin/gmg_commands/util.py b/mediagoblin/gmg_commands/util.py index 63e39ca9..4a4a9a74 100644 --- a/mediagoblin/gmg_commands/util.py +++ b/mediagoblin/gmg_commands/util.py @@ -18,6 +18,8 @@ from mediagoblin import app import getpass +import six + def setup_app(args): """ @@ -33,7 +35,7 @@ def prompt_if_not_set(variable, text, password=False): """ if variable is None: if not password: - variable=raw_input(text + u' ') + variable = six.moves.input(text + u' ') else: variable=getpass.getpass(text + u' ') |