diff options
Diffstat (limited to 'mediagoblin/db/sql')
| -rw-r--r-- | mediagoblin/db/sql/convert.py | 11 | ||||
| -rw-r--r-- | mediagoblin/db/sql/extratypes.py | 28 | ||||
| -rw-r--r-- | mediagoblin/db/sql/migrations.py | 17 | ||||
| -rw-r--r-- | mediagoblin/db/sql/models.py | 46 | ||||
| -rw-r--r-- | mediagoblin/db/sql/util.py | 271 |
5 files changed, 356 insertions, 17 deletions
diff --git a/mediagoblin/db/sql/convert.py b/mediagoblin/db/sql/convert.py index f6575be9..36d6fc7f 100644 --- a/mediagoblin/db/sql/convert.py +++ b/mediagoblin/db/sql/convert.py @@ -56,7 +56,7 @@ def convert_users(mk_db): copy_attrs(entry, new_entry, ('username', 'email', 'created', 'pw_hash', 'email_verified', 'status', 'verification_key', 'is_admin', 'url', - 'bio', 'bio_html', + 'bio', 'fp_verification_key', 'fp_token_expire',)) # new_entry.fp_verification_expire = entry.fp_token_expire @@ -77,9 +77,9 @@ def convert_media_entries(mk_db): new_entry = MediaEntry() copy_attrs(entry, new_entry, ('title', 'slug', 'created', - 'description', 'description_html', + 'description', 'media_type', 'state', 'license', - 'fail_error', + 'fail_error', 'fail_metadata', 'queued_task_id',)) copy_reference_attr(entry, new_entry, "uploader") @@ -133,7 +133,7 @@ def convert_media_comments(mk_db): new_entry = MediaComment() copy_attrs(entry, new_entry, ('created', - 'content', 'content_html',)) + 'content',)) copy_reference_attr(entry, new_entry, "media_entry") copy_reference_attr(entry, new_entry, "author") @@ -148,8 +148,7 @@ def convert_media_comments(mk_db): def main(): global_config, app_config = setup_global_and_app_config("mediagoblin.ini") - sql_conn, sql_db = sql_connect({'sql_engine': 'sqlite:///mediagoblin.db'}) - + sql_conn, sql_db = sql_connect(app_config) mk_conn, mk_db = mongo_connect(app_config) Base.metadata.create_all(sql_db.engine) diff --git a/mediagoblin/db/sql/extratypes.py b/mediagoblin/db/sql/extratypes.py index 3a594728..8e078f14 100644 --- a/mediagoblin/db/sql/extratypes.py +++ b/mediagoblin/db/sql/extratypes.py @@ -15,7 +15,8 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. -from sqlalchemy.types import TypeDecorator, Unicode +from sqlalchemy.types import TypeDecorator, Unicode, VARCHAR +import json class PathTupleWithSlashes(TypeDecorator): @@ -35,3 +36,28 @@ class PathTupleWithSlashes(TypeDecorator): if value is not None: value = tuple(value.split('/')) return value + + +# The following class and only this one class is in very +# large parts based on example code from sqlalchemy. +# +# The original copyright notice and license follows: +# Copyright (C) 2005-2011 the SQLAlchemy authors and contributors <see AUTHORS file> +# +# This module is part of SQLAlchemy and is released under +# the MIT License: http://www.opensource.org/licenses/mit-license.php +# +class JSONEncoded(TypeDecorator): + "Represents an immutable structure as a json-encoded string." + + impl = VARCHAR + + def process_bind_param(self, value, dialect): + if value is not None: + value = json.dumps(value) + return value + + def process_result_value(self, value, dialect): + if value is not None: + value = json.loads(value) + return value diff --git a/mediagoblin/db/sql/migrations.py b/mediagoblin/db/sql/migrations.py new file mode 100644 index 00000000..98d0d0aa --- /dev/null +++ b/mediagoblin/db/sql/migrations.py @@ -0,0 +1,17 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +MIGRATIONS = {} diff --git a/mediagoblin/db/sql/models.py b/mediagoblin/db/sql/models.py index 9d06f79c..699dbf33 100644 --- a/mediagoblin/db/sql/models.py +++ b/mediagoblin/db/sql/models.py @@ -29,9 +29,16 @@ from sqlalchemy.orm.collections import attribute_mapped_collection from sqlalchemy.sql.expression import desc from sqlalchemy.ext.associationproxy import association_proxy -from mediagoblin.db.sql.extratypes import PathTupleWithSlashes +from mediagoblin.db.sql.extratypes import PathTupleWithSlashes, JSONEncoded from mediagoblin.db.sql.base import Base, DictReadAttrProxy -from mediagoblin.db.mixin import UserMixin, MediaEntryMixin +from mediagoblin.db.mixin import UserMixin, MediaEntryMixin, MediaCommentMixin + +# It's actually kind of annoying how sqlalchemy-migrate does this, if +# I understand it right, but whatever. Anyway, don't remove this :P +# +# We could do migration calls more manually instead of relying on +# this import-based meddling... +from migrate import changeset class SimpleFieldAlias(object): @@ -64,7 +71,6 @@ class User(Base, UserMixin): is_admin = Column(Boolean, default=False, nullable=False) url = Column(Unicode) bio = Column(UnicodeText) # ?? - bio_html = Column(UnicodeText) # ?? fp_verification_key = Column(Unicode) fp_token_expire = Column(DateTime) @@ -86,14 +92,13 @@ class MediaEntry(Base, MediaEntryMixin): slug = Column(Unicode) created = Column(DateTime, nullable=False, default=datetime.datetime.now) description = Column(UnicodeText) # ?? - description_html = Column(UnicodeText) # ?? media_type = Column(Unicode, nullable=False) state = Column(Unicode, default=u'unprocessed', nullable=False) # or use sqlalchemy.types.Enum? license = Column(Unicode) fail_error = Column(Unicode) - fail_metadata = Column(UnicodeText) + fail_metadata = Column(JSONEncoded) queued_media_file = Column(PathTupleWithSlashes) @@ -209,10 +214,12 @@ class MediaTag(Base): creator=Tag.find_or_new ) - def __init__(self, name, slug): + def __init__(self, name=None, slug=None): Base.__init__(self) - self.name = name - self.tag_helper = Tag.find_or_new(slug) + if name is not None: + self.name = name + if slug is not None: + self.tag_helper = Tag.find_or_new(slug) @property def dict_view(self): @@ -220,7 +227,7 @@ class MediaTag(Base): return DictReadAttrProxy(self) -class MediaComment(Base): +class MediaComment(Base, MediaCommentMixin): __tablename__ = "media_comments" id = Column(Integer, primary_key=True) @@ -229,13 +236,32 @@ class MediaComment(Base): author = Column(Integer, ForeignKey('users.id'), nullable=False) created = Column(DateTime, nullable=False, default=datetime.datetime.now) content = Column(UnicodeText, nullable=False) - content_html = Column(UnicodeText) get_author = relationship(User) _id = SimpleFieldAlias("id") +MODELS = [ + User, MediaEntry, Tag, MediaTag, MediaComment] + + +###################################################### +# Special, migrations-tracking table +# +# Not listed in MODELS because this is special and not +# really migrated, but used for migrations (for now) +###################################################### + +class MigrationData(Base): + __tablename__ = "migrations" + + name = Column(Unicode, primary_key=True) + version = Column(Integer, nullable=False, default=0) + +###################################################### + + def show_table_init(engine_uri): if engine_uri is None: engine_uri = 'sqlite:///:memory:' diff --git a/mediagoblin/db/sql/util.py b/mediagoblin/db/sql/util.py new file mode 100644 index 00000000..08602414 --- /dev/null +++ b/mediagoblin/db/sql/util.py @@ -0,0 +1,271 @@ +# GNU MediaGoblin -- federated, autonomous media hosting +# Copyright (C) 2011 MediaGoblin contributors. See AUTHORS. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + + +import sys + +def _simple_printer(string): + """ + Prints a string, but without an auto \n at the end. + """ + sys.stdout.write(string) + sys.stdout.flush() + + +class MigrationManager(object): + """ + Migration handling tool. + + Takes information about a database, lets you update the database + to the latest migrations, etc. + """ + + def __init__(self, name, models, migration_registry, session, + printer=_simple_printer): + """ + Args: + - name: identifier of this section of the database + - session: session we're going to migrate + - migration_registry: where we should find all migrations to + run + """ + self.name = name + self.models = models + self.session = session + self.migration_registry = migration_registry + self._sorted_migrations = None + self.printer = printer + + # For convenience + from mediagoblin.db.sql.models import MigrationData + + self.migration_model = MigrationData + self.migration_table = MigrationData.__table__ + + @property + def sorted_migrations(self): + """ + Sort migrations if necessary and store in self._sorted_migrations + """ + if not self._sorted_migrations: + self._sorted_migrations = sorted( + self.migration_registry.items(), + # sort on the key... the migration number + key=lambda migration_tuple: migration_tuple[0]) + + return self._sorted_migrations + + @property + def migration_data(self): + """ + Get the migration row associated with this object, if any. + """ + return self.session.query( + self.migration_model).filter_by(name=self.name).first() + + @property + def latest_migration(self): + """ + Return a migration number for the latest migration, or 0 if + there are no migrations. + """ + if self.sorted_migrations: + return self.sorted_migrations[-1][0] + else: + # If no migrations have been set, we start at 0. + return 0 + + @property + def database_current_migration(self): + """ + Return the current migration in the database. + """ + # If the table doesn't even exist, return None. + if not self.migration_table.exists(self.session.bind): + return None + + # Also return None if self.migration_data is None. + if self.migration_data is None: + return None + + return self.migration_data.version + + def set_current_migration(self, migration_number=None): + """ + Set the migration in the database to migration_number + (or, the latest available) + """ + self.migration_data.version = migration_number or self.latest_migration + self.session.commit() + + def migrations_to_run(self): + """ + Get a list of migrations to run still, if any. + + Note that this will fail if there's no migration record for + this class! + """ + assert self.database_current_migration is not None + + db_current_migration = self.database_current_migration + + return [ + (migration_number, migration_func) + for migration_number, migration_func in self.sorted_migrations + if migration_number > db_current_migration] + + + def init_tables(self): + """ + Create all tables relative to this package + """ + # sanity check before we proceed, none of these should be created + for model in self.models: + # Maybe in the future just print out a "Yikes!" or something? + assert not model.__table__.exists(self.session.bind) + + self.migration_model.metadata.create_all( + self.session.bind, + tables=[model.__table__ for model in self.models]) + + def create_new_migration_record(self): + """ + Create a new migration record for this migration set + """ + migration_record = self.migration_model( + name=self.name, + version=self.latest_migration) + self.session.add(migration_record) + self.session.commit() + + def dry_run(self): + """ + Print out a dry run of what we would have upgraded. + """ + if self.database_current_migration is None: + self.printer( + u'~> Woulda initialized: %s\n' % self.name_for_printing()) + return u'inited' + + migrations_to_run = self.migrations_to_run() + if migrations_to_run: + self.printer( + u'~> Woulda updated %s:\n' % self.name_for_printing()) + + for migration_number, migration_func in migrations_to_run(): + self.printer( + u' + Would update %s, "%s"\n' % ( + migration_number, migration_func.func_name)) + + return u'migrated' + + def name_for_printing(self): + if self.name == u'__main__': + return u"main mediagoblin tables" + else: + # TODO: Use the friendlier media manager "human readable" name + return u'media type "%s"' % self.name + + def init_or_migrate(self): + """ + Initialize the database or migrate if appropriate. + + Returns information about whether or not we initialized + ('inited'), migrated ('migrated'), or did nothing (None) + """ + assure_migrations_table_setup(self.session) + + # Find out what migration number, if any, this database data is at, + # and what the latest is. + migration_number = self.database_current_migration + + # Is this our first time? Is there even a table entry for + # this identifier? + # If so: + # - create all tables + # - create record in migrations registry + # - print / inform the user + # - return 'inited' + if migration_number is None: + self.printer(u"-> Initializing %s... " % self.name_for_printing()) + + self.init_tables() + # auto-set at latest migration number + self.create_new_migration_record() + + self.printer(u"done.\n") + self.set_current_migration() + return u'inited' + + # Run migrations, if appropriate. + migrations_to_run = self.migrations_to_run() + if migrations_to_run: + self.printer( + u'-> Updating %s:\n' % self.name_for_printing()) + for migration_number, migration_func in migrations_to_run: + self.printer( + u' + Running migration %s, "%s"... ' % ( + migration_number, migration_func.func_name)) + migration_func(self.session) + self.printer('done.\n') + + self.set_current_migration() + return u'migrated' + + # Otherwise return None. Well it would do this anyway, but + # for clarity... ;) + return None + + +class RegisterMigration(object): + """ + Tool for registering migrations + + Call like: + + @RegisterMigration(33) + def update_dwarves(database): + [...] + + This will register your migration with the default migration + registry. Alternately, to specify a very specific + migration_registry, you can pass in that as the second argument. + + Note, the number of your migration should NEVER be 0 or less than + 0. 0 is the default "no migrations" state! + """ + def __init__(self, migration_number, migration_registry): + assert migration_number > 0, "Migration number must be > 0!" + assert migration_number not in migration_registry, \ + "Duplicate migration numbers detected! That's not allowed!" + + self.migration_number = migration_number + self.migration_registry = migration_registry + + def __call__(self, migration): + self.migration_registry[self.migration_number] = migration + return migration + + +def assure_migrations_table_setup(db): + """ + Make sure the migrations table is set up in the database. + """ + from mediagoblin.db.sql.models import MigrationData + + if not MigrationData.__table__.exists(db.bind): + MigrationData.metadata.create_all( + db.bind, tables=[MigrationData.__table__]) |