diff options
Diffstat (limited to 'mediagoblin/db')
| -rw-r--r-- | mediagoblin/db/migration_tools.py | 58 | ||||
| -rw-r--r-- | mediagoblin/db/migrations.py | 11 | ||||
| -rw-r--r-- | mediagoblin/db/migrations/README | 57 | ||||
| -rw-r--r-- | mediagoblin/db/migrations/env.py | 71 | ||||
| -rw-r--r-- | mediagoblin/db/migrations/script.py.mako | 22 | ||||
| -rw-r--r-- | mediagoblin/db/migrations/versions/.gitkeep | 0 | ||||
| -rw-r--r-- | mediagoblin/db/models.py | 42 | ||||
| -rw-r--r-- | mediagoblin/db/open.py | 4 |
8 files changed, 245 insertions, 20 deletions
diff --git a/mediagoblin/db/migration_tools.py b/mediagoblin/db/migration_tools.py index e39070c3..fae98643 100644 --- a/mediagoblin/db/migration_tools.py +++ b/mediagoblin/db/migration_tools.py @@ -14,14 +14,68 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. +from __future__ import unicode_literals + +import logging +import os + +from alembic import command +from alembic.config import Config +from alembic.migration import MigrationContext + +from mediagoblin.db.base import Base from mediagoblin.tools.common import simple_printer from sqlalchemy import Table from sqlalchemy.sql import select +log = logging.getLogger(__name__) + + class TableAlreadyExists(Exception): pass +class AlembicMigrationManager(object): + + def __init__(self, session): + root_dir = os.path.abspath(os.path.dirname(os.path.dirname( + os.path.dirname(__file__)))) + alembic_cfg_path = os.path.join(root_dir, 'alembic.ini') + self.alembic_cfg = Config(alembic_cfg_path) + self.session = session + + def get_current_revision(self): + context = MigrationContext.configure(self.session.bind) + return context.get_current_revision() + + def upgrade(self, version): + return command.upgrade(self.alembic_cfg, version or 'head') + + def downgrade(self, version): + if isinstance(version, int) or version is None or version.isdigit(): + version = 'base' + return command.downgrade(self.alembic_cfg, version) + + def stamp(self, revision): + return command.stamp(self.alembic_cfg, revision=revision) + + def init_tables(self): + Base.metadata.create_all(self.session.bind) + # load the Alembic configuration and generate the + # version table, "stamping" it with the most recent rev: + # XXX: we need to find a better way to detect current installations + # using sqlalchemy-migrate because we don't have to create all table + # for them + command.stamp(self.alembic_cfg, 'head') + + def init_or_migrate(self, version=None): + # XXX: we need to call this method when we ditch + # sqlalchemy-migrate entirely + # if self.get_current_revision() is None: + # self.init_tables() + self.upgrade(version) + + class MigrationManager(object): """ Migration handling tool. @@ -39,7 +93,7 @@ class MigrationManager(object): - migration_registry: where we should find all migrations to run """ - self.name = unicode(name) + self.name = name self.models = models self.foundations = foundations self.session = session @@ -230,7 +284,7 @@ class MigrationManager(object): for migration_number, migration_func in migrations_to_run: self.printer( u' + Running migration %s, "%s"... ' % ( - migration_number, migration_func.func_name)) + migration_number, migration_func.__name__)) migration_func(self.session) self.set_current_migration(migration_number) self.printer('done.\n') diff --git a/mediagoblin/db/migrations.py b/mediagoblin/db/migrations.py index c1a73795..31b8333e 100644 --- a/mediagoblin/db/migrations.py +++ b/mediagoblin/db/migrations.py @@ -17,13 +17,18 @@ import datetime import uuid +import six + +if six.PY2: + import migrate + from sqlalchemy import (MetaData, Table, Column, Boolean, SmallInteger, Integer, Unicode, UnicodeText, DateTime, ForeignKey, Date, Index) from sqlalchemy.exc import ProgrammingError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.sql import and_ -from migrate.changeset.constraint import UniqueConstraint +from sqlalchemy.schema import UniqueConstraint from mediagoblin.db.extratypes import JSONEncoded, MutationDict from mediagoblin.db.migration_tools import ( @@ -249,7 +254,7 @@ def mediaentry_new_slug_era(db): for row in db.execute(media_table.select()): # no slug, try setting to an id if not row.slug: - append_garbage_till_unique(row, unicode(row.id)) + append_garbage_till_unique(row, six.text_type(row.id)) # has "=" or ":" in it... we're getting rid of those elif u"=" in row.slug or u":" in row.slug: append_garbage_till_unique( @@ -278,7 +283,7 @@ def unique_collections_slug(db): existing_slugs[row.creator].append(row.slug) for row_id in slugs_to_change: - new_slug = unicode(uuid.uuid4()) + new_slug = six.text_type(uuid.uuid4()) db.execute(collection_table.update(). where(collection_table.c.id == row_id). values(slug=new_slug)) diff --git a/mediagoblin/db/migrations/README b/mediagoblin/db/migrations/README new file mode 100644 index 00000000..93d85eff --- /dev/null +++ b/mediagoblin/db/migrations/README @@ -0,0 +1,57 @@ +Migration Guide +--------------- + +Alembic comes with a CLI called ``alembic``. + +Create a Migration +^^^^^^^^^^^^^^^^^^ + +Lets create our first migration:: + + $ alembic revision -m "add favourite_band field" + Generating + /your/gmg/path/mediagoblin/db/migrations/versions/1e3793de36a_add_favourite_band_field.py ... done + +By default, migration files have two methods: ``upgrade`` and ``downgrade``. +Alembic will invoke these methods to apply the migrations to your current +database. + +Now, we need to edit our newly created migration file +``1e3793de36a_add_favourite_band_field.py`` to add a new column ``favourite_band`` +to ``core__users`` table:: + + def upgrade(): + op.add_column('core__users', sa.Column('favourite_band', sa.Unicode(100))) + + + def downgrade(): + op.drop_column('core__users', 'favourite_band') + +.. note:: + + Alembic can also generate `automatic migrations <http://alembic.readthedocs.org/en/latest/tutorial.html#auto-generating-migrations>`__. + +Then we can run ``gmg dbupdate`` to apply the new migration:: + + $ gmg dbupdate + INFO [alembic.migration] Context impl SQLiteImpl. + INFO [alembic.migration] Will assume non-transactional DDL. + INFO [alembic.migration] Running upgrade None -> 1e3793de36a, add favourite band field + +If you want to revert that migration, simply run:: + + $ alembic downgrade -1 + +.. warning:: + + Currently, Alembic cannot do ``DROP COLUMN``, ``ALTER COLUMN`` etc. + operations in SQLite. Please see https://bitbucket.org/zzzeek/alembic/issue/21/column-renames-not-supported-on-sqlite + for detailed information. + +Glossary +^^^^^^^^ + +* ``alembic.ini``: The Alembic configuration file. The ``alembic`` CLI will + look that file everytime it invaked. +* ``mediagoblin/db/migrations/versions/``: Alembic will add new migration files + to this directory. diff --git a/mediagoblin/db/migrations/env.py b/mediagoblin/db/migrations/env.py new file mode 100644 index 00000000..712b6164 --- /dev/null +++ b/mediagoblin/db/migrations/env.py @@ -0,0 +1,71 @@ +from __future__ import with_statement +from alembic import context +from sqlalchemy import engine_from_config, pool +from logging.config import fileConfig + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = None + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure(url=url, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + engine = engine_from_config( + config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool) + + connection = engine.connect() + context.configure( + connection=connection, + target_metadata=target_metadata + ) + + try: + with context.begin_transaction(): + context.run_migrations() + finally: + connection.close() + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() + diff --git a/mediagoblin/db/migrations/script.py.mako b/mediagoblin/db/migrations/script.py.mako new file mode 100644 index 00000000..95702017 --- /dev/null +++ b/mediagoblin/db/migrations/script.py.mako @@ -0,0 +1,22 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/mediagoblin/db/migrations/versions/.gitkeep b/mediagoblin/db/migrations/versions/.gitkeep new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/mediagoblin/db/migrations/versions/.gitkeep diff --git a/mediagoblin/db/models.py b/mediagoblin/db/models.py index ffd7b4f1..0069c85a 100644 --- a/mediagoblin/db/models.py +++ b/mediagoblin/db/models.py @@ -18,9 +18,10 @@ TODO: indexes on foreignkeys, where useful. """ +from __future__ import print_function + import logging import datetime -import base64 from sqlalchemy import Column, Integer, Unicode, UnicodeText, DateTime, \ Boolean, ForeignKey, UniqueConstraint, PrimaryKeyConstraint, \ @@ -40,17 +41,11 @@ from mediagoblin.db.mixin import UserMixin, MediaEntryMixin, \ from mediagoblin.tools.files import delete_media_files from mediagoblin.tools.common import import_component -# It's actually kind of annoying how sqlalchemy-migrate does this, if -# I understand it right, but whatever. Anyway, don't remove this :P -# -# We could do migration calls more manually instead of relying on -# this import-based meddling... -from migrate import changeset +import six _log = logging.getLogger(__name__) - class User(Base, UserMixin): """ TODO: We should consider moving some rarely used fields @@ -225,6 +220,8 @@ class RequestToken(Base): created = Column(DateTime, nullable=False, default=datetime.datetime.now) updated = Column(DateTime, nullable=False, default=datetime.datetime.now) + get_client = relationship(Client) + class AccessToken(Base): """ Model for representing the access tokens @@ -238,6 +235,8 @@ class AccessToken(Base): created = Column(DateTime, nullable=False, default=datetime.datetime.now) updated = Column(DateTime, nullable=False, default=datetime.datetime.now) + get_requesttoken = relationship(RequestToken) + class NonceTimestamp(Base): """ @@ -350,7 +349,7 @@ class MediaEntry(Base, MediaEntryMixin): return the value of the key. """ media_file = MediaFile.query.filter_by(media_entry=self.id, - name=unicode(file_key)).first() + name=six.text_type(file_key)).first() if media_file: if metadata_key: @@ -363,11 +362,11 @@ class MediaEntry(Base, MediaEntryMixin): Update the file_metadata of a MediaFile. """ media_file = MediaFile.query.filter_by(media_entry=self.id, - name=unicode(file_key)).first() + name=six.text_type(file_key)).first() file_metadata = media_file.file_metadata or {} - for key, value in kwargs.iteritems(): + for key, value in six.iteritems(kwargs): file_metadata[key] = value media_file.file_metadata = file_metadata @@ -392,7 +391,7 @@ class MediaEntry(Base, MediaEntryMixin): media_data.get_media_entry = self else: # Update old media data - for field, value in kwargs.iteritems(): + for field, value in six.iteritems(kwargs): setattr(media_data, field, value) @memoized_property @@ -421,7 +420,7 @@ class MediaEntry(Base, MediaEntryMixin): # Delete all related files/attachments try: delete_media_files(self) - except OSError, error: + except OSError as error: # Returns list of files we failed to delete _log.error('No such files from the user "{1}" to delete: ' '{0}'.format(str(error), self.get_uploader)) @@ -733,6 +732,14 @@ class Collection(Base, CollectionMixin): return CollectionItem.query.filter_by( collection=self.id).order_by(order_col) + def __repr__(self): + safe_title = self.title.encode('ascii', 'replace') + return '<{classname} #{id}: {title} by {creator}>'.format( + id=self.id, + classname=self.__class__.__name__, + creator=self.creator, + title=safe_title) + class CollectionItem(Base, CollectionItemMixin): __tablename__ = "core__collection_items" @@ -762,6 +769,13 @@ class CollectionItem(Base, CollectionItemMixin): """A dict like view on this object""" return DictReadAttrProxy(self) + def __repr__(self): + return '<{classname} #{id}: Entry {entry} in {collection}>'.format( + id=self.id, + classname=self.__class__.__name__, + collection=self.collection, + entry=self.media_entry) + class ProcessingMetaData(Base): __tablename__ = 'core__processing_metadata' @@ -1287,7 +1301,7 @@ def show_table_init(engine_uri): if __name__ == '__main__': from sys import argv - print repr(argv) + print(repr(argv)) if len(argv) == 2: uri = argv[1] else: diff --git a/mediagoblin/db/open.py b/mediagoblin/db/open.py index 4ff0945f..34f0bffa 100644 --- a/mediagoblin/db/open.py +++ b/mediagoblin/db/open.py @@ -18,6 +18,8 @@ from sqlalchemy import create_engine, event import logging +import six + from mediagoblin.db.base import Base, Session from mediagoblin import mg_globals @@ -28,7 +30,7 @@ class DatabaseMaster(object): def __init__(self, engine): self.engine = engine - for k, v in Base._decl_class_registry.iteritems(): + for k, v in six.iteritems(Base._decl_class_registry): setattr(self, k, v) def commit(self): |