view src/ltpdarepo/upgrade.py @ 121:41137a8a2b4c

Extend database schema upgrade test and fix upgrade accordingly.
author Daniele Nicolodi <daniele@grinta.net>
date Tue, 11 Oct 2011 18:28:28 +0200
parents 2843a5c5e607
children 9cccafa2c753
line wrap: on
line source

import logging
from functools import partial

import MySQLdb as mysql

from .config import HOSTNAME, DATABASE, USERNAME, PASSWORD

# upgrade steps register
steps = []


def register(r0, r1, func=None):
    # register upgrade functions
    if func == None:
        return partial(register, r0, r1)
    steps.append((r0, r1, func))
    return func


def upgrade(fromrev=None):
    """run database schema upgrade steps"""

    conn = mysql.connect(host=HOSTNAME, db=DATABASE, user=USERNAME, passwd=PASSWORD, charset='utf8')
    curs = conn.cursor()
    logger = logging.getLogger(__name__)

    # current schema version
    curs.execute("""SELECT value+0 FROM options WHERE name='version'""")
    schema = curs.fetchone()[0]

    if fromrev is not None:
        logger.info("force upgrade from v%s", schema)
        schema = fromrev

    # filter applicable upgrade steps
    todo = filter(lambda x: x[0] >= schema, steps)

    # iter upgrade steps
    for fromr, tor, step in sorted(todo):

        # run upgrade step
        logger.info("upgrading database schema from v%s to v%s", fromr, tor)
        step(conn)

        # update schema version
        curs.execute("""UPDATE options SET value=%s
                        WHERE name='version'""", str(tor))

    conn.commit()
    conn.close()


@register(2.4, 2.5)
def upgrade_24_to_25(conn):
    curs = conn.cursor()

    # make mysql beheave: use sql strict mode
    curs.execute("""SET GLOBAL sql_mode='STRICT_TRANS_TABLES'""")

    # consolidate privileges: there is no need to specify grants
    # both for 'localhost' and for '%' hosts. drop privileges granted
    # for hosts different than '%'
    curs.execute("""DELETE mysql.db FROM mysql.db, users
                    WHERE User=username AND Host <> '%'""")

    # drop privileges granted explicitly on 'transactions' tables
    curs.execute("""DELETE mysql.tables_priv FROM mysql.tables_priv, users
                    WHERE User=username AND Table_name='transactions'""")

    # drop privileges granted on 'test' database
    curs.execute("""DELETE mysql.db FROM mysql.db, users
                    WHERE User=username AND Db='test'""")

    # consolidate users accounts: there is no need to specify user
    # accounts both for 'localhost' and for '127.0.0.1' hosts. drop
    # user accounts for '127.0.0.1'
    curs.execute("""DELETE mysql.user FROM mysql.user, users
                    WHERE User=username AND Host='127.0.0.1'""")

    # reload grant tables
    curs.execute("FLUSH PRIVILEGES")

    # drop unused tables
    curs.execute("DROP TABLE IF EXISTS user_access")
    curs.execute("DROP TABLE IF EXISTS user_hosts")

    # drop 'password' column from users table in administrative
    # database: authentication is done using mysql database
    curs.execute("ALTER TABLE users DROP COLUMN password")

    # for each registered database
    curs.execute("SELECT db_name FROM available_dbs")
    databases = [row[0] for row in curs.fetchall()]
    for db in databases:
        # replace 'users' table with a view
        curs.execute("""DROP TABLE IF EXISTS `%s`.users""" % db)
        curs.execute("""CREATE VIEW `%s`.users AS
                        SELECT id, username FROM `%s`.users""" % (db, DATABASE))

    conn.commit()


@register(2.5, 2.6)
def upgrade_25_to_26(conn):
    curs = conn.cursor()

    # crerate "queries" table
    curs.execute("""CREATE TABLE IF NOT EXISTS queries (
                      id INT NOT NULL AUTO_INCREMENT,
                      title TEXT NOT NULL,
                      db TEXT NOT NULL,                             
                      query TEXT NOT NULL,
                      PRIMARY KEY (id)) CHARSET=utf8""")

    conn.commit()


@register(2.6, 2.7)
def upgrade_26_to_27(conn):
    logger = logging.getLogger(__name__)
    curs = conn.cursor()

    # store current database name
    curs.execute("SELECT DATABASE()")
    current = curs.fetchone()[0]

    # for each registered database with the old version
    curs.execute("SELECT db_name FROM available_dbs WHERE version < 2")
    databases = [row[0] for row in curs.fetchall()]
    for db in databases:
        try:
            # perform the big bad database schema upgrade
            logger.info("  upgrading database %s", db)
            curs.execute("USE `%s`" % db)
            _transmogrify(conn)
        except Exception:
            import traceback
            traceback.print_exc()
        else:
            # update version information
            curs.execute("USE `%s`" % current)
            curs.execute("UPDATE available_dbs SET version=2")

    # switch back to default database
    curs.execute("USE `%s`" % current)


def _transmogrify(conn):
    """the big bad database schema upgrade"""

    logger = logging.getLogger(__name__)
    curs = conn.cursor()

    # upgrade "objmeta" table
    curs.execute("ALTER TABLE objmeta DROP COLUMN id")
    curs.execute("ALTER TABLE objmeta MODIFY obj_id INT(11) UNSIGNED NOT NULL")

    # list of object types in obtained with
    # grep classdef classes/\@* -r | grep ltpda_ | grep -v Hidden | awk '{print $2}' | sort
    curs.execute("""ALTER TABLE objmeta MODIFY obj_type ENUM('ao', 'collection',
                    'filterbank', 'matrix', 'mfir', 'miir', 'parfrac', 'pest',
                    'plist', 'pzmodel', 'rational', 'smodel', 'ssm', 'timespan') NOT NULL""")

    # upgrade "bobjs" table
    curs.execute("ALTER TABLE bobjs DROP COLUMN id")
    curs.execute("ALTER TABLE bobjs MODIFY obj_id INT(11) UNSIGNED NOT NULL")
    curs.execute("ALTER TABLE bobjs DROP INDEX object_index")

    # upgrade "xxdata" tables
    for table in ('cdata', 'fsdata', 'tsdata', 'xydata'):
        # construct new table name
        newtable = '%s%d' % (table, 2)

        # upgrade
        curs.execute("DROP TABLE IF EXISTS %s" % newtable)
        curs.execute("CREATE TABLE %s SELECT * FROM %s" % (newtable, table))
        curs.execute("ALTER TABLE %s ADD COLUMN obj_id INT(11) UNSIGNED NOT NULL FIRST" % newtable)
        curs.execute("UPDATE %s AS new, ao SET new.obj_id=ao.obj_id WHERE new.id=ao.data_id" % newtable)

        # remove columns
        curs.execute("ALTER TABLE %s DROP COLUMN id" % newtable)

        # drop old table
        curs.execute("DROP TABLE %s" % table)
        curs.execute("CREATE TABLE %s SELECT * FROM %s" % (table, newtable))
        curs.execute("DROP TABLE %s" % newtable)

    # upgrade "cdata" table
    curs.execute("ALTER TABLE cdata DROP COLUMN xunits")

    # upgrade "ao" table
    curs.execute("ALTER TABLE ao DROP COLUMN id")
    curs.execute("ALTER TABLE ao MODIFY obj_id INT(11) UNSIGNED NOT NULL")
    curs.execute("ALTER TABLE ao DROP COLUMN mfilename")
    curs.execute("ALTER TABLE ao DROP COLUMN mdlfilename")
    curs.execute("ALTER TABLE ao DROP COLUMN data_id")
    curs.execute("""ALTER TABLE ao MODIFY data_type
                    ENUM('cdata', 'tsdata', 'fsdata', 'xydata') NOT NULL""")

    # upgrade "xxir" tables
    for table in ('mfir', 'miir'):
        curs.execute("ALTER TABLE %s MODIFY obj_id INT(11) UNSIGNED NOT NULL" % table)
        curs.execute("ALTER TABLE %s DROP COLUMN id" % table)

    # use "utf8" encoding
    curs.execute("SHOW TABLES")
    for table in [d[0] for d in curs.fetchall()]:
        dcurs = conn.cursor(mysql.cursors.DictCursor)
        dcurs.execute("DESCRIBE %s" % table)
        columns = [d['Field'] for d in dcurs.fetchall() if d['Type'] == 'text']
        for col in columns:
            curs.execute("ALTER TABLE %s MODIFY %s TEXT CHARACTER SET utf8" % (table, col))

    # upgrade to innodb storage
    curs.execute("SHOW TABLES")
    tables = [t[0] for t in curs.fetchall()]
    conn.autocommit(False)
    for table in tables:
        # "users" is a view and not a table
        if table in ('users', ):
            continue
        curs = conn.cursor()
        curs.execute("ALTER TABLE %s ENGINE = InnoDB" % table)
        # commit after each table conversion
        conn.commit()

    # for all tables
    curs.execute("SHOW TABLES")
    tables = [t[0] for t in curs.fetchall()]
    for table in tables:
        # except "objs", "collections" and "transactions"
        if table in ('collections', 'objs', 'transactions', 'users'):
            continue
        # add primary key
        curs.execute("ALTER TABLE %s ADD PRIMARY KEY (obj_id)" % table)
        # add foreign keys
        logger.info("    add foreign key to table %s", table)
        curs.execute("""ALTER TABLE %s ADD FOREIGN KEY (obj_id)
                        REFERENCES objs(id) ON DELETE CASCADE""" % table)
        conn.commit()


@register(2.7, 2.8)
def upgrade_27_to_28(conn):
    curs = conn.cursor()

    # for each registered database
    curs.execute("SELECT db_name FROM available_dbs")
    databases = [row[0] for row in curs.fetchall()]
    for db in databases:
        # add 'toffset' field to 'tsdata' table
        curs.execute("""ALTER TABLE `%s`.tsdata
                        ADD COLUMN toffset BIGINT NOT NULL DEFAULT 0""" % db)

    conn.commit()