From 61de77fc7eae1f844944bd692d13bf27a1fda6fe Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Sep 02 2015 15:17:52 +0000 Subject: [PATCH 1/3] Add SQL primary key and indexes Signed-off-by: Patrick Uiterwijk Reviewed-by: Rob Crittenden --- diff --git a/ipsilon/providers/openid/store.py b/ipsilon/providers/openid/store.py index e759bca..9b2bc4f 100644 --- a/ipsilon/providers/openid/store.py +++ b/ipsilon/providers/openid/store.py @@ -1,6 +1,6 @@ # Copyright (C) 2014 Ipsilon project Contributors, for license see COPYING -from ipsilon.util.data import Store, UNIQUE_DATA_COLUMNS +from ipsilon.util.data import Store, UNIQUE_DATA_TABLE from openid import oidutil from openid.association import Association @@ -79,7 +79,7 @@ class OpenIDStore(Store, OpenIDStoreInterface): self.del_unique_data('association', iden) def _initialize_schema(self): - q = self._query(self._db, 'association', UNIQUE_DATA_COLUMNS, + q = self._query(self._db, 'association', UNIQUE_DATA_TABLE, trans=False) q.create() diff --git a/ipsilon/util/data.py b/ipsilon/util/data.py index c0fe4ab..e92aae4 100644 --- a/ipsilon/util/data.py +++ b/ipsilon/util/data.py @@ -6,6 +6,7 @@ from ipsilon.util.log import Log from sqlalchemy import create_engine from sqlalchemy import MetaData, Table, Column, Text from sqlalchemy.pool import QueuePool, SingletonThreadPool +from sqlalchemy.schema import PrimaryKeyConstraint, Index from sqlalchemy.sql import select, and_ import ConfigParser import os @@ -13,9 +14,15 @@ import uuid import logging -CURRENT_SCHEMA_VERSION = 1 -OPTIONS_COLUMNS = ['name', 'option', 'value'] -UNIQUE_DATA_COLUMNS = ['uuid', 'name', 'value'] +CURRENT_SCHEMA_VERSION = 2 +OPTIONS_TABLE = {'columns': ['name', 'option', 'value'], + 'primary_key': ('name', 'option'), + 'indexes': [('name',)] + } +UNIQUE_DATA_TABLE = {'columns': ['uuid', 'name', 'value'], + 'primary_key': ('uuid', 'name'), + 'indexes': [('uuid',)] + } class DatabaseError(Exception): @@ -74,16 +81,27 @@ class SqlStore(Log): class SqlQuery(Log): - def __init__(self, db_obj, table, columns, trans=True): + def __init__(self, db_obj, table, table_def, trans=True): self._db = db_obj self._con = self._db.connection() self._trans = self._con.begin() if trans else None - self._table = self._get_table(table, columns) - - def _get_table(self, name, columns): - table = Table(name, MetaData(self._db.engine())) - for c in columns: - table.append_column(Column(c, Text())) + self._table = self._get_table(table, table_def) + + def _get_table(self, name, table_def): + if isinstance(table_def, list): + table_def = {'columns': table_def, + 'indexes': [], + 'primary_key': None} + table_creation = [] + for col_name in table_def['columns']: + table_creation.append(Column(col_name, Text())) + if table_def['primary_key']: + table_creation.append(PrimaryKeyConstraint( + *table_def['primary_key'])) + for index in table_def['indexes']: + idx_name = 'idx_%s_%s' % (name, '_'.join(index)) + table_creation.append(Index(idx_name, *index)) + table = Table(name, MetaData(self._db.engine()), *table_creation) return table def _where(self, kvfilter): @@ -159,7 +177,12 @@ class FileStore(Log): class FileQuery(Log): - def __init__(self, fstore, table, columns, trans=True): + def __init__(self, fstore, table, table_def, trans=True): + # We don't need indexes in a FileQuery, so drop that info + if isinstance(table_def, dict): + columns = table_def['columns'] + else: + columns = table_def self._fstore = fstore self._config = fstore.get_config() self._section = table @@ -278,7 +301,7 @@ class Store(Log): # We are storing multiple versions: one per class # That way, we can support plugins with differing schema versions from # the main codebase, and even in the same database. - q = self._query(self._db, 'dbinfo', OPTIONS_COLUMNS, trans=False) + q = self._query(self._db, 'dbinfo', OPTIONS_TABLE, trans=False) q.create() cls_name = self.__class__.__name__ current_version = self.load_options('dbinfo').get('%s_schema' @@ -387,7 +410,7 @@ class Store(Log): kvfilter = dict() if name: kvfilter['name'] = name - options = self._load_data(table, OPTIONS_COLUMNS, kvfilter) + options = self._load_data(table, OPTIONS_TABLE, kvfilter) if name and name in options: return options[name] return options @@ -396,7 +419,7 @@ class Store(Log): curvals = dict() q = None try: - q = self._query(self._db, table, OPTIONS_COLUMNS) + q = self._query(self._db, table, OPTIONS_TABLE) rows = q.select({'name': name}, ['option', 'value']) for row in rows: curvals[row[0]] = row[1] @@ -419,7 +442,7 @@ class Store(Log): kvfilter = {'name': name} q = None try: - q = self._query(self._db, table, OPTIONS_COLUMNS) + q = self._query(self._db, table, OPTIONS_TABLE) if options is None: q.delete(kvfilter) else: @@ -437,7 +460,7 @@ class Store(Log): newid = str(uuid.uuid4()) q = None try: - q = self._query(self._db, table, UNIQUE_DATA_COLUMNS) + q = self._query(self._db, table, UNIQUE_DATA_TABLE) for name in data: q.insert((newid, name, data[name])) q.commit() @@ -456,12 +479,12 @@ class Store(Log): kvfilter['name'] = name if value: kvfilter['value'] = value - return self._load_data(table, UNIQUE_DATA_COLUMNS, kvfilter) + return self._load_data(table, UNIQUE_DATA_TABLE, kvfilter) def save_unique_data(self, table, data): q = None try: - q = self._query(self._db, table, UNIQUE_DATA_COLUMNS) + q = self._query(self._db, table, UNIQUE_DATA_TABLE) for uid in data: curvals = dict() rows = q.select({'uuid': uid}, ['name', 'value']) @@ -490,7 +513,7 @@ class Store(Log): def del_unique_data(self, table, uuidval): kvfilter = {'uuid': uuidval} try: - q = self._query(self._db, table, UNIQUE_DATA_COLUMNS, trans=False) + q = self._query(self._db, table, UNIQUE_DATA_TABLE, trans=False) q.delete(kvfilter) except Exception, e: # pylint: disable=broad-except self.error("Failed to delete data from %s: [%s]" % (table, e)) @@ -498,7 +521,7 @@ class Store(Log): def _reset_data(self, table): q = None try: - q = self._query(self._db, table, UNIQUE_DATA_COLUMNS) + q = self._query(self._db, table, UNIQUE_DATA_TABLE) q.drop() q.create() q.commit() @@ -536,7 +559,7 @@ class AdminStore(Store): 'info_config', 'login_config', 'provider_config']: - q = self._query(self._db, table, OPTIONS_COLUMNS, trans=False) + q = self._query(self._db, table, OPTIONS_TABLE, trans=False) q.create() def _upgrade_schema(self, old_version): @@ -561,7 +584,7 @@ class UserStore(Store): return self.load_options(plugin+"_data", user) def _initialize_schema(self): - q = self._query(self._db, 'users', OPTIONS_COLUMNS, trans=False) + q = self._query(self._db, 'users', OPTIONS_TABLE, trans=False) q.create() def _upgrade_schema(self, old_version): @@ -574,7 +597,7 @@ class TranStore(Store): super(TranStore, self).__init__('transactions.db') def _initialize_schema(self): - q = self._query(self._db, 'transactions', UNIQUE_DATA_COLUMNS, + q = self._query(self._db, 'transactions', UNIQUE_DATA_TABLE, trans=False) q.create() @@ -588,7 +611,7 @@ class SAML2SessionStore(Store): super(SAML2SessionStore, self).__init__(database_url=database_url) self.table = 'saml2_sessions' # pylint: disable=protected-access - table = SqlQuery(self._db, self.table, UNIQUE_DATA_COLUMNS)._table + table = SqlQuery(self._db, self.table, UNIQUE_DATA_TABLE)._table table.create(checkfirst=True) def _get_unique_id_from_column(self, name, value): @@ -609,7 +632,7 @@ class SAML2SessionStore(Store): def remove_expired_sessions(self): # pylint: disable=protected-access - table = SqlQuery(self._db, self.table, UNIQUE_DATA_COLUMNS)._table + table = SqlQuery(self._db, self.table, UNIQUE_DATA_TABLE)._table sel = select([table.columns.uuid]). \ where(and_(table.c.name == 'expiration_time', table.c.value <= datetime.datetime.now())) @@ -665,7 +688,7 @@ class SAML2SessionStore(Store): self._reset_data(self.table) def _initialize_schema(self): - q = self._query(self._db, self.table, UNIQUE_DATA_COLUMNS, + q = self._query(self._db, self.table, UNIQUE_DATA_TABLE, trans=False) q.create() diff --git a/ipsilon/util/sessions.py b/ipsilon/util/sessions.py index b870319..ef059d1 100644 --- a/ipsilon/util/sessions.py +++ b/ipsilon/util/sessions.py @@ -10,12 +10,15 @@ except ImportError: import pickle -SESSION_COLUMNS = ['id', 'data', 'expiration_time'] +SESSION_TABLE = {'columns': ['id', 'data', 'expiration_time'], + 'primary_key': ('id', ), + 'indexes': [('expiration_time',)] + } class SessionStore(Store): def _initialize_schema(self): - q = self._query(self._db, 'sessions', SESSION_COLUMNS, + q = self._query(self._db, 'sessions', SESSION_TABLE, trans=False) q.create() @@ -44,12 +47,12 @@ class SqlSession(Session): cls._db = cls._store._db def _exists(self): - q = SqlQuery(self._db, 'sessions', SESSION_COLUMNS) + q = SqlQuery(self._db, 'sessions', SESSION_TABLE) result = q.select({'id': self.id}) return True if result.fetchone() else False def _load(self): - q = SqlQuery(self._db, 'sessions', SESSION_COLUMNS) + q = SqlQuery(self._db, 'sessions', SESSION_TABLE) result = q.select({'id': self.id}) r = result.fetchone() if r: @@ -59,7 +62,7 @@ class SqlSession(Session): def _save(self, expiration_time): q = None try: - q = SqlQuery(self._db, 'sessions', SESSION_COLUMNS, trans=True) + q = SqlQuery(self._db, 'sessions', SESSION_TABLE, trans=True) q.delete({'id': self.id}) data = pickle.dumps((self._data, expiration_time), self._proto) q.insert((self.id, base64.b64encode(data), expiration_time)) @@ -70,7 +73,7 @@ class SqlSession(Session): raise def _delete(self): - q = SqlQuery(self._db, 'sessions', SESSION_COLUMNS) + q = SqlQuery(self._db, 'sessions', SESSION_TABLE) q.delete({'id': self.id}) # copy what RamSession does for now From cff71af03913b5b0987171205ef0c460b2f6fff8 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Sep 02 2015 15:17:55 +0000 Subject: [PATCH 2/3] Implement database upgrade for indexes This implements upgrading from schema version 1 to 2. Signed-off-by: Patrick Uiterwijk Reviewed-by: Rob Crittenden --- diff --git a/ipsilon/providers/openid/store.py b/ipsilon/providers/openid/store.py index 9b2bc4f..40e8b8e 100644 --- a/ipsilon/providers/openid/store.py +++ b/ipsilon/providers/openid/store.py @@ -84,4 +84,14 @@ class OpenIDStore(Store, OpenIDStoreInterface): q.create() def _upgrade_schema(self, old_version): - raise NotImplementedError() + if old_version == 1: + # In schema version 2, we added indexes and primary keys + # pylint: disable=protected-access + table = self._query(self._db, 'association', UNIQUE_DATA_TABLE, + trans=False)._table + self._db.add_constraint(table.primary_key) + for index in table.indexes: + self._db.add_index(index) + return 2 + else: + raise NotImplementedError() diff --git a/ipsilon/util/data.py b/ipsilon/util/data.py index e92aae4..79b7156 100644 --- a/ipsilon/util/data.py +++ b/ipsilon/util/data.py @@ -6,7 +6,8 @@ from ipsilon.util.log import Log from sqlalchemy import create_engine from sqlalchemy import MetaData, Table, Column, Text from sqlalchemy.pool import QueuePool, SingletonThreadPool -from sqlalchemy.schema import PrimaryKeyConstraint, Index +from sqlalchemy.schema import (PrimaryKeyConstraint, Index, AddConstraint, + CreateIndex) from sqlalchemy.sql import select, and_ import ConfigParser import os @@ -29,7 +30,16 @@ class DatabaseError(Exception): pass -class SqlStore(Log): +class BaseStore(Log): + # Some helper functions used for upgrades + def add_constraint(self, table): + raise NotImplementedError() + + def add_index(self, index): + raise NotImplementedError() + + +class SqlStore(BaseStore): __instances = {} @classmethod @@ -61,6 +71,18 @@ class SqlStore(Log): self._dbengine = create_engine(engine_name, **pool_args) self.is_readonly = False + def add_constraint(self, constraint): + if self._dbengine.dialect.name != 'sqlite': + # It is impossible to add constraints to a pre-existing table for + # SQLite + # source: http://www.sqlite.org/omitted.html + create_constraint = AddConstraint(constraint, bind=self._dbengine) + create_constraint.execute() + + def add_index(self, index): + add_index = CreateIndex(index, bind=self._dbengine) + add_index.execute() + def debug(self, fact): if self.db_conn_log: super(SqlStore, self).debug(fact) @@ -151,7 +173,7 @@ class SqlQuery(Log): self._con.execute(self._table.delete(self._where(kvfilter))) -class FileStore(Log): +class FileStore(BaseStore): def __init__(self, name): self._filename = name @@ -174,6 +196,12 @@ class FileStore(Log): self._config.read(self._filename) return self._config + def add_constraint(self, table): + raise NotImplementedError() + + def add_index(self, index): + raise NotImplementedError() + class FileQuery(Log): @@ -350,6 +378,7 @@ class Store(Log): # themselves. # They might implement downgrading if that's feasible, or just throw # NotImplementedError + # Should return the new schema version raise NotImplementedError() def upgrade_database(self): @@ -361,8 +390,17 @@ class Store(Log): self._store_new_schema_version(self._code_schema_version()) elif old_schema_version != self._code_schema_version(): # Upgrade from old_schema_version to code_schema_version - self._upgrade_schema(old_schema_version) - self._store_new_schema_version(self._code_schema_version()) + self.debug('Upgrading from schema version %i' % old_schema_version) + new_version = self._upgrade_schema(old_schema_version) + if not new_version: + error = ('Schema upgrade error: %s did not provide a ' + + 'new schema version number!' % + self.__class__.__name__) + self.error(error) + raise Exception(error) + self._store_new_schema_version(new_version) + # Check if we are now up-to-date + self.upgrade_database() @property def is_readonly(self): @@ -563,7 +601,21 @@ class AdminStore(Store): q.create() def _upgrade_schema(self, old_version): - raise NotImplementedError() + if old_version == 1: + # In schema version 2, we added indexes and primary keys + for table in ['config', + 'info_config', + 'login_config', + 'provider_config']: + # pylint: disable=protected-access + table = self._query(self._db, table, OPTIONS_TABLE, + trans=False)._table + self._db.add_constraint(table.primary_key) + for index in table.indexes: + self._db.add_index(index) + return 2 + else: + raise NotImplementedError() class UserStore(Store): @@ -588,7 +640,17 @@ class UserStore(Store): q.create() def _upgrade_schema(self, old_version): - raise NotImplementedError() + if old_version == 1: + # In schema version 2, we added indexes and primary keys + # pylint: disable=protected-access + table = self._query(self._db, 'users', OPTIONS_TABLE, + trans=False)._table + self._db.add_constraint(table.primary_key) + for index in table.indexes: + self._db.add_index(index) + return 2 + else: + raise NotImplementedError() class TranStore(Store): @@ -602,7 +664,17 @@ class TranStore(Store): q.create() def _upgrade_schema(self, old_version): - raise NotImplementedError() + if old_version == 1: + # In schema version 2, we added indexes and primary keys + # pylint: disable=protected-access + table = self._query(self._db, 'transactions', UNIQUE_DATA_TABLE, + trans=False)._table + self._db.add_constraint(table.primary_key) + for index in table.indexes: + self._db.add_index(index) + return 2 + else: + raise NotImplementedError() class SAML2SessionStore(Store): @@ -693,4 +765,14 @@ class SAML2SessionStore(Store): q.create() def _upgrade_schema(self, old_version): - raise NotImplementedError() + if old_version == 1: + # In schema version 2, we added indexes and primary keys + # pylint: disable=protected-access + table = self._query(self._db, self.table, UNIQUE_DATA_TABLE, + trans=False)._table + self._db.add_constraint(table.primary_key) + for index in table.indexes: + self._db.add_index(index) + return 2 + else: + raise NotImplementedError() diff --git a/ipsilon/util/sessions.py b/ipsilon/util/sessions.py index ef059d1..8df3b4d 100644 --- a/ipsilon/util/sessions.py +++ b/ipsilon/util/sessions.py @@ -23,7 +23,17 @@ class SessionStore(Store): q.create() def _upgrade_schema(self, old_version): - raise NotImplementedError() + if old_version == 1: + # In schema version 2, we added indexes and primary keys + # pylint: disable=protected-access + table = self._query(self._db, 'sessions', SESSION_TABLE, + trans=False)._table + self._db.add_constraint(table.primary_key) + for index in table.indexes: + self._db.add_index(index) + return 2 + else: + raise NotImplementedError() class SqlSession(Session): From ee2cd3b48ed3903dbb35726eb9992dd13f826f48 Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Sep 02 2015 15:19:23 +0000 Subject: [PATCH 3/3] Add test suite for database upgrades Signed-off-by: Patrick Uiterwijk Reviewed-by: Rob Crittenden --- diff --git a/Makefile b/Makefile index d15ca4d..ddfdf14 100644 --- a/Makefile +++ b/Makefile @@ -104,6 +104,7 @@ tests: wrappers PYTHONPATH=./ ./tests/tests.py --test=ldap PYTHONPATH=./ ./tests/tests.py --test=ldapdown PYTHONPATH=./ ./tests/tests.py --test=openid + PYTHONPATH=./ ./tests/tests.py --test=dbupgrades test: lp-test unittests tests diff --git a/tests/blobs/old_dbs/v1/adminconfig.sqlite.dump b/tests/blobs/old_dbs/v1/adminconfig.sqlite.dump new file mode 100644 index 0000000..d8e4d0a --- /dev/null +++ b/tests/blobs/old_dbs/v1/adminconfig.sqlite.dump @@ -0,0 +1,65 @@ +PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE dbinfo ( + name TEXT NOT NULL, + option TEXT NOT NULL, + value TEXT +); +INSERT INTO "dbinfo" VALUES('AdminStore_schema','version','1'); +CREATE TABLE config ( + name TEXT NOT NULL, + option TEXT NOT NULL, + value TEXT +); +CREATE TABLE info_config ( + name TEXT NOT NULL, + option TEXT NOT NULL, + value TEXT +); +CREATE TABLE login_config ( + name TEXT NOT NULL, + option TEXT NOT NULL, + value TEXT +); +INSERT INTO "login_config" VALUES('global','enabled','testauth'); +CREATE TABLE provider_config ( + name TEXT NOT NULL, + option TEXT NOT NULL, + value TEXT +); +INSERT INTO "provider_config" VALUES('openid','endpoint url','http://127.0.0.11:45081/idp_v1/openid/'); +INSERT INTO "provider_config" VALUES('openid','database url','openid.sqlite'); +INSERT INTO "provider_config" VALUES('openid','identity url template','http://127.0.0.11:45081/idp_v1/openid/id/%(username)s'); +INSERT INTO "provider_config" VALUES('openid','enabled extensions',''); +INSERT INTO "provider_config" VALUES('global','enabled','openid,persona,saml2'); +INSERT INTO "provider_config" VALUES('persona','allowed domains','127.0.0.11:45081'); +INSERT INTO "provider_config" VALUES('persona','issuer domain','127.0.0.11:45081'); +INSERT INTO "provider_config" VALUES('persona','idp key file','persona/persona.key'); +INSERT INTO "provider_config" VALUES('saml2','idp nameid salt','6c78ae3b33db4fe4886edb1679490821'); +INSERT INTO "provider_config" VALUES('saml2','idp metadata validity','1825'); +INSERT INTO "provider_config" VALUES('saml2','idp certificate file','saml2/idp.pem'); +INSERT INTO "provider_config" VALUES('saml2','idp key file','saml2/idp.key'); +INSERT INTO "provider_config" VALUES('saml2','session database url','saml2.sessions.db.sqlite'); +INSERT INTO "provider_config" VALUES('saml2','idp metadata file','metadata.xml'); +INSERT INTO "provider_config" VALUES('saml2','idp storage path','saml2'); +CREATE TABLE testauth_data ( + uuid TEXT NOT NULL, + name TEXT NOT NULL, + value TEXT +); +CREATE TABLE openid_data ( + uuid TEXT NOT NULL, + name TEXT NOT NULL, + value TEXT +); +CREATE TABLE persona_data ( + uuid TEXT NOT NULL, + name TEXT NOT NULL, + value TEXT +); +CREATE TABLE saml2_data ( + uuid TEXT NOT NULL, + name TEXT NOT NULL, + value TEXT +); +COMMIT; diff --git a/tests/blobs/old_dbs/v1/openid.sqlite.dump b/tests/blobs/old_dbs/v1/openid.sqlite.dump new file mode 100644 index 0000000..4618785 --- /dev/null +++ b/tests/blobs/old_dbs/v1/openid.sqlite.dump @@ -0,0 +1,14 @@ +PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE dbinfo ( + name TEXT NOT NULL, + option TEXT NOT NULL, + value TEXT +); +INSERT INTO "dbinfo" VALUES('OpenIDStore_schema','version','1'); +CREATE TABLE association ( + uuid TEXT NOT NULL, + name TEXT NOT NULL, + value TEXT +); +COMMIT; diff --git a/tests/blobs/old_dbs/v1/saml2.sessions.db.sqlite.dump b/tests/blobs/old_dbs/v1/saml2.sessions.db.sqlite.dump new file mode 100644 index 0000000..1e8a694 --- /dev/null +++ b/tests/blobs/old_dbs/v1/saml2.sessions.db.sqlite.dump @@ -0,0 +1,14 @@ +PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE saml2_sessions ( + uuid TEXT NOT NULL, + name TEXT NOT NULL, + value TEXT +); +CREATE TABLE dbinfo ( + name TEXT NOT NULL, + option TEXT NOT NULL, + value TEXT +); +INSERT INTO "dbinfo" VALUES('SAML2SessionStore_schema','version','1'); +COMMIT; diff --git a/tests/blobs/old_dbs/v1/transactions.sqlite.dump b/tests/blobs/old_dbs/v1/transactions.sqlite.dump new file mode 100644 index 0000000..b803e78 --- /dev/null +++ b/tests/blobs/old_dbs/v1/transactions.sqlite.dump @@ -0,0 +1,14 @@ +PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE dbinfo ( + name TEXT NOT NULL, + option TEXT NOT NULL, + value TEXT +); +INSERT INTO "dbinfo" VALUES('TranStore_schema','version','1'); +CREATE TABLE transactions ( + uuid TEXT NOT NULL, + name TEXT NOT NULL, + value TEXT +); +COMMIT; diff --git a/tests/blobs/old_dbs/v1/userprefs.sqlite.dump b/tests/blobs/old_dbs/v1/userprefs.sqlite.dump new file mode 100644 index 0000000..a20af5e --- /dev/null +++ b/tests/blobs/old_dbs/v1/userprefs.sqlite.dump @@ -0,0 +1,14 @@ +PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE dbinfo ( + name TEXT NOT NULL, + option TEXT NOT NULL, + value TEXT +); +INSERT INTO "dbinfo" VALUES('UserStore_schema','version','1'); +CREATE TABLE users ( + name TEXT NOT NULL, + option TEXT NOT NULL, + value TEXT +); +COMMIT; diff --git a/tests/dbupgrades.py b/tests/dbupgrades.py new file mode 100755 index 0000000..73d04fb --- /dev/null +++ b/tests/dbupgrades.py @@ -0,0 +1,149 @@ +#!/usr/bin/python +# +# Copyright (C) 2014 Ipsilon project Contributors, for license see COPYING + +from helpers.common import IpsilonTestBase # pylint: disable=relative-import +from helpers.http import HttpSessions # pylint: disable=relative-import +import os +import pwd +import sys +import signal +import subprocess +import ipsilon.util.data + +idp_g = {'TEMPLATES': '${TESTDIR}/templates/install', + 'CONFDIR': '${TESTDIR}/etc', + 'DATADIR': '${TESTDIR}/lib', + 'HTTPDCONFD': '${TESTDIR}/${NAME}/conf.d', + 'STATICDIR': '${ROOTDIR}', + 'BINDIR': '${ROOTDIR}/ipsilon', + 'WSGI_SOCKET_PREFIX': '${TESTDIR}/${NAME}/logs/wsgi'} + + +idp_a = {'hostname': '${ADDRESS}:${PORT}', + 'admin_user': '${TEST_USER}', + 'system_user': '${TEST_USER}', + 'instance': '${NAME}', + 'secure': 'no', + 'testauth': 'yes', + 'pam': 'no', + 'gssapi': 'no', + 'ipa': 'no', + 'server_debugging': 'True'} + + +class IpsilonTest(IpsilonTestBase): + + def __init__(self): + super(IpsilonTest, self).__init__('dbupgrades', __file__) + + def setup_servers(self, env=None): + pass + + def test_upgrade_from(self, env, old_version): + # Setup IDP Server + print "Installing IDP server to test upgrade from %i" % old_version + name = 'idp_v%i' % old_version + addr = '127.0.0.%i' % (10 + old_version) + port = str(45080 + old_version) + idp = self.generate_profile(idp_g, idp_a, name, addr, port) + conf = self.setup_idp_server(idp, name, addr, port, env) + + # Move database of old_version into place + cfgfile = os.path.join(self.testdir, 'etc', name, 'ipsilon.conf') + db_indir = os.path.join(self.rootdir, 'tests', 'blobs', 'old_dbs', + 'v%i' % old_version) + db_outdir = os.path.join(self.testdir, 'lib', name) + + if old_version > 0: + for database in ['adminconfig', + 'openid', + 'saml2.sessions.db', + 'transactions', + 'userprefs']: + db_in = os.path.join(db_indir, '%s.sqlite.dump' % database) + db_out = os.path.join(db_outdir, '%s.sqlite' % database) + os.unlink(db_out) + cmd = ['/bin/sqlite3', db_out, '.read %s' % db_in] + subprocess.check_call(cmd) + + # Upgrade that database + cmd = [os.path.join(self.rootdir, + 'ipsilon/install/ipsilon-upgrade-database'), + cfgfile] + subprocess.check_call(cmd, + cwd=os.path.join(self.testdir, 'lib', name), + env=env) + + # Check some version-specific changes, to see if the upgrade went OK + if old_version == 0: + # Check all features in a newly created database + # Let's verify if at least one index was created + test_db = os.path.join(db_outdir, 'adminconfig.sqlite') + p = subprocess.Popen(['/bin/sqlite3', test_db, '.dump'], + stdout=subprocess.PIPE) + output, _ = p.communicate() + if p.returncode: + print 'Sqlite dump failed' + sys.exit(1) + if 'CREATE INDEX' not in output: + raise Exception('Database upgrade did not introduce index') + if 'PRIMARY KEY' not in output: + raise Exception('Database upgrade did not introduce primary ' + + 'key') + elif old_version == 1: + # In 1 -> 2, we added indexes and primary keys + # Let's verify if at least one index was created + test_db = os.path.join(db_outdir, 'adminconfig.sqlite') + p = subprocess.Popen(['/bin/sqlite3', test_db, '.dump'], + stdout=subprocess.PIPE) + output, _ = p.communicate() + if p.returncode: + print 'Sqlite dump failed' + sys.exit(1) + if 'CREATE INDEX' not in output: + raise Exception('Database upgrade did not introduce index') + # SQLite did not support creating primary keys, so we can't test + + # Start the httpd server + http_server = self.start_http_server(conf, env) + + # Now attempt to use the upgraded database + exe = self.execname + if exe.endswith('c'): + exe = exe[:-1] + exe = [exe] + exe.append(str(old_version)) + exe.append(name) + exe.append('%s:%s' % (addr, port)) + exit_code = subprocess.call(exe, env=env) + if exit_code: + sys.exit(exit_code) + + # Now kill the last http server + os.killpg(http_server.pid, signal.SIGTERM) + self.processes.remove(http_server) + + def run(self, env): + for version in range(ipsilon.util.data.CURRENT_SCHEMA_VERSION): + self.test_upgrade_from(env, version) + + +if __name__ == '__main__': + from_version = sys.argv[1] + idpname = sys.argv[2] + url = sys.argv[3] + + user = pwd.getpwuid(os.getuid())[0] + + sess = HttpSessions() + sess.add_server(idpname, 'http://%s' % url, user, + 'ipsilon') + + print "dbupgrades: From v%s: Authenticate to IDP ..." % from_version, + try: + sess.auth_to_idp(idpname) + except Exception, e: # pylint: disable=broad-except + print >> sys.stderr, " ERROR: %s" % repr(e) + sys.exit(1) + print " SUCCESS" diff --git a/tests/helpers/common.py b/tests/helpers/common.py index eadfdc3..a7021e2 100755 --- a/tests/helpers/common.py +++ b/tests/helpers/common.py @@ -177,6 +177,7 @@ class IpsilonTestBase(object): p = subprocess.Popen(['/usr/sbin/httpd', '-DFOREGROUND', '-f', conf], env=env, preexec_fn=os.setsid) self.processes.append(p) + return p def start_pgdb_server(self, datadir, rundir, log, addr, port, env): p = subprocess.Popen(['/usr/bin/pg_ctl', 'start', '-D', datadir, '-o',