alihan/fn-serverless
1
0
Fork 0
mirror of https://github.com/fnproject/fn.git synced 2022-10-28 21:29:17 +03:00
Code Issues Packages Projects Releases Wiki Activity

migratex api uses tx now instead of db (#939)

Browse Source 
* migratex api uses tx now instead of db

we want to be able to do external queries outside of the migration itself
inside of the same transaction for version checking. if we don't do this, we
risk the case where we set the version to the latest but we don't run the
table creates at all, so we have a db that thinks it's up to date but doesn't
even have any tables, and on subsequent boots if a migration slides in then
the migrations will run when there are no tables. it was unlikely, but now
it's dead.

* tx friendly table exists check

the previous existence checker for dbs was relying on getting back errors
about the db not existing. if we use this in a tx, it makes the whole tx
invalid for postgres. so, now we have count the table queries which return a 1
or a 0 instead of a 1 or an error so that we can check existence inside of a
transaction. voila.
This commit is contained in:
Reed Allman
2018-04-13 15:21:54 -07:00
committed by GitHub
parent 312fd8ec12
commit a481191db2
4 changed files with 131 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
api/datastore/sql/migratex/migrate.go
View File

@@ -12,6 +12,7 @@ import (
"github.com/go-sql-driver/mysql"
"github.com/jmoiron/sqlx"
"github.com/lib/pq"
"github.com/sirupsen/logrus"
)
var (
@@ -76,25 +77,19 @@ func (m MigFields) Version() int64 { return m.Versi
// TODO instance must have `multiStatements` set to true ?
func Up(ctx context.Context, db *sqlx.DB, migs []Migration) error {
return migrate(ctx, db, migs, true)
func Up(ctx context.Context, tx *sqlx.Tx, migs []Migration) error {
return migrate(ctx, tx, migs, true)
}
func Down(ctx context.Context, db *sqlx.DB, migs []Migration) error {
return migrate(ctx, db, migs, false)
func Down(ctx context.Context, tx *sqlx.Tx, migs []Migration) error {
return migrate(ctx, tx, migs, false)
}
func migrate(ctx context.Context, db *sqlx.DB, migs []Migration, up bool) error {
var curVersion int64 // could be NilVersion, is ok
err := tx(ctx, db, func(tx *sqlx.Tx) error {
var dirty bool
var err error
curVersion, dirty, err = Version(ctx, tx)
if dirty {
return dirtyErr(curVersion)
}
return err
})
func migrate(ctx context.Context, tx *sqlx.Tx, migs []Migration, up bool) error {
curVersion, dirty, err := Version(ctx, tx)
if dirty {
return dirtyErr(curVersion)
}
if err != nil {
return err
}
@@ -119,7 +114,7 @@ func migrate(ctx context.Context, db *sqlx.DB, migs []Migration, up bool) error
// XXX(reed): we could more gracefully handle concurrent databases trying to
// run migrations here by handling error and feeding back the version.
// get something working mode for now...
err := run(ctx, db, m, up)
err := run(ctx, tx, m, up)
if err != nil {
return err
}
@@ -129,19 +124,6 @@ func migrate(ctx context.Context, db *sqlx.DB, migs []Migration, up bool) error
return nil
}
func tx(ctx context.Context, db *sqlx.DB, f func(*sqlx.Tx) error) error {
tx, err := db.BeginTxx(ctx, nil)
if err != nil {
return err
}
err = f(tx)
if err != nil {
tx.Rollback()
return err
}
return tx.Commit()
}
func withLock(ctx context.Context, tx *sqlx.Tx, f func(*sqlx.Tx) error) error {
err := lock(ctx, tx)
if err != nil {
@@ -188,45 +170,43 @@ func (m MultiError) Error() string {
return strings.Join(strs, "\n")
}
func run(ctx context.Context, db *sqlx.DB, m Migration, up bool) error {
return tx(ctx, db, func(tx *sqlx.Tx) error {
return withLock(ctx, tx, func(tx *sqlx.Tx) error {
// within the transaction, we need to check the version and ensure this
// migration has not already been applied.
curVersion, dirty, err := Version(ctx, tx)
if dirty {
return dirtyErr(curVersion)
}
func run(ctx context.Context, tx *sqlx.Tx, m Migration, up bool) error {
return withLock(ctx, tx, func(tx *sqlx.Tx) error {
// within the transaction, we need to check the version and ensure this
// migration has not already been applied.
curVersion, dirty, err := Version(ctx, tx)
if dirty {
return dirtyErr(curVersion)
}
// enforce monotonicity
if up && curVersion != NilVersion && m.Version() != curVersion+1 {
return fmt.Errorf("non-contiguous migration attempted up: %v != %v", m.Version(), curVersion+1)
} else if !up && m.Version() != curVersion { // down is always unraveling
return fmt.Errorf("non-contiguous migration attempted down: %v != %v", m.Version(), curVersion)
}
// enforce monotonicity
if up && curVersion != NilVersion && m.Version() != curVersion+1 {
return fmt.Errorf("non-contiguous migration attempted up: %v != %v", m.Version(), curVersion+1)
} else if !up && m.Version() != curVersion { // down is always unraveling
return fmt.Errorf("non-contiguous migration attempted down: %v != %v", m.Version(), curVersion)
}
// TODO is this robust enough? we could check
version := m.Version()
if !up {
version = m.Version() - 1
}
// TODO is this robust enough? we could check
version := m.Version()
if !up {
version = m.Version() - 1
}
// TODO we don't need the dirty bit anymore since we're using transactions?
err = SetVersion(ctx, tx, version, true)
// TODO we don't need the dirty bit anymore since we're using transactions?
err = SetVersion(ctx, tx, version, true)
if up {
err = m.Up(ctx, tx)
} else {
err = m.Down(ctx, tx)
}
if up {
err = m.Up(ctx, tx)
} else {
err = m.Down(ctx, tx)
}
if err != nil {
return migrateErr(version, up, err)
}
if err != nil {
return migrateErr(version, up, err)
}
err = SetVersion(ctx, tx, version, false)
return err
})
err = SetVersion(ctx, tx, version, false)
return err
})
}
@@ -297,7 +277,8 @@ func unlock(ctx context.Context, tx *sqlx.Tx) error {
func SetVersion(ctx context.Context, tx *sqlx.Tx, version int64, dirty bool) error {
err := ensureVersionTable(ctx, tx)
if err != nil {
return nil
logrus.WithError(err).Error("error ensuring version table")
return err
}
// TODO need to handle down migration better
@@ -305,12 +286,14 @@ func SetVersion(ctx context.Context, tx *sqlx.Tx, version int64, dirty bool) err
// this just nukes the whole table which is kinda lame.
query := tx.Rebind("DELETE FROM " + MigrationsTable)
if _, err := tx.Exec(query); err != nil {
logrus.WithError(err).Error("error deleting version table")
return err
}
if version >= 0 {
query = tx.Rebind(`INSERT INTO ` + MigrationsTable + ` (version, dirty) VALUES (?, ?)`)
if _, err := tx.ExecContext(ctx, query, version, dirty); err != nil {
logrus.WithError(err).Error("error updating version table")
return err
}
}

24
api/datastore/sql/migratex/migrate_test.go
View File

@@ -38,7 +38,14 @@ func TestMigrateUp(t *testing.T) {
ctx := context.Background()
err = tx(ctx, db, func(tx *sqlx.Tx) error {
do := func() error {
tx, err := db.Beginx()
if err != nil {
return err
}
defer tx.Commit()
version, dirty, err := Version(ctx, tx)
if version != NilVersion || err != nil || dirty {
return fmt.Errorf("version err: %v %v", err, dirty)
@@ -48,7 +55,7 @@ func TestMigrateUp(t *testing.T) {
return errors.New("found existing version in db, nuke it")
}
err = Up(ctx, db, []Migration{x})
err = Up(ctx, tx, []Migration{x})
if err != nil {
return err
}
@@ -61,7 +68,15 @@ func TestMigrateUp(t *testing.T) {
if version != x.Version() {
return errors.New("version did not update, migration should have ran.")
}
return nil
}
err = do()
if err != nil {
t.Fatalf("couldn't run migrations: %v", err)
}
do = func() error {
// make sure the table is there.
// TODO find a db agnostic way of doing this.
// query := db.Rebind(`SELECT foo FROM sqlite_master WHERE type = 'table'`)
@@ -76,9 +91,10 @@ func TestMigrateUp(t *testing.T) {
return fmt.Errorf("migration version worked but migration didn't work: %v", result)
}
return nil
})
}
err = do()
if err != nil {
t.Fatalf("bad things happened: %v", err)
t.Fatalf("migration check failed: %v", err)
}
}