mirror of
https://github.com/fnproject/fn.git
synced 2022-10-28 21:29:17 +03:00
* adds migrations closes #57 migrations only run if the database is not brand new. brand new databases will contain all the right fields when CREATE TABLE is called, this is for readability mostly more than efficiency (do not want to have to go through all of the database migrations to ascertain what columns a table has). upon startup of a new database, the migrations will be analyzed and the highest version set, so that future migrations will be run. this should also avoid running through all the migrations, which could bork db's easily enough (if the user just exits from impatience, say). otherwise, all migrations that a db has not yet seen will be run against it upon startup, this should be seamless to the user whether they had a db that had 0 migrations run on it before or N. this means users will not have to explicitly run any migrations on their dbs nor see any errors when we upgrade the db (so long as things go well). if migrations do not go so well, users will have to manually repair dbs (this is the intention of the `migrate` library and it seems sane), this should be rare, and I'm unsure myself how best to resolve not having gone through this myself, I would assume it will require running down migrations and then manually updating the migration field; in any case, docs once one of us has to go through this. migrations are written to files and checked into version control, and then use go-bindata to generate those files into go code and compiled in to be consumed by the migrate library (so that we don't have to put migration files on any servers) -- this is also in vcs. this seems to work ok. I don't like having to use the separate go-bindata tool but it wasn't really hard to install and then go generate takes care of the args. adding migrations should be relatively rare anyway, but tried to make it pretty painless. 1 migration to add created_at to the route is done here as an example of how to do migrations, as well as testing these things ;) -- `created_at` will be `0001-01-01T00:00:00.000Z` for any existing routes after a user runs this version. could spend the extra time adding 'today's date to any outstanding records, but that's not really accurate, the main thing is nobody will have to nuke their db with the migrations in place & we don't have any prod clusters really to worry about. all future routes will correctly have `created_at` set, and plan to add other timestamps but wanted to keep this patch as small as possible so only did routes.created_at. there are tests that a spankin new db will work as expected as well as a db after running all down & up migrations works. the latter tests only run on mysql and postgres, since sqlite3 does not like ALTER TABLE DROP COLUMN; up migrations will need to be tested manually for sqlite3 only, but in theory if they are simple and work on postgres and mysql, there is a good likelihood of success; the new migration from this patch works on sqlite3 fine. for now, we need to use `github.com/rdallman/migrate` to move forward, as getting integrated into upstream is proving difficult due to `github.com/go-sql-driver/mysql` being broken on master (yay dependencies). Fortunately for us, we vendor a version of the `mysql` bindings that actually works, thus, we are capable of using the `mattes/migrate` library with success due to that. this also will require go1.9 to use the new `database/sql.Conn` type, CI has been updated accordingly. some doc fixes too from testing.. and of course updated all deps. anyway, whew. this should let us add fields to the db without busting everybody's dbs. open to feedback on better ways, but this was overall pretty simple despite futzing with mysql. * add migrate pkg to deps, update deps use rdallman/migrate until we resolve in mattes land * add README in migrations package * add ref to mattes lib
339 lines
8.9 KiB
Go
339 lines
8.9 KiB
Go
package cockroachdb
|
|
|
|
import (
|
|
"database/sql"
|
|
"fmt"
|
|
"io"
|
|
"io/ioutil"
|
|
nurl "net/url"
|
|
|
|
"github.com/cockroachdb/cockroach-go/crdb"
|
|
"github.com/lib/pq"
|
|
"github.com/mattes/migrate"
|
|
"github.com/mattes/migrate/database"
|
|
"regexp"
|
|
"strconv"
|
|
"context"
|
|
)
|
|
|
|
func init() {
|
|
db := CockroachDb{}
|
|
database.Register("cockroach", &db)
|
|
database.Register("cockroachdb", &db)
|
|
database.Register("crdb-postgres", &db)
|
|
}
|
|
|
|
var DefaultMigrationsTable = "schema_migrations"
|
|
var DefaultLockTable = "schema_lock"
|
|
|
|
var (
|
|
ErrNilConfig = fmt.Errorf("no config")
|
|
ErrNoDatabaseName = fmt.Errorf("no database name")
|
|
)
|
|
|
|
type Config struct {
|
|
MigrationsTable string
|
|
LockTable string
|
|
ForceLock bool
|
|
DatabaseName string
|
|
}
|
|
|
|
type CockroachDb struct {
|
|
db *sql.DB
|
|
isLocked bool
|
|
|
|
// Open and WithInstance need to guarantee that config is never nil
|
|
config *Config
|
|
}
|
|
|
|
func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) {
|
|
if config == nil {
|
|
return nil, ErrNilConfig
|
|
}
|
|
|
|
if err := instance.Ping(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
query := `SELECT current_database()`
|
|
var databaseName string
|
|
if err := instance.QueryRow(query).Scan(&databaseName); err != nil {
|
|
return nil, &database.Error{OrigErr: err, Query: []byte(query)}
|
|
}
|
|
|
|
if len(databaseName) == 0 {
|
|
return nil, ErrNoDatabaseName
|
|
}
|
|
|
|
config.DatabaseName = databaseName
|
|
|
|
if len(config.MigrationsTable) == 0 {
|
|
config.MigrationsTable = DefaultMigrationsTable
|
|
}
|
|
|
|
if len(config.LockTable) == 0 {
|
|
config.LockTable = DefaultLockTable
|
|
}
|
|
|
|
px := &CockroachDb{
|
|
db: instance,
|
|
config: config,
|
|
}
|
|
|
|
if err := px.ensureVersionTable(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if err := px.ensureLockTable(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return px, nil
|
|
}
|
|
|
|
func (c *CockroachDb) Open(url string) (database.Driver, error) {
|
|
purl, err := nurl.Parse(url)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// As Cockroach uses the postgres protocol, and 'postgres' is already a registered database, we need to replace the
|
|
// connect prefix, with the actual protocol, so that the library can differentiate between the implementations
|
|
re := regexp.MustCompile("^(cockroach(db)?|crdb-postgres)")
|
|
connectString := re.ReplaceAllString(migrate.FilterCustomQuery(purl).String(), "postgres")
|
|
|
|
db, err := sql.Open("postgres", connectString)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
migrationsTable := purl.Query().Get("x-migrations-table")
|
|
if len(migrationsTable) == 0 {
|
|
migrationsTable = DefaultMigrationsTable
|
|
}
|
|
|
|
lockTable := purl.Query().Get("x-lock-table")
|
|
if len(lockTable) == 0 {
|
|
lockTable = DefaultLockTable
|
|
}
|
|
|
|
forceLockQuery := purl.Query().Get("x-force-lock")
|
|
forceLock, err := strconv.ParseBool(forceLockQuery)
|
|
if err != nil {
|
|
forceLock = false
|
|
}
|
|
|
|
px, err := WithInstance(db, &Config{
|
|
DatabaseName: purl.Path,
|
|
MigrationsTable: migrationsTable,
|
|
LockTable: lockTable,
|
|
ForceLock: forceLock,
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return px, nil
|
|
}
|
|
|
|
func (c *CockroachDb) Close() error {
|
|
return c.db.Close()
|
|
}
|
|
|
|
// Locking is done manually with a separate lock table. Implementing advisory locks in CRDB is being discussed
|
|
// See: https://github.com/cockroachdb/cockroach/issues/13546
|
|
func (c *CockroachDb) Lock() error {
|
|
err := crdb.ExecuteTx(context.Background(), c.db, nil, func(tx *sql.Tx) error {
|
|
aid, err := database.GenerateAdvisoryLockId(c.config.DatabaseName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
query := "SELECT * FROM " + c.config.LockTable + " WHERE lock_id = $1"
|
|
rows, err := tx.Query(query, aid)
|
|
if err != nil {
|
|
return database.Error{OrigErr: err, Err: "failed to fetch migration lock", Query: []byte(query)}
|
|
}
|
|
defer rows.Close()
|
|
|
|
// If row exists at all, lock is present
|
|
locked := rows.Next()
|
|
if locked && !c.config.ForceLock {
|
|
return database.Error{Err: "lock could not be acquired; already locked", Query: []byte(query)}
|
|
}
|
|
|
|
query = "INSERT INTO " + c.config.LockTable + " (lock_id) VALUES ($1)"
|
|
if _, err := tx.Exec(query, aid) ; err != nil {
|
|
return database.Error{OrigErr: err, Err: "failed to set migration lock", Query: []byte(query)}
|
|
}
|
|
|
|
return nil
|
|
})
|
|
|
|
if err != nil {
|
|
return err
|
|
} else {
|
|
c.isLocked = true
|
|
return nil
|
|
}
|
|
}
|
|
|
|
// Locking is done manually with a separate lock table. Implementing advisory locks in CRDB is being discussed
|
|
// See: https://github.com/cockroachdb/cockroach/issues/13546
|
|
func (c *CockroachDb) Unlock() error {
|
|
aid, err := database.GenerateAdvisoryLockId(c.config.DatabaseName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// In the event of an implementation (non-migration) error, it is possible for the lock to not be released. Until
|
|
// a better locking mechanism is added, a manual purging of the lock table may be required in such circumstances
|
|
query := "DELETE FROM " + c.config.LockTable + " WHERE lock_id = $1"
|
|
if _, err := c.db.Exec(query, aid); err != nil {
|
|
if e, ok := err.(*pq.Error); ok {
|
|
// 42P01 is "UndefinedTableError" in CockroachDB
|
|
// https://github.com/cockroachdb/cockroach/blob/master/pkg/sql/pgwire/pgerror/codes.go
|
|
if e.Code == "42P01" {
|
|
// On drops, the lock table is fully removed; This is fine, and is a valid "unlocked" state for the schema
|
|
c.isLocked = false
|
|
return nil
|
|
}
|
|
}
|
|
return database.Error{OrigErr: err, Err: "failed to release migration lock", Query: []byte(query)}
|
|
}
|
|
|
|
c.isLocked = false
|
|
return nil
|
|
}
|
|
|
|
func (c *CockroachDb) Run(migration io.Reader) error {
|
|
migr, err := ioutil.ReadAll(migration)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// run migration
|
|
query := string(migr[:])
|
|
if _, err := c.db.Exec(query); err != nil {
|
|
return database.Error{OrigErr: err, Err: "migration failed", Query: migr}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (c *CockroachDb) SetVersion(version int, dirty bool) error {
|
|
return crdb.ExecuteTx(context.Background(), c.db, nil, func(tx *sql.Tx) error {
|
|
if _, err := tx.Exec( `TRUNCATE "` + c.config.MigrationsTable + `"`); err != nil {
|
|
return err
|
|
}
|
|
|
|
if version >= 0 {
|
|
if _, err := tx.Exec(`INSERT INTO "` + c.config.MigrationsTable + `" (version, dirty) VALUES ($1, $2)`, version, dirty); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
})
|
|
}
|
|
|
|
func (c *CockroachDb) Version() (version int, dirty bool, err error) {
|
|
query := `SELECT version, dirty FROM "` + c.config.MigrationsTable + `" LIMIT 1`
|
|
err = c.db.QueryRow(query).Scan(&version, &dirty)
|
|
|
|
switch {
|
|
case err == sql.ErrNoRows:
|
|
return database.NilVersion, false, nil
|
|
|
|
case err != nil:
|
|
if e, ok := err.(*pq.Error); ok {
|
|
// 42P01 is "UndefinedTableError" in CockroachDB
|
|
// https://github.com/cockroachdb/cockroach/blob/master/pkg/sql/pgwire/pgerror/codes.go
|
|
if e.Code == "42P01" {
|
|
return database.NilVersion, false, nil
|
|
}
|
|
}
|
|
return 0, false, &database.Error{OrigErr: err, Query: []byte(query)}
|
|
|
|
default:
|
|
return version, dirty, nil
|
|
}
|
|
}
|
|
|
|
func (c *CockroachDb) Drop() error {
|
|
// select all tables in current schema
|
|
query := `SELECT table_name FROM information_schema.tables WHERE table_schema=(SELECT current_schema())`
|
|
tables, err := c.db.Query(query)
|
|
if err != nil {
|
|
return &database.Error{OrigErr: err, Query: []byte(query)}
|
|
}
|
|
defer tables.Close()
|
|
|
|
// delete one table after another
|
|
tableNames := make([]string, 0)
|
|
for tables.Next() {
|
|
var tableName string
|
|
if err := tables.Scan(&tableName); err != nil {
|
|
return err
|
|
}
|
|
if len(tableName) > 0 {
|
|
tableNames = append(tableNames, tableName)
|
|
}
|
|
}
|
|
|
|
if len(tableNames) > 0 {
|
|
// delete one by one ...
|
|
for _, t := range tableNames {
|
|
query = `DROP TABLE IF EXISTS ` + t + ` CASCADE`
|
|
if _, err := c.db.Exec(query); err != nil {
|
|
return &database.Error{OrigErr: err, Query: []byte(query)}
|
|
}
|
|
}
|
|
if err := c.ensureVersionTable(); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (c *CockroachDb) ensureVersionTable() error {
|
|
// check if migration table exists
|
|
var count int
|
|
query := `SELECT COUNT(1) FROM information_schema.tables WHERE table_name = $1 AND table_schema = (SELECT current_schema()) LIMIT 1`
|
|
if err := c.db.QueryRow(query, c.config.MigrationsTable).Scan(&count); err != nil {
|
|
return &database.Error{OrigErr: err, Query: []byte(query)}
|
|
}
|
|
if count == 1 {
|
|
return nil
|
|
}
|
|
|
|
// if not, create the empty migration table
|
|
query = `CREATE TABLE "` + c.config.MigrationsTable + `" (version INT NOT NULL PRIMARY KEY, dirty BOOL NOT NULL)`
|
|
if _, err := c.db.Exec(query); err != nil {
|
|
return &database.Error{OrigErr: err, Query: []byte(query)}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
|
|
func (c *CockroachDb) ensureLockTable() error {
|
|
// check if lock table exists
|
|
var count int
|
|
query := `SELECT COUNT(1) FROM information_schema.tables WHERE table_name = $1 AND table_schema = (SELECT current_schema()) LIMIT 1`
|
|
if err := c.db.QueryRow(query, c.config.LockTable).Scan(&count); err != nil {
|
|
return &database.Error{OrigErr: err, Query: []byte(query)}
|
|
}
|
|
if count == 1 {
|
|
return nil
|
|
}
|
|
|
|
// if not, create the empty lock table
|
|
query = `CREATE TABLE "` + c.config.LockTable + `" (lock_id INT NOT NULL PRIMARY KEY)`
|
|
if _, err := c.db.Exec(query); err != nil {
|
|
return &database.Error{OrigErr: err, Query: []byte(query)}
|
|
}
|
|
|
|
return nil
|
|
}
|