mirror of https://gerrit.hackerspace.pl/hscloud
go/mirko: add SQL migrations machinery
This uses github.com/golang-migrate/migrate and adds a Source that allows using go_embed data files. We also provide a test/example. Change-Id: Icd2b6c7f7d0f728073b3fdf39b432b33ce61a3cdchanges/03/3/1
parent
2316ac0e99
commit
30317b4278
50
WORKSPACE
50
WORKSPACE
|
@ -600,6 +600,56 @@ go_repository(
|
|||
importpath = "github.com/matttproud/golang_protobuf_extensions",
|
||||
)
|
||||
|
||||
go_repository(
|
||||
name = "com_github_jmoiron_sqlx",
|
||||
commit = "38398a30ed8516ffda617a04c822de09df8a3ec5",
|
||||
importpath = "github.com/jmoiron/sqlx",
|
||||
)
|
||||
|
||||
go_repository(
|
||||
name = "com_github_lib_pq",
|
||||
commit = "3427c32cb71afc948325f299f040e53c1dd78979",
|
||||
importpath = "github.com/lib/pq",
|
||||
)
|
||||
|
||||
go_repository(
|
||||
name = "com_github_gchaincl_sqlhooks",
|
||||
commit = "1932c8dd22f2283687586008bf2d58c2c5c014d0",
|
||||
importpath = "github.com/gchaincl/sqlhooks",
|
||||
)
|
||||
|
||||
go_repository(
|
||||
name = "com_github_golang_migrate_migrate_v4",
|
||||
commit = "e93eaeb3fe21ce2ccc1365277a01863e6bc84d9c",
|
||||
importpath = "github.com/golang-migrate/migrate/v4",
|
||||
remote = "https://github.com/golang-migrate/migrate",
|
||||
vcs = "git",
|
||||
)
|
||||
|
||||
go_repository(
|
||||
name = "com_github_hashicorp_go_multierror",
|
||||
commit = "bdca7bb83f603b80ef756bb953fe1dafa9cd00a2",
|
||||
importpath = "github.com/hashicorp/go-multierror",
|
||||
)
|
||||
|
||||
go_repository(
|
||||
name = "com_github_hashicorp_errwrap",
|
||||
commit = "8a6fb523712970c966eefc6b39ed2c5e74880354",
|
||||
importpath = "github.com/hashicorp/errwrap",
|
||||
)
|
||||
|
||||
go_repository(
|
||||
name = "com_github_cockroachdb_cockroach_go",
|
||||
commit = "e0a95dfd547cc9c3ebaaba1a12c2afe4bf621ac5",
|
||||
importpath = "github.com/cockroachdb/cockroach-go",
|
||||
)
|
||||
|
||||
go_repository(
|
||||
name = "com_github_jackc_pgx",
|
||||
commit = "6954c15ad0bd3c9aa6dd1b190732b020379beb28",
|
||||
importpath = "github.com/jackc/pgx",
|
||||
)
|
||||
|
||||
go_repository(
|
||||
name = "com_github_golang_collections_go_datastructures",
|
||||
commit = "59788d5eb2591d3497ffb8fafed2f16fe00e7775",
|
||||
|
|
|
@ -5,13 +5,16 @@ go_library(
|
|||
srcs = [
|
||||
"kubernetes.go",
|
||||
"mirko.go",
|
||||
"sql_migrations.go",
|
||||
],
|
||||
importpath = "code.hackerspace.pl/hscloud/go/mirko",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//go/pki:go_default_library",
|
||||
"//go/statusz:go_default_library",
|
||||
"@com_github_gchaincl_sqlhooks//:go_default_library",
|
||||
"@com_github_golang_glog//:go_default_library",
|
||||
"@com_github_golang_migrate_migrate_v4//source:go_default_library",
|
||||
"@io_k8s_client_go//kubernetes:go_default_library",
|
||||
"@io_k8s_client_go//rest:go_default_library",
|
||||
"@org_golang_google_grpc//:go_default_library",
|
||||
|
|
|
@ -0,0 +1,172 @@
|
|||
package mirko
|
||||
|
||||
// Migration support via github.com/golang-migrations/migrate for go_embed data in Bazel.
|
||||
// For example usage, see go/mirko/tests/sql.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4/source"
|
||||
)
|
||||
|
||||
func NewMigrationsFromBazel(data map[string][]byte) (source.Driver, error) {
|
||||
migrations := make(map[uint]*migration)
|
||||
|
||||
for k, v := range data {
|
||||
parts := strings.Split(k, ".")
|
||||
errInvalid := fmt.Errorf("invalid migration filename: %q", k)
|
||||
|
||||
if len(parts) != 3 {
|
||||
return nil, errInvalid
|
||||
}
|
||||
if parts[2] != "sql" {
|
||||
return nil, errInvalid
|
||||
}
|
||||
if parts[1] != "up" && parts[1] != "down" {
|
||||
return nil, errInvalid
|
||||
}
|
||||
direction := parts[1]
|
||||
|
||||
nameParts := strings.SplitN(parts[0], "_", 2)
|
||||
if len(nameParts) != 2 {
|
||||
return nil, errInvalid
|
||||
}
|
||||
|
||||
name := nameParts[1]
|
||||
|
||||
version32, err := strconv.ParseUint(nameParts[0], 10, 32)
|
||||
if err != nil {
|
||||
return nil, errInvalid
|
||||
}
|
||||
version := uint(version32)
|
||||
|
||||
m, ok := migrations[version]
|
||||
if !ok {
|
||||
migrations[version] = &migration{
|
||||
version: version,
|
||||
name: name,
|
||||
}
|
||||
m = migrations[version]
|
||||
} else {
|
||||
if m.name != name {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("migration version %d exists under diffrent names (%q vs %q)", version, name, m.name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if direction == "up" {
|
||||
m.up = v
|
||||
} else {
|
||||
m.down = v
|
||||
}
|
||||
}
|
||||
|
||||
var first uint
|
||||
for version, migration := range migrations {
|
||||
if migration.up == nil {
|
||||
return nil, fmt.Errorf("migration version %d has no up file", version)
|
||||
}
|
||||
if migration.down == nil {
|
||||
return nil, fmt.Errorf("migration version %d has no down file", version)
|
||||
}
|
||||
if first == 0 {
|
||||
first = version
|
||||
}
|
||||
if version < first {
|
||||
first = version
|
||||
}
|
||||
}
|
||||
|
||||
if first == 0 {
|
||||
return nil, fmt.Errorf("no migrations, or lowest migration version is 0")
|
||||
}
|
||||
|
||||
return &migrationSource{
|
||||
migrations: migrations,
|
||||
first: first,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type migrationSource struct {
|
||||
migrations map[uint]*migration
|
||||
first uint
|
||||
}
|
||||
|
||||
type migration struct {
|
||||
version uint
|
||||
name string
|
||||
up []byte
|
||||
down []byte
|
||||
}
|
||||
|
||||
func (s *migrationSource) Open(url string) (source.Driver, error) {
|
||||
if url != "" {
|
||||
return nil, fmt.Errorf("bazel migration source is not configure via an URL")
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *migrationSource) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *migrationSource) First() (uint, error) {
|
||||
return s.first, nil
|
||||
}
|
||||
|
||||
func (s *migrationSource) Prev(version uint) (uint, error) {
|
||||
var prev uint
|
||||
for ver, _ := range s.migrations {
|
||||
if ver > prev && ver < version {
|
||||
prev = ver
|
||||
}
|
||||
}
|
||||
if prev == 0 {
|
||||
return 0, os.ErrNotExist
|
||||
}
|
||||
return prev, nil
|
||||
}
|
||||
|
||||
func (s *migrationSource) Next(version uint) (uint, error) {
|
||||
var next uint
|
||||
for ver, _ := range s.migrations {
|
||||
if ver <= version {
|
||||
continue
|
||||
}
|
||||
if next == 0 {
|
||||
next = ver
|
||||
}
|
||||
if ver < next {
|
||||
next = ver
|
||||
}
|
||||
}
|
||||
if next <= version {
|
||||
return 0, os.ErrNotExist
|
||||
}
|
||||
return next, nil
|
||||
}
|
||||
|
||||
func (s *migrationSource) ReadUp(version uint) (io.ReadCloser, string, error) {
|
||||
m, ok := s.migrations[version]
|
||||
if !ok {
|
||||
return nil, "", os.ErrNotExist
|
||||
}
|
||||
|
||||
return ioutil.NopCloser(bytes.NewReader(m.up)), m.name, nil
|
||||
}
|
||||
|
||||
func (s *migrationSource) ReadDown(version uint) (io.ReadCloser, string, error) {
|
||||
m, ok := s.migrations[version]
|
||||
if !ok {
|
||||
return nil, "", os.ErrNotExist
|
||||
}
|
||||
|
||||
return ioutil.NopCloser(bytes.NewReader(m.down)), m.name, nil
|
||||
}
|
|
@ -0,0 +1,10 @@
|
|||
load("@io_bazel_rules_go//go:def.bzl", "go_test")
|
||||
|
||||
go_test(
|
||||
name = "go_default_test",
|
||||
srcs = ["sql_test.go"],
|
||||
deps = [
|
||||
"//go/mirko/tests/sql/migrations:go_default_library",
|
||||
"@com_github_golang_migrate_migrate_v4//database/sqlite3:go_default_library",
|
||||
],
|
||||
)
|
|
@ -0,0 +1 @@
|
|||
DROP TABLE users;
|
|
@ -0,0 +1,4 @@
|
|||
CREATE TABLE users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
username STRING
|
||||
);
|
|
@ -0,0 +1 @@
|
|||
DELETE FROM users WHERE username = 'q3k';
|
|
@ -0,0 +1 @@
|
|||
INSERT INTO users (username) VALUES ("q3k");
|
|
@ -0,0 +1,7 @@
|
|||
ALTER TABLE users RENAME TO users_old;
|
||||
CREATE TABLE users (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
username STRING
|
||||
);
|
||||
INSERT INTO users (username) SELECT (username) FROM users_old;
|
||||
DROP TABLE users_old;
|
|
@ -0,0 +1 @@
|
|||
ALTER TABLE users ADD cool BOOLEAN NULL;
|
|
@ -0,0 +1,23 @@
|
|||
load("@io_bazel_rules_go//go:def.bzl", "go_library")
|
||||
load("@io_bazel_rules_go//extras:embed_data.bzl", "go_embed_data")
|
||||
|
||||
go_embed_data(
|
||||
name = "migrations_data",
|
||||
srcs = glob(["*.sql"]),
|
||||
package = "migrations",
|
||||
flatten = True,
|
||||
)
|
||||
|
||||
go_library(
|
||||
name = "go_default_library",
|
||||
srcs = [
|
||||
"migrations.go",
|
||||
":migrations_data", # keep
|
||||
],
|
||||
importpath = "code.hackerspace.pl/hscloud/go/mirko/tests/sql/migrations",
|
||||
visibility = ["//go/mirko/tests/sql:__subpackages__"],
|
||||
deps = [
|
||||
"//go/mirko:go_default_library",
|
||||
"@com_github_golang_migrate_migrate_v4//:go_default_library",
|
||||
],
|
||||
)
|
|
@ -0,0 +1,17 @@
|
|||
package migrations
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"code.hackerspace.pl/hscloud/go/mirko"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
)
|
||||
|
||||
func New(dburl string) (*migrate.Migrate, error) {
|
||||
source, err := mirko.NewMigrationsFromBazel(Data)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not create migrations: %v", err)
|
||||
}
|
||||
return migrate.NewWithSourceInstance("bazel", source, dburl)
|
||||
}
|
|
@ -0,0 +1,38 @@
|
|||
package sql
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
_ "github.com/golang-migrate/migrate/v4/database/sqlite3"
|
||||
|
||||
"code.hackerspace.pl/hscloud/go/mirko/tests/sql/migrations"
|
||||
)
|
||||
|
||||
// TestOkay goes up and down fully through migrations.
|
||||
func TestOkay(t *testing.T) {
|
||||
m, err := migrations.New("sqlite3://:memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("migrations.New: %v", err)
|
||||
}
|
||||
|
||||
err = m.Up()
|
||||
if err != nil {
|
||||
t.Fatalf("m.Up() failed: %v", err)
|
||||
}
|
||||
|
||||
vers, dirty, err := m.Version()
|
||||
if err != nil {
|
||||
t.Fatalf("m.Version() failed: %v", err)
|
||||
}
|
||||
if dirty {
|
||||
t.Errorf("database migration shouldn't be dirty")
|
||||
}
|
||||
if want, got := uint(1564669988), vers; want != got {
|
||||
t.Errorf("got database version %d, want %d", want, got)
|
||||
}
|
||||
|
||||
err = m.Down()
|
||||
if err != nil {
|
||||
t.Fatalf("m.Down() failed: %v", err)
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue