go/mirko: add SQL migrations machinery

This uses github.com/golang-migrate/migrate and adds a Source that
allows using go_embed data files.

We also provide a test/example.

Change-Id: Icd2b6c7f7d0f728073b3fdf39b432b33ce61a3cd

WORKSPACE

@@ -600,6 +600,56 @@ go_repository(
     importpath = "github.com/matttproud/golang_protobuf_extensions",
 )
 
+go_repository(
+    name = "com_github_jmoiron_sqlx",
+    commit = "38398a30ed8516ffda617a04c822de09df8a3ec5",
+    importpath = "github.com/jmoiron/sqlx",
+)
+
+go_repository(
+    name = "com_github_lib_pq",
+    commit = "3427c32cb71afc948325f299f040e53c1dd78979",
+    importpath = "github.com/lib/pq",
+)
+
+go_repository(
+    name = "com_github_gchaincl_sqlhooks",
+    commit = "1932c8dd22f2283687586008bf2d58c2c5c014d0",
+    importpath = "github.com/gchaincl/sqlhooks",
+)
+
+go_repository(
+    name = "com_github_golang_migrate_migrate_v4",
+    commit = "e93eaeb3fe21ce2ccc1365277a01863e6bc84d9c",
+    importpath = "github.com/golang-migrate/migrate/v4",
+    remote = "https://github.com/golang-migrate/migrate",
+    vcs = "git",
+)
+
+go_repository(
+    name = "com_github_hashicorp_go_multierror",
+    commit = "bdca7bb83f603b80ef756bb953fe1dafa9cd00a2",
+    importpath = "github.com/hashicorp/go-multierror",
+)
+
+go_repository(
+    name = "com_github_hashicorp_errwrap",
+    commit = "8a6fb523712970c966eefc6b39ed2c5e74880354",
+    importpath = "github.com/hashicorp/errwrap",
+)
+
+go_repository(
+    name = "com_github_cockroachdb_cockroach_go",
+    commit = "e0a95dfd547cc9c3ebaaba1a12c2afe4bf621ac5",
+    importpath = "github.com/cockroachdb/cockroach-go",
+)
+
+go_repository(
+    name = "com_github_jackc_pgx",
+    commit = "6954c15ad0bd3c9aa6dd1b190732b020379beb28",
+    importpath = "github.com/jackc/pgx",
+)
+
 go_repository(
     name = "com_github_golang_collections_go_datastructures",
     commit = "59788d5eb2591d3497ffb8fafed2f16fe00e7775",

go/mirko/BUILD.bazel

@@ -5,13 +5,16 @@ go_library(
     srcs = [
         "kubernetes.go",
         "mirko.go",
+        "sql_migrations.go",
     ],
     importpath = "code.hackerspace.pl/hscloud/go/mirko",
     visibility = ["//visibility:public"],
     deps = [
         "//go/pki:go_default_library",
         "//go/statusz:go_default_library",
+        "@com_github_gchaincl_sqlhooks//:go_default_library",
         "@com_github_golang_glog//:go_default_library",
+        "@com_github_golang_migrate_migrate_v4//source:go_default_library",
         "@io_k8s_client_go//kubernetes:go_default_library",
         "@io_k8s_client_go//rest:go_default_library",
         "@org_golang_google_grpc//:go_default_library",

go/mirko/sql_migrations.go

@@ -0,0 +1,172 @@
+package mirko
+
+// Migration support via github.com/golang-migrations/migrate for go_embed data in Bazel.
+// For example usage, see go/mirko/tests/sql.
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/golang-migrate/migrate/v4/source"
+)
+
+func NewMigrationsFromBazel(data map[string][]byte) (source.Driver, error) {
+	migrations := make(map[uint]*migration)
+
+	for k, v := range data {
+		parts := strings.Split(k, ".")
+		errInvalid := fmt.Errorf("invalid migration filename: %q", k)
+
+		if len(parts) != 3 {
+			return nil, errInvalid
+		}
+		if parts[2] != "sql" {
+			return nil, errInvalid
+		}
+		if parts[1] != "up" && parts[1] != "down" {
+			return nil, errInvalid
+		}
+		direction := parts[1]
+
+		nameParts := strings.SplitN(parts[0], "_", 2)
+		if len(nameParts) != 2 {
+			return nil, errInvalid
+		}
+
+		name := nameParts[1]
+
+		version32, err := strconv.ParseUint(nameParts[0], 10, 32)
+		if err != nil {
+			return nil, errInvalid
+		}
+		version := uint(version32)
+
+		m, ok := migrations[version]
+		if !ok {
+			migrations[version] = &migration{
+				version: version,
+				name:    name,
+			}
+			m = migrations[version]
+		} else {
+			if m.name != name {
+				if err != nil {
+					return nil, fmt.Errorf("migration version %d exists under diffrent names (%q vs %q)", version, name, m.name)
+				}
+			}
+		}
+
+		if direction == "up" {
+			m.up = v
+		} else {
+			m.down = v
+		}
+	}
+
+	var first uint
+	for version, migration := range migrations {
+		if migration.up == nil {
+			return nil, fmt.Errorf("migration version %d has no up file", version)
+		}
+		if migration.down == nil {
+			return nil, fmt.Errorf("migration version %d has no down file", version)
+		}
+		if first == 0 {
+			first = version
+		}
+		if version < first {
+			first = version
+		}
+	}
+
+	if first == 0 {
+		return nil, fmt.Errorf("no migrations, or lowest migration version is 0")
+	}
+
+	return &migrationSource{
+		migrations: migrations,
+		first:      first,
+	}, nil
+}
+
+type migrationSource struct {
+	migrations map[uint]*migration
+	first      uint
+}
+
+type migration struct {
+	version uint
+	name    string
+	up      []byte
+	down    []byte
+}
+
+func (s *migrationSource) Open(url string) (source.Driver, error) {
+	if url != "" {
+		return nil, fmt.Errorf("bazel migration source is not configure via an URL")
+	}
+	return s, nil
+}
+
+func (s *migrationSource) Close() error {
+	return nil
+}
+
+func (s *migrationSource) First() (uint, error) {
+	return s.first, nil
+}
+
+func (s *migrationSource) Prev(version uint) (uint, error) {
+	var prev uint
+	for ver, _ := range s.migrations {
+		if ver > prev && ver < version {
+			prev = ver
+		}
+	}
+	if prev == 0 {
+		return 0, os.ErrNotExist
+	}
+	return prev, nil
+}
+
+func (s *migrationSource) Next(version uint) (uint, error) {
+	var next uint
+	for ver, _ := range s.migrations {
+		if ver <= version {
+			continue
+		}
+		if next == 0 {
+			next = ver
+		}
+		if ver < next {
+			next = ver
+		}
+	}
+	if next <= version {
+		return 0, os.ErrNotExist
+	}
+	return next, nil
+}
+
+func (s *migrationSource) ReadUp(version uint) (io.ReadCloser, string, error) {
+	m, ok := s.migrations[version]
+	if !ok {
+		return nil, "", os.ErrNotExist
+	}
+
+	return ioutil.NopCloser(bytes.NewReader(m.up)), m.name, nil
+}
+
+func (s *migrationSource) ReadDown(version uint) (io.ReadCloser, string, error) {
+	m, ok := s.migrations[version]
+	if !ok {
+		return nil, "", os.ErrNotExist
+	}
+
+	return ioutil.NopCloser(bytes.NewReader(m.down)), m.name, nil
+}

go/mirko/tests/sql/BUILD.bazel

@@ -0,0 +1,10 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_test")
+
+go_test(
+    name = "go_default_test",
+    srcs = ["sql_test.go"],
+    deps = [
+        "//go/mirko/tests/sql/migrations:go_default_library",
+        "@com_github_golang_migrate_migrate_v4//database/sqlite3:go_default_library",
+    ],
+)

go/mirko/tests/sql/migrations/1564669958_one.down.sql

@@ -0,0 +1 @@
+DROP TABLE users;

go/mirko/tests/sql/migrations/1564669958_one.up.sql

@@ -0,0 +1,4 @@
+CREATE TABLE users (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    username STRING
+);

go/mirko/tests/sql/migrations/1564669976_two.down.sql

@@ -0,0 +1 @@
+DELETE FROM users WHERE username = 'q3k';

go/mirko/tests/sql/migrations/1564669976_two.up.sql

@@ -0,0 +1 @@
+INSERT INTO users (username) VALUES ("q3k");

go/mirko/tests/sql/migrations/1564669988_three.down.sql

@@ -0,0 +1,7 @@
+ALTER TABLE users RENAME TO users_old;
+CREATE TABLE users (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    username STRING
+);
+INSERT INTO users (username) SELECT (username) FROM users_old;
+DROP TABLE users_old;

go/mirko/tests/sql/migrations/1564669988_three.up.sql

@@ -0,0 +1 @@
+ALTER TABLE users ADD cool BOOLEAN NULL;

go/mirko/tests/sql/migrations/BUILD.bazel

@@ -0,0 +1,23 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//extras:embed_data.bzl", "go_embed_data")
+
+go_embed_data(
+    name = "migrations_data",
+    srcs = glob(["*.sql"]),
+    package = "migrations",
+    flatten = True,
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "migrations.go",
+        ":migrations_data",  # keep
+    ],
+    importpath = "code.hackerspace.pl/hscloud/go/mirko/tests/sql/migrations",
+    visibility = ["//go/mirko/tests/sql:__subpackages__"],
+    deps = [
+        "//go/mirko:go_default_library",
+        "@com_github_golang_migrate_migrate_v4//:go_default_library",
+    ],
+)

go/mirko/tests/sql/migrations/migrations.go

@@ -0,0 +1,17 @@
+package migrations
+
+import (
+	"fmt"
+
+	"code.hackerspace.pl/hscloud/go/mirko"
+
+	"github.com/golang-migrate/migrate/v4"
+)
+
+func New(dburl string) (*migrate.Migrate, error) {
+	source, err := mirko.NewMigrationsFromBazel(Data)
+	if err != nil {
+		return nil, fmt.Errorf("could not create migrations: %v", err)
+	}
+	return migrate.NewWithSourceInstance("bazel", source, dburl)
+}

go/mirko/tests/sql/sql_test.go

@@ -0,0 +1,38 @@
+package sql
+
+import (
+	"testing"
+
+	_ "github.com/golang-migrate/migrate/v4/database/sqlite3"
+
+	"code.hackerspace.pl/hscloud/go/mirko/tests/sql/migrations"
+)
+
+// TestOkay goes up and down fully through migrations.
+func TestOkay(t *testing.T) {
+	m, err := migrations.New("sqlite3://:memory:")
+	if err != nil {
+		t.Fatalf("migrations.New: %v", err)
+	}
+
+	err = m.Up()
+	if err != nil {
+		t.Fatalf("m.Up() failed: %v", err)
+	}
+
+	vers, dirty, err := m.Version()
+	if err != nil {
+		t.Fatalf("m.Version() failed: %v", err)
+	}
+	if dirty {
+		t.Errorf("database migration shouldn't be dirty")
+	}
+	if want, got := uint(1564669988), vers; want != got {
+		t.Errorf("got database version %d, want %d", want, got)
+	}
+
+	err = m.Down()
+	if err != nil {
+		t.Fatalf("m.Down() failed: %v", err)
+	}
+}