From fc62e0c0731c91f0df0c363fcee613f2adf24a2f Mon Sep 17 00:00:00 2001 From: Ronoaldo Pereira Date: Sun, 3 Dec 2023 15:03:54 -0300 Subject: [PATCH] fixes: pure-go build, iterator for postgres (#80) * fixs: pure-go build, iterator for postgres - Uses modernc.org/sqlite package - Properly used row filters for Iterator - Small updates for easy local testing Refs #77 * doc: updated Iterator behavior. --- auth/auth_migrate_test.go | 4 +- auth/auth_sqlite_test.go | 10 ++-- block/block.go | 6 +- block/block_migrate_test.go | 4 +- block/block_postgres.go | 64 +++++++++++++++----- block/block_postgres_test.go | 18 ++++++ block/block_sqlite.go | 2 +- block/block_sqlite_test.go | 4 +- block/block_test.go | 78 ++++++++++++++++--------- context.go | 8 ++- context_test.go | 2 +- docker-compose.yml | 5 +- go.mod | 31 +++++++--- go.sum | 66 +++++++++++++++++++-- mod_storage/mod_storage_migrate_test.go | 4 +- mod_storage/mod_storage_sqlite_test.go | 4 +- player/player_search_test.go | 3 +- player/player_test.go | 8 +-- player/playermetadata_test.go | 2 +- wal/wal_test.go | 6 +- 20 files changed, 243 insertions(+), 86 deletions(-) diff --git a/auth/auth_migrate_test.go b/auth/auth_migrate_test.go index b342330..6e6583d 100644 --- a/auth/auth_migrate_test.go +++ b/auth/auth_migrate_test.go @@ -4,7 +4,7 @@ import ( "database/sql" "testing" - _ "github.com/mattn/go-sqlite3" + _ "modernc.org/sqlite" "github.com/minetest-go/mtdb/auth" "github.com/minetest-go/mtdb/types" @@ -13,7 +13,7 @@ import ( func TestMigrateAuthSQlite(t *testing.T) { // open db - db, err := sql.Open("sqlite3", ":memory:") + db, err := sql.Open("sqlite", ":memory:") assert.NoError(t, err) assert.NoError(t, auth.MigrateAuthDB(db, types.DATABASE_SQLITE)) diff --git a/auth/auth_sqlite_test.go b/auth/auth_sqlite_test.go index c8aa404..61be286 100644 --- a/auth/auth_sqlite_test.go +++ b/auth/auth_sqlite_test.go @@ -5,7 +5,7 @@ import ( "os" "testing" - _ "github.com/mattn/go-sqlite3" + _ "modernc.org/sqlite" "github.com/minetest-go/mtdb/auth" "github.com/minetest-go/mtdb/types" @@ -15,7 +15,7 @@ import ( func TestEmptySQliteRepo(t *testing.T) { // open db - db, err := sql.Open("sqlite3", ":memory:") + db, err := sql.Open("sqlite", ":memory:") assert.NoError(t, err) repo := auth.NewAuthRepository(db, types.DATABASE_SQLITE) assert.NotNil(t, repo) @@ -33,7 +33,7 @@ func TestSQliteRepo(t *testing.T) { copyFileContents("testdata/auth.wal.sqlite", dbfile.Name()) // open db - db, err := sql.Open("sqlite3", "file:"+dbfile.Name()) + db, err := sql.Open("sqlite", "file:"+dbfile.Name()) assert.NoError(t, err) repo := auth.NewAuthRepository(db, types.DATABASE_SQLITE) assert.NotNil(t, repo) @@ -99,7 +99,7 @@ func TestSQlitePrivRepo(t *testing.T) { copyFileContents("testdata/auth.wal.sqlite", dbfile.Name()) // open db - db, err := sql.Open("sqlite3", "file:"+dbfile.Name()) + db, err := sql.Open("sqlite", "file:"+dbfile.Name()) assert.NoError(t, err) repo := auth.NewPrivilegeRepository(db, types.DATABASE_SQLITE) assert.NotNil(t, repo) @@ -155,7 +155,7 @@ func TestSqliteAuthRepo(t *testing.T) { dbfile, err := os.CreateTemp(os.TempDir(), "auth.sqlite") assert.NoError(t, err) assert.NotNil(t, dbfile) - db, err := sql.Open("sqlite3", "file:"+dbfile.Name()) + db, err := sql.Open("sqlite", "file:"+dbfile.Name()) assert.NoError(t, err) assert.NoError(t, auth.MigrateAuthDB(db, types.DATABASE_SQLITE)) assert.NoError(t, wal.EnableWAL(db)) diff --git a/block/block.go b/block/block.go index a4733ed..ef2ffb6 100644 --- a/block/block.go +++ b/block/block.go @@ -27,6 +27,8 @@ func (b *Block) String() string { } // BlockRepository implementes data access layer for the Minetest map data. +// All positions are in mapblock coordinates, as described here: +// https://github.com/minetest/minetest/blob/master/doc/lua_api.md#mapblock-coordinates type BlockRepository interface { types.Backup @@ -34,8 +36,8 @@ type BlockRepository interface { GetByPos(x, y, z int) (*Block, error) // Iterator returns a channel to fetch all data from the starting position - // X,Y,Z, with the map blocks sorted by position ascending. Sorting is done - // by X,Y and Z coordinates. + // X,Y,Z (exclusive), with the map blocks sorted by position ascending. + // Sorting is done by Z, Y, X to keep consistency with Sqlite map format. Iterator(x, y, z int) (chan *Block, types.Closer, error) // Update upserts the provided map block in the database, using the position diff --git a/block/block_migrate_test.go b/block/block_migrate_test.go index 5d77e66..a133b44 100644 --- a/block/block_migrate_test.go +++ b/block/block_migrate_test.go @@ -4,7 +4,7 @@ import ( "database/sql" "testing" - _ "github.com/mattn/go-sqlite3" + _ "modernc.org/sqlite" "github.com/minetest-go/mtdb/block" "github.com/minetest-go/mtdb/types" @@ -13,7 +13,7 @@ import ( func TestMigrateBlockSQlite(t *testing.T) { // open db - db, err := sql.Open("sqlite3", ":memory:") + db, err := sql.Open("sqlite", ":memory:") assert.NoError(t, err) assert.NoError(t, block.MigrateBlockDB(db, types.DATABASE_SQLITE)) diff --git a/block/block_postgres.go b/block/block_postgres.go index 7a193f6..66306e5 100644 --- a/block/block_postgres.go +++ b/block/block_postgres.go @@ -30,51 +30,85 @@ func (repo *postgresBlockRepository) GetByPos(x, y, z int) (*Block, error) { return entry, err } +// IteratorBatchSize is a default value to be used while batching over Iterators. +var IteratorBatchSize = 4096 + +// iteratorQuery uses the keyset pagination as described in this SO question +// https://stackoverflow.com/q/57504274 +// +// The WHERE (args) >= (params) works in a way to keep the sorting during comparision +// without the need to implement this as an offset/cursor +var iteratorQuery = `-- Query blocks to iterate over +SELECT posX, posY, posZ, data +FROM blocks +WHERE (posZ, posY, posX) > ($3, $2, $1) +ORDER BY posZ, posY, posX +LIMIT $4` + func (repo *postgresBlockRepository) Iterator(x, y, z int) (chan *Block, types.Closer, error) { - rows, err := repo.db.Query(` - SELECT posX, posY, posZ, data - FROM blocks - WHERE posX >= $1 AND posY >= $2 AND posZ >= $3 - ORDER BY posX, posY, posZ - `, x, y, z) + // logging setup + l := logrus.WithField("iterating_from", []int{x, y, z}) + count := int64(0) + page := 0 + pageSize := 0 + + l.Debug("running database query") + rows, err := repo.db.Query(iteratorQuery, x, y, z, IteratorBatchSize) if err != nil { return nil, nil, err } - l := logrus.WithField("iterating_from", []int{x, y, z}) - ch := make(chan *Block) + ch := make(chan *Block, IteratorBatchSize) done := make(types.WhenDone, 1) - count := int64(0) + lastPos := Block{} // Spawn go routine to fetch rows and send to channel go func() { defer close(ch) defer rows.Close() - l.Debug("Retrieving database rows ...") + l.Debug("retrieving database rows ...") for { select { case <-done: // We can now return, we are done - l.Debugf("Iterator closed by caller. Finishing up...") + l.Debugf("iterator closed by caller; finishing up...") return default: if rows.Next() { // Debug progress while fetching rows every 100's count++ + pageSize++ if count%100 == 0 { - l.Debugf("Retrieved %d records so far", count) + l.Debugf("retrieved %d records so far", count) } // Fetch and send to channel b := &Block{} if err = rows.Scan(&b.PosX, &b.PosY, &b.PosZ, &b.Data); err != nil { - l.Errorf("Failed to read next item from iterator: %v; aborting", err) + l.Errorf("failed to read next item from iterator: %v; aborting", err) return } + lastPos.PosX, lastPos.PosY, lastPos.PosZ = b.PosX, b.PosY, b.PosZ ch <- b } else { - l.Debug("Iterator finished, closing up rows and channel") - return + f := logrus.Fields{"last_pos": lastPos, "last_page_size": pageSize, "page": page} + if pageSize > 0 { + page++ + // If the previous batch is > 0, restart the query from last position + if err := rows.Close(); err != nil { + l.WithField("err", err).Warning("error closing previous batch") + } + rows, err = repo.db.Query(iteratorQuery, lastPos.PosX, lastPos.PosY, lastPos.PosZ, IteratorBatchSize) + if err != nil { + l.WithField("err", err).Warning("error restarting query") + return + } + pageSize = 0 + l.WithFields(f).Debug("batch finished, restarting next batch") + } else { + l.WithFields(f).Debug("iterator finished, closing up rows and channel") + return + } } } } diff --git a/block/block_postgres_test.go b/block/block_postgres_test.go index 372a7e3..d5028c5 100644 --- a/block/block_postgres_test.go +++ b/block/block_postgres_test.go @@ -63,6 +63,24 @@ func TestPostgresMaxConnections(t *testing.T) { } func TestPostgresIterator(t *testing.T) { + logToTesting(t) + blocks_repo, _ := setupPostgress(t) + testBlocksRepositoryIterator(t, blocks_repo) +} + +func TestPostgresIteratorBatches(t *testing.T) { + logToTesting(t) + + oldSize := block.IteratorBatchSize + setUp := func() { + block.IteratorBatchSize = 1 + } + tearDown := func() { + block.IteratorBatchSize = oldSize + } + + setUp() + defer tearDown() blocks_repo, _ := setupPostgress(t) testBlocksRepositoryIterator(t, blocks_repo) } diff --git a/block/block_sqlite.go b/block/block_sqlite.go index a016e00..f688e16 100644 --- a/block/block_sqlite.go +++ b/block/block_sqlite.go @@ -78,7 +78,7 @@ func (repo *sqliteBlockRepository) Iterator(x, y, z int) (chan *Block, types.Clo rows, err := repo.db.Query(` SELECT pos, data FROM blocks - WHERE pos >= $1 + WHERE pos > $1 ORDER BY pos `, pos) if err != nil { diff --git a/block/block_sqlite_test.go b/block/block_sqlite_test.go index ee35fd1..b24b4df 100644 --- a/block/block_sqlite_test.go +++ b/block/block_sqlite_test.go @@ -5,7 +5,7 @@ import ( "os" "testing" - _ "github.com/mattn/go-sqlite3" + _ "modernc.org/sqlite" "github.com/minetest-go/mtdb/block" "github.com/minetest-go/mtdb/types" @@ -17,7 +17,7 @@ func setupSqlite(t *testing.T) (block.BlockRepository, *sql.DB) { dbfile, err := os.CreateTemp(os.TempDir(), "map.sqlite") assert.NoError(t, err) assert.NotNil(t, dbfile) - db, err := sql.Open("sqlite3", "file:"+dbfile.Name()) + db, err := sql.Open("sqlite", "file:"+dbfile.Name()) assert.NoError(t, err) assert.NoError(t, wal.EnableWAL(db)) diff --git a/block/block_test.go b/block/block_test.go index 7d23519..90f58d5 100644 --- a/block/block_test.go +++ b/block/block_test.go @@ -82,25 +82,36 @@ func testBlocksRepository(t *testing.T, block_repo block.BlockRepository) { } func testBlocksRepositoryIterator(t *testing.T, blocks_repo block.BlockRepository) { - negX, negY, negZ := block.AsBlockPos(-32000, -32000, -32000) - posX, posY, posZ := block.AsBlockPos(32000, 32000, 32000) - testData := []block.Block{ - {negX, negY, negZ, []byte("negative")}, - {0, 0, 0, []byte("zero")}, - {posX, posY, posZ, []byte("positive")}, + // For readability, octals < 8 were used so they don't change the actuall value. + {-2000, -2000, -2000, []byte("negative[0]")}, + {-1999, -0004, -2000, []byte("negative[1]")}, + {+0000, +0000, +0000, []byte("zero")}, + {-2001, +0000, +0001, []byte("negative[2]")}, + {+0001, +0002, +2000, []byte("positive[0]")}, + {+2000, +2000, +2000, []byte("positive[1]")}, + } + setUp := func() { + for i := range testData { + b := testData[i] + blocks_repo.Update(&b) + } } - for i := range testData { - b := testData[i] - blocks_repo.Update(&b) + tearDown := func() { + for _, b := range testData { + blocks_repo.Delete(b.PosX, b.PosY, b.PosZ) + } } - // logrus.SetLevel(logrus.DebugLevel) + + setUp() + defer tearDown() // Helper function to loop over all channel data // TODO(ronoaldo): simplify this so we don't have // to work so verbose. Perhaps implement a wrapper // to the channel? - consumeAll := func(it chan *block.Block) int { + consumeAll := func(tc string, it chan *block.Block) int { + t.Logf("Test Case: %s", tc) t.Logf("consumeAll: fetching data from iterator") count := 0 for { @@ -121,28 +132,37 @@ func testBlocksRepositoryIterator(t *testing.T, blocks_repo block.BlockRepositor } } - // Fetch from neg -> pos, retrieves all three blocks - it, _, err := blocks_repo.Iterator(negX, negY, negZ) - if assert.NoError(t, err) { - assert.Equal(t, 3, consumeAll(it)) + type testCase struct { + x int + y int + z int + name string + expected int } - // Fetch from zero -> pos, retrieves two blocks - it, _, err = blocks_repo.Iterator(0, 0, 0) - if assert.NoError(t, err) { - assert.Equal(t, 2, consumeAll(it)) + // Sort order should be (z, y, x) to keep consistency with how sqlite + // in minetest works (using a single pos field with z+y+x summed and byte-shifted) + testCases := []testCase{ + {-2001, -2001, -2001, "starting from -2000,-2000,-2000 should return 6", 6}, + {-2000, -2000, -2000, "starting from -2000,-2000,-2000 should return 5", 5}, + {-0001, -0001, -0001, "starting from -0001,-0001,-0001 should return 4", 4}, + {+0000, +0000, +0000, "starting from +0000,+0000,+0000 should return 3", 3}, + {+0000, +0000, +0001, "starting from +0000,+0000,+0001 should return 2", 2}, + {+0000, +0000, +1999, "starting from +2000,+2000,+1999 should return 2", 2}, + {+1999, +1999, +1999, "starting from +1999,+1999,+1999 should return 2", 2}, + {+2000, +2000, +2000, "starting from +2000,+2000,+2000 should return 0", 0}, + {+2000, +2000, +2001, "starting from +2000,+2000,+2001 should return 0", 0}, + {+2000, +2001, +2000, "starting from +2000,+2001,+2000 should return 0", 0}, + {+2001, +2000, +2000, "starting from +2001,+2000,+2000 should return 0", 0}, } - // Fetch from zero +1 -> pos, retrieves only one - it, _, err = blocks_repo.Iterator(0, 0, 1) - if assert.NoError(t, err) { - assert.Equal(t, 1, consumeAll(it)) - } - - // Fetch from 2000,2000,2000, retrieves zero blocks - it, _, err = blocks_repo.Iterator(posX+1, posY+1, posZ+1) - if assert.NoError(t, err) { - assert.Equal(t, 0, consumeAll(it)) + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + it, _, err := blocks_repo.Iterator(tc.x, tc.y, tc.z) + if assert.NoError(t, err) { + assert.Equal(t, tc.expected, consumeAll(tc.name, it)) + } + }) } } diff --git a/context.go b/context.go index 5f29828..6d32797 100644 --- a/context.go +++ b/context.go @@ -3,10 +3,10 @@ package mtdb import ( "archive/zip" "database/sql" + "fmt" "path" _ "github.com/lib/pq" - _ "github.com/mattn/go-sqlite3" "github.com/minetest-go/mtdb/auth" "github.com/minetest-go/mtdb/block" "github.com/minetest-go/mtdb/mod_storage" @@ -15,6 +15,7 @@ import ( "github.com/minetest-go/mtdb/wal" "github.com/minetest-go/mtdb/worldconfig" "github.com/sirupsen/logrus" + _ "modernc.org/sqlite" ) type Context struct { @@ -78,7 +79,7 @@ func connectAndMigrate(t types.DatabaseType, sqliteConn, psqlConn string, migFn default: // default to sqlite datasource = sqliteConn - dbtype = "sqlite3" + dbtype = "sqlite" } if t == types.DATABASE_POSTGRES && datasource == "" { @@ -136,6 +137,9 @@ func New(world_dir string) (*Context, error) { } if ctx.map_db != nil { ctx.Blocks = block.NewBlockRepository(ctx.map_db, dbtype) + if ctx.Blocks == nil { + return nil, fmt.Errorf("invalid repository dbtype: %v", dbtype) + } ctx.backuprepos = append(ctx.backuprepos, ctx.Blocks) } diff --git a/context_test.go b/context_test.go index e413129..4e814e1 100644 --- a/context_test.go +++ b/context_test.go @@ -9,7 +9,7 @@ import ( "testing" _ "github.com/lib/pq" - _ "github.com/mattn/go-sqlite3" + _ "modernc.org/sqlite" "github.com/minetest-go/mtdb" "github.com/stretchr/testify/assert" diff --git a/docker-compose.yml b/docker-compose.yml index e9906bb..6827bb1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,7 @@ services: POSTGRES_PASSWORD: enter volumes: - "postgres:/var/lib/postgresql/data" + command: -c port=9432 integration-test: image: golang:1.21.4 @@ -20,8 +21,10 @@ services: - PGPASSWORD=enter - PGHOST=postgres - PGDATABASE=postgres - - PGPORT=5432 + - PGPORT=9432 - LOGLEVEL=debug + - CGO_ENABLED=0 + - GO_FLAGS=-count=1 working_dir: /data command: ["go", "test", "./...", "-v", "-cover"] diff --git a/go.mod b/go.mod index 5f79072..44f6893 100644 --- a/go.mod +++ b/go.mod @@ -2,16 +2,33 @@ module github.com/minetest-go/mtdb go 1.19 -require ( - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/mattn/go-sqlite3 v1.14.18 - github.com/pmezard/go-difflib v1.0.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect -) - require ( github.com/lib/pq v1.10.9 github.com/sirupsen/logrus v1.9.3 github.com/stretchr/testify v1.8.4 + modernc.org/sqlite v1.27.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/mod v0.3.0 // indirect golang.org/x/sys v0.11.0 // indirect + golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78 // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + lukechampine.com/uint128 v1.2.0 // indirect + modernc.org/cc/v3 v3.40.0 // indirect + modernc.org/ccgo/v3 v3.16.13 // indirect + modernc.org/libc v1.29.0 // indirect + modernc.org/mathutil v1.6.0 // indirect + modernc.org/memory v1.7.2 // indirect + modernc.org/opt v0.1.3 // indirect + modernc.org/strutil v1.1.3 // indirect + modernc.org/token v1.0.1 // indirect ) diff --git a/go.sum b/go.sum index 16a2873..1c6424c 100644 --- a/go.sum +++ b/go.sum @@ -1,25 +1,83 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= -github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI= -github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78 h1:M8tBwCtWD/cZV9DZpFYRUgaymAYAr+aIUTWzDaM3uPs= +golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI= +lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= +modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= +modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= +modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= +modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= +modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= +modernc.org/libc v1.29.0 h1:tTFRFq69YKCF2QyGNuRUQxKBm1uZZLubf6Cjh/pVHXs= +modernc.org/libc v1.29.0/go.mod h1:DaG/4Q3LRRdqpiLyP0C2m1B8ZMGkQ+cCgOIjEtQlYhQ= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= +modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.27.0 h1:MpKAHoyYB7xqcwnUwkuD+npwEa0fojF0B5QRbN+auJ8= +modernc.org/sqlite v1.27.0/go.mod h1:Qxpazz0zH8Z1xCFyi5GSL3FzbtZ3fvbjmywNogldEW0= +modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= +modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= +modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY= +modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg= +modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY= diff --git a/mod_storage/mod_storage_migrate_test.go b/mod_storage/mod_storage_migrate_test.go index c529dc4..d32d5a1 100644 --- a/mod_storage/mod_storage_migrate_test.go +++ b/mod_storage/mod_storage_migrate_test.go @@ -4,7 +4,7 @@ import ( "database/sql" "testing" - _ "github.com/mattn/go-sqlite3" + _ "modernc.org/sqlite" "github.com/minetest-go/mtdb/mod_storage" "github.com/minetest-go/mtdb/types" @@ -13,7 +13,7 @@ import ( func TestMigrateModStorageSQlite(t *testing.T) { // open db - db, err := sql.Open("sqlite3", ":memory:") + db, err := sql.Open("sqlite", ":memory:") assert.NoError(t, err) assert.NoError(t, mod_storage.MigrateModStorageDB(db, types.DATABASE_SQLITE)) diff --git a/mod_storage/mod_storage_sqlite_test.go b/mod_storage/mod_storage_sqlite_test.go index dd0857b..def83d1 100644 --- a/mod_storage/mod_storage_sqlite_test.go +++ b/mod_storage/mod_storage_sqlite_test.go @@ -7,7 +7,7 @@ import ( "os" "testing" - _ "github.com/mattn/go-sqlite3" + _ "modernc.org/sqlite" "github.com/minetest-go/mtdb/mod_storage" "github.com/minetest-go/mtdb/types" @@ -22,7 +22,7 @@ func TestModStorageSQliteRepo(t *testing.T) { copyFileContents("testdata/mod_storage.sqlite", dbfile.Name()) // open db - db, err := sql.Open("sqlite3", "file:"+dbfile.Name()) + db, err := sql.Open("sqlite", "file:"+dbfile.Name()) assert.NoError(t, err) repo := mod_storage.NewModStorageRepository(db, types.DATABASE_SQLITE) assert.NotNil(t, repo) diff --git a/player/player_search_test.go b/player/player_search_test.go index 3f664a2..693feb1 100644 --- a/player/player_search_test.go +++ b/player/player_search_test.go @@ -104,7 +104,7 @@ func testPlayerSearch(t *testing.T, repo *player.PlayerRepository) { func TestPlayerSearchSQLite(t *testing.T) { // open db - db, err := sql.Open("sqlite3", ":memory:") + db, err := sql.Open("sqlite", ":memory:") assert.NoError(t, err) assert.NoError(t, player.MigratePlayerDB(db, types.DATABASE_SQLITE)) @@ -117,6 +117,7 @@ func TestPlayerSearchPostgres(t *testing.T) { db, err := getPostgresDB(t) assert.NoError(t, err) assert.NotNil(t, db) + db.Exec("delete from player") assert.NoError(t, player.MigratePlayerDB(db, types.DATABASE_POSTGRES)) repo := player.NewPlayerRepository(db, types.DATABASE_POSTGRES) diff --git a/player/player_test.go b/player/player_test.go index 5252a3e..667a351 100644 --- a/player/player_test.go +++ b/player/player_test.go @@ -10,7 +10,7 @@ import ( "testing" _ "github.com/lib/pq" - _ "github.com/mattn/go-sqlite3" + _ "modernc.org/sqlite" "github.com/minetest-go/mtdb/player" "github.com/minetest-go/mtdb/types" @@ -48,7 +48,7 @@ func testRepository(t *testing.T, repo *player.PlayerRepository) { func TestSQliteMigratePlayer(t *testing.T) { // open db - db, err := sql.Open("sqlite3", ":memory:") + db, err := sql.Open("sqlite", ":memory:") assert.NoError(t, err) assert.NoError(t, player.MigratePlayerDB(db, types.DATABASE_SQLITE)) @@ -62,7 +62,7 @@ func TestSqlitePlayerRepo(t *testing.T) { copyFileContents("testdata/players.sqlite", dbfile.Name()) // open db - db, err := sql.Open("sqlite3", "file:"+dbfile.Name()) + db, err := sql.Open("sqlite", "file:"+dbfile.Name()) assert.NoError(t, err) assert.NoError(t, player.MigratePlayerDB(db, types.DATABASE_SQLITE)) repo := player.NewPlayerRepository(db, types.DATABASE_SQLITE) @@ -117,7 +117,7 @@ func TestSqlitePlayerRepo(t *testing.T) { func TestSQlitePlayerRepo2(t *testing.T) { // open db - db, err := sql.Open("sqlite3", ":memory:") + db, err := sql.Open("sqlite", ":memory:") assert.NoError(t, err) assert.NoError(t, player.MigratePlayerDB(db, types.DATABASE_SQLITE)) diff --git a/player/playermetadata_test.go b/player/playermetadata_test.go index ad7c723..ad545e6 100644 --- a/player/playermetadata_test.go +++ b/player/playermetadata_test.go @@ -51,7 +51,7 @@ func testPlayerMetadata(t *testing.T, repo *player.PlayerMetadataRepository, pre func TestPlayerMetadataSQlite(t *testing.T) { dbfile, err := os.CreateTemp(os.TempDir(), "playermetadata.sqlite") assert.NoError(t, err) - db, err := sql.Open("sqlite3", dbfile.Name()) + db, err := sql.Open("sqlite", dbfile.Name()) assert.NoError(t, err) assert.NoError(t, player.MigratePlayerDB(db, types.DATABASE_SQLITE)) diff --git a/wal/wal_test.go b/wal/wal_test.go index c1549c1..079ce81 100644 --- a/wal/wal_test.go +++ b/wal/wal_test.go @@ -5,7 +5,7 @@ import ( "os" "testing" - _ "github.com/mattn/go-sqlite3" + _ "modernc.org/sqlite" "github.com/minetest-go/mtdb/wal" "github.com/stretchr/testify/assert" @@ -17,7 +17,7 @@ func TestCheckJournalModeDelete(t *testing.T) { assert.NotNil(t, dbfile) copyFileContents("testdata/auth.sqlite", dbfile.Name()) - db, err := sql.Open("sqlite3", "file:"+dbfile.Name()+"?mode=ro") + db, err := sql.Open("sqlite", "file:"+dbfile.Name()+"?mode=ro") assert.NoError(t, err) assert.Error(t, wal.EnableWAL(db)) } @@ -28,7 +28,7 @@ func TestCheckJournalModeWal(t *testing.T) { assert.NotNil(t, dbfile) copyFileContents("testdata/auth.wal.sqlite", dbfile.Name()) - db, err := sql.Open("sqlite3", "file:"+dbfile.Name()+"?mode=ro") + db, err := sql.Open("sqlite", "file:"+dbfile.Name()+"?mode=ro") assert.NoError(t, err) assert.NoError(t, wal.EnableWAL(db)) }