From 3ca3e04340660490c3f8ef23f4a3a2ada5ee1e2d Mon Sep 17 00:00:00 2001 From: Mats Linander Date: Sun, 17 Dec 2023 14:33:52 +0000 Subject: [PATCH] storage: use connection string for postgres --- gtfs_test.go | 9 ++ realtime_integration_test.go | 2 - realtime_test.go | 10 +- static_bench_test.go | 31 ++--- static_integration_test.go | 17 +-- static_test.go | 214 ++++++++--------------------------- storage/postgres.go | 31 ++--- storage/storage_test.go | 21 ++-- whitebox_test.go | 126 +++++++++++++++++++++ 9 files changed, 228 insertions(+), 233 deletions(-) create mode 100644 gtfs_test.go create mode 100644 whitebox_test.go diff --git a/gtfs_test.go b/gtfs_test.go new file mode 100644 index 0000000..b10a625 --- /dev/null +++ b/gtfs_test.go @@ -0,0 +1,9 @@ +package gtfs_test + +// Most tests in this package run against the in-memory and storage +// backends by default. If PostgresConnStr is set, they'll also run +// against postgres. + +const ( + PostgresConnStr = "" // "postgres://postgres:mysecretpassword@localhost:5432/gtfs?sslmode=disable" +) diff --git a/realtime_integration_test.go b/realtime_integration_test.go index 23d57ea..4340bcf 100644 --- a/realtime_integration_test.go +++ b/realtime_integration_test.go @@ -72,8 +72,6 @@ func TestRealtimeIntegrationNYCFerryDelaysOnER(t *testing.T) { }, }, departures) - fmt.Println("Failure below") - // Makes sure delay got propagated to Dumbo. // // Minor note: the original schedule has this departing 11:13, diff --git a/realtime_test.go b/realtime_test.go index f76eff8..b88c8c9 100644 --- a/realtime_test.go +++ b/realtime_test.go @@ -29,17 +29,9 @@ func staticAndReaderFromFiles(t *testing.T, backend string, files map[string][]s s, err = storage.NewSQLiteStorage() require.NoError(t, err) } else if backend == "postgres" { - s, err = storage.NewPSQLStorage(storage.PSQLConfig{ - Host: "localhost", - Port: 5432, - User: "postgres", - Password: "mysecretpassword", - DBName: "gtfs", - ClearDB: true, - }) + s, err = storage.NewPSQLStorage(PostgresConnStr, true) require.NoError(t, err) } else { - t.Fatalf("Unknown backend: %s", backend) } diff --git a/static_bench_test.go b/static_bench_test.go index e374a58..69797e9 100644 --- a/static_bench_test.go +++ b/static_bench_test.go @@ -1,15 +1,16 @@ -package gtfs +package gtfs_test import ( "io/ioutil" "testing" "time" + "tidbyt.dev/gtfs" "tidbyt.dev/gtfs/parse" "tidbyt.dev/gtfs/storage" ) -func loadFeed(b *testing.B, backend string, filename string) *Static { +func loadFeed(b *testing.B, backend string, filename string) *gtfs.Static { var s storage.Storage var err error if backend == "memory" { @@ -20,14 +21,7 @@ func loadFeed(b *testing.B, backend string, filename string) *Static { b.Error(err) } } else if backend == "postgres" { - s, err = storage.NewPSQLStorage(storage.PSQLConfig{ - Host: "localhost", - Port: 5432, - User: "postgres", - Password: "mysecretpassword", - DBName: "gtfs", - ClearDB: false, - }) + s, err = storage.NewPSQLStorage(PostgresConnStr, true) if err != nil { b.Error(err) } @@ -58,7 +52,7 @@ func loadFeed(b *testing.B, backend string, filename string) *Static { b.Error(err) } - static, err := NewStatic(reader, metadata) + static, err := gtfs.NewStatic(reader, metadata) if err != nil { b.Error(err) } @@ -83,7 +77,12 @@ func benchNearbyStops(b *testing.B, backend string) { func benchDepartures(b *testing.B, backend string) { static := loadFeed(b, backend, "testdata/caltrain_20160406.zip") - when := time.Date(2020, 2, 3, 8, 0, 0, 0, static.location) + tz, err := time.LoadLocation("America/Los_Angeles") + if err != nil { + b.Error(err) + } + + when := time.Date(2020, 2, 3, 8, 0, 0, 0, tz) window := 1 * time.Hour stops, err := static.NearbyStops(40.734673, -73.989951, 0, nil) @@ -118,8 +117,10 @@ func BenchmarkGTFSStatic(b *testing.B) { b.Run(test.Name+"_sqlite", func(b *testing.B) { test.Bench(b, "sqlite") }) - b.Run(test.Name+"_postgres", func(b *testing.B) { - test.Bench(b, "postgres") - }) + if PostgresConnStr != "" { + b.Run(test.Name+"_postgres", func(b *testing.B) { + test.Bench(b, "postgres") + }) + } } } diff --git a/static_integration_test.go b/static_integration_test.go index f5ce299..f3adc9f 100644 --- a/static_integration_test.go +++ b/static_integration_test.go @@ -22,14 +22,7 @@ func loadFeed2(t *testing.T, backend string, filename string) (*gtfs.Static, sto s, err = storage.NewSQLiteStorage() require.NoError(t, err) } else if backend == "postgres" { - s, err = storage.NewPSQLStorage(storage.PSQLConfig{ - Host: "localhost", - Port: 5432, - User: "postgres", - Password: "mysecretpassword", - DBName: "gtfs", - ClearDB: true, - }) + s, err = storage.NewPSQLStorage(PostgresConnStr, true) require.NoError(t, err) } else { t.Fatalf("unknown backend %q", backend) @@ -247,8 +240,10 @@ func TestStaticIntegration(t *testing.T) { t.Run(test.Name+"_memory", func(t *testing.T) { test.Test(t, "memory") }) - // t.Run(test.Name+"_postgres", func(t *testing.T) { - // test.Test(t, "postgres") - // }) + if PostgresConnStr != "" { + t.Run(test.Name+"_postgres", func(t *testing.T) { + test.Test(t, "postgres") + }) + } } } diff --git a/static_test.go b/static_test.go index 1239320..8759409 100644 --- a/static_test.go +++ b/static_test.go @@ -1,4 +1,4 @@ -package gtfs +package gtfs_test import ( "archive/zip" @@ -11,11 +11,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "tidbyt.dev/gtfs" "tidbyt.dev/gtfs/parse" "tidbyt.dev/gtfs/storage" ) -func staticFromFiles(t *testing.T, backend string, files map[string][]string) *Static { +func staticFromFiles(t *testing.T, backend string, files map[string][]string) *gtfs.Static { var s storage.Storage var err error if backend == "memory" { @@ -24,14 +25,7 @@ func staticFromFiles(t *testing.T, backend string, files map[string][]string) *S s, err = storage.NewSQLiteStorage() require.NoError(t, err) } else if backend == "postgres" { - s, err = storage.NewPSQLStorage(storage.PSQLConfig{ - Host: "localhost", - Port: 5432, - User: "postgres", - Password: "mysecretpassword", - DBName: "gtfs", - ClearDB: true, - }) + s, err = storage.NewPSQLStorage(PostgresConnStr, true) require.NoError(t, err) } else { @@ -78,7 +72,7 @@ func staticFromFiles(t *testing.T, backend string, files map[string][]string) *S reader, err := s.GetReader("test") require.NoError(t, err) - static, err := NewStatic(reader, metadata) + static, err := gtfs.NewStatic(reader, metadata) require.NoError(t, err) return static @@ -146,7 +140,7 @@ func testStaticDeparturesWindowing(t *testing.T, backend string) { // station should have 2 L train departures departures, err := g.Departures("14", time.Date(2020, 2, 4, 6, 0, 0, 0, time.UTC), 30*time.Minute, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "14", RouteID: "L", @@ -168,7 +162,7 @@ func testStaticDeparturesWindowing(t *testing.T, backend string) { // boundary of the window. departures, err = g.Departures("14", time.Date(2020, 2, 4, 6, 10, 0, 0, time.UTC), 50*time.Minute, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "14", RouteID: "L", @@ -216,7 +210,7 @@ func testStaticDeparturesWindowing(t *testing.T, backend string) { // Start window at 6:30 and earlier departures are cut departures, err = g.Departures("14", time.Date(2020, 2, 4, 6, 30, 0, 0, time.UTC), 50*time.Minute, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "14", RouteID: "L", @@ -250,17 +244,17 @@ func testStaticDeparturesWindowing(t *testing.T, backend string) { // Push window past last departure and we get nothing departures, err = g.Departures("14", time.Date(2020, 2, 4, 6, 51, 0, 0, time.UTC), 50*time.Minute, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{}, departures) + assert.Equal(t, []gtfs.Departure{}, departures) // Non-existent stop also gives us nothing departures, err = g.Departures("FOO", time.Date(2020, 2, 4, 6, 30, 0, 0, time.UTC), 50*time.Minute, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{}, departures) + assert.Equal(t, []gtfs.Departure{}, departures) // But a large enough window reaches next day's departures. departures, err = g.Departures("14", time.Date(2020, 2, 4, 6, 51, 0, 0, time.UTC), duration(23, 50, 0), -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "14", RouteID: "L", @@ -294,7 +288,7 @@ func testStaticDeparturesWindowing(t *testing.T, backend string) { // Outside of calendar, we get nothing (Jan 1st 2021 was a Friday) departures, err = g.Departures("14", time.Date(2021, 1, 1, 6, 30, 0, 0, time.UTC), 50*time.Minute, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{}, departures) + assert.Equal(t, []gtfs.Departure{}, departures) } func testStaticDeparturesWeekendSchedule(t *testing.T, backend string) { @@ -343,7 +337,7 @@ func testStaticDeparturesWeekendSchedule(t *testing.T, backend string) { // Feb 14th is a Friday, so weekday schedule applies. departures, err := g.Departures("6a", time.Date(2020, 2, 14, 9, 0, 0, 0, time.UTC), 20*time.Minute, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "6a", RouteID: "L", @@ -357,7 +351,7 @@ func testStaticDeparturesWeekendSchedule(t *testing.T, backend string) { // Feb 15th will be on weekend schedule departures, err = g.Departures("6a", time.Date(2020, 2, 15, 9, 0, 0, 0, time.UTC), 20*time.Minute, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "6a", RouteID: "L", @@ -371,7 +365,7 @@ func testStaticDeparturesWeekendSchedule(t *testing.T, backend string) { // Window spanning from 14th into 15th can capture stops from both days departures, err = g.Departures("6a", time.Date(2020, 2, 14, 9, 29, 0, 0, time.UTC), 24*time.Hour-1*time.Second, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "6a", RouteID: "L", @@ -428,7 +422,7 @@ func testStaticDeparturesTimezones(t *testing.T, backend string) { // Querying using the transit agency's time zone departures, err := g.Departures("6a", time.Date(2020, 2, 3, 9, 0, 0, 0, tzNYC), 20*time.Minute, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "6a", RouteID: "L", @@ -442,7 +436,7 @@ func testStaticDeparturesTimezones(t *testing.T, backend string) { // Querying using UTC, which in February 2020 is NYC+5 departures, err = g.Departures("6a", time.Date(2020, 2, 3, 14, 0, 0, 0, time.UTC), 20*time.Minute, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "6a", RouteID: "L", @@ -457,7 +451,7 @@ func testStaticDeparturesTimezones(t *testing.T, backend string) { // large enough window departures, err = g.Departures("6a", time.Date(2020, 2, 2, 22, 0, 0, 0, time.UTC), 20*time.Hour, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "6a", RouteID: "L", @@ -505,7 +499,7 @@ func testStaticDeparturesOvernightTrip(t *testing.T, backend string) { // but is still part of the feb 9 trip. departures, err := g.Departures("3a", time.Date(2020, 2, 9, 23, 30, 0, 0, tzNYC), 2*time.Hour, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "3a", RouteID: "L", @@ -519,7 +513,7 @@ func testStaticDeparturesOvernightTrip(t *testing.T, backend string) { // It's also there if we query for departures on the 10th departures, err = g.Departures("3a", time.Date(2020, 2, 10, 0, 15, 0, 0, tzNYC), 20*time.Minute, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "3a", RouteID: "L", @@ -534,7 +528,7 @@ func testStaticDeparturesOvernightTrip(t *testing.T, backend string) { // NYC+5) departures, err = g.Departures("3a", time.Date(2020, 2, 10, 4, 30, 0, 0, time.UTC), 2*time.Hour, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "3a", RouteID: "L", @@ -546,7 +540,7 @@ func testStaticDeparturesOvernightTrip(t *testing.T, backend string) { }, departures) departures, err = g.Departures("3a", time.Date(2020, 2, 10, 5, 15, 0, 0, time.UTC), 20*time.Minute, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "3a", RouteID: "L", @@ -605,7 +599,7 @@ func testStaticDeparturesCalendarDateOverride(t *testing.T, backend string) { // disabled. departures, err := g.Departures("8a", time.Date(2020, 2, 9, 22, 0, 0, 0, tzNYC), 2*time.Hour, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "8a", RouteID: "L", @@ -616,24 +610,24 @@ func testStaticDeparturesCalendarDateOverride(t *testing.T, backend string) { }, departures) departures, err = g.Departures("8a", time.Date(2020, 2, 8, 22, 0, 0, 0, tzNYC), 5*time.Hour, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{}, departures) + assert.Equal(t, []gtfs.Departure{}, departures) departures, err = g.Departures("3a", time.Date(2020, 2, 8, 22, 0, 0, 0, tzNYC), 5*time.Hour, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{}, departures) + assert.Equal(t, []gtfs.Departure{}, departures) // The trips from the 16th are also disabled, including spill // over into the 17th. The 15th is still up though, including // spill over into the 16th. departures, err = g.Departures("8a", time.Date(2020, 2, 16, 22, 0, 0, 0, tzNYC), 5*time.Hour, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{}, departures) + assert.Equal(t, []gtfs.Departure{}, departures) departures, err = g.Departures("3a", time.Date(2020, 2, 16, 22, 0, 0, 0, tzNYC), 5*time.Hour, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{}, departures) + assert.Equal(t, []gtfs.Departure{}, departures) departures, err = g.Departures("8a", time.Date(2020, 2, 15, 22, 0, 0, 0, tzNYC), 5*time.Hour, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "8a", RouteID: "L", @@ -644,7 +638,7 @@ func testStaticDeparturesCalendarDateOverride(t *testing.T, backend string) { }, departures) departures, err = g.Departures("3a", time.Date(2020, 2, 15, 22, 0, 0, 0, tzNYC), 5*time.Hour, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "3a", RouteID: "L", @@ -658,7 +652,7 @@ func testStaticDeparturesCalendarDateOverride(t *testing.T, backend string) { // into the the 25th. 25th remains disabled. departures, err = g.Departures("8a", time.Date(2020, 2, 24, 22, 0, 0, 0, tzNYC), 5*time.Hour, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "8a", RouteID: "L", @@ -669,7 +663,7 @@ func testStaticDeparturesCalendarDateOverride(t *testing.T, backend string) { }, departures) departures, err = g.Departures("3a", time.Date(2020, 2, 24, 22, 0, 0, 0, tzNYC), 5*time.Hour, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "3a", RouteID: "L", @@ -680,7 +674,7 @@ func testStaticDeparturesCalendarDateOverride(t *testing.T, backend string) { }, departures) departures, err = g.Departures("8a", time.Date(2020, 2, 25, 22, 0, 0, 0, tzNYC), 5*time.Hour, -1, "", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{}, departures) + assert.Equal(t, []gtfs.Departure{}, departures) } // Real world schedules seem to provide departure_time for all stops, @@ -724,7 +718,7 @@ func testStaticDeparturesNoDepartureFromFinalStop(t *testing.T, backend string) assert.Equal(t, nil, err) assert.Equal(t, 1, len(departures)) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "14", RouteID: "L", @@ -736,7 +730,7 @@ func testStaticDeparturesNoDepartureFromFinalStop(t *testing.T, backend string) departures, err = g.Departures("3a", time.Date(2020, 2, 9, 23, 0, 0, 0, tzNYC), 2*time.Hour, -1, "", -1, nil) assert.Equal(t, nil, err) - assert.Equal(t, []Departure{}, departures) + assert.Equal(t, []gtfs.Departure{}, departures) } func testStaticDeparturesFiltering(t *testing.T, backend string) { @@ -802,7 +796,7 @@ func testStaticDeparturesFiltering(t *testing.T, backend string) { departures, err := g.Departures("alpha", time.Date(2020, 3, 14, 0, 0, 0, 0, tzNYC), longDuration, 1, "", -1, nil) assert.NoError(t, err) assert.Equal(t, 1, len(departures)) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "alpha", RouteID: "RouteA", @@ -815,18 +809,18 @@ func testStaticDeparturesFiltering(t *testing.T, backend string) { // Specifying non-existent route and/or direction -> no results departures, err = g.Departures("alpha", time.Date(2020, 3, 14, 0, 0, 0, 0, tzNYC), longDuration, 1, "", 1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{}, departures) + assert.Equal(t, []gtfs.Departure{}, departures) departures, err = g.Departures("alpha", time.Date(2020, 3, 14, 0, 0, 0, 0, tzNYC), longDuration, 1, "RouteC", 0, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{}, departures) + assert.Equal(t, []gtfs.Departure{}, departures) departures, err = g.Departures("alpha", time.Date(2020, 3, 14, 0, 0, 0, 0, tzNYC), longDuration, 1, "RouteC", -1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{}, departures) + assert.Equal(t, []gtfs.Departure{}, departures) // The beta stop has departures in 2 direction departures, err = g.Departures("beta", time.Date(2020, 3, 14, 0, 0, 0, 0, tzNYC), longDuration, 1, "", 0, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "beta", RouteID: "RouteA", @@ -838,7 +832,7 @@ func testStaticDeparturesFiltering(t *testing.T, backend string) { departures, err = g.Departures("beta", time.Date(2020, 3, 14, 0, 0, 0, 0, tzNYC), longDuration, 1, "", 1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "beta", RouteID: "RouteA", @@ -851,7 +845,7 @@ func testStaticDeparturesFiltering(t *testing.T, backend string) { // Pushing start time back discards earlier departures departures, err = g.Departures("beta", time.Date(2020, 3, 14, 12, 0, 0, 0, tzNYC), longDuration, 1, "", 0, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "beta", RouteID: "RouteA", @@ -862,7 +856,7 @@ func testStaticDeparturesFiltering(t *testing.T, backend string) { }, departures) departures, err = g.Departures("beta", time.Date(2020, 3, 14, 12, 0, 0, 0, tzNYC), longDuration, 1, "RouteA", 1, nil) assert.NoError(t, err) - assert.Equal(t, []Departure{ + assert.Equal(t, []gtfs.Departure{ { StopID: "beta", RouteID: "RouteA", @@ -987,7 +981,7 @@ func testStaticDeparturesWithParentStations(t *testing.T, backend string) { }, }) - getDeps := func(stopID string) []Departure { + getDeps := func(stopID string) []gtfs.Departure { // Feb 3rd is a Monday. departures, err := g.Departures( stopID, @@ -1031,122 +1025,10 @@ func TestStatic(t *testing.T) { t.Run(fmt.Sprintf("%s SQLite", test.Name), func(t *testing.T) { test.Test(t, "sqlite") }) - // t.Run(fmt.Sprintf("%s Postgres", test.Name), func(t *testing.T) { - // test.Test(t, "postgres") - // }) - } -} - -func TestStaticRangePerDate(t *testing.T) { - tzET, err := time.LoadLocation("America/New_York") - require.NoError(t, err) - - // Eastern daylight savings started March 12th, 2023. At 2PM - // it became 3PM. - - // Eastern standard time started November 5th, 2023. At 2AM - // it became 1AM. - - for _, tc := range []struct { - Name string - Start time.Time - Window time.Duration - Max time.Duration - Expected []span - }{ - { - "simple", - time.Date(2023, 2, 3, 6, 0, 0, 0, tzET), - 30 * time.Minute, - 1 * time.Hour, - []span{{"20230203", "060000", "063000"}}, - }, - - { - "past midnight", - time.Date(2023, 2, 3, 6, 0, 0, 0, tzET), - 30 * time.Hour, - 1 * time.Hour, - []span{ - {"20230203", "060000", ""}, - {"20230204", "", "120000"}, - }, - }, - - { - "past midnight, with change to daylight savings time", - time.Date(2023, 3, 11, 6, 0, 0, 0, tzET), - 30 * time.Hour, - 1 * time.Hour, - []span{ - {"20230311", "060000", ""}, - {"20230312", "", "130000"}, - }, - }, - - { - "past midnight, with change to standard time", - time.Date(2023, 11, 4, 6, 0, 0, 0, tzET), - 30 * time.Hour, - 1 * time.Hour, - []span{ - {"20231104", "060000", ""}, - {"20231105", "", "110000"}, - }, - }, - - { - "multiple days", - time.Date(2023, 2, 3, 6, 0, 0, 0, tzET), - 49 * time.Hour, - 1 * time.Hour, - []span{ - {"20230203", "060000", ""}, - {"20230204", "", ""}, - {"20230205", "", "070000"}, - }, - }, - - { - "maxTrip indicating overflow from previous day", - time.Date(2023, 2, 3, 6, 0, 0, 0, tzET), - 2 * time.Hour, - (24 + 7) * time.Hour, - []span{ - {"20230202", "300000", ""}, - {"20230203", "060000", "080000"}, - }, - }, - - { - "overflow precisely touching range", - time.Date(2023, 2, 3, 6, 0, 0, 0, tzET), - 2 * time.Hour, - (24 + 6) * time.Hour, - []span{ - {"20230202", "300000", ""}, - {"20230203", "060000", "080000"}, - }, - }, - - { - "multi day with overflow reaching end of range", - time.Date(2023, 2, 3, 6, 0, 0, 0, tzET), - (48+18)*time.Hour + 30*time.Minute, - (24 + 1) * time.Hour, - []span{ - {"20230203", "060000", ""}, - {"20230204", "", ""}, - {"20230205", "", "243000"}, - {"20230206", "", "003000"}, - }, - }, - - // TODO: write a test of overflow with DST change - } { - t.Run(tc.Name, func(t *testing.T) { - spans := rangePerDate(tc.Start, tc.Window, tc.Max) - assert.Equal(t, tc.Expected, spans) - }) + if PostgresConnStr != "" { + t.Run(fmt.Sprintf("%s Postgres", test.Name), func(t *testing.T) { + test.Test(t, "postgres") + }) + } } } diff --git a/storage/postgres.go b/storage/postgres.go index 8c127eb..1ede273 100644 --- a/storage/postgres.go +++ b/storage/postgres.go @@ -15,19 +15,7 @@ const ( PSQLStopTimeBatchSize = 5000 ) -type PSQLConfig struct { - Host string - Port int - User string - Password string - DBName string - - ClearDB bool -} - type PSQLStorage struct { - PSQLConfig - db *sql.DB } @@ -43,18 +31,22 @@ type PSQLFeedReader struct { db *sql.DB } -func NewPSQLStorage(cfg PSQLConfig) (*PSQLStorage, error) { - connStr := fmt.Sprintf( - "host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", - cfg.Host, cfg.Port, cfg.User, cfg.Password, cfg.DBName, - ) +// Creates a new Postgres Storage using the provided connection string. +// +// If clearDB is true, the database will be cleared on startup. You +// probably only want this for testing. +func NewPSQLStorage(connStr string, clearDB bool) (*PSQLStorage, error) { db, err := sql.Open("postgres", connStr) if err != nil { return nil, fmt.Errorf("failed to open db: %w", err) } - if cfg.ClearDB { + if db.Ping() != nil { + return nil, fmt.Errorf("failed to ping db: %w", err) + } + + if clearDB { _, err = db.Exec(` DROP TABLE IF EXISTS feed; DROP TABLE IF EXISTS agency; @@ -91,8 +83,7 @@ CREATE TABLE IF NOT EXISTS feed ( } return &PSQLStorage{ - PSQLConfig: cfg, - db: db, + db: db, }, nil } diff --git a/storage/storage_test.go b/storage/storage_test.go index 0040850..10447ef 100644 --- a/storage/storage_test.go +++ b/storage/storage_test.go @@ -17,6 +17,14 @@ import ( "tidbyt.dev/gtfs/storage" ) +// Tests of the storage implementations. The in-memory and sqlite +// implementations are always run, while postgres require the +// PostgresConnStr below to be set. + +const ( + PostgresConnStr = "" // "postgres://postgres:mysecretpassword@localhost:5432/gtfs?sslmode=disable" +) + type StorageBuilder func() (storage.Storage, error) func readerFromFiles(t *testing.T, sb StorageBuilder, files map[string][]string) storage.FeedReader { @@ -2071,19 +2079,12 @@ func TestStorage(t *testing.T) { return storage.NewSQLiteStorage(storage.SQLiteConfig{OnDisk: true, Directory: dir}) }) }) - /* + if PostgresConnStr != "" { t.Run(fmt.Sprintf("%s Postgres", test.Name), func(t *testing.T) { test.Test(t, func() (storage.Storage, error) { - return storage.NewPSQLStorage(storage.PSQLConfig{ - Host: "localhost", - Port: 5432, - User: "postgres", - Password: "mysecretpassword", - DBName: "gtfs", - ClearDB: true, - }) + return storage.NewPSQLStorage(PostgresConnStr, true) }) }) - */ + } } } diff --git a/whitebox_test.go b/whitebox_test.go new file mode 100644 index 0000000..597c11a --- /dev/null +++ b/whitebox_test.go @@ -0,0 +1,126 @@ +package gtfs + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Don't love this, but some internal functions are finicky and need +// testing. + +func TestStaticRangePerDate(t *testing.T) { + tzET, err := time.LoadLocation("America/New_York") + require.NoError(t, err) + + // Eastern daylight savings started March 12th, 2023. At 2PM + // it became 3PM. + + // Eastern standard time started November 5th, 2023. At 2AM + // it became 1AM. + + for _, tc := range []struct { + Name string + Start time.Time + Window time.Duration + Max time.Duration + Expected []span + }{ + { + "simple", + time.Date(2023, 2, 3, 6, 0, 0, 0, tzET), + 30 * time.Minute, + 1 * time.Hour, + []span{{"20230203", "060000", "063000"}}, + }, + + { + "past midnight", + time.Date(2023, 2, 3, 6, 0, 0, 0, tzET), + 30 * time.Hour, + 1 * time.Hour, + []span{ + {"20230203", "060000", ""}, + {"20230204", "", "120000"}, + }, + }, + + { + "past midnight, with change to daylight savings time", + time.Date(2023, 3, 11, 6, 0, 0, 0, tzET), + 30 * time.Hour, + 1 * time.Hour, + []span{ + {"20230311", "060000", ""}, + {"20230312", "", "130000"}, + }, + }, + + { + "past midnight, with change to standard time", + time.Date(2023, 11, 4, 6, 0, 0, 0, tzET), + 30 * time.Hour, + 1 * time.Hour, + []span{ + {"20231104", "060000", ""}, + {"20231105", "", "110000"}, + }, + }, + + { + "multiple days", + time.Date(2023, 2, 3, 6, 0, 0, 0, tzET), + 49 * time.Hour, + 1 * time.Hour, + []span{ + {"20230203", "060000", ""}, + {"20230204", "", ""}, + {"20230205", "", "070000"}, + }, + }, + + { + "maxTrip indicating overflow from previous day", + time.Date(2023, 2, 3, 6, 0, 0, 0, tzET), + 2 * time.Hour, + (24 + 7) * time.Hour, + []span{ + {"20230202", "300000", ""}, + {"20230203", "060000", "080000"}, + }, + }, + + { + "overflow precisely touching range", + time.Date(2023, 2, 3, 6, 0, 0, 0, tzET), + 2 * time.Hour, + (24 + 6) * time.Hour, + []span{ + {"20230202", "300000", ""}, + {"20230203", "060000", "080000"}, + }, + }, + + { + "multi day with overflow reaching end of range", + time.Date(2023, 2, 3, 6, 0, 0, 0, tzET), + (48+18)*time.Hour + 30*time.Minute, + (24 + 1) * time.Hour, + []span{ + {"20230203", "060000", ""}, + {"20230204", "", ""}, + {"20230205", "", "243000"}, + {"20230206", "", "003000"}, + }, + }, + + // TODO: write a test of overflow with DST change + } { + t.Run(tc.Name, func(t *testing.T) { + spans := rangePerDate(tc.Start, tc.Window, tc.Max) + assert.Equal(t, tc.Expected, spans) + }) + } +}