Skip to content

Commit

Permalink
tests: unify most static helpers (#8)
Browse files Browse the repository at this point in the history
This gets rid of a lot repeated code for loading static data in unit testing.
  • Loading branch information
matslina authored Dec 17, 2023
1 parent a5ecd9f commit 1d7b82c
Show file tree
Hide file tree
Showing 6 changed files with 120 additions and 243 deletions.
104 changes: 103 additions & 1 deletion gtfs_test.go
Original file line number Diff line number Diff line change
@@ -1,9 +1,111 @@
package gtfs_test

// Most tests in this package run against the in-memory and storage
// Helpers and configuration for tests.
//
// Many tests in this package run against the in-memory and sqlite
// backends by default. If PostgresConnStr is set, they'll also run
// against postgres.

import (
"archive/zip"
"bytes"
"io/ioutil"
"strings"
"testing"

"github.com/stretchr/testify/require"

"tidbyt.dev/gtfs"
"tidbyt.dev/gtfs/parse"
"tidbyt.dev/gtfs/storage"
)

const (
PostgresConnStr = "" // "postgres://postgres:mysecretpassword@localhost:5432/gtfs?sslmode=disable"
)

func GFSTest_BuildStorage(t testing.TB, backend string) storage.Storage {
var s storage.Storage
var err error
if backend == "memory" {
s = storage.NewMemoryStorage()
} else if backend == "sqlite" {
s, err = storage.NewSQLiteStorage()
require.NoError(t, err)
} else if backend == "postgres" {
s, err = storage.NewPSQLStorage(PostgresConnStr, true)
require.NoError(t, err)
}
require.NotEqual(t, nil, s, "unknown backend %q", backend)

return s
}

func GTFSTest_LoadStatic(t testing.TB, backend string, buf *bytes.Buffer) (*gtfs.Static, storage.FeedReader) {
s := GFSTest_BuildStorage(t, backend)

// Parse buf into storage
feedWriter, err := s.GetWriter("test")
require.NoError(t, err)

metadata, err := parse.ParseStatic(feedWriter, buf.Bytes())
require.NoError(t, err)

require.NoError(t, feedWriter.Close())

// Create Static
reader, err := s.GetReader("test")
require.NoError(t, err)

static, err := gtfs.NewStatic(reader, metadata)
require.NoError(t, err)

return static, reader
}

func GTFSTest_LoadStaticFile(t testing.TB, backend string, filename string) (*gtfs.Static, storage.FeedReader) {
buf, err := ioutil.ReadFile(filename)
require.NoError(t, err)

return GTFSTest_LoadStatic(t, backend, bytes.NewBuffer(buf))
}

func GTFSTest_BuildStatic(
t testing.TB,
backend string,
files map[string][]string,
) (*gtfs.Static, storage.FeedReader) {

// Fill in missing files with (mostly blank) dummy data.
if files["agency.txt"] == nil {
files["agency.txt"] = []string{"agency_timezone,agency_name,agency_url", "UTC,FooAgency,http://example.com"}
}
if files["calendar.txt"] == nil && files["calendar_dates.txt"] == nil {
files["calendar.txt"] = []string{"service_id"}
}
if files["routes.txt"] == nil {
files["routes.txt"] = []string{"route_id"}
}
if files["trips.txt"] == nil {
files["trips.txt"] = []string{"trip_id"}
}
if files["stops.txt"] == nil {
files["stops.txt"] = []string{"stop_id"}
}
if files["stop_times.txt"] == nil {
files["stop_times.txt"] = []string{"stop_id"}
}

// Create zip
buf := &bytes.Buffer{}
w := zip.NewWriter(buf)
for filename, content := range files {
f, err := w.Create(filename)
require.NoError(t, err)
_, err = f.Write([]byte(strings.Join(content, "\n")))
require.NoError(t, err)
}
require.NoError(t, w.Close())

return GTFSTest_LoadStatic(t, backend, buf)
}
2 changes: 1 addition & 1 deletion realtime_integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import (
)

func loadNYCFerryRealtime(t *testing.T, suffix string) *gtfs.Realtime {
static, reader := loadFeed2(t, "sqlite", "testdata/nycferry_static.zip")
static, reader := GTFSTest_LoadStaticFile(t, "sqlite", "testdata/nycferry_static.zip")

buf, err := ioutil.ReadFile(fmt.Sprintf("testdata/nycferry_realtime_%s", suffix))
require.NoError(t, err)
Expand Down
72 changes: 3 additions & 69 deletions realtime_test.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,8 @@
package gtfs_test

import (
"archive/zip"
"bytes"
"context"
"fmt"
"strings"
"testing"
"time"

Expand All @@ -15,72 +12,9 @@ import (
proto "google.golang.org/protobuf/proto"

"tidbyt.dev/gtfs"
"tidbyt.dev/gtfs/parse"
"tidbyt.dev/gtfs/storage"
)

// TODO: really annoying to have these helpers all over the tests...
func staticAndReaderFromFiles(t *testing.T, backend string, files map[string][]string) (*gtfs.Static, storage.FeedReader) {
var s storage.Storage
var err error
if backend == "memory" {
s = storage.NewMemoryStorage()
} else if backend == "sqlite" {
s, err = storage.NewSQLiteStorage()
require.NoError(t, err)
} else if backend == "postgres" {
s, err = storage.NewPSQLStorage(PostgresConnStr, true)
require.NoError(t, err)
} else {
t.Fatalf("Unknown backend: %s", backend)
}

if files["agency.txt"] == nil {
files["agency.txt"] = []string{"agency_timezone,agency_name,agency_url", "UTC,FooAgency,http://example.com"}
}
if files["calendar.txt"] == nil && files["calendar_dates.txt"] == nil {
files["calendar.txt"] = []string{"service_id"}
}
if files["routes.txt"] == nil {
files["routes.txt"] = []string{"route_id"}
}
if files["trips.txt"] == nil {
files["trips.txt"] = []string{"trip_id"}
}
if files["stops.txt"] == nil {
files["stops.txt"] = []string{"stop_id"}
}
if files["stop_times.txt"] == nil {
files["stop_times.txt"] = []string{"stop_id"}
}

buf := &bytes.Buffer{}
w := zip.NewWriter(buf)
for filename, content := range files {
f, err := w.Create(filename)
require.NoError(t, err)
_, err = f.Write([]byte(strings.Join(content, "\n")))
require.NoError(t, err)
}
require.NoError(t, w.Close())

feedWriter, err := s.GetWriter("test")
require.NoError(t, err)

metadata, err := parse.ParseStatic(feedWriter, buf.Bytes())
require.NoError(t, err)

require.NoError(t, feedWriter.Close())

reader, err := s.GetReader("test")
require.NoError(t, err)

static, err := gtfs.NewStatic(reader, metadata)
require.NoError(t, err)

return static, reader
}

// Helpers for building gtfs-realtime feeds
type StopUpdate struct {
ArrivalSet bool
Expand Down Expand Up @@ -188,7 +122,7 @@ func buildFeed(t *testing.T, tripUpdates []TripUpdate) [][]byte {
// A simple Static fixture. Trips t1 and t2 cover the same three
// stops s1-s3. Trip t3 covers z1-z2. Full service all days of 2020.
func SimpleStaticFixture(t *testing.T) (*gtfs.Static, storage.FeedReader) {
static, reader := staticAndReaderFromFiles(t, "memory", map[string][]string{
static, reader := GTFSTest_BuildStatic(t, "memory", map[string][]string{
"calendar.txt": {
"service_id,start_date,end_date,monday,tuesday,wednesday,thursday,friday,saturday,sunday",
"everyday,20200101,20210101,1,1,1,1,1,1,1",
Expand Down Expand Up @@ -863,7 +797,7 @@ func TestRealtimeTimeWindowing(t *testing.T) {
func TestRealtimeTripWithLoop(t *testing.T) {
// This static schedule has t1 running from s1 to s2, and then
// 3 loops s3-s5, and finally end of the trip at s3.
static, reader := staticAndReaderFromFiles(t, "memory", map[string][]string{
static, reader := GTFSTest_BuildStatic(t, "memory", map[string][]string{
// A weekdays only schedule
"calendar.txt": {
"service_id,start_date,end_date,monday,tuesday,wednesday,thursday,friday,saturday,sunday",
Expand Down Expand Up @@ -1033,7 +967,7 @@ func TestRealtimeDepartureFiltering(t *testing.T) {
// Two routes: bus going center to south and rail going center
// to east. Each route has two trips: one heading out, one
// heading in.
static, reader := staticAndReaderFromFiles(t, "memory", map[string][]string{
static, reader := GTFSTest_BuildStatic(t, "memory", map[string][]string{
// A weekdays only schedule
"calendar.txt": {
"service_id,start_date,end_date,monday,tuesday,wednesday,thursday,friday,saturday,sunday",
Expand Down
59 changes: 2 additions & 57 deletions static_bench_test.go
Original file line number Diff line number Diff line change
@@ -1,67 +1,12 @@
package gtfs_test

import (
"io/ioutil"
"testing"
"time"

"tidbyt.dev/gtfs"
"tidbyt.dev/gtfs/parse"
"tidbyt.dev/gtfs/storage"
)

func loadFeed(b *testing.B, backend string, filename string) *gtfs.Static {
var s storage.Storage
var err error
if backend == "memory" {
s = storage.NewMemoryStorage()
} else if backend == "sqlite" {
s, err = storage.NewSQLiteStorage()
if err != nil {
b.Error(err)
}
} else if backend == "postgres" {
s, err = storage.NewPSQLStorage(PostgresConnStr, true)
if err != nil {
b.Error(err)
}
} else {
b.Error("unknown backend")
}

content, err := ioutil.ReadFile(filename)
if err != nil {
b.Error(err)
}

writer, err := s.GetWriter("benchmarking")
if err != nil {
b.Error(err)
}
metadata, err := parse.ParseStatic(writer, content)
if err != nil {
b.Error(err)
}
err = writer.Close()
if err != nil {
b.Error(err)
}

reader, err := s.GetReader("benchmarking")
if err != nil {
b.Error(err)
}

static, err := gtfs.NewStatic(reader, metadata)
if err != nil {
b.Error(err)
}

return static
}

func benchNearbyStops(b *testing.B, backend string) {
static := loadFeed(b, backend, "testdata/caltrain_20160406.zip")
static, _ := GTFSTest_LoadStaticFile(b, backend, "testdata/caltrain_20160406.zip")

b.ResetTimer()

Expand All @@ -75,7 +20,7 @@ func benchNearbyStops(b *testing.B, backend string) {
}

func benchDepartures(b *testing.B, backend string) {
static := loadFeed(b, backend, "testdata/caltrain_20160406.zip")
static, _ := GTFSTest_LoadStaticFile(b, backend, "testdata/caltrain_20160406.zip")

tz, err := time.LoadLocation("America/Los_Angeles")
if err != nil {
Expand Down
40 changes: 2 additions & 38 deletions static_integration_test.go
Original file line number Diff line number Diff line change
@@ -1,59 +1,23 @@
package gtfs_test

import (
"io/ioutil"
"testing"
"time"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"tidbyt.dev/gtfs"
"tidbyt.dev/gtfs/parse"
"tidbyt.dev/gtfs/storage"
)

func loadFeed2(t *testing.T, backend string, filename string) (*gtfs.Static, storage.FeedReader) {
var s storage.Storage
var err error
if backend == "memory" {
s = storage.NewMemoryStorage()
} else if backend == "sqlite" {
s, err = storage.NewSQLiteStorage()
require.NoError(t, err)
} else if backend == "postgres" {
s, err = storage.NewPSQLStorage(PostgresConnStr, true)
require.NoError(t, err)
} else {
t.Fatalf("unknown backend %q", backend)
}

content, err := ioutil.ReadFile(filename)
require.NoError(t, err)

writer, err := s.GetWriter("benchmarking")
require.NoError(t, err)
metadata, err := parse.ParseStatic(writer, content)
require.NoError(t, err)
err = writer.Close()
require.NoError(t, err)

reader, err := s.GetReader("benchmarking")
require.NoError(t, err)

static, err := gtfs.NewStatic(reader, metadata)
require.NoError(t, err)

return static, reader
}

func testGTFSStaticIntegrationNearbyStops(t *testing.T, backend string) {
if testing.Short() {
t.Skip("loading MTA dump is slow")
}

// This is a giant GTFS file from the MTA
g, _ := loadFeed2(t, backend, "testdata/mta_static.zip")
g, _ := GTFSTest_LoadStaticFile(t, backend, "testdata/mta_static.zip")

// The 4 nearest stops for 544 Park Ave, BK. There are other
// stops with the same coordinates, but they all have
Expand Down Expand Up @@ -107,7 +71,7 @@ func testGTFSStaticIntegrationDepartures(t *testing.T, backend string) {
}

// This is a giant GTFS file from the MTA
g, _ := loadFeed2(t, backend, "testdata/mta_static.zip")
g, _ := GTFSTest_LoadStaticFile(t, backend, "testdata/mta_static.zip")

// Let's look at the G33S stop, also known as "Bedford -
// Nostrand Avs". Between 22:50 and 23:10 there are are 6
Expand Down
Loading

0 comments on commit 1d7b82c

Please sign in to comment.