Skip to content

Commit

Permalink
tr/traffic-event-bz: refactoring and testing
Browse files Browse the repository at this point in the history
  • Loading branch information
@clezag
clezag committed Dec 30, 2024
1 parent 55a379b commit 2b79257
Show file tree
Hide file tree
Showing 6 changed files with 151 additions and 81 deletions.
7 changes: 6 additions & 1 deletion transformers/traffic-event-prov-bz/src/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go 1.23.4
require (
github.com/gofrs/uuid/v5 v5.3.0
github.com/kelseyhightower/envconfig v1.4.0
github.com/noi-techpark/go-bdp-client v1.0.1-0.20241223144230-9cf8308a0358
github.com/noi-techpark/go-bdp-client v1.0.1-0.20241230091831-3fd536c600ac
github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20241217152708-001fb6f116e0
github.com/twpayne/go-geom v1.5.7
gotest.tools/v3 v3.5.1
Expand All @@ -17,6 +17,11 @@ require (
github.com/klauspost/compress v1.17.11 // indirect
github.com/montanaflynn/stats v0.7.1 // indirect
github.com/rabbitmq/amqp091-go v1.10.0 // indirect
github.com/tidwall/gjson v1.18.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
github.com/tidwall/sjson v1.2.5 // indirect
github.com/wI2L/jsondiff v0.6.1 // indirect
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
github.com/xdg-go/scram v1.1.2 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
Expand Down
16 changes: 14 additions & 2 deletions transformers/traffic-event-prov-bz/src/go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,26 @@ github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IX
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
github.com/noi-techpark/go-bdp-client v1.0.1-0.20241223144230-9cf8308a0358 h1:DEg4/ybQ8ryyM6llXAN9iS0IYJRr+JlaU/0HeMI1EGQ=
github.com/noi-techpark/go-bdp-client v1.0.1-0.20241223144230-9cf8308a0358/go.mod h1:aooKwED49M7Au+9Y/o8wW/4yggIvaVRHc0JJvPnS10c=
github.com/noi-techpark/go-bdp-client v1.0.1-0.20241230091831-3fd536c600ac h1:1fAeA0dplz+uRhi8CQcdzTh1ORqMUBA+mDIQSf12QyA=
github.com/noi-techpark/go-bdp-client v1.0.1-0.20241230091831-3fd536c600ac/go.mod h1:aooKwED49M7Au+9Y/o8wW/4yggIvaVRHc0JJvPnS10c=
github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20241217152708-001fb6f116e0 h1:+jstfT0wGjInqpaimKTcQofYio83PJxM6CyqjK920zs=
github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20241217152708-001fb6f116e0/go.mod h1:x/eTKLIBpI0ITcXvfBYxPVc6PHAQN237Uvv0afM0bOg=
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
github.com/twpayne/go-geom v1.5.7 h1:7fdceDUr03/MP7rAKOaTV6x9njMiQdxB/D0PDzMTCDc=
github.com/twpayne/go-geom v1.5.7/go.mod h1:y4fTAQtLedXW8eG2Yo4tYrIGN1yIwwKkmA+K3iSHKBA=
github.com/wI2L/jsondiff v0.6.1 h1:ISZb9oNWbP64LHnu4AUhsMF5W0FIj5Ok3Krip9Shqpw=
github.com/wI2L/jsondiff v0.6.1/go.mod h1:KAEIojdQq66oJiHhDyQez2x+sRit0vIzC9KeK0yizxM=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
Expand Down
137 changes: 76 additions & 61 deletions transformers/traffic-event-prov-bz/src/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,74 +31,17 @@ func main() {
b := bdplib.FromEnv()

tr.ListenFromEnv(env, func(r *dto.Raw[string]) error {
dtos := []trafficEvent{}
if err := json.Unmarshal([]byte(r.Rawdata), &dtos); err != nil {
dtos, err := unmarshalRawJson(r.Rawdata)
if err != nil {
return fmt.Errorf("could not unmarshal the raw payload json: %w", err)
}
events := []bdplib.Event{}
for _, d := range dtos {
j, err := makeUUIDJson(d)
e, err := mapEvent(d)
if err != nil {
return err
}
uuid := makeUUID(j)
e := bdplib.Event{}
e.Uuid = uuid
e.EventSeriesUuid = uuid
e.Category = fmt.Sprintf("%s_%s | %s_%s", d.TycodeIt, d.SubTycodeIt, d.TycodeDe, d.SubTycodeDe)
e.Origin = b.Origin
e.Name = strconv.Itoa(d.MessageID)
e.Description = fmt.Sprintf("%s | %s", d.DescriptionIt, d.DescriptionDe)

if d.X != nil && d.Y != nil {
wkt, err := point2WKT(*d.X, *d.Y)
if err != nil {
return fmt.Errorf("error creating point wkt: %w", err)
}
e.WktGeometry = wkt
}

beginDate, err := time.Parse(dayDateFormat, d.BeginDate)
if err != nil {
return fmt.Errorf("error parsing BeginDate (%s): %w", d.BeginDate, err)
}
e.EventStart = beginDate.UTC().UnixMilli()

if d.EndDate != nil {
endDate, err := time.Parse(dayDateFormat, *d.EndDate)
if err != nil {
return fmt.Errorf("error parsing EndDate (%s): %w", *d.EndDate, err)
}
e.EventEnd = endDate.UTC().UnixMilli() + 1 // +1 because we exclude the upper bound.
}
e.MetaData = map[string]any{}
e.MetaData["json_featuretype"] = d.JSONFeaturetype
e.MetaData["publisherDateTime"] = d.PublishDateTime
e.MetaData["tycodeValue"] = d.TycodeValue
e.MetaData["tycodeDe"] = d.TycodeDe
e.MetaData["tycodeIt"] = d.TycodeIt
e.MetaData["subTycodeValue"] = d.SubTycodeValue
e.MetaData["subTycodeDe"] = d.SubTycodeDe
e.MetaData["subTycodeIt"] = d.SubTycodeIt
e.MetaData["placeDe"] = d.PlaceDe
e.MetaData["placeIt"] = d.PlaceIt
e.MetaData["actualMail"] = d.ActualMail
e.MetaData["messageId"] = d.MessageID
e.MetaData["messageStatus"] = d.MessageStatus
e.MetaData["messageZoneId"] = d.MessageZoneID
e.MetaData["messageZoneDescDe"] = d.MessageZoneDescDe
e.MetaData["messageZoneDescIt"] = d.MessageZoneDescIt
e.MetaData["messageGradId"] = d.MessageGradID
e.MetaData["messageGradDescDe"] = d.MessageGradDescDe
e.MetaData["messageGradDescIt"] = d.MessageGradDescIt
e.MetaData["messageStreetId"] = d.MessageStreetID
e.MetaData["messageStreetWapDescDe"] = d.MessageStreetWapDescDe
e.MetaData["messageStreetWapDescIt"] = d.MessageStreetWapDescIt
e.MetaData["messageStreetInternetDescDe"] = d.MessageStreetInternetDescDe
e.MetaData["messageStreetInternetDescIt"] = d.MessageStreetInternetDescIt
e.MetaData["messageStreetNr"] = d.MessageStreetNr
e.MetaData["messageStreetHierarchie"] = d.MessageStreetHierarchie

events = append(events, e)
}

Expand All @@ -107,6 +50,78 @@ func main() {
})
}

func unmarshalRawJson(s string) ([]trafficEvent, error) {
dtos := []trafficEvent{}
err := json.Unmarshal([]byte(s), &dtos)
return dtos, err
}

func mapEvent(d trafficEvent) (bdplib.Event, error) {
e := bdplib.Event{}
j, err := makeUUIDJson(d)
if err != nil {
return e, err
}
uuid := makeUUID(j)
e.Uuid = uuid
e.EventSeriesUuid = uuid
e.Category = fmt.Sprintf("%s_%s | %s_%s", d.TycodeIt, d.SubTycodeIt, d.TycodeDe, d.SubTycodeDe)
e.Name = strconv.Itoa(d.MessageID)
e.Description = fmt.Sprintf("%s | %s", d.DescriptionIt, d.DescriptionDe)

if d.X != nil && d.Y != nil {
wkt, err := point2WKT(*d.X, *d.Y)
if err != nil {
return e, fmt.Errorf("error creating point wkt: %w", err)
}
e.WktGeometry = wkt
}

beginDate, err := time.Parse(dayDateFormat, d.BeginDate)
if err != nil {
return e, fmt.Errorf("error parsing BeginDate (%s): %w", d.BeginDate, err)
}
e.EventStart = beginDate.UTC().UnixMilli()

if d.EndDate != nil && *d.EndDate != "" {
endDate, err := time.Parse(dayDateFormat, *d.EndDate)
if err != nil {
return e, fmt.Errorf("error parsing EndDate (%s): %w", *d.EndDate, err)
}
e.EventEnd = endDate.UTC().UnixMilli() + 1 // +1 because we exclude the upper bound.
}

e.MetaData = map[string]any{}
e.MetaData["json_featuretype"] = d.JSONFeaturetype
e.MetaData["publisherDateTime"] = d.PublishDateTime
e.MetaData["tycodeValue"] = d.TycodeValue
e.MetaData["tycodeDe"] = d.TycodeDe
e.MetaData["tycodeIt"] = d.TycodeIt
e.MetaData["subTycodeValue"] = d.SubTycodeValue
e.MetaData["subTycodeDe"] = d.SubTycodeDe
e.MetaData["subTycodeIt"] = d.SubTycodeIt
e.MetaData["placeDe"] = d.PlaceDe
e.MetaData["placeIt"] = d.PlaceIt
e.MetaData["actualMail"] = d.ActualMail
e.MetaData["messageId"] = d.MessageID
e.MetaData["messageStatus"] = d.MessageStatus
e.MetaData["messageZoneId"] = d.MessageZoneID
e.MetaData["messageZoneDescDe"] = d.MessageZoneDescDe
e.MetaData["messageZoneDescIt"] = d.MessageZoneDescIt
e.MetaData["messageGradId"] = d.MessageGradID
e.MetaData["messageGradDescDe"] = d.MessageGradDescDe
e.MetaData["messageGradDescIt"] = d.MessageGradDescIt
e.MetaData["messageStreetId"] = d.MessageStreetID
e.MetaData["messageStreetWapDescDe"] = d.MessageStreetWapDescDe
e.MetaData["messageStreetWapDescIt"] = d.MessageStreetWapDescIt
e.MetaData["messageStreetInternetDescDe"] = d.MessageStreetInternetDescDe
e.MetaData["messageStreetInternetDescIt"] = d.MessageStreetInternetDescIt
e.MetaData["messageStreetNr"] = d.MessageStreetNr
e.MetaData["messageStreetHierarchie"] = d.MessageStreetHierarchie

return e, nil
}

type trafficEvent struct {
JSONFeaturetype string `json:"json_featuretype"`
PublishDateTime string `json:"publishDateTime"`
Expand Down Expand Up @@ -218,7 +233,7 @@ func makeUUIDJson(e trafficEvent) (string, error) {
return "", fmt.Errorf("cannot parse beginDate: %w", err)
}
u.BeginDate = &begin
if e.EndDate != nil {
if e.EndDate != nil && *e.EndDate != "" {
end, err := toDate(*e.EndDate)
if err != nil {
return "", fmt.Errorf("cannot parse endDate: %w", err)
Expand Down
70 changes: 53 additions & 17 deletions transformers/traffic-event-prov-bz/src/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,31 +5,67 @@
package main

import (
"encoding/json"
"os"
"testing"

"github.com/noi-techpark/go-bdp-client/bdplib"
"github.com/wI2L/jsondiff"
"gotest.tools/v3/assert"
)

func Test_makeUUIDJson(t *testing.T) {
func testDateJson(t *testing.T, bd string, ed string, x float64, y float64, expectedJson string) {
e := trafficEvent{}
e.BeginDate = "2025-01-07"
e.EndDate = "2025-02-13"
e.X = 11.1893940941287
e.Y = 46.6715162831429
testString := "{\"beginDate\":{\"year\":2025,\"month\":\"JANUARY\",\"dayOfWeek\":\"TUESDAY\",\"leapYear\":false,\"dayOfMonth\":7,\"monthValue\":1,\"era\":\"CE\",\"dayOfYear\":7,\"chronology\":{\"calendarType\":\"iso8601\",\"id\":\"ISO\"}},\"endDate\":{\"year\":2025,\"month\":\"FEBRUARY\",\"dayOfWeek\":\"THURSDAY\",\"leapYear\":false,\"dayOfMonth\":13,\"monthValue\":2,\"era\":\"CE\",\"dayOfYear\":44,\"chronology\":{\"calendarType\":\"iso8601\",\"id\":\"ISO\"}},\"X\":11.1893940941287,\"Y\":46.6715162831429}"
e.BeginDate = bd
e.EndDate = &ed

e.X = &x
e.Y = &y

uuidJson, err := makeUUIDJson(e)
assert.NilError(t, err, "failed creating json")
assert.Equal(t, uuidJson, testString)
assert.Equal(t, makeUUID(uuidJson), "c14c2e9b-5044-5255-b422-c790cd95495d")
assert.Equal(t, uuidJson, expectedJson)
}

func Test_makeUUIDJson(t *testing.T) {
testDateJson(t, "2025-01-07", "2025-02-13", 11.1893940941287, 46.6715162831429, "{\"beginDate\":{\"year\":2025,\"month\":\"JANUARY\",\"dayOfWeek\":\"TUESDAY\",\"leapYear\":false,\"dayOfMonth\":7,\"monthValue\":1,\"era\":\"CE\",\"dayOfYear\":7,\"chronology\":{\"calendarType\":\"iso8601\",\"id\":\"ISO\"}},\"endDate\":{\"year\":2025,\"month\":\"FEBRUARY\",\"dayOfWeek\":\"THURSDAY\",\"leapYear\":false,\"dayOfMonth\":13,\"monthValue\":2,\"era\":\"CE\",\"dayOfYear\":44,\"chronology\":{\"calendarType\":\"iso8601\",\"id\":\"ISO\"}},\"X\":11.1893940941287,\"Y\":46.6715162831429}")

// handle null date
e.BeginDate = "2024-09-30"
e.EndDate = ""
e.X = 11.4555831531882
e.Y = 46.4466206755139
testString = "{\"beginDate\":{\"year\":2024,\"month\":\"SEPTEMBER\",\"dayOfWeek\":\"MONDAY\",\"leapYear\":true,\"dayOfMonth\":30,\"monthValue\":9,\"era\":\"CE\",\"dayOfYear\":274,\"chronology\":{\"calendarType\":\"iso8601\",\"id\":\"ISO\"}},\"endDate\":null,\"X\":11.4555831531882,\"Y\":46.4466206755139}"
uuidJson, err = makeUUIDJson(e)
assert.NilError(t, err, "failed creating json")
assert.Equal(t, uuidJson, testString)
assert.Equal(t, makeUUID(uuidJson), "477bcef1-82ed-5665-ada9-bc1905619e12")
testDateJson(t, "2024-09-30", "", 11.4555831531882, 46.4466206755139, "{\"beginDate\":{\"year\":2024,\"month\":\"SEPTEMBER\",\"dayOfWeek\":\"MONDAY\",\"leapYear\":true,\"dayOfMonth\":30,\"monthValue\":9,\"era\":\"CE\",\"dayOfYear\":274,\"chronology\":{\"calendarType\":\"iso8601\",\"id\":\"ISO\"}},\"endDate\":null,\"X\":11.4555831531882,\"Y\":46.4466206755139}")
}

func Test_mapping(t *testing.T) {
// In and out JSONS are taken from the previous data collector. This is to confirm compatibility
f, err := os.ReadFile("testdata/in.json")
assert.NilError(t, err, "failed loading source events file")
in, err := unmarshalRawJson(string(f))
assert.NilError(t, err, "failed unmarshalling testing input")
evs := []bdplib.Event{}
for _, e := range in {
ev, err := mapEvent(e)
assert.NilError(t, err, "failed mapping event")
evs = append(evs, ev)
}
out, err := os.ReadFile("testdata/out.json")
assert.NilError(t, err, "failed loading target events file")
referenceEvs := []bdplib.Event{}
err = json.Unmarshal(out, &referenceEvs)
assert.NilError(t, err, "failed unmarshalling target events file")
referenceMap := map[string]bdplib.Event{}
for _, e := range referenceEvs {
referenceMap[e.Name] = e
}

for _, e := range evs {
diff, err := jsondiff.Compare(e, referenceMap[e.Name], jsondiff.Equivalent(), jsondiff.Ignores("/origin", "/uuid", "/eventSeriesUuid"))
assert.NilError(t, err, "error diffing jsons")
if len(diff) > 0 {
t.Error("Unexpected difference between input and output:")
t.Log(diff)
s, _ := json.Marshal(e)
t.Log(string(s))
s, _ = json.Marshal(referenceMap[e.Name])
t.Log(string(s))
}
}
}
1 change: 1 addition & 0 deletions transformers/traffic-event-prov-bz/src/testdata/in.json
View file

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions transformers/traffic-event-prov-bz/src/testdata/out.json
View file

Large diffs are not rendered by default.

0 comments on commit 2b79257

Please sign in to comment.