diff --git a/transformers/traffic-event-prov-bz/src/go.mod b/transformers/traffic-event-prov-bz/src/go.mod index 28b0285..e3db49c 100644 --- a/transformers/traffic-event-prov-bz/src/go.mod +++ b/transformers/traffic-event-prov-bz/src/go.mod @@ -3,13 +3,16 @@ module opendatahub.com/tr-traffic-event-prov-bz go 1.23.4 require ( + github.com/gofrs/uuid/v5 v5.3.0 github.com/kelseyhightower/envconfig v1.4.0 github.com/noi-techpark/go-bdp-client v1.0.1-0.20241223144230-9cf8308a0358 github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20241217152708-001fb6f116e0 + gotest.tools/v3 v3.5.1 ) require ( github.com/golang/snappy v0.0.4 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/klauspost/compress v1.17.11 // indirect github.com/montanaflynn/stats v0.7.1 // indirect github.com/rabbitmq/amqp091-go v1.10.0 // indirect diff --git a/transformers/traffic-event-prov-bz/src/go.sum b/transformers/traffic-event-prov-bz/src/go.sum index 03674aa..b4466f7 100644 --- a/transformers/traffic-event-prov-bz/src/go.sum +++ b/transformers/traffic-event-prov-bz/src/go.sum @@ -1,5 +1,7 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gofrs/uuid/v5 v5.3.0 h1:m0mUMr+oVYUdxpMLgSYCZiXe7PuVPnI94+OMeVBNedk= +github.com/gofrs/uuid/v5 v5.3.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -58,3 +60,5 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= diff --git a/transformers/traffic-event-prov-bz/src/main.go b/transformers/traffic-event-prov-bz/src/main.go index 2d92ce3..92984dc 100644 --- a/transformers/traffic-event-prov-bz/src/main.go +++ b/transformers/traffic-event-prov-bz/src/main.go @@ -5,9 +5,13 @@ package main import ( + "crypto/sha1" "encoding/json" "fmt" + "strings" + "time" + "github.com/gofrs/uuid/v5" "github.com/kelseyhightower/envconfig" "github.com/noi-techpark/go-bdp-client/bdplib" "github.com/noi-techpark/go-opendatahub-ingest/dto" @@ -24,7 +28,7 @@ func main() { b := bdplib.FromEnv() tr.ListenFromEnv(env, func(r *dto.Raw[string]) error { - dtos := trafficEvent{} + dtos := []trafficEvent{} if err := json.Unmarshal([]byte(r.Rawdata), &dtos); err != nil { return fmt.Errorf("could not unmarshal the raw payload json: %w", err) } @@ -39,11 +43,89 @@ func main() { } -func getUuidFields(e trafficEvent) map[string]any { - return nil +// All this strange UUID stuff is just there to replicate the UUID generation originally done in Java, and thus maintain primary key compatibility. +// Unfortunately, it uses quite particular behavior that is elaborate to replicate. +// In essence, it generates a JSON from a few fields, and then calculates a v5 UUID (with namespace = null) + +// TODO: make UUID generation more sane, but make sure to migrate all existing events in DB, an clear up if changing the UUIDs could lead to problems + +type UUIDMap struct { + BeginDate *UUIDDate `json:"beginDate"` + EndDate *UUIDDate `json:"endDate"` + X float64 `json:"X"` + Y float64 `json:"Y"` +} +type UUIDDate struct { + Year int `json:"year"` + Month string `json:"month"` + DayOfWeek string `json:"dayOfWeek"` + LeapYear bool `json:"leapYear"` + DayOfMonth int `json:"dayOfMonth"` + MonthValue int `json:"monthValue"` + Era string `json:"era"` + DayOfYear int `json:"dayOfYear"` + Chronology struct { + CalendarType string `json:"calendarType"` + ID string `json:"id"` + } `json:"chronology"` +} + +func toDate(s string) (UUIDDate, error) { + d, err := time.Parse("2006-01-02", s) + if err != nil { + return UUIDDate{}, err + } + ret := UUIDDate{} + ret.Year = d.Year() + ret.Month = strings.ToUpper(d.Month().String()) + ret.DayOfWeek = strings.ToUpper(d.Weekday().String()) + ret.LeapYear = ret.Year%4 == 0 && ret.Year%100 != 0 || ret.Year%400 == 0 + ret.DayOfMonth = d.Day() + ret.MonthValue = int(d.Month()) + ret.Era = "CE" + ret.DayOfYear = d.YearDay() + ret.Chronology.CalendarType = "iso8601" + ret.Chronology.ID = "ISO" + + return ret, nil +} + +func makeUUID(name string) string { + // Have to do the v5 UUID ourselves, because the one from library does not support a nil namespace. The code is copied from there, sans the namespace part + hash := sha1.New() + hash.Write([]byte(name)) + u := uuid.UUID{} + copy(u[:], hash.Sum(nil)) + u.SetVersion(uuid.V5) + u.SetVariant(uuid.VariantRFC9562) + return u.String() +} + +func makeUUIDJson(e trafficEvent) (string, error) { + u := UUIDMap{} + begin, err := toDate(e.BeginDate) + if err != nil { + return "", fmt.Errorf("cannot parse beginDate: %w", err) + } + u.BeginDate = &begin + if e.EndDate != "" { + end, err := toDate(e.EndDate) + if err != nil { + return "", fmt.Errorf("cannot parse endDate: %w", err) + } + u.EndDate = &end + } + u.X = e.X + u.Y = e.Y + jsonBytes, err := json.Marshal(u) + if err != nil { + return "", fmt.Errorf("cannot marshal uuid json: %w", err) + } + jsonString := string(jsonBytes[:]) + return jsonString, nil } -type trafficEvent []struct { +type trafficEvent struct { JSONFeaturetype string `json:"json_featuretype"` PublishDateTime string `json:"publishDateTime"` BeginDate string `json:"beginDate"` diff --git a/transformers/traffic-event-prov-bz/src/main_test.go b/transformers/traffic-event-prov-bz/src/main_test.go new file mode 100644 index 0000000..e6678c1 --- /dev/null +++ b/transformers/traffic-event-prov-bz/src/main_test.go @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: 2024 NOI Techpark +// +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "testing" + + "gotest.tools/v3/assert" +) + +func Test_makeUUIDJson(t *testing.T) { + e := trafficEvent{} + e.BeginDate = "2025-01-07" + e.EndDate = "2025-02-13" + e.X = 11.1893940941287 + e.Y = 46.6715162831429 + testString := "{\"beginDate\":{\"year\":2025,\"month\":\"JANUARY\",\"dayOfWeek\":\"TUESDAY\",\"leapYear\":false,\"dayOfMonth\":7,\"monthValue\":1,\"era\":\"CE\",\"dayOfYear\":7,\"chronology\":{\"calendarType\":\"iso8601\",\"id\":\"ISO\"}},\"endDate\":{\"year\":2025,\"month\":\"FEBRUARY\",\"dayOfWeek\":\"THURSDAY\",\"leapYear\":false,\"dayOfMonth\":13,\"monthValue\":2,\"era\":\"CE\",\"dayOfYear\":44,\"chronology\":{\"calendarType\":\"iso8601\",\"id\":\"ISO\"}},\"X\":11.1893940941287,\"Y\":46.6715162831429}" + uuidJson, err := makeUUIDJson(e) + assert.NilError(t, err, "failed creating json") + assert.Equal(t, uuidJson, testString) + assert.Equal(t, makeUUID(uuidJson), "c14c2e9b-5044-5255-b422-c790cd95495d") + + // handle null date + e.BeginDate = "2024-09-30" + e.EndDate = "" + e.X = 11.4555831531882 + e.Y = 46.4466206755139 + testString = "{\"beginDate\":{\"year\":2024,\"month\":\"SEPTEMBER\",\"dayOfWeek\":\"MONDAY\",\"leapYear\":true,\"dayOfMonth\":30,\"monthValue\":9,\"era\":\"CE\",\"dayOfYear\":274,\"chronology\":{\"calendarType\":\"iso8601\",\"id\":\"ISO\"}},\"endDate\":null,\"X\":11.4555831531882,\"Y\":46.4466206755139}" + uuidJson, err = makeUUIDJson(e) + assert.NilError(t, err, "failed creating json") + assert.Equal(t, uuidJson, testString) + assert.Equal(t, makeUUID(uuidJson), "477bcef1-82ed-5665-ada9-bc1905619e12") +}