diff --git a/transformers/traffic-event-prov-bz/src/go.mod b/transformers/traffic-event-prov-bz/src/go.mod index e3db49c..e512261 100644 --- a/transformers/traffic-event-prov-bz/src/go.mod +++ b/transformers/traffic-event-prov-bz/src/go.mod @@ -7,6 +7,7 @@ require ( github.com/kelseyhightower/envconfig v1.4.0 github.com/noi-techpark/go-bdp-client v1.0.1-0.20241223144230-9cf8308a0358 github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20241217152708-001fb6f116e0 + github.com/twpayne/go-geom v1.5.7 gotest.tools/v3 v3.5.1 ) diff --git a/transformers/traffic-event-prov-bz/src/go.sum b/transformers/traffic-event-prov-bz/src/go.sum index b4466f7..a4bc8e7 100644 --- a/transformers/traffic-event-prov-bz/src/go.sum +++ b/transformers/traffic-event-prov-bz/src/go.sum @@ -1,3 +1,7 @@ +github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZP/GkPY= +github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= +github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc= +github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gofrs/uuid/v5 v5.3.0 h1:m0mUMr+oVYUdxpMLgSYCZiXe7PuVPnI94+OMeVBNedk= @@ -6,6 +10,8 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= @@ -18,6 +24,8 @@ github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20241217152708-001fb6f116e0 github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20241217152708-001fb6f116e0/go.mod h1:x/eTKLIBpI0ITcXvfBYxPVc6PHAQN237Uvv0afM0bOg= github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/twpayne/go-geom v1.5.7 h1:7fdceDUr03/MP7rAKOaTV6x9njMiQdxB/D0PDzMTCDc= +github.com/twpayne/go-geom v1.5.7/go.mod h1:y4fTAQtLedXW8eG2Yo4tYrIGN1yIwwKkmA+K3iSHKBA= github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= diff --git a/transformers/traffic-event-prov-bz/src/main.go b/transformers/traffic-event-prov-bz/src/main.go index 92984dc..11a593a 100644 --- a/transformers/traffic-event-prov-bz/src/main.go +++ b/transformers/traffic-event-prov-bz/src/main.go @@ -8,6 +8,7 @@ import ( "crypto/sha1" "encoding/json" "fmt" + "strconv" "strings" "time" @@ -17,6 +18,8 @@ import ( "github.com/noi-techpark/go-opendatahub-ingest/dto" "github.com/noi-techpark/go-opendatahub-ingest/ms" "github.com/noi-techpark/go-opendatahub-ingest/tr" + "github.com/twpayne/go-geom" + "github.com/twpayne/go-geom/encoding/wkt" ) var env tr.Env @@ -33,14 +36,119 @@ func main() { return fmt.Errorf("could not unmarshal the raw payload json: %w", err) } events := []bdplib.Event{} - for d, _ := range dtos { - fmt.Print(d) + for _, d := range dtos { + j, err := makeUUIDJson(d) + if err != nil { + return err + } + uuid := makeUUID(j) + e := bdplib.Event{} + e.Uuid = uuid + e.EventSeriesUuid = uuid + e.Category = fmt.Sprintf("%s_%s | %s_%s", d.TycodeIt, d.SubTycodeIt, d.TycodeDe, d.SubTycodeDe) + e.Origin = b.Origin + e.Name = strconv.Itoa(d.MessageID) + e.Description = fmt.Sprintf("%s | %s", d.DescriptionIt, d.DescriptionDe) + + if d.X != nil && d.Y != nil { + wkt, err := point2WKT(*d.X, *d.Y) + if err != nil { + return fmt.Errorf("error creating point wkt: %w", err) + } + e.WktGeometry = wkt + } + + beginDate, err := time.Parse(dayDateFormat, d.BeginDate) + if err != nil { + return fmt.Errorf("error parsing BeginDate (%s): %w", d.BeginDate, err) + } + e.EventStart = beginDate.UTC().UnixMilli() + + if d.EndDate != nil { + endDate, err := time.Parse(dayDateFormat, *d.EndDate) + if err != nil { + return fmt.Errorf("error parsing EndDate (%s): %w", *d.EndDate, err) + } + e.EventEnd = endDate.UTC().UnixMilli() + 1 // +1 because we exclude the upper bound. + } + e.MetaData = map[string]any{} + e.MetaData["json_featuretype"] = d.JSONFeaturetype + e.MetaData["publisherDateTime"] = d.PublishDateTime + e.MetaData["tycodeValue"] = d.TycodeValue + e.MetaData["tycodeDe"] = d.TycodeDe + e.MetaData["tycodeIt"] = d.TycodeIt + e.MetaData["subTycodeValue"] = d.SubTycodeValue + e.MetaData["subTycodeDe"] = d.SubTycodeDe + e.MetaData["subTycodeIt"] = d.SubTycodeIt + e.MetaData["placeDe"] = d.PlaceDe + e.MetaData["placeIt"] = d.PlaceIt + e.MetaData["actualMail"] = d.ActualMail + e.MetaData["messageId"] = d.MessageID + e.MetaData["messageStatus"] = d.MessageStatus + e.MetaData["messageZoneId"] = d.MessageZoneID + e.MetaData["messageZoneDescDe"] = d.MessageZoneDescDe + e.MetaData["messageZoneDescIt"] = d.MessageZoneDescIt + e.MetaData["messageGradId"] = d.MessageGradID + e.MetaData["messageGradDescDe"] = d.MessageGradDescDe + e.MetaData["messageGradDescIt"] = d.MessageGradDescIt + e.MetaData["messageStreetId"] = d.MessageStreetID + e.MetaData["messageStreetWapDescDe"] = d.MessageStreetWapDescDe + e.MetaData["messageStreetWapDescIt"] = d.MessageStreetWapDescIt + e.MetaData["messageStreetInternetDescDe"] = d.MessageStreetInternetDescDe + e.MetaData["messageStreetInternetDescIt"] = d.MessageStreetInternetDescIt + e.MetaData["messageStreetNr"] = d.MessageStreetNr + e.MetaData["messageStreetHierarchie"] = d.MessageStreetHierarchie + + events = append(events, e) } b.SyncEvents(events) return nil }) +} + +type trafficEvent struct { + JSONFeaturetype string `json:"json_featuretype"` + PublishDateTime string `json:"publishDateTime"` + BeginDate string `json:"beginDate"` + EndDate *string `json:"endDate"` + DescriptionDe string `json:"descriptionDe"` + DescriptionIt string `json:"descriptionIt"` + TycodeValue string `json:"tycodeValue"` + TycodeDe string `json:"tycodeDe"` + TycodeIt string `json:"tycodeIt"` + SubTycodeValue string `json:"subTycodeValue"` + SubTycodeDe string `json:"subTycodeDe"` + SubTycodeIt string `json:"subTycodeIt"` + PlaceDe string `json:"placeDe"` + PlaceIt string `json:"placeIt"` + ActualMail int `json:"actualMail"` + MessageID int `json:"messageId"` + MessageStatus int `json:"messageStatus"` + MessageZoneID int `json:"messageZoneId"` + MessageZoneDescDe string `json:"messageZoneDescDe"` + MessageZoneDescIt string `json:"messageZoneDescIt"` + MessageGradID int `json:"messageGradId"` + MessageGradDescDe string `json:"messageGradDescDe"` + MessageGradDescIt string `json:"messageGradDescIt"` + MessageStreetID int `json:"messageStreetId"` + MessageStreetWapDescDe string `json:"messageStreetWapDescDe"` + MessageStreetWapDescIt string `json:"messageStreetWapDescIt"` + MessageStreetInternetDescDe string `json:"messageStreetInternetDescDe"` + MessageStreetInternetDescIt string `json:"messageStreetInternetDescIt"` + MessageStreetNr string `json:"messageStreetNr"` + MessageStreetHierarchie int `json:"messageStreetHierarchie"` + MessageTypeID int `json:"messageTypeId"` + MessageTypeDescDe string `json:"messageTypeDescDe"` + MessageTypeDescIt string `json:"messageTypeDescIt"` + X *float64 `json:"X"` + Y *float64 `json:"Y"` +} +func point2WKT(x float64, y float64) (string, error) { + p := geom.NewPointFlat(geom.XY, []float64{x, y}) + p.SetSRID(4326) + return wkt.Marshal(p) } // All this strange UUID stuff is just there to replicate the UUID generation originally done in Java, and thus maintain primary key compatibility. @@ -52,8 +160,8 @@ func main() { type UUIDMap struct { BeginDate *UUIDDate `json:"beginDate"` EndDate *UUIDDate `json:"endDate"` - X float64 `json:"X"` - Y float64 `json:"Y"` + X *float64 `json:"X"` + Y *float64 `json:"Y"` } type UUIDDate struct { Year int `json:"year"` @@ -70,8 +178,10 @@ type UUIDDate struct { } `json:"chronology"` } +const dayDateFormat = "2006-01-02" + func toDate(s string) (UUIDDate, error) { - d, err := time.Parse("2006-01-02", s) + d, err := time.Parse(dayDateFormat, s) if err != nil { return UUIDDate{}, err } @@ -108,8 +218,8 @@ func makeUUIDJson(e trafficEvent) (string, error) { return "", fmt.Errorf("cannot parse beginDate: %w", err) } u.BeginDate = &begin - if e.EndDate != "" { - end, err := toDate(e.EndDate) + if e.EndDate != nil { + end, err := toDate(*e.EndDate) if err != nil { return "", fmt.Errorf("cannot parse endDate: %w", err) } @@ -124,41 +234,3 @@ func makeUUIDJson(e trafficEvent) (string, error) { jsonString := string(jsonBytes[:]) return jsonString, nil } - -type trafficEvent struct { - JSONFeaturetype string `json:"json_featuretype"` - PublishDateTime string `json:"publishDateTime"` - BeginDate string `json:"beginDate"` - EndDate string `json:"endDate"` - DescriptionDe string `json:"descriptionDe"` - DescriptionIt string `json:"descriptionIt"` - TycodeValue string `json:"tycodeValue"` - TycodeDe string `json:"tycodeDe"` - TycodeIt string `json:"tycodeIt"` - SubTycodeValue string `json:"subTycodeValue"` - SubTycodeDe string `json:"subTycodeDe"` - SubTycodeIt string `json:"subTycodeIt"` - PlaceDe string `json:"placeDe"` - PlaceIt string `json:"placeIt"` - ActualMail int `json:"actualMail"` - MessageID int `json:"messageId"` - MessageStatus int `json:"messageStatus"` - MessageZoneID int `json:"messageZoneId"` - MessageZoneDescDe string `json:"messageZoneDescDe"` - MessageZoneDescIt string `json:"messageZoneDescIt"` - MessageGradID int `json:"messageGradId"` - MessageGradDescDe string `json:"messageGradDescDe"` - MessageGradDescIt string `json:"messageGradDescIt"` - MessageStreetID int `json:"messageStreetId"` - MessageStreetWapDescDe string `json:"messageStreetWapDescDe"` - MessageStreetWapDescIt string `json:"messageStreetWapDescIt"` - MessageStreetInternetDescDe string `json:"messageStreetInternetDescDe"` - MessageStreetInternetDescIt string `json:"messageStreetInternetDescIt"` - MessageStreetNr string `json:"messageStreetNr"` - MessageStreetHierarchie int `json:"messageStreetHierarchie"` - MessageTypeID int `json:"messageTypeId"` - MessageTypeDescDe string `json:"messageTypeDescDe"` - MessageTypeDescIt string `json:"messageTypeDescIt"` - X float64 `json:"X"` - Y float64 `json:"Y"` -}