Skip to content

Commit

Permalink
tr/traffic-events-bz: wip migrated transformer, all fields mapped
Browse files Browse the repository at this point in the history
  • Loading branch information
@clezag
clezag committed Dec 27, 2024
1 parent a16bc46 commit 55a379b
Show file tree
Hide file tree
Showing 3 changed files with 126 additions and 45 deletions.
1 change: 1 addition & 0 deletions transformers/traffic-event-prov-bz/src/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ require (
github.com/kelseyhightower/envconfig v1.4.0
github.com/noi-techpark/go-bdp-client v1.0.1-0.20241223144230-9cf8308a0358
github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20241217152708-001fb6f116e0
github.com/twpayne/go-geom v1.5.7
gotest.tools/v3 v3.5.1
)

Expand Down
8 changes: 8 additions & 0 deletions transformers/traffic-event-prov-bz/src/go.sum
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
github.com/alecthomas/assert/v2 v2.10.0 h1:jjRCHsj6hBJhkmhznrCzoNpbA3zqy0fYiUcYZP/GkPY=
github.com/alecthomas/assert/v2 v2.10.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gofrs/uuid/v5 v5.3.0 h1:m0mUMr+oVYUdxpMLgSYCZiXe7PuVPnI94+OMeVBNedk=
Expand All @@ -6,6 +10,8 @@ github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
Expand All @@ -18,6 +24,8 @@ github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20241217152708-001fb6f116e0
github.com/noi-techpark/go-opendatahub-ingest v0.0.0-20241217152708-001fb6f116e0/go.mod h1:x/eTKLIBpI0ITcXvfBYxPVc6PHAQN237Uvv0afM0bOg=
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/twpayne/go-geom v1.5.7 h1:7fdceDUr03/MP7rAKOaTV6x9njMiQdxB/D0PDzMTCDc=
github.com/twpayne/go-geom v1.5.7/go.mod h1:y4fTAQtLedXW8eG2Yo4tYrIGN1yIwwKkmA+K3iSHKBA=
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
Expand Down
162 changes: 117 additions & 45 deletions transformers/traffic-event-prov-bz/src/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"crypto/sha1"
"encoding/json"
"fmt"
"strconv"
"strings"
"time"

Expand All @@ -17,6 +18,8 @@ import (
"github.com/noi-techpark/go-opendatahub-ingest/dto"
"github.com/noi-techpark/go-opendatahub-ingest/ms"
"github.com/noi-techpark/go-opendatahub-ingest/tr"
"github.com/twpayne/go-geom"
"github.com/twpayne/go-geom/encoding/wkt"
)

var env tr.Env
Expand All @@ -33,14 +36,119 @@ func main() {
return fmt.Errorf("could not unmarshal the raw payload json: %w", err)
}
events := []bdplib.Event{}
for d, _ := range dtos {
fmt.Print(d)
for _, d := range dtos {
j, err := makeUUIDJson(d)
if err != nil {
return err
}
uuid := makeUUID(j)
e := bdplib.Event{}
e.Uuid = uuid
e.EventSeriesUuid = uuid
e.Category = fmt.Sprintf("%s_%s | %s_%s", d.TycodeIt, d.SubTycodeIt, d.TycodeDe, d.SubTycodeDe)
e.Origin = b.Origin
e.Name = strconv.Itoa(d.MessageID)
e.Description = fmt.Sprintf("%s | %s", d.DescriptionIt, d.DescriptionDe)

if d.X != nil && d.Y != nil {
wkt, err := point2WKT(*d.X, *d.Y)
if err != nil {
return fmt.Errorf("error creating point wkt: %w", err)
}
e.WktGeometry = wkt
}

beginDate, err := time.Parse(dayDateFormat, d.BeginDate)
if err != nil {
return fmt.Errorf("error parsing BeginDate (%s): %w", d.BeginDate, err)
}
e.EventStart = beginDate.UTC().UnixMilli()

if d.EndDate != nil {
endDate, err := time.Parse(dayDateFormat, *d.EndDate)
if err != nil {
return fmt.Errorf("error parsing EndDate (%s): %w", *d.EndDate, err)
}
e.EventEnd = endDate.UTC().UnixMilli() + 1 // +1 because we exclude the upper bound.
}
e.MetaData = map[string]any{}
e.MetaData["json_featuretype"] = d.JSONFeaturetype
e.MetaData["publisherDateTime"] = d.PublishDateTime
e.MetaData["tycodeValue"] = d.TycodeValue
e.MetaData["tycodeDe"] = d.TycodeDe
e.MetaData["tycodeIt"] = d.TycodeIt
e.MetaData["subTycodeValue"] = d.SubTycodeValue
e.MetaData["subTycodeDe"] = d.SubTycodeDe
e.MetaData["subTycodeIt"] = d.SubTycodeIt
e.MetaData["placeDe"] = d.PlaceDe
e.MetaData["placeIt"] = d.PlaceIt
e.MetaData["actualMail"] = d.ActualMail
e.MetaData["messageId"] = d.MessageID
e.MetaData["messageStatus"] = d.MessageStatus
e.MetaData["messageZoneId"] = d.MessageZoneID
e.MetaData["messageZoneDescDe"] = d.MessageZoneDescDe
e.MetaData["messageZoneDescIt"] = d.MessageZoneDescIt
e.MetaData["messageGradId"] = d.MessageGradID
e.MetaData["messageGradDescDe"] = d.MessageGradDescDe
e.MetaData["messageGradDescIt"] = d.MessageGradDescIt
e.MetaData["messageStreetId"] = d.MessageStreetID
e.MetaData["messageStreetWapDescDe"] = d.MessageStreetWapDescDe
e.MetaData["messageStreetWapDescIt"] = d.MessageStreetWapDescIt
e.MetaData["messageStreetInternetDescDe"] = d.MessageStreetInternetDescDe
e.MetaData["messageStreetInternetDescIt"] = d.MessageStreetInternetDescIt
e.MetaData["messageStreetNr"] = d.MessageStreetNr
e.MetaData["messageStreetHierarchie"] = d.MessageStreetHierarchie

events = append(events, e)
}

b.SyncEvents(events)
return nil
})
}

type trafficEvent struct {
JSONFeaturetype string `json:"json_featuretype"`
PublishDateTime string `json:"publishDateTime"`
BeginDate string `json:"beginDate"`
EndDate *string `json:"endDate"`
DescriptionDe string `json:"descriptionDe"`
DescriptionIt string `json:"descriptionIt"`
TycodeValue string `json:"tycodeValue"`
TycodeDe string `json:"tycodeDe"`
TycodeIt string `json:"tycodeIt"`
SubTycodeValue string `json:"subTycodeValue"`
SubTycodeDe string `json:"subTycodeDe"`
SubTycodeIt string `json:"subTycodeIt"`
PlaceDe string `json:"placeDe"`
PlaceIt string `json:"placeIt"`
ActualMail int `json:"actualMail"`
MessageID int `json:"messageId"`
MessageStatus int `json:"messageStatus"`
MessageZoneID int `json:"messageZoneId"`
MessageZoneDescDe string `json:"messageZoneDescDe"`
MessageZoneDescIt string `json:"messageZoneDescIt"`
MessageGradID int `json:"messageGradId"`
MessageGradDescDe string `json:"messageGradDescDe"`
MessageGradDescIt string `json:"messageGradDescIt"`
MessageStreetID int `json:"messageStreetId"`
MessageStreetWapDescDe string `json:"messageStreetWapDescDe"`
MessageStreetWapDescIt string `json:"messageStreetWapDescIt"`
MessageStreetInternetDescDe string `json:"messageStreetInternetDescDe"`
MessageStreetInternetDescIt string `json:"messageStreetInternetDescIt"`
MessageStreetNr string `json:"messageStreetNr"`
MessageStreetHierarchie int `json:"messageStreetHierarchie"`
MessageTypeID int `json:"messageTypeId"`
MessageTypeDescDe string `json:"messageTypeDescDe"`
MessageTypeDescIt string `json:"messageTypeDescIt"`
X *float64 `json:"X"`
Y *float64 `json:"Y"`
}

func point2WKT(x float64, y float64) (string, error) {
p := geom.NewPointFlat(geom.XY, []float64{x, y})
p.SetSRID(4326)
return wkt.Marshal(p)
}

// All this strange UUID stuff is just there to replicate the UUID generation originally done in Java, and thus maintain primary key compatibility.
Expand All @@ -52,8 +160,8 @@ func main() {
type UUIDMap struct {
BeginDate *UUIDDate `json:"beginDate"`
EndDate *UUIDDate `json:"endDate"`
X float64 `json:"X"`
Y float64 `json:"Y"`
X *float64 `json:"X"`
Y *float64 `json:"Y"`
}
type UUIDDate struct {
Year int `json:"year"`
Expand All @@ -70,8 +178,10 @@ type UUIDDate struct {
} `json:"chronology"`
}

const dayDateFormat = "2006-01-02"

func toDate(s string) (UUIDDate, error) {
d, err := time.Parse("2006-01-02", s)
d, err := time.Parse(dayDateFormat, s)
if err != nil {
return UUIDDate{}, err
}
Expand Down Expand Up @@ -108,8 +218,8 @@ func makeUUIDJson(e trafficEvent) (string, error) {
return "", fmt.Errorf("cannot parse beginDate: %w", err)
}
u.BeginDate = &begin
if e.EndDate != "" {
end, err := toDate(e.EndDate)
if e.EndDate != nil {
end, err := toDate(*e.EndDate)
if err != nil {
return "", fmt.Errorf("cannot parse endDate: %w", err)
}
Expand All @@ -124,41 +234,3 @@ func makeUUIDJson(e trafficEvent) (string, error) {
jsonString := string(jsonBytes[:])
return jsonString, nil
}

type trafficEvent struct {
JSONFeaturetype string `json:"json_featuretype"`
PublishDateTime string `json:"publishDateTime"`
BeginDate string `json:"beginDate"`
EndDate string `json:"endDate"`
DescriptionDe string `json:"descriptionDe"`
DescriptionIt string `json:"descriptionIt"`
TycodeValue string `json:"tycodeValue"`
TycodeDe string `json:"tycodeDe"`
TycodeIt string `json:"tycodeIt"`
SubTycodeValue string `json:"subTycodeValue"`
SubTycodeDe string `json:"subTycodeDe"`
SubTycodeIt string `json:"subTycodeIt"`
PlaceDe string `json:"placeDe"`
PlaceIt string `json:"placeIt"`
ActualMail int `json:"actualMail"`
MessageID int `json:"messageId"`
MessageStatus int `json:"messageStatus"`
MessageZoneID int `json:"messageZoneId"`
MessageZoneDescDe string `json:"messageZoneDescDe"`
MessageZoneDescIt string `json:"messageZoneDescIt"`
MessageGradID int `json:"messageGradId"`
MessageGradDescDe string `json:"messageGradDescDe"`
MessageGradDescIt string `json:"messageGradDescIt"`
MessageStreetID int `json:"messageStreetId"`
MessageStreetWapDescDe string `json:"messageStreetWapDescDe"`
MessageStreetWapDescIt string `json:"messageStreetWapDescIt"`
MessageStreetInternetDescDe string `json:"messageStreetInternetDescDe"`
MessageStreetInternetDescIt string `json:"messageStreetInternetDescIt"`
MessageStreetNr string `json:"messageStreetNr"`
MessageStreetHierarchie int `json:"messageStreetHierarchie"`
MessageTypeID int `json:"messageTypeId"`
MessageTypeDescDe string `json:"messageTypeDescDe"`
MessageTypeDescIt string `json:"messageTypeDescIt"`
X float64 `json:"X"`
Y float64 `json:"Y"`
}

0 comments on commit 55a379b

Please sign in to comment.