From b43039bab7f3708e5a533fec671c48ad87171e0c Mon Sep 17 00:00:00 2001 From: Clemens Zagler Date: Mon, 18 Nov 2024 16:09:00 +0100 Subject: [PATCH] tr/echarging-ocpi migrate to new shared go lib --- transformers/echarging-ocpi/src/go.mod | 18 ++-- transformers/echarging-ocpi/src/go.sum | 24 +++-- transformers/echarging-ocpi/src/listen.go | 112 ---------------------- transformers/echarging-ocpi/src/main.go | 23 +++-- transformers/echarging-ocpi/src/mq.go | 103 -------------------- 5 files changed, 39 insertions(+), 241 deletions(-) delete mode 100644 transformers/echarging-ocpi/src/listen.go delete mode 100644 transformers/echarging-ocpi/src/mq.go diff --git a/transformers/echarging-ocpi/src/go.mod b/transformers/echarging-ocpi/src/go.mod index d5671bf..2c8d501 100644 --- a/transformers/echarging-ocpi/src/go.mod +++ b/transformers/echarging-ocpi/src/go.mod @@ -1,24 +1,28 @@ module opendatahub.com/tr-echarging-ocpi -go 1.22.7 +go 1.23.3 require ( github.com/noi-techpark/go-bdp-client v1.0.0 github.com/noi-techpark/go-timeseries-client v0.0.0-20241031132739-c45d758598b3 github.com/rabbitmq/amqp091-go v1.10.0 - go.mongodb.org/mongo-driver v1.16.0 ) require ( github.com/golang/snappy v0.0.4 // indirect - github.com/kelseyhightower/envconfig v1.4.0 github.com/klauspost/compress v1.13.6 // indirect github.com/montanaflynn/stats v0.7.1 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect github.com/xdg-go/stringprep v1.0.4 // indirect - github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect - golang.org/x/crypto v0.22.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/text v0.14.0 // indirect + github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect + go.mongodb.org/mongo-driver v1.17.1 // indirect + golang.org/x/crypto v0.26.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/text v0.17.0 // indirect +) + +require ( + github.com/kelseyhightower/envconfig v1.4.0 + github.com/noi-techpark/go-odh-ingest v0.0.0-20241118150711-064a68bb7637 ) diff --git a/transformers/echarging-ocpi/src/go.sum b/transformers/echarging-ocpi/src/go.sum index fbe98a1..8b09afd 100644 --- a/transformers/echarging-ocpi/src/go.sum +++ b/transformers/echarging-ocpi/src/go.sum @@ -12,6 +12,10 @@ github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8 github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/noi-techpark/go-bdp-client v1.0.0 h1:mXRGzGehKaFVLl7gKhbXQyzL7SkvFRgGJs265mVxO5o= github.com/noi-techpark/go-bdp-client v1.0.0/go.mod h1:aooKwED49M7Au+9Y/o8wW/4yggIvaVRHc0JJvPnS10c= +github.com/noi-techpark/go-odh-ingest v0.0.0-20241118150519-e059257d6fbb h1:Y/FbZ2DwpbE4va6xyA87CuTCG1sumT1TfwVbX9Ll9uI= +github.com/noi-techpark/go-odh-ingest v0.0.0-20241118150519-e059257d6fbb/go.mod h1:xLK95jcd59jN70ld9ROLOd+GL5eB32OX8KAHSvxKSFw= +github.com/noi-techpark/go-odh-ingest v0.0.0-20241118150711-064a68bb7637 h1:ySjsTPQEBGFwHeZoUpV5/Va0mX0+t8+8SLzn4wIGANU= +github.com/noi-techpark/go-odh-ingest v0.0.0-20241118150711-064a68bb7637/go.mod h1:sXfAQqXO2vzJwO/E41V6iReVpD9Uy+NNdZm9Md71n/8= github.com/noi-techpark/go-timeseries-client v0.0.0-20241031132739-c45d758598b3 h1:991izyA4IYAgOMZzBKc917V9m138vo6ZnkSNcxjzneA= github.com/noi-techpark/go-timeseries-client v0.0.0-20241031132739-c45d758598b3/go.mod h1:HzbXTeKGUegflWeRfgwfQFduX7P7YrZydBfVzeW0D4s= github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= @@ -22,25 +26,25 @@ github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d h1:splanxYIlg+5LfHAM6xpdFEAYOk8iySO56hMFq6uLyA= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM= +github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.mongodb.org/mongo-driver v1.16.0 h1:tpRsfBJMROVHKpdGyc1BBEzzjDUWjItxbVSZ8Ls4BQ4= -go.mongodb.org/mongo-driver v1.16.0/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= +go.mongodb.org/mongo-driver v1.17.1 h1:Wic5cJIwJgSpBhe3lx3+/RybR5PiYRMpVFgO7cOHyIM= +go.mongodb.org/mongo-driver v1.17.1/go.mod h1:wwWm/+BuOddhcq3n68LKRmgk2wXzmF6s0SFOa0GINL4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -52,8 +56,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= diff --git a/transformers/echarging-ocpi/src/listen.go b/transformers/echarging-ocpi/src/listen.go deleted file mode 100644 index 05827ab..0000000 --- a/transformers/echarging-ocpi/src/listen.go +++ /dev/null @@ -1,112 +0,0 @@ -// SPDX-FileCopyrightText: 2024 NOI Techpark -// -// SPDX-License-Identifier: AGPL-3.0-or-later - -package main - -import ( - "context" - "encoding/json" - "fmt" - "log/slog" - "os" - "time" - - "github.com/rabbitmq/amqp091-go" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/bson/primitive" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" -) - -type raw[Rawtype any] struct { - Provider string - Timestamp time.Time - Rawdata Rawtype -} -type incoming struct { - Id string - Db string - Collection string -} - -func getMongo[Rawtype any](m incoming) (*raw[Rawtype], error) { - c, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(os.Getenv("MONGO_URI"))) - if err != nil { - return nil, err - } - defer c.Disconnect(context.TODO()) - id, err := primitive.ObjectIDFromHex(m.Id) - if err != nil { - return nil, err - } - r := &raw[Rawtype]{} - if err := c.Database(m.Db).Collection(m.Collection).FindOne(context.TODO(), bson.M{"_id": id}).Decode(r); err != nil { - return nil, err - } - return r, nil -} - -func getRawFrame[Rawtype any](m incoming) (*raw[Rawtype], error) { - raw, err := getMongo[Rawtype](m) - if err != nil { - return nil, fmt.Errorf("error getting raw from mongo: %w", err) - } - - slog.Debug("Dumping raw data", "dto", raw) - return raw, nil -} - -func msgReject(d *amqp091.Delivery) { - if err := d.Reject(false); err != nil { - slog.Error("error rejecting already errored message", "err", err) - panic(err) - } -} - -// Default Listen function for typical transformer with one queue -func Listen[Rawtype any](handler func(*raw[Rawtype]) error) { - r, err := RabbitConnect(os.Getenv("MQ_LISTEN_URI")) - if err != nil { - panic(err) - } - mq, err := r.Consume( - os.Getenv("MQ_LISTEN_EXCHANGE"), - os.Getenv("MQ_LISTEN_QUEUE"), - os.Getenv("MQ_LISTEN_KEY"), - os.Getenv("MQ_LISTEN_CONSUMER"), - ) - if err != nil { - panic(err) - } - HandleQueue(mq, handler) -} - -func HandleQueue[Rawtype any](mq <-chan amqp091.Delivery, handler func(*raw[Rawtype]) error) { - for msg := range mq { - slog.Debug("Received a message", "body", msg.Body) - - msgBody := incoming{} - if err := json.Unmarshal(msg.Body, &msgBody); err != nil { - slog.Error("Error unmarshalling mq message", "err", err) - msgReject(&msg) - continue - } - - rawFrame, err := getRawFrame[Rawtype](msgBody) - if err != nil { - slog.Error("Cannot get mongo raw data", "err", err, "msg", msgBody) - msgReject(&msg) - continue - } - - err = handler(rawFrame) - if err != nil { - slog.Error("Error during handling of message", "err", err) - msgReject(&msg) - continue - } - - failOnError(msg.Ack(false), "Could not ACK elaborated msg") - } -} diff --git a/transformers/echarging-ocpi/src/main.go b/transformers/echarging-ocpi/src/main.go index 869da97..e1f1a03 100644 --- a/transformers/echarging-ocpi/src/main.go +++ b/transformers/echarging-ocpi/src/main.go @@ -13,6 +13,10 @@ import ( "github.com/kelseyhightower/envconfig" "github.com/noi-techpark/go-bdp-client/bdplib" + "github.com/noi-techpark/go-odh-ingest/dto" + "github.com/noi-techpark/go-odh-ingest/mq" + "github.com/noi-techpark/go-odh-ingest/ms" + "github.com/noi-techpark/go-odh-ingest/tr" "github.com/noi-techpark/go-timeseries-client/odhts" "github.com/rabbitmq/amqp091-go" ) @@ -38,9 +42,12 @@ func syncDataTypes(b *bdplib.Bdp) { } var cfg struct { + ms.Env + MQ_URI string MQ_CONSUMER string MQ_EXCHANGE string + MONGO_URI string // for data incoming from echarging-ocpi pushes MQ_PUSH_QUEUE string @@ -51,8 +58,6 @@ var cfg struct { MQ_POLL_KEY string NINJA_URL string - - LOG_LEVEL string } type EVSERaw struct { @@ -74,27 +79,27 @@ var locDataMu = sync.Mutex{} func main() { envconfig.MustProcess("", &cfg) - initLogging(cfg.LOG_LEVEL) + ms.InitLog(cfg.LOGLEVEL) b := bdplib.FromEnv() setupNinja() syncDataTypes(b) - rabbit, err := RabbitConnect(cfg.MQ_URI) + rabbit, err := mq.Connect(cfg.MQ_URI, cfg.MQ_CONSUMER) failOnError(err, "failed connecting to rabbitmq") defer rabbit.Close() - rabbit.OnClose(func(err amqp091.Error) { + rabbit.OnClose(func(err *amqp091.Error) { slog.Error("rabbitmq connection closed unexpectedly") panic(err) }) - pushMQ, err := rabbit.Consume(cfg.MQ_EXCHANGE, cfg.MQ_PUSH_QUEUE, cfg.MQ_PUSH_KEY, cfg.MQ_CONSUMER+"-push") + pushMQ, err := rabbit.Consume(cfg.MQ_EXCHANGE, cfg.MQ_PUSH_QUEUE, cfg.MQ_PUSH_KEY) failOnError(err, "failed creating push queue") // Handle push updates, coming via OCPI endpoint - go HandleQueue(pushMQ, func(r *raw[EVSERaw]) error { + go tr.HandleQueue(pushMQ, cfg.MONGO_URI, func(r *dto.Raw[EVSERaw]) error { plugData := b.CreateDataMap() plugData.AddRecord(stationId(r.Rawdata.Params.Evse_uid, b.Origin), dtPlugStatus.Name, bdplib.CreateRecord(r.Timestamp.UnixMilli(), r.Rawdata.Body.Status, period)) @@ -134,11 +139,11 @@ func main() { return nil }) - pullMQ, err := rabbit.Consume(cfg.MQ_EXCHANGE, cfg.MQ_POLL_QUEUE, cfg.MQ_POLL_KEY, cfg.MQ_CONSUMER+"-pull") + pullMQ, err := rabbit.Consume(cfg.MQ_EXCHANGE, cfg.MQ_POLL_QUEUE, cfg.MQ_POLL_KEY) failOnError(err, "failed creating poll queue") // Handle full station details, coming a few times a day via REST poller - go HandleQueue(pullMQ, func(r *raw[[]OCPILocations]) error { + go tr.HandleQueue(pullMQ, cfg.MONGO_URI, func(r *dto.Raw[[]OCPILocations]) error { stations := []bdplib.Station{} locationData := b.CreateDataMap() plugs := []bdplib.Station{} diff --git a/transformers/echarging-ocpi/src/mq.go b/transformers/echarging-ocpi/src/mq.go deleted file mode 100644 index 2dff10c..0000000 --- a/transformers/echarging-ocpi/src/mq.go +++ /dev/null @@ -1,103 +0,0 @@ -// SPDX-FileCopyrightText: 2024 NOI Techpark -// -// SPDX-License-Identifier: AGPL-3.0-or-later - -package main - -import ( - "encoding/json" - "fmt" - "time" - - amqp "github.com/rabbitmq/amqp091-go" -) - -type mqMsg struct { - Provider string `json:"provider"` - Timestamp time.Time `json:"timestamp"` - Rawdata any `json:"rawdata"` -} - -type RabbitC struct { - Con *amqp.Connection - Ch *amqp.Channel -} - -func (r *RabbitC) Close() { - if r.Ch != nil && !r.Ch.IsClosed() { - _ = r.Ch.Close() - } - if r.Con != nil && !r.Con.IsClosed() { - _ = r.Con.Close() - } -} - -func (r *RabbitC) OnClose(handler func(amqp.Error)) { - r.Con.NotifyClose(func() chan *amqp.Error { - notifyClose := make(chan *amqp.Error) - go func() { - err := <-notifyClose - handler(*err) - }() - return notifyClose - }()) -} - -func RabbitConnect(url string) (RabbitC, error) { - r := RabbitC{} - con, err := amqp.Dial(url) - if err != nil { - return r, err - } - - ch, err := con.Channel() - if err != nil { - return r, err - } - // Default prefetch of 10 messages - ch.Qos(10, 0, false) - - r.Ch = ch - r.Con = con - - return r, nil -} - -func (r *RabbitC) Publish(msg mqMsg, exchange string) error { - payload, err := json.Marshal(msg) - if err != nil { - return fmt.Errorf("error marshalling message to json: %w", err) - } - - err = r.Ch.Publish( - exchange, // exchange - msg.Provider, // routing key - false, // mandatory - false, // immediate - amqp.Publishing{ - ContentType: "application/json", - Body: payload, - Headers: amqp.Table{"provider": msg.Provider}, - }) - - if err != nil { - return fmt.Errorf("error sending amqp msg: %w", err) - } - return nil -} - -func (r *RabbitC) Consume(exchange string, queue string, routingkey string, consumer string) (<-chan amqp.Delivery, error) { - q, err := r.Ch.QueueDeclare(queue, true, false, false, false, nil) - if err != nil { - return nil, fmt.Errorf("failed to declare queue %s: %w", queue, err) - } - err = r.Ch.QueueBind(q.Name, routingkey, exchange, false, nil) - if err != nil { - return nil, fmt.Errorf("failed binding queue %s: %w", queue, err) - } - mq, err := r.Ch.Consume(q.Name, consumer, false, false, false, false, nil) - if err != nil { - return nil, fmt.Errorf("failed registering consumer for queue %s: %w", queue, err) - } - return mq, nil -}