From 584b165eb3abbff5b2be3d3d947cc80078d5aea5 Mon Sep 17 00:00:00 2001 From: Jeroen Bobbeldijk Date: Thu, 15 Jun 2023 15:58:18 +0200 Subject: [PATCH 1/3] process the images in a subprocess to prevent crashing the main application on segfaults --- LICENSE | 21 ++ README.md | 37 +++- go.mod | 22 +- go.sum | 89 +++++++- goheif.go | 60 ++++- goheif_test.go | 34 ++- heic2jpg/main.go | 15 +- heif/heif.go | 2 +- libde265/libde265.go | 302 ++++++++++++++++---------- libde265/plugin/plugin.go | 256 ++++++++++++++++++++++ libde265/requests/requests.go | 23 ++ libde265/responses/responses.go | 20 ++ libde265/shared/libde265_interface.go | 197 +++++++++++++++++ libde265/worker_example/main.go | 7 + 14 files changed, 948 insertions(+), 137 deletions(-) create mode 100644 LICENSE create mode 100644 libde265/plugin/plugin.go create mode 100644 libde265/requests/requests.go create mode 100644 libde265/responses/responses.go create mode 100644 libde265/shared/libde265_interface.go create mode 100644 libde265/worker_example/main.go diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c4d5af5 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Klippa App BV + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 369d6b3..3b77028 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,39 @@ Intel and ARM supported ## Install -```go get github.com/jdeng/goheif``` +```go get github.com/klippa-app/goheif``` - Code Sample + +First make a worker package/binary in the libde265_plugin directory. + +``` +package main + +import "github.com/klippa-app/goheif/libde265/plugin" + +func main() { + plugin.StartPlugin() +} ``` + +Then use the worker file/binary in your program. +If you want to make it run through go, use the example below. + +You can also go build `libde265_plugin/main.go` and then reference it in BinPath, this is the advices run method for deployments. + +``` +package main + +func init() { + err := goheif.Init(Config{Lib265Config: libde265.Config{ + Command: libde265.Command{ + BinPath: "go", + Args: []string{"run", "libde265_plugin/main.go"}, + }, + }}) +} + func main() { flag.Parse() ... @@ -51,7 +80,9 @@ func main() { - Some minor bugfixes - A few new box parsers, noteably 'iref' and 'hvcC' -- Include libde265's source code (SSE by default enabled) and a simple golang binding +- Includes libde265 using pkg-config and a simple golang binding + +- Processes the images in a subprocess to prevent crashing the main application on segfaults - A Utility `heic2jpg` to illustrate the usage. @@ -66,7 +97,5 @@ func main() { - libde265 (https://github.com/strukturag/libde265) - implementation learnt from libheif (https://github.com/strukturag/libheif) -## TODO -- Upstream the changes to heif? diff --git a/go.mod b/go.mod index 8ca0a78..b342a23 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,25 @@ -module github.com/jdeng/goheif +module github.com/klippa-app/goheif -go 1.16 +go 1.20 require ( + github.com/google/uuid v1.3.0 + github.com/hashicorp/go-hclog v1.5.0 + github.com/hashicorp/go-plugin v1.4.10 github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd ) + +require ( + github.com/fatih/color v1.13.0 // indirect + github.com/golang/protobuf v1.3.4 // indirect + github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect + github.com/mattn/go-colorable v0.1.12 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 // indirect + github.com/oklog/run v1.0.0 // indirect + golang.org/x/net v0.0.0-20190311183353-d8887717615a // indirect + golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 // indirect + golang.org/x/text v0.3.0 // indirect + google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 // indirect + google.golang.org/grpc v1.27.1 // indirect +) diff --git a/go.sum b/go.sum index d30e55c..c8ea309 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,89 @@ -github.com/jdeng/goheif v0.0.0-20210309200126-b184a7b446fa h1:ISwtQHwIaKiwhFFmBOIib1o1jH3UvtKPnsEo45zsVj0= -github.com/jdeng/goheif v0.0.0-20210309200126-b184a7b446fa/go.mod h1:aKVJoQ0cc9K5Xb058XSnnAxXLliR97qbSqWBlm5ca1E= +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.4 h1:87PNWwrRvUSnqS4dlcBU/ftvOIBep4sYuBLlh6rX2wk= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= +github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-plugin v1.4.10 h1:xUbmA4jC6Dq163/fWcp8P3JuHilrHHMLNRxzGQJ9hNk= +github.com/hashicorp/go-plugin v1.4.10/go.mod h1:6/1TEzT0eQznvI/gV2CM29DLSkAK/e58mUWKVsPaph0= +github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M= +github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77 h1:7GoSOOW2jpsfkntVKaS2rAr1TJqfcxotyaUcuxoZSzg= +github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6 h1:nonptSpoQ4vQjyraW20DXPAglgQfVnM9ZC6MmNLMR60= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/goheif.go b/goheif.go index c6b0011..34459dd 100644 --- a/goheif.go +++ b/goheif.go @@ -2,14 +2,15 @@ package goheif import ( "bytes" + "errors" "fmt" "image" "image/color" "io" - "io/ioutil" + "sync" - "github.com/jdeng/goheif/heif" - "github.com/jdeng/goheif/libde265" + "github.com/klippa-app/goheif/heif" + "github.com/klippa-app/goheif/libde265" ) // SafeEncoding uses more memory but seems to make @@ -64,16 +65,11 @@ func decodeHevcItem(dec *libde265.Decoder, hf *heif.File, item *heif.Item) (*ima dec.Reset() dec.Push(hdr) - tile, err := dec.DecodeImage(data) + ycc, err := dec.DecodeImage(data) if err != nil { return nil, err } - ycc, ok := tile.(*image.YCbCr) - if !ok { - return nil, fmt.Errorf("Tile is not YCbCr") - } - return ycc, nil } @@ -83,6 +79,10 @@ func ExtractExif(ra io.ReaderAt) ([]byte, error) { } func Decode(r io.Reader) (image.Image, error) { + if !isInitialized { + return nil, NotInitializedError + } + ra, err := asReaderAt(r) if err != nil { return nil, err @@ -188,6 +188,10 @@ func Decode(r io.Reader) (image.Image, error) { func DecodeConfig(r io.Reader) (image.Config, error) { var config image.Config + if !isInitialized { + return config, NotInitializedError + } + ra, err := asReaderAt(r) if err != nil { return config, err @@ -218,7 +222,7 @@ func asReaderAt(r io.Reader) (io.ReaderAt, error) { return ra, nil } - b, err := ioutil.ReadAll(r) + b, err := io.ReadAll(r) if err != nil { return nil, err } @@ -226,8 +230,42 @@ func asReaderAt(r io.Reader) (io.ReaderAt, error) { return bytes.NewReader(b), nil } +var NotInitializedError = errors.New("goheif was not initialized, you must call the Init() method") +var isInitialized = false +var initLock = sync.Mutex{} + +type Config struct { + Lib265Config libde265.Config +} + +func Init(config Config) error { + initLock.Lock() + defer initLock.Unlock() + if isInitialized { + return nil + } + err := libde265.Init(config.Lib265Config) + if err != nil { + return err + } + isInitialized = true + + return nil +} + +func DeInit() { + initLock.Lock() + defer initLock.Unlock() + + if !isInitialized { + return + } + + libde265.DeInit() + isInitialized = true +} + func init() { - libde265.Init() // they check for "ftyp" at the 5th bytes, let's do the same... // https://github.com/strukturag/libheif/blob/master/libheif/heif.cc#L94 image.RegisterFormat("heic", "????ftyp", Decode, DecodeConfig) diff --git a/goheif_test.go b/goheif_test.go index 9ff1273..bd6db35 100644 --- a/goheif_test.go +++ b/goheif_test.go @@ -6,9 +6,29 @@ import ( "io" "io/ioutil" "testing" + + "github.com/klippa-app/goheif/libde265" ) +func initLib() error { + err := Init(Config{Lib265Config: libde265.Config{ + Command: libde265.Command{ + BinPath: "go", + Args: []string{"run", "libde265/worker_example/main.go"}, + }, + }}) + if err != nil { + return err + } + return nil +} + func TestFormatRegistered(t *testing.T) { + err := initLib() + if err != nil { + t.Fatal(err) + } + b, err := ioutil.ReadFile("testdata/camel.heic") if err != nil { t.Fatal(err) @@ -29,10 +49,18 @@ func TestFormatRegistered(t *testing.T) { } func BenchmarkSafeEncoding(b *testing.B) { + err := initLib() + if err != nil { + b.Fatal(err) + } benchEncoding(b, true) } func BenchmarkRegularEncoding(b *testing.B) { + err := initLib() + if err != nil { + b.Fatal(err) + } benchEncoding(b, false) } @@ -54,7 +82,11 @@ func benchEncoding(b *testing.B, safe bool) { b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - Decode(r) + _, err = Decode(r) + if err != nil { + b.Fatal(err) + } + r.Seek(0, io.SeekStart) } } diff --git a/heic2jpg/main.go b/heic2jpg/main.go index 28a1661..36bc695 100644 --- a/heic2jpg/main.go +++ b/heic2jpg/main.go @@ -8,7 +8,8 @@ import ( "log" "os" - "github.com/jdeng/goheif" + "github.com/klippa-app/goheif" + "github.com/klippa-app/goheif/libde265" ) // Skip Writer for exif writing @@ -59,6 +60,18 @@ func newWriterExif(w io.Writer, exif []byte) (io.Writer, error) { return writer, nil } +func init() { + err := goheif.Init(goheif.Config{Lib265Config: libde265.Config{ + Command: libde265.Command{ + BinPath: "go", + Args: []string{"run", "libde265/worker_example/main.go"}, + }, + }}) + if err != nil { + log.Fatalf("could not start libde265 worker: %s", err.Error()) + } +} + func main() { flag.Parse() if flag.NArg() != 2 { diff --git a/heif/heif.go b/heif/heif.go index cc332b8..421ea71 100644 --- a/heif/heif.go +++ b/heif/heif.go @@ -27,7 +27,7 @@ import ( "io" "log" - "github.com/jdeng/goheif/heif/bmff" + "github.com/klippa-app/goheif/heif/bmff" ) // File represents a HEIF file. diff --git a/libde265/libde265.go b/libde265/libde265.go index b6a01c1..e727d5a 100644 --- a/libde265/libde265.go +++ b/libde265/libde265.go @@ -1,45 +1,138 @@ package libde265 -// #cgo pkg-config: libde265 -// #include -// #include -// #include "libde265/de265.h" -import "C" - import ( - "fmt" + "errors" "image" - "unsafe" + "log" + "os" + "os/exec" + "time" + + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-plugin" + + "github.com/klippa-app/goheif/libde265/requests" + "github.com/klippa-app/goheif/libde265/shared" ) type Decoder struct { - ctx unsafe.Pointer - hasImage bool + id string safeEncode bool } -func Init() { - C.de265_init() +var client *plugin.Client +var gRPCClient plugin.ClientProtocol +var libde265plugin shared.Libde265 +var currentConfig Config + +type Config struct { + Command Command } -func Fini() { - C.de265_free() +type Command struct { + BinPath string + Args []string + + // StartTimeout is the timeout to wait for the plugin to say it + // has started successfully. + StartTimeout time.Duration } -func NewDecoder(opts ...Option) (*Decoder, error) { - p := C.de265_new_decoder() - if p == nil { - return nil, fmt.Errorf("Unable to create decoder") +func Init(config Config) error { + if client != nil { + return nil } - dec := &Decoder{ctx: p, hasImage: false} - for _, opt := range opts { - opt(dec) + currentConfig = config + + return startPlugin() +} + +func DeInit() { + gRPCClient.Close() + gRPCClient = nil + client.Kill() + client = nil + libde265plugin = nil +} + +func startPlugin() error { + var handshakeConfig = plugin.HandshakeConfig{ + ProtocolVersion: 1, + MagicCookieKey: "BASIC_PLUGIN", + MagicCookieValue: "libde265", } - return dec, nil + // pluginMap is the map of plugins we can dispense. + var pluginMap = map[string]plugin.Plugin{ + "libde265": &shared.Libde265Plugin{}, + } + + logger := hclog.New(&hclog.LoggerOptions{ + Name: "plugin", + Output: os.Stdout, + Level: hclog.Debug, + }) + + client := plugin.NewClient(&plugin.ClientConfig{ + HandshakeConfig: handshakeConfig, + Plugins: pluginMap, + Cmd: exec.Command(currentConfig.Command.BinPath, currentConfig.Command.Args...), + Logger: logger, + }) + + rpcClient, err := client.Client() + if err != nil { + log.Fatal(err) + } + + gRPCClient = rpcClient + + raw, err := rpcClient.Dispense("libde265") + if err != nil { + log.Fatal(err) + } + + pluginInstance := raw.(shared.Libde265) + pong, err := pluginInstance.Ping() + if err != nil { + return err + } + + if pong != "Pong" { + return errors.New("Wrong ping/pong result") + } + + libde265plugin = pluginInstance + + return nil +} + +func checkPlugin() error { + pong, err := libde265plugin.Ping() + if err != nil { + log.Printf("restarting libde265 plugin due to wrong pong result: %s", err.Error()) + err = startPlugin() + if err != nil { + log.Printf("could not restart libde265 plugin: %s", err.Error()) + return err + } + } + + if pong != "Pong" { + log.Printf("restarting libde265 plugin due to wrong pong result: %s", pong) + err = startPlugin() + if err != nil { + log.Printf("could not restart libde265 plugin: %s", err.Error()) + return err + } + } + + return nil } +var NotInitializedError = errors.New("libde265 was not initialized, you must call the Init() method") + type Option func(*Decoder) func WithSafeEncoding(b bool) Option { @@ -48,119 +141,98 @@ func WithSafeEncoding(b bool) Option { } } -func (dec *Decoder) Free() { - dec.Reset() - C.de265_free_decoder(dec.ctx) -} +func NewDecoder(opts ...Option) (*Decoder, error) { + if libde265plugin == nil { + return nil, NotInitializedError + } -func (dec *Decoder) Reset() { - if dec.ctx != nil && dec.hasImage { - C.de265_release_next_picture(dec.ctx) - dec.hasImage = false + err := checkPlugin() + if err != nil { + return nil, errors.New("could not check or start plugin") } - C.de265_reset(dec.ctx) -} + dec := &Decoder{} + for _, opt := range opts { + opt(dec) + } -func (dec *Decoder) Push(data []byte) error { - var pos int - totalSize := len(data) - for pos < totalSize { - if pos+4 > totalSize { - return fmt.Errorf("Invalid NAL data") - } + newDecoder, err := libde265plugin.NewDecoder(&requests.NewDecoder{ + SafeEncode: dec.safeEncode, + }) + if err != nil { + return nil, err + } - nalSize := uint32(data[pos])<<24 | uint32(data[pos+1])<<16 | uint32(data[pos+2])<<8 | uint32(data[pos+3]) - pos += 4 + dec.id = newDecoder.ID + return dec, nil +} - if pos+int(nalSize) > totalSize { - return fmt.Errorf("Invalid NAL size: %d", nalSize) - } +func (dec *Decoder) Free() error { + if libde265plugin == nil { + return NotInitializedError + } + + err := checkPlugin() + if err != nil { + return errors.New("could not check or start plugin") + } - C.de265_push_NAL(dec.ctx, unsafe.Pointer(&data[pos]), C.int(nalSize), C.de265_PTS(0), nil) - pos += int(nalSize) + _, err = libde265plugin.CloseDecoder(&requests.CloseDecoder{ID: dec.id}) + if err != nil { + return err } return nil } -func (dec *Decoder) DecodeImage(data []byte) (image.Image, error) { - if dec.hasImage { - fmt.Printf("previous image may leak") +func (dec *Decoder) Reset() error { + if libde265plugin == nil { + return NotInitializedError } - if len(data) > 0 { - if err := dec.Push(data); err != nil { - return nil, err - } + err := checkPlugin() + if err != nil { + return errors.New("could not check or start plugin") } - if ret := C.de265_flush_data(dec.ctx); ret != C.DE265_OK { - return nil, fmt.Errorf("flush_data error") + _, err = libde265plugin.ResetDecoder(&requests.ResetDecoder{ID: dec.id}) + if err != nil { + return err } - var more C.int = 1 - for more != 0 { - if decerr := C.de265_decode(dec.ctx, &more); decerr != C.DE265_OK { - return nil, fmt.Errorf("decode error") - } + return nil +} - for { - warning := C.de265_get_warning(dec.ctx) - if warning == C.DE265_OK { - break - } - fmt.Printf("warning: %v\n", C.GoString(C.de265_get_error_text(warning))) - } +func (dec *Decoder) Push(data []byte) error { + if libde265plugin == nil { + return NotInitializedError + } - if img := C.de265_get_next_picture(dec.ctx); img != nil { - dec.hasImage = true // lazy release - - width := C.de265_get_image_width(img, 0) - height := C.de265_get_image_height(img, 0) - - var ystride, cstride C.int - y := C.de265_get_image_plane(img, 0, &ystride) - cb := C.de265_get_image_plane(img, 1, &cstride) - cheight := C.de265_get_image_height(img, 1) - cr := C.de265_get_image_plane(img, 2, &cstride) - // crh := C.de265_get_image_height(img, 2) - - // sanity check - if int(height)*int(ystride) >= int(1<<30) { - return nil, fmt.Errorf("image too big") - } - - var r image.YCbCrSubsampleRatio - switch chroma := C.de265_get_chroma_format(img); chroma { - case C.de265_chroma_420: - r = image.YCbCrSubsampleRatio420 - case C.de265_chroma_422: - r = image.YCbCrSubsampleRatio422 - case C.de265_chroma_444: - r = image.YCbCrSubsampleRatio444 - } - ycc := &image.YCbCr{ - YStride: int(ystride), - CStride: int(cstride), - SubsampleRatio: r, - Rect: image.Rectangle{Min: image.Point{0, 0}, Max: image.Point{int(width), int(height)}}, - } - if dec.safeEncode { - ycc.Y = C.GoBytes(unsafe.Pointer(y), C.int(height*ystride)) - ycc.Cb = C.GoBytes(unsafe.Pointer(cb), C.int(cheight*cstride)) - ycc.Cr = C.GoBytes(unsafe.Pointer(cr), C.int(cheight*cstride)) - } else { - ycc.Y = (*[1 << 30]byte)(unsafe.Pointer(y))[:int(height)*int(ystride)] - ycc.Cb = (*[1 << 30]byte)(unsafe.Pointer(cb))[:int(cheight)*int(cstride)] - ycc.Cr = (*[1 << 30]byte)(unsafe.Pointer(cr))[:int(cheight)*int(cstride)] - } - - //C.de265_release_next_picture(dec.ctx) - - return ycc, nil - } + err := checkPlugin() + if err != nil { + return errors.New("could not check or start plugin") } - return nil, fmt.Errorf("No picture") + _, err = libde265plugin.PushDecoder(&requests.PushDecoder{ID: dec.id, Data: data}) + if err != nil { + return err + } + return nil +} + +func (dec *Decoder) DecodeImage(data []byte) (*image.YCbCr, error) { + if libde265plugin == nil { + return nil, NotInitializedError + } + + err := checkPlugin() + if err != nil { + return nil, errors.New("could not check or start plugin") + } + + resp, err := libde265plugin.RenderDecoder(&requests.RenderDecoder{ID: dec.id, Data: data}) + if err != nil { + return nil, err + } + return resp.Image, nil } diff --git a/libde265/plugin/plugin.go b/libde265/plugin/plugin.go new file mode 100644 index 0000000..65bf015 --- /dev/null +++ b/libde265/plugin/plugin.go @@ -0,0 +1,256 @@ +package plugin + +// #cgo pkg-config: libde265 +// #include +// #include +// #include "libde265/de265.h" +import "C" + +import ( + "errors" + "fmt" + "image" + "sync" + "unsafe" + + "github.com/klippa-app/goheif/libde265/requests" + "github.com/klippa-app/goheif/libde265/responses" + "github.com/klippa-app/goheif/libde265/shared" + + "github.com/google/uuid" + "github.com/hashicorp/go-plugin" +) + +var handshakeConfig = plugin.HandshakeConfig{ + ProtocolVersion: 1, + MagicCookieKey: "BASIC_PLUGIN", + MagicCookieValue: "libde265", +} + +func StartPlugin() { + var pluginMap = map[string]plugin.Plugin{ + "libde265": &shared.Libde265Plugin{Impl: &libde265Implementation{ + Decoders: map[string]*decoder{}, + DecodersLock: sync.Mutex{}, + }}, + } + + plugin.Serve(&plugin.ServeConfig{ + HandshakeConfig: handshakeConfig, + Plugins: pluginMap, + }) +} + +type libde265Implementation struct { + Decoders map[string]*decoder + DecodersLock sync.Mutex +} + +func (l *libde265Implementation) Ping() (string, error) { + return "Pong", nil +} + +func (l *libde265Implementation) getDecoder(id string) (*decoder, error) { + l.DecodersLock.Lock() + defer l.DecodersLock.Unlock() + + val, ok := l.Decoders[id] + if !ok { + return nil, errors.New("could not find decoder") + } + + return val, nil +} + +func (l *libde265Implementation) NewDecoder(request *requests.NewDecoder) (*responses.NewDecoder, error) { + p := C.de265_new_decoder() + if p == nil { + return nil, fmt.Errorf("Unable to create decoder") + } + + dec := &decoder{ctx: p, hasImage: request.SafeEncode} + newID := uuid.New().String() + l.DecodersLock.Lock() + defer l.DecodersLock.Unlock() + l.Decoders[newID] = dec + + return &responses.NewDecoder{ID: newID}, nil +} + +func (l *libde265Implementation) CloseDecoder(request *requests.CloseDecoder) (*responses.CloseDecoder, error) { + dec, err := l.getDecoder(request.ID) + if err != nil { + return nil, err + } + + dec.Free() + l.DecodersLock.Lock() + defer l.DecodersLock.Unlock() + delete(l.Decoders, request.ID) + + return &responses.CloseDecoder{}, nil +} + +func (l *libde265Implementation) ResetDecoder(request *requests.ResetDecoder) (*responses.ResetDecoder, error) { + dec, err := l.getDecoder(request.ID) + if err != nil { + return nil, err + } + + dec.Reset() + + return &responses.ResetDecoder{}, nil +} + +func (l *libde265Implementation) PushDecoder(request *requests.PushDecoder) (*responses.PushDecoder, error) { + dec, err := l.getDecoder(request.ID) + if err != nil { + return nil, err + } + + err = dec.Push(request.Data) + if err != nil { + return nil, err + } + + return &responses.PushDecoder{}, nil +} + +func (l *libde265Implementation) RenderDecoder(request *requests.RenderDecoder) (*responses.RenderDecoder, error) { + dec, err := l.getDecoder(request.ID) + if err != nil { + return nil, err + } + + img, err := dec.DecodeImage(request.Data) + if err != nil { + return nil, err + } + + return &responses.RenderDecoder{ + Image: img, + }, nil +} + +type decoder struct { + ctx unsafe.Pointer + hasImage bool + safeEncode bool +} + +func (dec *decoder) Free() { + dec.Reset() + C.de265_free_decoder(dec.ctx) +} + +func (dec *decoder) Reset() { + if dec.ctx != nil && dec.hasImage { + C.de265_release_next_picture(dec.ctx) + dec.hasImage = false + } + + C.de265_reset(dec.ctx) +} + +func (dec *decoder) Push(data []byte) error { + var pos int + totalSize := len(data) + for pos < totalSize { + if pos+4 > totalSize { + return fmt.Errorf("Invalid NAL data") + } + + nalSize := uint32(data[pos])<<24 | uint32(data[pos+1])<<16 | uint32(data[pos+2])<<8 | uint32(data[pos+3]) + pos += 4 + + if pos+int(nalSize) > totalSize { + return fmt.Errorf("Invalid NAL size: %d", nalSize) + } + + C.de265_push_NAL(dec.ctx, unsafe.Pointer(&data[pos]), C.int(nalSize), C.de265_PTS(0), nil) + pos += int(nalSize) + } + + return nil +} + +func (dec *decoder) DecodeImage(data []byte) (*image.YCbCr, error) { + if dec.hasImage { + fmt.Printf("previous image may leak") + } + + if len(data) > 0 { + if err := dec.Push(data); err != nil { + return nil, err + } + } + + if ret := C.de265_flush_data(dec.ctx); ret != C.DE265_OK { + return nil, fmt.Errorf("flush_data error") + } + + var more C.int = 1 + for more != 0 { + if decerr := C.de265_decode(dec.ctx, &more); decerr != C.DE265_OK { + return nil, fmt.Errorf("decode error") + } + + for { + warning := C.de265_get_warning(dec.ctx) + if warning == C.DE265_OK { + break + } + fmt.Printf("warning: %v\n", C.GoString(C.de265_get_error_text(warning))) + } + + if img := C.de265_get_next_picture(dec.ctx); img != nil { + dec.hasImage = true // lazy release + + width := C.de265_get_image_width(img, 0) + height := C.de265_get_image_height(img, 0) + + var ystride, cstride C.int + y := C.de265_get_image_plane(img, 0, &ystride) + cb := C.de265_get_image_plane(img, 1, &cstride) + cheight := C.de265_get_image_height(img, 1) + cr := C.de265_get_image_plane(img, 2, &cstride) + // crh := C.de265_get_image_height(img, 2) + + // sanity check + if int(height)*int(ystride) >= int(1<<30) { + return nil, fmt.Errorf("image too big") + } + + var r image.YCbCrSubsampleRatio + switch chroma := C.de265_get_chroma_format(img); chroma { + case C.de265_chroma_420: + r = image.YCbCrSubsampleRatio420 + case C.de265_chroma_422: + r = image.YCbCrSubsampleRatio422 + case C.de265_chroma_444: + r = image.YCbCrSubsampleRatio444 + } + ycc := &image.YCbCr{ + YStride: int(ystride), + CStride: int(cstride), + SubsampleRatio: r, + Rect: image.Rectangle{Min: image.Point{0, 0}, Max: image.Point{int(width), int(height)}}, + } + if dec.safeEncode { + ycc.Y = C.GoBytes(unsafe.Pointer(y), C.int(height*ystride)) + ycc.Cb = C.GoBytes(unsafe.Pointer(cb), C.int(cheight*cstride)) + ycc.Cr = C.GoBytes(unsafe.Pointer(cr), C.int(cheight*cstride)) + } else { + ycc.Y = (*[1 << 30]byte)(unsafe.Pointer(y))[:int(height)*int(ystride)] + ycc.Cb = (*[1 << 30]byte)(unsafe.Pointer(cb))[:int(cheight)*int(cstride)] + ycc.Cr = (*[1 << 30]byte)(unsafe.Pointer(cr))[:int(cheight)*int(cstride)] + } + + //C.de265_release_next_picture(dec.ctx) + + return ycc, nil + } + } + + return nil, fmt.Errorf("No picture") +} diff --git a/libde265/requests/requests.go b/libde265/requests/requests.go new file mode 100644 index 0000000..cd9fc42 --- /dev/null +++ b/libde265/requests/requests.go @@ -0,0 +1,23 @@ +package requests + +type NewDecoder struct { + SafeEncode bool +} + +type CloseDecoder struct { + ID string +} + +type ResetDecoder struct { + ID string +} + +type PushDecoder struct { + ID string + Data []byte +} + +type RenderDecoder struct { + ID string + Data []byte +} diff --git a/libde265/responses/responses.go b/libde265/responses/responses.go new file mode 100644 index 0000000..cc1580b --- /dev/null +++ b/libde265/responses/responses.go @@ -0,0 +1,20 @@ +package responses + +import "image" + +type NewDecoder struct { + ID string +} + +type CloseDecoder struct { +} + +type ResetDecoder struct { +} + +type PushDecoder struct { +} + +type RenderDecoder struct { + Image *image.YCbCr +} diff --git a/libde265/shared/libde265_interface.go b/libde265/shared/libde265_interface.go new file mode 100644 index 0000000..2eafce4 --- /dev/null +++ b/libde265/shared/libde265_interface.go @@ -0,0 +1,197 @@ +package shared + +import ( + "fmt" + "net/rpc" + + "github.com/klippa-app/goheif/libde265/requests" + "github.com/klippa-app/goheif/libde265/responses" + + "github.com/hashicorp/go-plugin" +) + +type Libde265 interface { + Ping() (string, error) + NewDecoder(*requests.NewDecoder) (*responses.NewDecoder, error) + CloseDecoder(*requests.CloseDecoder) (*responses.CloseDecoder, error) + PushDecoder(*requests.PushDecoder) (*responses.PushDecoder, error) + ResetDecoder(*requests.ResetDecoder) (*responses.ResetDecoder, error) + RenderDecoder(*requests.RenderDecoder) (*responses.RenderDecoder, error) +} + +type Libde265RPC struct{ client *rpc.Client } + +func (g *Libde265RPC) Ping() (string, error) { + var resp string + err := g.client.Call("Plugin.Ping", new(interface{}), &resp) + if err != nil { + return "", err + } + + return resp, nil +} + +func (g *Libde265RPC) NewDecoder(request *requests.NewDecoder) (*responses.NewDecoder, error) { + resp := &responses.NewDecoder{} + err := g.client.Call("Plugin.NewDecoder", request, resp) + if err != nil { + return nil, err + } + + return resp, nil +} + +func (g *Libde265RPC) CloseDecoder(request *requests.CloseDecoder) (*responses.CloseDecoder, error) { + resp := &responses.CloseDecoder{} + err := g.client.Call("Plugin.CloseDecoder", request, resp) + if err != nil { + return nil, err + } + + return resp, nil +} + +func (g *Libde265RPC) PushDecoder(request *requests.PushDecoder) (*responses.PushDecoder, error) { + resp := &responses.PushDecoder{} + err := g.client.Call("Plugin.PushDecoder", request, resp) + if err != nil { + return nil, err + } + + return resp, nil +} + +func (g *Libde265RPC) ResetDecoder(request *requests.ResetDecoder) (*responses.ResetDecoder, error) { + resp := &responses.ResetDecoder{} + err := g.client.Call("Plugin.ResetDecoder", request, resp) + if err != nil { + return nil, err + } + + return resp, nil +} + +func (g *Libde265RPC) RenderDecoder(request *requests.RenderDecoder) (*responses.RenderDecoder, error) { + resp := &responses.RenderDecoder{} + err := g.client.Call("Plugin.RenderDecoder", request, resp) + if err != nil { + return nil, err + } + + return resp, nil +} + +type Libde265RPCServer struct { + Impl Libde265 +} + +func (s *Libde265RPCServer) Ping(args interface{}, resp *string) error { + var err error + *resp, err = s.Impl.Ping() + if err != nil { + return err + } + return nil +} + +func (s *Libde265RPCServer) NewDecoder(request *requests.NewDecoder, resp *responses.NewDecoder) (err error) { + defer func() { + if panicError := recover(); panicError != nil { + err = fmt.Errorf("panic occurred in %s: %v", "NewDecoder", panicError) + } + }() + + implResp, err := s.Impl.NewDecoder(request) + if err != nil { + return err + } + + // Overwrite the target address of resp to the target address of implResp. + *resp = *implResp + + return nil +} + +func (s *Libde265RPCServer) CloseDecoder(request *requests.CloseDecoder, resp *responses.CloseDecoder) (err error) { + defer func() { + if panicError := recover(); panicError != nil { + err = fmt.Errorf("panic occurred in %s: %v", "CloseDecoder", panicError) + } + }() + + implResp, err := s.Impl.CloseDecoder(request) + if err != nil { + return err + } + + // Overwrite the target address of resp to the target address of implResp. + *resp = *implResp + + return nil +} + +func (s *Libde265RPCServer) PushDecoder(request *requests.PushDecoder, resp *responses.PushDecoder) (err error) { + defer func() { + if panicError := recover(); panicError != nil { + err = fmt.Errorf("panic occurred in %s: %v", "PushDecoder", panicError) + } + }() + + implResp, err := s.Impl.PushDecoder(request) + if err != nil { + return err + } + + // Overwrite the target address of resp to the target address of implResp. + *resp = *implResp + + return nil +} + +func (s *Libde265RPCServer) ResetDecoder(request *requests.ResetDecoder, resp *responses.ResetDecoder) (err error) { + defer func() { + if panicError := recover(); panicError != nil { + err = fmt.Errorf("panic occurred in %s: %v", "ResetDecoder", panicError) + } + }() + + implResp, err := s.Impl.ResetDecoder(request) + if err != nil { + return err + } + + // Overwrite the target address of resp to the target address of implResp. + *resp = *implResp + + return nil +} + +func (s *Libde265RPCServer) RenderDecoder(request *requests.RenderDecoder, resp *responses.RenderDecoder) (err error) { + defer func() { + if panicError := recover(); panicError != nil { + err = fmt.Errorf("panic occurred in %s: %v", "RenderDecoder", panicError) + } + }() + + implResp, err := s.Impl.RenderDecoder(request) + if err != nil { + return err + } + + // Overwrite the target address of resp to the target address of implResp. + *resp = *implResp + + return nil +} + +type Libde265Plugin struct { + Impl Libde265 +} + +func (p *Libde265Plugin) Server(*plugin.MuxBroker) (interface{}, error) { + return &Libde265RPCServer{Impl: p.Impl}, nil +} + +func (Libde265Plugin) Client(b *plugin.MuxBroker, c *rpc.Client) (interface{}, error) { + return &Libde265RPC{client: c}, nil +} diff --git a/libde265/worker_example/main.go b/libde265/worker_example/main.go new file mode 100644 index 0000000..df2bc96 --- /dev/null +++ b/libde265/worker_example/main.go @@ -0,0 +1,7 @@ +package main + +import "github.com/klippa-app/goheif/libde265/plugin" + +func main() { + plugin.StartPlugin() +} From 2280cfba7938b95c9562e20d1f09a33a66dea87f Mon Sep 17 00:00:00 2001 From: Jeroen Bobbeldijk Date: Thu, 15 Jun 2023 16:00:42 +0200 Subject: [PATCH 2/3] Fix starttimeout option --- libde265/libde265.go | 1 + 1 file changed, 1 insertion(+) diff --git a/libde265/libde265.go b/libde265/libde265.go index e727d5a..ad2edee 100644 --- a/libde265/libde265.go +++ b/libde265/libde265.go @@ -79,6 +79,7 @@ func startPlugin() error { Plugins: pluginMap, Cmd: exec.Command(currentConfig.Command.BinPath, currentConfig.Command.Args...), Logger: logger, + StartTimeout: currentConfig.Command.StartTimeout, }) rpcClient, err := client.Client() From c5010c8d1c8d5111d423a4e8393b2e1e182022f2 Mon Sep 17 00:00:00 2001 From: Jeroen Bobbeldijk Date: Thu, 15 Jun 2023 16:45:27 +0200 Subject: [PATCH 3/3] Add renderfile --- goheif.go | 6 +- goheif_test.go | 69 ++++++++- libde265/libde265.go | 34 ++++- libde265/plugin/plugin.go | 212 +++++++++++++++++++++++++- libde265/requests/requests.go | 18 ++- libde265/responses/responses.go | 4 + libde265/shared/libde265_interface.go | 29 ++++ 7 files changed, 362 insertions(+), 10 deletions(-) diff --git a/goheif.go b/goheif.go index 34459dd..c7f2cd9 100644 --- a/goheif.go +++ b/goheif.go @@ -15,7 +15,7 @@ import ( // SafeEncoding uses more memory but seems to make // the library safer to use in containers. -var SafeEncoding bool +var SafeEncoding = true type gridBox struct { columns, rows int @@ -64,8 +64,8 @@ func decodeHevcItem(dec *libde265.Decoder, hf *heif.File, item *heif.Item) (*ima } dec.Reset() - dec.Push(hdr) - ycc, err := dec.DecodeImage(data) + dec.Push(&hdr) + ycc, err := dec.DecodeImage(&data) if err != nil { return nil, err } diff --git a/goheif_test.go b/goheif_test.go index bd6db35..a0e4953 100644 --- a/goheif_test.go +++ b/goheif_test.go @@ -3,8 +3,11 @@ package goheif import ( "bytes" "image" + _ "image/jpeg" + _ "image/png" "io" "io/ioutil" + "os" "testing" "github.com/klippa-app/goheif/libde265" @@ -29,7 +32,7 @@ func TestFormatRegistered(t *testing.T) { t.Fatal(err) } - b, err := ioutil.ReadFile("testdata/camel.heic") + b, err := os.ReadFile("testdata/camel.heic") if err != nil { t.Fatal(err) } @@ -48,6 +51,70 @@ func TestFormatRegistered(t *testing.T) { } } +func TestRenderJPEG(t *testing.T) { + err := initLib() + if err != nil { + t.Fatal(err) + } + + b, err := os.ReadFile("testdata/camel.heic") + if err != nil { + t.Fatal(err) + } + + renderedFile, err := libde265.RenderFile(&b, libde265.RenderOptions{ + OutputFormat: libde265.RenderFileOutputFormatJPG, + }) + if err != nil { + t.Fatal(err) + } + + img, dec, err := image.Decode(bytes.NewReader(*renderedFile)) + if err != nil { + t.Fatalf("unable to decode jpeg image: %s", err) + } + + if got, want := dec, "jpeg"; got != want { + t.Errorf("unexpected decoder: got %s, want %s", got, want) + } + + if w, h := img.Bounds().Dx(), img.Bounds().Dy(); w != 1596 || h != 1064 { + t.Errorf("unexpected decoded image size: got %dx%d, want 1596x1064", w, h) + } +} + +func TestRenderPNG(t *testing.T) { + err := initLib() + if err != nil { + t.Fatal(err) + } + + b, err := os.ReadFile("testdata/camel.heic") + if err != nil { + t.Fatal(err) + } + + renderedFile, err := libde265.RenderFile(&b, libde265.RenderOptions{ + OutputFormat: libde265.RenderFileOutputFormatPNG, + }) + if err != nil { + t.Fatal(err) + } + + img, dec, err := image.Decode(bytes.NewReader(*renderedFile)) + if err != nil { + t.Fatalf("unable to decode jpeg image: %s", err) + } + + if got, want := dec, "png"; got != want { + t.Errorf("unexpected decoder: got %s, want %s", got, want) + } + + if w, h := img.Bounds().Dx(), img.Bounds().Dy(); w != 1596 || h != 1064 { + t.Errorf("unexpected decoded image size: got %dx%d, want 1596x1064", w, h) + } +} + func BenchmarkSafeEncoding(b *testing.B) { err := initLib() if err != nil { diff --git a/libde265/libde265.go b/libde265/libde265.go index ad2edee..52bb960 100644 --- a/libde265/libde265.go +++ b/libde265/libde265.go @@ -204,7 +204,7 @@ func (dec *Decoder) Reset() error { return nil } -func (dec *Decoder) Push(data []byte) error { +func (dec *Decoder) Push(data *[]byte) error { if libde265plugin == nil { return NotInitializedError } @@ -221,7 +221,7 @@ func (dec *Decoder) Push(data []byte) error { return nil } -func (dec *Decoder) DecodeImage(data []byte) (*image.YCbCr, error) { +func (dec *Decoder) DecodeImage(data *[]byte) (*image.YCbCr, error) { if libde265plugin == nil { return nil, NotInitializedError } @@ -237,3 +237,33 @@ func (dec *Decoder) DecodeImage(data []byte) (*image.YCbCr, error) { } return resp.Image, nil } + +type RenderFileOutputFormat string // The file format to render output as. + +const ( + RenderFileOutputFormatJPG RenderFileOutputFormat = "jpg" // Render the file as a JPEG file. + RenderFileOutputFormatPNG RenderFileOutputFormat = "png" // Render the file as a PNG file. +) + +type RenderOptions struct { + OutputFormat RenderFileOutputFormat // The format to output the image as + MaxFileSize int64 // The maximum filesize, if jpg is chosen as output format, it will try to compress it until it fits + SafeEncoding bool // Whether to use safe encoding. +} + +func RenderFile(data *[]byte, options RenderOptions) (*[]byte, error) { + if libde265plugin == nil { + return nil, NotInitializedError + } + + err := checkPlugin() + if err != nil { + return nil, errors.New("could not check or start plugin") + } + + resp, err := libde265plugin.RenderFile(&requests.RenderFile{Data: data, OutputFormat: requests.RenderFileOutputFormat(options.OutputFormat), MaxFileSize: options.MaxFileSize, SafeEncoding: options.SafeEncoding}) + if err != nil { + return nil, err + } + return resp.Output, nil +} diff --git a/libde265/plugin/plugin.go b/libde265/plugin/plugin.go index 65bf015..e31b523 100644 --- a/libde265/plugin/plugin.go +++ b/libde265/plugin/plugin.go @@ -7,12 +7,16 @@ package plugin import "C" import ( + "bytes" "errors" "fmt" "image" + "image/jpeg" + "image/png" "sync" "unsafe" + "github.com/klippa-app/goheif/heif" "github.com/klippa-app/goheif/libde265/requests" "github.com/klippa-app/goheif/libde265/responses" "github.com/klippa-app/goheif/libde265/shared" @@ -108,7 +112,7 @@ func (l *libde265Implementation) PushDecoder(request *requests.PushDecoder) (*re return nil, err } - err = dec.Push(request.Data) + err = dec.Push(*request.Data) if err != nil { return nil, err } @@ -122,7 +126,7 @@ func (l *libde265Implementation) RenderDecoder(request *requests.RenderDecoder) return nil, err } - img, err := dec.DecodeImage(request.Data) + img, err := dec.DecodeImage(*request.Data) if err != nil { return nil, err } @@ -132,6 +136,210 @@ func (l *libde265Implementation) RenderDecoder(request *requests.RenderDecoder) }, nil } +type gridBox struct { + columns, rows int + width, height int +} + +func newGridBox(data []byte) (*gridBox, error) { + if len(data) < 8 { + return nil, fmt.Errorf("invalid data") + } + // version := data[0] + flags := data[1] + rows := int(data[2]) + 1 + columns := int(data[3]) + 1 + + var width, height int + if (flags & 1) != 0 { + if len(data) < 12 { + return nil, fmt.Errorf("invalid data") + } + + width = int(data[4])<<24 | int(data[5])<<16 | int(data[6])<<8 | int(data[7]) + height = int(data[8])<<24 | int(data[9])<<16 | int(data[10])<<8 | int(data[11]) + } else { + width = int(data[4])<<8 | int(data[5]) + height = int(data[6])<<8 | int(data[7]) + } + + return &gridBox{columns: columns, rows: rows, width: width, height: height}, nil +} + +func (l *libde265Implementation) decodeHevcItem(decoderID string, hf *heif.File, item *heif.Item) (*image.YCbCr, error) { + if item.Info.ItemType != "hvc1" { + return nil, fmt.Errorf("Unsupported item type: %s", item.Info.ItemType) + } + + hvcc, ok := item.HevcConfig() + if !ok { + return nil, fmt.Errorf("No hvcC") + } + + hdr := hvcc.AsHeader() + data, err := hf.GetItemData(item) + if err != nil { + return nil, err + } + + dec, err := l.getDecoder(decoderID) + if err != nil { + return nil, err + } + + dec.Reset() + dec.Push(hdr) + ycc, err := dec.DecodeImage(data) + if err != nil { + return nil, err + } + + return ycc, nil +} + +func (l *libde265Implementation) RenderFile(request *requests.RenderFile) (*responses.RenderFile, error) { + ra := bytes.NewReader(*request.Data) + hf := heif.Open(ra) + + it, err := hf.PrimaryItem() + if err != nil { + return nil, err + } + + width, height, ok := it.SpatialExtents() + if !ok { + return nil, fmt.Errorf("No dimension") + } + + if it.Info == nil { + return nil, fmt.Errorf("No item info") + } + + resp, err := l.NewDecoder(&requests.NewDecoder{SafeEncode: request.SafeEncoding}) + if err != nil { + return nil, err + } + decoderID := resp.ID + + defer l.CloseDecoder(&requests.CloseDecoder{ID: decoderID}) + + var outImage *image.YCbCr + if it.Info.ItemType == "hvc1" { + outImage, err = l.decodeHevcItem(decoderID, hf, it) + if err != nil { + return nil, err + } + } else { + if it.Info.ItemType != "grid" { + return nil, fmt.Errorf("No grid") + } + + data, err := hf.GetItemData(it) + if err != nil { + return nil, err + } + + grid, err := newGridBox(data) + if err != nil { + return nil, err + } + + dimg := it.Reference("dimg") + if dimg == nil { + return nil, fmt.Errorf("No dimg") + } + + if len(dimg.ToItemIDs) != grid.columns*grid.rows { + return nil, fmt.Errorf("Tiles number not matched") + } + + var tileWidth, tileHeight int + for i, y := 0, 0; y < grid.rows; y += 1 { + for x := 0; x < grid.columns; x += 1 { + id := dimg.ToItemIDs[i] + item, err := hf.ItemByID(id) + if err != nil { + return nil, err + } + + ycc, err := l.decodeHevcItem(decoderID, hf, item) + if err != nil { + return nil, err + } + + rect := ycc.Bounds() + if tileWidth == 0 { + tileWidth, tileHeight = rect.Dx(), rect.Dy() + width, height := tileWidth*grid.columns, tileHeight*grid.rows + outImage = image.NewYCbCr(image.Rectangle{image.Pt(0, 0), image.Pt(width, height)}, ycc.SubsampleRatio) + } + + if tileWidth != rect.Dx() || tileHeight != rect.Dy() { + return nil, fmt.Errorf("Inconsistent tile dimensions") + } + + // copy y stride data + for i := 0; i < rect.Dy(); i += 1 { + copy(outImage.Y[(y*tileHeight+i)*outImage.YStride+x*ycc.YStride:], ycc.Y[i*ycc.YStride:(i+1)*ycc.YStride]) + } + + // height of c strides + cHeight := len(ycc.Cb) / ycc.CStride + + // copy c stride data + for i := 0; i < cHeight; i += 1 { + copy(outImage.Cb[(y*cHeight+i)*outImage.CStride+x*ycc.CStride:], ycc.Cb[i*ycc.CStride:(i+1)*ycc.CStride]) + copy(outImage.Cr[(y*cHeight+i)*outImage.CStride+x*ycc.CStride:], ycc.Cr[i*ycc.CStride:(i+1)*ycc.CStride]) + } + + i += 1 + } + } + + //crop to actual size when applicable + outImage.Rect = image.Rectangle{image.Pt(0, 0), image.Pt(width, height)} + } + + var imgBuf bytes.Buffer + if request.OutputFormat == requests.RenderFileOutputFormatJPG { + var opt jpeg.Options + opt.Quality = 95 + + for { + err := jpeg.Encode(&imgBuf, outImage, &opt) + if err != nil { + return nil, err + } + + if request.MaxFileSize == 0 || int64(imgBuf.Len()) < request.MaxFileSize { + break + } + + opt.Quality -= 10 + + if opt.Quality <= 45 { + return nil, errors.New("image would exceed maximum filesize") + } + + imgBuf.Reset() + } + } else if request.OutputFormat == requests.RenderFileOutputFormatPNG { + err := png.Encode(&imgBuf, outImage) + if err != nil { + return nil, err + } + + if request.MaxFileSize != 0 && int64(imgBuf.Len()) > request.MaxFileSize { + return nil, errors.New("image would exceed maximum filesize") + } + } else { + return nil, errors.New("invalid output format given") + } + + output := imgBuf.Bytes() + return &responses.RenderFile{Output: &output}, nil +} + type decoder struct { ctx unsafe.Pointer hasImage bool diff --git a/libde265/requests/requests.go b/libde265/requests/requests.go index cd9fc42..759a3e4 100644 --- a/libde265/requests/requests.go +++ b/libde265/requests/requests.go @@ -14,10 +14,24 @@ type ResetDecoder struct { type PushDecoder struct { ID string - Data []byte + Data *[]byte } type RenderDecoder struct { ID string - Data []byte + Data *[]byte +} + +type RenderFileOutputFormat string // The file format to render output as. + +const ( + RenderFileOutputFormatJPG RenderFileOutputFormat = "jpg" // Render the file as a JPEG file. + RenderFileOutputFormatPNG RenderFileOutputFormat = "png" // Render the file as a PNG file. +) + +type RenderFile struct { + Data *[]byte // The file data. + OutputFormat RenderFileOutputFormat // The format to output the image as + MaxFileSize int64 // The maximum filesize, if jpg is chosen as output format, it will try to compress it until it fits + SafeEncoding bool // Whether to use safe encoding. } diff --git a/libde265/responses/responses.go b/libde265/responses/responses.go index cc1580b..d303ba1 100644 --- a/libde265/responses/responses.go +++ b/libde265/responses/responses.go @@ -18,3 +18,7 @@ type PushDecoder struct { type RenderDecoder struct { Image *image.YCbCr } + +type RenderFile struct { + Output *[]byte +} diff --git a/libde265/shared/libde265_interface.go b/libde265/shared/libde265_interface.go index 2eafce4..4d9cbe7 100644 --- a/libde265/shared/libde265_interface.go +++ b/libde265/shared/libde265_interface.go @@ -17,6 +17,7 @@ type Libde265 interface { PushDecoder(*requests.PushDecoder) (*responses.PushDecoder, error) ResetDecoder(*requests.ResetDecoder) (*responses.ResetDecoder, error) RenderDecoder(*requests.RenderDecoder) (*responses.RenderDecoder, error) + RenderFile(*requests.RenderFile) (*responses.RenderFile, error) } type Libde265RPC struct{ client *rpc.Client } @@ -81,6 +82,16 @@ func (g *Libde265RPC) RenderDecoder(request *requests.RenderDecoder) (*responses return resp, nil } +func (g *Libde265RPC) RenderFile(request *requests.RenderFile) (*responses.RenderFile, error) { + resp := &responses.RenderFile{} + err := g.client.Call("Plugin.RenderFile", request, resp) + if err != nil { + return nil, err + } + + return resp, nil +} + type Libde265RPCServer struct { Impl Libde265 } @@ -184,6 +195,24 @@ func (s *Libde265RPCServer) RenderDecoder(request *requests.RenderDecoder, resp return nil } +func (s *Libde265RPCServer) RenderFile(request *requests.RenderFile, resp *responses.RenderFile) (err error) { + defer func() { + if panicError := recover(); panicError != nil { + err = fmt.Errorf("panic occurred in %s: %v", "RenderFile", panicError) + } + }() + + implResp, err := s.Impl.RenderFile(request) + if err != nil { + return err + } + + // Overwrite the target address of resp to the target address of implResp. + *resp = *implResp + + return nil +} + type Libde265Plugin struct { Impl Libde265 }