replace czlib with go-libdeflate

.github/workflows/main.yml

@@ -12,7 +12,7 @@ jobs:
       - uses: actions/checkout@v3
 
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: '1.21'
 
@@ -29,4 +29,4 @@ jobs:
       - name: codecov
         uses: codecov/codecov-action@v1
         with:
-          file: ./profile.cov
+          file: ./profile.cov

README.md

@@ -144,9 +144,9 @@ BenchmarkChange_UnmarshalJSON-12     287707        317723        +10.43%
 ## CGO and zlib
 
 OSM PBF data comes in blocks, each block is zlib compressed. Decompressing this
-data takes about 33% of the total read time. [DataDog/czlib](https://github.com/DataDog/czlib) is
-used to speed this process.
-See [osmpbf/README.md](osmpbf#using-cgoczlib-for-decompression) for more details.
+data takes about 33% of the total read time. [4kills/go-libdeflate](https://github.com/4kills/go-libdeflate) is
+used to speed up decompressing.
+See [osmpbf/README.md](osmpbf#using-libdeflate-for-decompression) for more details.
 
 As a result, a C compiler is necessary to install this module. On macOS this may require
 installing pkg-config using something like `brew install pkg-config`

go.mod

@@ -1,12 +1,13 @@
 module github.com/paulmach/osm
 
-go 1.13
+go 1.19
 
 require (
-	github.com/datadog/czlib v0.0.0-20160811164712-4bc9a24e37f2
-	github.com/paulmach/orb v0.1.3
+	github.com/4kills/go-libdeflate/v2 v2.0.3
+	github.com/paulmach/orb v0.10.0
 	github.com/paulmach/protoscan v0.2.1
 	golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/protobuf v1.27.1
 )
+
+require go.mongodb.org/mongo-driver v1.12.1 // indirect

go.sum

@@ -1,17 +1,94 @@
-github.com/datadog/czlib v0.0.0-20160811164712-4bc9a24e37f2 h1:ISaMhBq2dagaoptFGUyywT5SzpysCbHofX3sCNw1djo=
-github.com/datadog/czlib v0.0.0-20160811164712-4bc9a24e37f2/go.mod h1:2yDaWzisHKoQoxm+EU4YgKBaD7g1M0pxy7THWG44Lro=
+github.com/4kills/go-libdeflate/v2 v2.0.3 h1:Y13oRUvtAXFJkcW4F0MnaQQB753a71sTutGrVbEAubQ=
+github.com/4kills/go-libdeflate/v2 v2.0.3/go.mod h1:hyouZv4OAhHaaMpYuejstUN0xOg8mA+yy75WE3Ty6SM=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/paulmach/orb v0.1.3 h1:Wa1nzU269Zv7V9paVEY1COWW8FCqv4PC/KJRbJSimpM=
-github.com/paulmach/orb v0.1.3/go.mod h1:VFlX/8C+IQ1p6FTRRKzKoOPJnvEtA5G0Veuqwbu//Vk=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/paulmach/orb v0.10.0 h1:guVYVqzxHE/CQ1KpfGO077TR0ATHSNjp4s6XGLn3W9s=
+github.com/paulmach/orb v0.10.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU=
 github.com/paulmach/protoscan v0.2.1 h1:rM0FpcTjUMvPUNk2BhPJrreDKetq43ChnL+x1sRg8O8=
 github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
+go.mongodb.org/mongo-driver v1.12.1 h1:nLkghSU8fQNaK7oUmDhQFsnrtcoNy7Z6LVFKsEecqgE=
+go.mongodb.org/mongo-driver v1.12.1/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w=
 golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

osmpbf/README.md

@@ -80,31 +80,39 @@ This package supports reading OSM PBF files where the ways have been annotated w
 
 Coordinates are stored in the `Lat` and `Lon` fields of each `WayNode`. There is no need to specify an explicit option; when the node locations are present on the ways, they are loaded automatically. For more info about the OSM PBF format extension, see [the original blog post](https://blog.jochentopf.com/2016-04-20-node-locations-on-ways.html).
 
-## Using cgo/czlib for decompression
+## Using cgo libdeflate for decompression
 
 OSM PBF files are a set of blocks that are zlib compressed. When using the pure golang
 implementation this can account for about 1/3 of the read time. When cgo is enabled
-the package will used [czlib](https://github.com/DataDog/czlib).
+the package [go-libdeflate](https://github.com/4kills/libdeflate) will used.
+
+Previous versions used the lib czlib based on zlib. libdeflate is more performant
+and more memory efficient for uncompressing.
 
 ```
 $ CGO_ENABLED=0 go test -bench . > disabled.txt
 $ CGO_ENABLED=1 go test -bench . > enabled.txt
 $ benchcmp disabled.txt enabled.txt
-benchmark                        old ns/op     new ns/op     delta
-BenchmarkLondon-12               312294630     229927205     -26.37%
-BenchmarkLondon_nodes-12         246562457     160021768     -35.10%
-BenchmarkLondon_ways-12          216803544     134747327     -37.85%
-BenchmarkLondon_relations-12     158722633     80560144      -49.24%
-
-benchmark                        old allocs     new allocs     delta
-BenchmarkLondon-12               2469128        2416804        -2.12%
-BenchmarkLondon_nodes-12         1056166        1003850        -4.95%
-BenchmarkLondon_ways-12          1845032        1792716        -2.84%
-BenchmarkLondon_relations-12     509090         456772         -10.28%
-
-benchmark                        old bytes     new bytes     delta
-BenchmarkLondon-12               963734544     954877896     -0.92%
-BenchmarkLondon_nodes-12         658337435     649482060     -1.35%
-BenchmarkLondon_ways-12          441674734     432819378     -2.00%
-BenchmarkLondon_relations-12     187941609     179086389     -4.71%
-```
+benchmark                              old ns/op     new ns/op     delta
+BenchmarkLondon-8                      361519289     275254714     -23.86%
+BenchmarkLondon_withFiltersTrue-8      392469042     263935960     -32.75%
+BenchmarkLondon_withFiltersFalse-8     310824940     200477972     -35.50%
+BenchmarkLondon_nodes-8                295277528     180614979     -38.83%
+BenchmarkLondon_ways-8                 257494509     140700970     -45.36%
+BenchmarkLondon_relations-8            189490128     75263200      -60.28%
+
+benchmark                              old allocs     new allocs     delta
+BenchmarkLondon-8                      4863784        4808526        -1.14%
+BenchmarkLondon_withFiltersTrue-8      4863786        4808515        -1.14%
+BenchmarkLondon_withFiltersFalse-8     1419995        1364724        -3.89%
+BenchmarkLondon_nodes-8                3450825        3395559        -1.60%
+BenchmarkLondon_ways-8                 1851359        1796099        -2.98%
+BenchmarkLondon_relations-8            515422         460152         -10.72%
+
+benchmark                              old bytes     new bytes     delta
+BenchmarkLondon-8                      947061317     924789892     -2.35%
+BenchmarkLondon_withFiltersTrue-8      947061146     924787588     -2.35%
+BenchmarkLondon_withFiltersFalse-8     388725836     366452840     -5.73%
+BenchmarkLondon_nodes-8                641663624     619391213     -3.47%
+BenchmarkLondon_ways-8                 460631859     438360054     -4.84%
+BenchmarkLondon_relations-8            206899749     184626277     -10.77%

osmpbf/decode.go

@@ -1,7 +1,6 @@
 package osmpbf
 
 import (
-	"bytes"
 	"context"
 	"encoding/binary"
 	"errors"
@@ -344,28 +343,8 @@ func getData(blob *osmpbf.Blob, data []byte) ([]byte, error) {
 		return blob.GetRaw(), nil
 
 	case blob.ZlibData != nil:
-		r, err := zlibReader(blob.GetZlibData())
-		if err != nil {
-			return nil, err
-		}
-
-		// using the bytes.Buffer allows for the preallocation of the necessary space.
-		l := blob.GetRawSize() + bytes.MinRead
-		if cap(data) < int(l) {
-			data = make([]byte, 0, l+l/10)
-		} else {
-			data = data[:0]
-		}
-		buf := bytes.NewBuffer(data)
-		if _, err = buf.ReadFrom(r); err != nil {
-			return nil, err
-		}
-
-		if buf.Len() != int(blob.GetRawSize()) {
-			return nil, fmt.Errorf("raw blob data size %d but expected %d", buf.Len(), blob.GetRawSize())
-		}
+		return decompress(blob.GetZlibData(), (int)(blob.GetRawSize()), data)
 
-		return buf.Bytes(), nil
 	default:
 		return nil, errors.New("unknown blob data")
 	}

osmpbf/zlib_cgo.go

@@ -1,14 +1,25 @@
+//go:build cgo
 // +build cgo
 
 package osmpbf
 
 import (
-	"bytes"
-	"io"
+	"fmt"
 
-	"github.com/datadog/czlib"
+	deflate "github.com/4kills/go-libdeflate/v2"
 )
 
-func zlibReader(data []byte) (io.ReadCloser, error) {
-	return czlib.NewReader(bytes.NewReader(data))
+func decompress(in []byte, size int, data []byte) ([]byte, error) {
+	if cap(data) > (int)(size) {
+		data = data[0:size]
+	} else {
+		data = nil
+	}
+
+	_, buf, err := deflate.DecompressZlib(in, data)
+	if len(buf) != int(size) {
+		return nil, fmt.Errorf("raw blob data size %d but expected %d", len(buf), size)
+	}
+
+	return buf, err
 }

osmpbf/zlib_go.go

@@ -1,13 +1,34 @@
+//go:build !cgo
 // +build !cgo
 
 package osmpbf
 
 import (
 	"bytes"
 	"compress/zlib"
-	"io"
+	"fmt"
 )
 
-func zlibReader(data []byte) (io.ReadCloser, error) {
-	return zlib.NewReader(bytes.NewReader(data))
+func decompress(in []byte, size int, data []byte) ([]byte, error) {
+	r, err := zlib.NewReader(bytes.NewReader(in))
+	if err != nil {
+		return nil, err
+	}
+
+	// using the bytes.Buffer allows for the preallocation of the necessary space.
+	l := size + bytes.MinRead
+	if cap(data) < int(l) {
+		data = make([]byte, 0, l+l/10)
+	} else {
+		data = data[:0]
+	}
+	buf := bytes.NewBuffer(data)
+	if _, err = buf.ReadFrom(r); err != nil {
+		return nil, err
+	}
+
+	if buf.Len() != int(size) {
+		return nil, fmt.Errorf("raw blob data size %d but expected %d", buf.Len(), size)
+	}
+	return buf.Bytes(), nil
 }