Skip to content

Commit

Permalink
Merge pull request #24 from flowrean/node-locations-on-ways
Browse files Browse the repository at this point in the history 
Add support for node locations on ways
  • Loading branch information
@paulmach
paulmach authored Apr 1, 2021
2 parents eeed6ca + 203c4db commit 75f29bc
Show file tree
Hide file tree
Showing 7 changed files with 681 additions and 317 deletions.
6 changes: 6 additions & 0 deletions osmpbf/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,12 @@ type Scanner struct {
}
```

### OSM PBF files with node locations on ways

This package supports reading OSM PBF files where the ways have been annotated with the coordinates of each node. Such files can be generated using [osmium](https://osmcode.org/osmium-tool), with the [add-locations-to-ways](https://docs.osmcode.org/osmium/latest/osmium-add-locations-to-ways.html) subcommand. This feature makes it possible to work with the ways and their geometries without having to keep all node locations in some index (which takes work and memory resources).

Coordinates are stored in the `Lat` and `Lon` fields of each `WayNode`. There is no need to specify an explicit option; when the node locations are present on the ways, they are loaded automatically. For more info about the OSM PBF format extension, see [the original blog post](https://blog.jochentopf.com/2016-04-20-node-locations-on-ways.html).

### Using cgo/czlib for decompression

OSM PBF files are a set of blocks that are zlib compressed. When using the pure golang
Expand Down
48 changes: 47 additions & 1 deletion osmpbf/decode_data.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ type dataDecoder struct {

// ways
nodes *protoscan.Iterator
wlats *protoscan.Iterator
wlons *protoscan.Iterator

// relations
roles *protoscan.Iterator
Expand Down Expand Up @@ -422,8 +424,12 @@ func (dec *dataDecoder) extractDenseNodes() error {

func (dec *dataDecoder) scanWays(data []byte) error {
st := dec.primitiveBlock.GetStringtable().GetS()
granularity := int64(dec.primitiveBlock.GetGranularity())
dateGranularity := int64(dec.primitiveBlock.GetDateGranularity())

latOffset := dec.primitiveBlock.GetLatOffset()
lonOffset := dec.primitiveBlock.GetLonOffset()

msg := protoscan.New(data)

way := &osm.Way{Visible: true}
Expand Down Expand Up @@ -503,7 +509,9 @@ func (dec *dataDecoder) scanWays(data []byte) error {
}

var prev, index int64
way.Nodes = make(osm.WayNodes, dec.nodes.Count(protoscan.WireTypeVarint))
if len(way.Nodes) == 0 {
way.Nodes = make(osm.WayNodes, dec.nodes.Count(protoscan.WireTypeVarint))
}
for dec.nodes.HasNext() {
v, err := dec.nodes.Sint64()
if err != nil {
Expand All @@ -513,6 +521,44 @@ func (dec *dataDecoder) scanWays(data []byte) error {
way.Nodes[index].ID = osm.NodeID(prev)
index++
}
case 9: // lat
dec.wlats, err = msg.Iterator(dec.wlats)
if err != nil {
return err
}

var prev, index int64
if len(way.Nodes) == 0 {
way.Nodes = make(osm.WayNodes, dec.wlats.Count(protoscan.WireTypeVarint))
}
for dec.wlats.HasNext() {
v, err := dec.wlats.Sint64()
if err != nil {
return err
}
prev = v + prev // delta encoding
way.Nodes[index].Lat = 1e-9 * float64(latOffset + (granularity * prev))
index++
}
case 10: // lon
dec.wlons, err = msg.Iterator(dec.wlons)
if err != nil {
return err
}

var prev, index int64
if len(way.Nodes) == 0 {
way.Nodes = make(osm.WayNodes, dec.wlons.Count(protoscan.WireTypeVarint))
}
for dec.wlons.HasNext() {
v, err := dec.wlons.Sint64()
if err != nil {
return err
}
prev = v + prev // delta encoding
way.Nodes[index].Lon = 1e-9 * float64(lonOffset + (granularity * prev))
index++
}
default:
msg.Skip()
}
Expand Down
173 changes: 124 additions & 49 deletions osmpbf/decode_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"
"io"
"math"
"net/http"
"os"
"reflect"
Expand All @@ -18,6 +19,11 @@ const (
// Originally downloaded from http://download.geofabrik.de/europe/great-britain/england/greater-london.html
London = "greater-london-140324.osm.pbf"
LondonURL = "https://gist.githubusercontent.com/paulmach/853d57b83d408480d3b148b07954c110/raw/853f33f4dbe4246915134f1cde8edb30241ecc10/greater-london-140324.osm.pbf"
// Created based on the above file, by running `osmium add-locations-to-ways`.
LondonLocations = "greater-london-140324-low.osm.pbf"
LondonLocationsURL = "https://drive.google.com/u/0/uc?id=1S_Fuya6hUsejDx5YqAhGHpJ8mnRndcEC&export=download"

coordinatesPrecision = 1e7
)

func parseTime(s string) time.Time {
Expand All @@ -28,31 +34,65 @@ func parseTime(s string) time.Time {
return t
}

func stripCoordinates(w *osm.Way) *osm.Way {
if w == nil {
return nil
}
ws := new(osm.Way)
*ws = *w
ws.Nodes = make(osm.WayNodes, len(w.Nodes))
for i, n := range w.Nodes {
n.Lat, n.Lon = 0, 0
ws.Nodes[i] = n
}
return ws
}

func roundCoordinates(w *osm.Way) {
if w == nil {
return
}
for i := range w.Nodes {
w.Nodes[i].Lat = math.Round(w.Nodes[i].Lat * coordinatesPrecision) / coordinatesPrecision
w.Nodes[i].Lon = math.Round(w.Nodes[i].Lon * coordinatesPrecision) / coordinatesPrecision
}
}

type OSMFileTest struct {
*testing.T
FileName string
FileURL string
ExpNode *osm.Node
ExpWay *osm.Way
ExpRel *osm.Relation
ExpNodeCount, ExpWayCount, ExpRelCount uint64
IDsExpOrder []string
}

var (
IDsExpectedOrder = []string{
// Start of dense nodes.
idsExpectedOrderNodes = []string{
"node/44", "node/47", "node/52", "node/58", "node/60",
"node/79", // Just because way/79 is already there
"node/2740703694", "node/2740703695", "node/2740703697",
"node/2740703699", "node/2740703701",
// End of dense nodes.

// Start of ways.
}
idsExpectedOrderWays = []string{
"way/73", "way/74", "way/75", "way/79", "way/482",
"way/268745428", "way/268745431", "way/268745434", "way/268745436",
"way/268745439",
// End of ways.

// Start of relations.
}
idsExpectedOrderRelations = []string{
"relation/69", "relation/94", "relation/152", "relation/245",
"relation/332", "relation/3593436", "relation/3595575",
"relation/3595798", "relation/3599126", "relation/3599127",
// End of relations
}
IDsExpectedOrderNoNodes = append(idsExpectedOrderWays, idsExpectedOrderRelations...)
IDsExpectedOrder = append(idsExpectedOrderNodes, IDsExpectedOrderNoNodes...)

IDs map[string]bool

enc uint64 = 2729006
encl uint64 = 244523
ewc uint64 = 459055
erc uint64 = 12833

Expand All @@ -75,19 +115,19 @@ var (
Visible: true,
}

ew = &osm.Way{
ewl = &osm.Way{
ID: 4257116,
Nodes: osm.WayNodes{
{ID: 21544864},
{ID: 333731851},
{ID: 333731852},
{ID: 333731850},
{ID: 333731855},
{ID: 333731858},
{ID: 333731854},
{ID: 108047},
{ID: 769984352},
{ID: 21544864},
{ID: 21544864, Lat: 51.5230531, Lon: -0.1408525},
{ID: 333731851, Lat: 51.5224309, Lon: -0.1402297},
{ID: 333731852, Lat: 51.5224107, Lon: -0.1401878},
{ID: 333731850, Lat: 51.522422, Lon: -0.1401375},
{ID: 333731855, Lat: 51.522792, Lon: -0.1392477},
{ID: 333731858, Lat: 51.5228209, Lon: -0.1392124},
{ID: 333731854, Lat: 51.5228579, Lon: -0.1392339},
{ID: 108047, Lat: 51.5234407, Lon: -0.1398771},
{ID: 769984352, Lat: 51.5232469, Lon: -0.1403648},
{ID: 21544864, Lat: 51.5230531, Lon: -0.1408525},
},
Tags: osm.Tags([]osm.Tag{
{Key: "area", Value: "yes"},
Expand All @@ -102,6 +142,8 @@ var (
Visible: true,
}

ew = stripCoordinates(ewl)

er = &osm.Relation{
ID: 7677,
Members: osm.Members{
Expand All @@ -128,45 +170,45 @@ func init() {
}
}

func downloadTestOSMFile(t *testing.T) {
if _, err := os.Stat(London); os.IsNotExist(err) {
out, err := os.Create(London)
func (ft *OSMFileTest) downloadTestOSMFile() {
if _, err := os.Stat(ft.FileName); os.IsNotExist(err) {
out, err := os.Create(ft.FileName)
if err != nil {
t.Fatal(err)
ft.T.Fatal(err)
}
defer out.Close()

resp, err := http.Get(LondonURL)
resp, err := http.Get(ft.FileURL)
if err != nil {
t.Fatal(err)
ft.T.Fatal(err)
}
defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
t.Fatalf("test status code invalid: %v", resp.StatusCode)
ft.T.Fatalf("test status code invalid: %v", resp.StatusCode)
}

if _, err := io.Copy(out, resp.Body); err != nil {
t.Fatal(err)
ft.T.Fatal(err)
}
} else if err != nil {
t.Fatal(err)
ft.T.Fatal(err)
}
}

func TestDecode(t *testing.T) {
downloadTestOSMFile(t)
func (ft *OSMFileTest) testDecode() {
ft.downloadTestOSMFile()

f, err := os.Open(London)
f, err := os.Open(ft.FileName)
if err != nil {
t.Fatal(err)
ft.T.Fatal(err)
}
defer f.Close()

d := newDecoder(context.Background(), &Scanner{}, f)
err = d.Start(runtime.GOMAXPROCS(-1))
if err != nil {
t.Fatal(err)
ft.T.Fatal(err)
}

var n *osm.Node
Expand All @@ -181,13 +223,13 @@ func TestDecode(t *testing.T) {
if err == io.EOF {
break
} else if err != nil {
t.Fatal(err)
ft.T.Fatal(err)
}

switch v := e.(type) {
case *osm.Node:
nc++
if v.ID == en.ID {
if v.ID == ft.ExpNode.ID {
n = v
}
id = fmt.Sprintf("node/%d", v.ID)
Expand All @@ -196,7 +238,7 @@ func TestDecode(t *testing.T) {
}
case *osm.Way:
wc++
if v.ID == ew.ID {
if v.ID == ft.ExpWay.ID {
w = v
}
id = fmt.Sprintf("way/%d", v.ID)
Expand All @@ -205,7 +247,7 @@ func TestDecode(t *testing.T) {
}
case *osm.Relation:
rc++
if v.ID == er.ID {
if v.ID == ft.ExpRel.ID {
r = v
}
id = fmt.Sprintf("relation/%d", v.ID)
Expand All @@ -216,22 +258,55 @@ func TestDecode(t *testing.T) {
}
d.Close()

if !reflect.DeepEqual(en, n) {
t.Errorf("\nExpected: %#v\nActual: %#v", en, n)
if !reflect.DeepEqual(ft.ExpNode, n) {
ft.T.Errorf("\nExpected: %#v\nActual: %#v", ft.ExpNode, n)
}
roundCoordinates(w)
if !reflect.DeepEqual(ft.ExpWay, w) {
ft.T.Errorf("\nExpected: %#v\nActual: %#v", ft.ExpWay, w)
}
if !reflect.DeepEqual(ew, w) {
t.Errorf("\nExpected: %#v\nActual: %#v", ew, w)
if !reflect.DeepEqual(ft.ExpRel, r) {
ft.T.Errorf("\nExpected: %#v\nActual: %#v", ft.ExpRel, r)
}
if !reflect.DeepEqual(er, r) {
t.Errorf("\nExpected: %#v\nActual: %#v", er, r)
if ft.ExpNodeCount != nc || ft.ExpWayCount != wc || ft.ExpRelCount != rc {
ft.T.Errorf("\nExpected %7d nodes, %7d ways, %7d relations\nGot %7d nodes, %7d ways, %7d relations.",
ft.ExpNodeCount, ft.ExpWayCount, ft.ExpRelCount, nc, wc, rc)
}
if enc != nc || ewc != wc || erc != rc {
t.Errorf("\nExpected %7d nodes, %7d ways, %7d relations\nGot %7d nodes, %7d ways, %7d relations.",
enc, ewc, erc, nc, wc, rc)
if !reflect.DeepEqual(ft.IDsExpOrder, idsOrder) {
ft.T.Errorf("\nExpected: %v\nGot: %v", ft.IDsExpOrder, idsOrder)
}
if !reflect.DeepEqual(IDsExpectedOrder, idsOrder) {
t.Errorf("\nExpected: %v\nGot: %v", IDsExpectedOrder, idsOrder)
}

func TestDecode(t *testing.T) {
ft := &OSMFileTest{
T: t,
FileName: London,
FileURL: LondonURL,
ExpNode: en,
ExpWay: ew,
ExpRel: er,
ExpNodeCount: enc,
ExpWayCount: ewc,
ExpRelCount: erc,
IDsExpOrder: IDsExpectedOrder,
}
ft.testDecode()
}

func TestDecodeLocations(t *testing.T) {
ft := &OSMFileTest{
T: t,
FileName: LondonLocations,
FileURL: LondonLocationsURL,
ExpNode: en,
ExpWay: ewl,
ExpRel: er,
ExpNodeCount: encl,
ExpWayCount: ewc,
ExpRelCount: erc,
IDsExpOrder: IDsExpectedOrderNoNodes,
}
ft.testDecode()
}

func TestDecode_Close(t *testing.T) {
Expand Down
Loading

0 comments on commit 75f29bc

Please sign in to comment.