lua interop

CMakeLists.txt

@@ -98,6 +98,7 @@ file(GLOB tilemaker_src_files
 	src/shp_processor.cpp
 	src/sorted_node_store.cpp
 	src/sorted_way_store.cpp
+	src/tag_map.cpp
 	src/tile_data.cpp
 	src/tilemaker.cpp
 	src/tile_worker.cpp

Makefile

@@ -123,6 +123,7 @@ tilemaker: \
 	src/shp_processor.o \
 	src/sorted_node_store.o \
 	src/sorted_way_store.o \
+	src/tag_map.o \
 	src/tile_data.o \
 	src/tilemaker.o \
 	src/tile_worker.o \

docs/CONFIGURATION.md

@@ -107,13 +107,16 @@ For example:
 
 ### Lua processing
 
-Your Lua file needs to supply 5 things:
+Your Lua file needs to supply a few things:
 
 1. `node_keys`, a list of those OSM keys which indicate that a node should be processed
-2. `init_function(name)` (optional), a function to initialize Lua logic
-2. `node_function(node)`, a function to process an OSM node and add it to layers
-3. `way_function(way)`, a function to process an OSM way and add it to layers
-3. `exit_function` (optional), a function to finalize Lua logic (useful to show statistics)
+2. `node_function()`, a function to process an OSM node and add it to layers
+3. `way_function()`, a function to process an OSM way and add it to layers
+4. (optional) `init_function(name)`, a function to initialize Lua logic
+5. (optional) `exit_function`, a function to finalize Lua logic (useful to show statistics)
+6. (optional) `relation_scan_function`, a function to determine whether your Lua file wishes to process the given relation 
+7. (optional) `relation_function`, a function to process an OSM relation and add it to layers
+8. (optional) `attribute_function`, a function to remap attributes from shapefiles
 
 `node_keys` is a simple list (or in Lua parlance, a 'table') of OSM tag keys. If a node has one of those keys, it will be processed by `node_function`; if not, it'll be skipped. For example, if you wanted to show highway crossings and railway stations, it should be `{ "highway", "railway" }`. (This avoids the need to process the vast majority of nodes which contain no important tags at all.)
 
@@ -127,28 +130,30 @@ Note the order: you write to a layer first, then set attributes after.
 
 To do that, you use these methods:
 
-* `node:Find(key)` or `way:Find(key)`: get the value for a tag, or the empty string if not present. For example, `way:Find("railway")` might return "rail" for a railway, "siding" for a siding, or "" if it isn't a railway at all.
-* `node:Holds(key)` or `way:Holds(key)`: returns true if that key exists, false otherwise.
-* `node:Layer("layer_name", false)` or `way:Layer("layer_name", is_area)`: write this node/way to the named layer. This is how you put objects in your vector tile. is_area (true/false) specifies whether a way should be treated as an area, or just as a linestring.
-* `way:LayerAsCentroid("layer_name")`: write a single centroid point for this way to the named layer (useful for labels and POIs).
-* `node:Attribute(key,value,minzoom)` or `node:Attribute(key,value,minzoom)`: add an attribute to the most recently written layer. Argument `minzoom` is optional, use it if you do not want to write the attribute on lower zoom levels.
-* `node:AttributeNumeric(key,value,minzoom)`, `node:AttributeBoolean(key,value,minzoom)` (and `way:`...): for numeric/boolean columns.
-* `node:Id()` or `way:Id()`: get the OSM ID of the current object.
-* `node:ZOrder(number)` or `way:ZOrder(number)`: Set a numeric value (default 0, 1-byte signed integer) used to sort features within a layer. Use this feature to ensure a proper rendering order if the rendering engine itself does not support sorting. Sorting is not supported across layers merged with `write_to`. Features with different z-order are not merged if `combine_below` or `combine_polygons_below` is used.
-* `node:MinZoom(zoom)` or `way:MinZoom(zoom)`: set the minimum zoom level (0-15) at which this object will be written. Note that the JSON layer configuration minimum still applies (so `:MinZoom(5)` will have no effect if your layer only starts at z6).
-* `way:Length()` and `way:Area()`: return the length (metres)/area (square metres) of the current object. Requires recent Boost.
-* `way:Centroid()`: return the lat/lon of the centre of the current object as a two-element Lua table (element 1 is lat, 2 is lon).
+* `Find(key)`: get the value for a tag, or the empty string if not present. For example, `Find("railway")` might return "rail" for a railway, "siding" for a siding, or "" if it isn't a railway at all.
+* `Holds(key)`: returns true if that key exists, false otherwise.
+* `Layer("layer_name", is_area)`: write this node/way to the named layer. This is how you put objects in your vector tile. is_area (true/false) specifies whether a way should be treated as an area, or just as a linestring.
+* `LayerAsCentroid("layer_name")`: write a single centroid point for this way to the named layer (useful for labels and POIs).
+* `Attribute(key,value,minzoom)`: add an attribute to the most recently written layer. Argument `minzoom` is optional, use it if you do not want to write the attribute on lower zoom levels.
+* `AttributeNumeric(key,value,minzoom)`, `AttributeBoolean(key,value,minzoom)`: for numeric/boolean columns.
+* `Id()`: get the OSM ID of the current object.
+* `ZOrder(number)`: Set a numeric value (default 0, 1-byte signed integer) used to sort features within a layer. Use this feature to ensure a proper rendering order if the rendering engine itself does not support sorting. Sorting is not supported across layers merged with `write_to`. Features with different z-order are not merged if `combine_below` or `combine_polygons_below` is used.
+* `MinZoom(zoom)`: set the minimum zoom level (0-15) at which this object will be written. Note that the JSON layer configuration minimum still applies (so `:MinZoom(5)` will have no effect if your layer only starts at z6).
+* `Length()` and `Area()`: return the length (metres)/area (square metres) of the current object. Requires recent Boost.
+* `Centroid()`: return the lat/lon of the centre of the current object as a two-element Lua table (element 1 is lat, 2 is lon).
 
 The simplest possible function, to include roads/paths and nothing else, might look like this:
 
-    function way_function(way)
-      local highway = way:Find("highway")
+```lua
+    function way_function()
+      local highway = Find("highway")
       if highway~="" then
-        way:Layer("roads", false)
-        way:Attribute("name", way:Find("name"))
-        way:Attribute("type", highway)
+        Layer("roads", false)
+        Attribute("name", Find("name"))
+        Attribute("type", highway)
       end
     end
+```
 
 Take a look at the supplied process.lua for a simple example, or the more complex OpenMapTiles-compatible script in `resources/`. You can specify another filename with the `--process` option.
 
@@ -197,11 +202,11 @@ When processing OSM objects with your Lua script, you can perform simple spatial
 
 You can then find out whether a node is within one of these polygons using the `Intersects` method:
 
-    if node:Intersects("countries") then print("Looks like it's on land"); end
+    if Intersects("countries") then print("Looks like it's on land"); end
 
 Or you can find out what country(/ies) the node is within using `FindIntersecting`, which returns a table:
 
-    names = node:FindIntersecting("countries")
+    names = FindIntersecting("countries")
     print(table.concat(name,","))
 
 To enable these functions, set `index` to true in your shapefile layer definition. `index_column` is not needed for `Intersects` but required for `FindIntersecting`.

docs/RELATIONS.md

@@ -22,26 +22,30 @@ This is a two-stage process: first, when reading relations, indicate that these
 
 To define which relations should be accepted, add a `relation_scan_function`:
 
-    function relation_scan_function(relation)
-      if relation:Find("type")=="route" and relation:Find("route")=="bicycle" then
-        local network = relation:Find("network")
-        if network=="ncn" then relation:Accept() end
+```lua
+    function relation_scan_function()
+      if Find("type")=="route" and Find("route")=="bicycle" then
+        local network = Find("network")
+        if network=="ncn" then Accept() end
       end
     end
+```
 
-This function takes the relation as its sole argument. Examine the tags using `relation:Find(key)` as normal. (You can also use `relation:Holds(key)` and `relation:Id()`.) If you want to use this relation, call `relation:Accept()`.
+Examine the tags using `Find(key)` as normal. (You can also use `Holds(key)` and `Id()`.) If you want to use this relation, call `Accept()`.
 
 #### Stage 2: accessing relations from ways
 
-Now that you've accepted the relations, they will be available from `way_function`. They are accessed using an iterator (`way:NextRelation()`) which reads each relation for that way in turn, returning nil when there are no more relations available. Once you have accessed a relation with the iterator, you can read its tags with `way:FindInRelation(key)`. For example:
+Now that you've accepted the relations, they will be available from `way_function`. They are accessed using an iterator (`NextRelation()`) which reads each relation for that way in turn, returning nil when there are no more relations available. Once you have accessed a relation with the iterator, you can read its tags with `FindInRelation(key)`. For example:
 
+```lua
     while true do
-      local rel = way:NextRelation()
+      local rel = NextRelation()
       if not rel then break end
-      print ("Part of route "..way:FindInRelation("ref"))
+      print ("Part of route "..FindInRelation("ref"))
     end
+```
 
-(Should you need to re-read the relations, you can reset the iterator with `way:RestartRelations()`.)
+(Should you need to re-read the relations, you can reset the iterator with `RestartRelations()`.)
 
 
 ### Writing relation geometries
@@ -52,13 +56,15 @@ First, make sure that you have accepted the relations using `relation_scan_funct
 
 Then write a `relation_function`, which works in the same way as `way_function` would:
 
-    function relation_function(relation)
-      if relation:Find("type")=="route" and relation:Find("route")=="bicycle" then
-        relation:Layer("bike_routes", false)
-        relation:Attribute("class", relation:Find("network"))
-        relation:Attribute("ref", relation:Find("ref"))
+```lua
+    function relation_function()
+      if Find("type")=="route" and Find("route")=="bicycle" then
+        Layer("bike_routes", false)
+        Attribute("class", Find("network"))
+        Attribute("ref", Find("ref"))
       end
     end
+```
 
 
 ### Not supported

include/attribute_store.h

@@ -183,11 +183,14 @@ struct AttributePair {
 #define SHARD_BITS 14
 #define ATTRIBUTE_SHARDS (1 << SHARD_BITS)
 
+class AttributeStore;
 class AttributePairStore {
 public:
 	AttributePairStore():
 		finalized(false),
-		pairsMutex(ATTRIBUTE_SHARDS)
+		pairsMutex(ATTRIBUTE_SHARDS),
+		lookups(0),
+		lookupsUncached(0)
 	{
 		// The "hot" shard has a capacity of 64K, the others are unbounded.
 		pairs.push_back(DequeMap<AttributePair>(1 << 16));
@@ -202,9 +205,10 @@ class AttributePairStore {
 	const AttributePair& getPairUnsafe(uint32_t i) const;
 	uint32_t addPair(AttributePair& pair, bool isHot);
 
-	std::vector<DequeMap<AttributePair>> pairs;
 
 private:
+	friend class AttributeStore;
+	std::vector<DequeMap<AttributePair>> pairs;
 	bool finalized;
 	// We refer to all attribute pairs by index.
 	//
@@ -214,6 +218,8 @@ class AttributePairStore {
 	// we suspect will be popular. It only ever has 64KB items,
 	// so that we can reference it with a short.
 	mutable std::vector<std::mutex> pairsMutex;
+	std::atomic<uint64_t> lookupsUncached;
+	std::atomic<uint64_t> lookups;
 };
 
 // AttributeSet is a set of AttributePairs
@@ -406,7 +412,8 @@ struct AttributeStore {
 		finalized(false),
 		sets(ATTRIBUTE_SHARDS),
 		setsMutex(ATTRIBUTE_SHARDS),
-		lookups(0) {
+		lookups(0),
+		lookupsUncached(0) {
 	}
 
 	AttributeKeyStore keyStore;
@@ -418,6 +425,7 @@ struct AttributeStore {
 	mutable std::vector<std::mutex> setsMutex;
 
 	mutable std::mutex mutex;
+	std::atomic<uint64_t> lookupsUncached;
 	std::atomic<uint64_t> lookups;
 };
 

include/deque_map.h

@@ -95,7 +95,11 @@ class DequeMap {
 		return -1;
 	}
 
-	const T& at(uint32_t index) const {
+	inline const T& operator[](uint32_t index) const {
+		return objects[index];
+	}
+
+	inline const T& at(uint32_t index) const {
 		return objects.at(index);
 	}
 

include/osm_lua_processing.h

@@ -18,6 +18,8 @@
 
 #include <boost/container/flat_map.hpp>
 
+class TagMap;
+
 // Lua
 extern "C" {
 	#include "lua.h"
@@ -33,6 +35,19 @@ extern bool verbose;
 class AttributeStore;
 class AttributeSet;
 
+// A string, which might be in `currentTags` as a value. If Lua
+// code refers to an absent value, it'll fallback to passing
+// it as a std::string.
+//
+// The intent is that Attribute("name", Find("name")) is a common
+// pattern, and we ought to avoid marshalling a string back and
+// forth from C++ to Lua when possible.
+struct PossiblyKnownTagValue {
+	bool found;
+	uint32_t index;
+	std::string fallback;
+};
+
 /**
 	\brief OsmLuaProcessing - converts OSM objects into OutputObjects.
 
@@ -76,13 +91,13 @@ class OsmLuaProcessing {
 	using tag_map_t = boost::container::flat_map<protozero::data_view, protozero::data_view, DataViewLessThan>;
 
 	// Scan non-MP relation
-	bool scanRelation(WayID id, const tag_map_t &tags);
+	bool scanRelation(WayID id, const TagMap& tags);
 
 	/// \brief We are now processing a significant node
-	void setNode(NodeID id, LatpLon node, const tag_map_t &tags);
+	void setNode(NodeID id, LatpLon node, const TagMap& tags);
 
 	/// \brief We are now processing a way
-	bool setWay(WayID wayId, LatpLonVec const &llVec, const tag_map_t &tags);
+	bool setWay(WayID wayId, LatpLonVec const &llVec, const TagMap& tags);
 
 	/** \brief We are now processing a relation
 	 * (note that we store relations as ways with artificial IDs, and that
@@ -93,7 +108,7 @@ class OsmLuaProcessing {
 		const PbfReader::Relation& relation,
 		const WayVec& outerWayVec,
 		const WayVec& innerWayVec,
-		const tag_map_t& tags,
+		const TagMap& tags,
 		bool isNativeMP,
 		bool isInnerOuter
 	);
@@ -103,12 +118,6 @@ class OsmLuaProcessing {
 	// Get the ID of the current object
 	std::string Id() const;
 
-	// Check if there's a value for a given key
-	bool Holds(const std::string& key) const;
-
-	// Get an OSM tag for a given key (or return empty string if none)
-	const std::string Find(const std::string& key) const;
-
 	// ----	Spatial queries called from Lua
 
 	// Find intersecting shapefile layer
@@ -219,6 +228,7 @@ class OsmLuaProcessing {
 	inline AttributeStore &getAttributeStore() { return attributeStore; }
 
 	struct luaProcessingException :std::exception {};
+	const TagMap* currentTags;
 
 private:
 	/// Internal: clear current cached state
@@ -239,6 +249,8 @@ class OsmLuaProcessing {
 		lastStoredGeometryId = 0;
 	}
 
+	void removeAttributeIfNeeded(const std::string& key);
+
 	const inline Point getPoint() {
 		return Point(lon/10000000.0,latp/10000000.0);
 	}
@@ -281,7 +293,7 @@ class OsmLuaProcessing {
 	class LayerDefinition &layers;
 
 	std::vector<std::pair<OutputObject, AttributeSet>> outputs;		// All output objects that have been created
-	const boost::container::flat_map<protozero::data_view, protozero::data_view, DataViewLessThan>* currentTags;
+	std::vector<std::string> outputKeys;
 	const PbfReader::Relation* currentRelation;
 	const std::vector<protozero::data_view>* stringTable;
 

include/osm_store.h

@@ -82,52 +82,56 @@ class RelationScanStore {
 
 private:
 	using tag_map_t = boost::container::flat_map<std::string, std::string>;
-	std::map<WayID, std::vector<std::pair<WayID, uint16_t>>> relationsForWays;
-	std::map<NodeID, std::vector<std::pair<WayID, uint16_t>>> relationsForNodes;
-	std::map<WayID, tag_map_t> relationTags;
-	mutable std::mutex mutex;
+	std::vector<std::map<WayID, std::vector<std::pair<WayID, uint16_t>>>> relationsForWays;
+	std::vector<std::map<NodeID, std::vector<std::pair<WayID, uint16_t>>>> relationsForNodes;
+	std::vector<std::map<WayID, tag_map_t>> relationTags;
+	mutable std::vector<std::mutex> mutex;
 	RelationRoles relationRoles;
 
 public:
+	RelationScanStore(): relationsForWays(128), relationsForNodes(128), relationTags(128), mutex(128) {}
 	void relation_contains_way(WayID relid, WayID wayid, std::string role) {
 		uint16_t roleId = relationRoles.getOrAddRole(role);
-		std::lock_guard<std::mutex> lock(mutex);
-		relationsForWays[wayid].emplace_back(std::make_pair(relid, roleId));
+		const size_t shard = wayid % mutex.size();
+		std::lock_guard<std::mutex> lock(mutex[shard]);
+		relationsForWays[shard][wayid].emplace_back(std::make_pair(relid, roleId));
 	}
 	void relation_contains_node(WayID relid, NodeID nodeId, std::string role) {
 		uint16_t roleId = relationRoles.getOrAddRole(role);
-		std::lock_guard<std::mutex> lock(mutex);
-		relationsForNodes[nodeId].emplace_back(std::make_pair(relid, roleId));
+		const size_t shard = nodeId % mutex.size();
+		std::lock_guard<std::mutex> lock(mutex[shard]);
+		relationsForNodes[shard][nodeId].emplace_back(std::make_pair(relid, roleId));
 	}
 	void store_relation_tags(WayID relid, const tag_map_t &tags) {
-		std::lock_guard<std::mutex> lock(mutex);
-		relationTags[relid] = tags;
+		const size_t shard = relid % mutex.size();
+		std::lock_guard<std::mutex> lock(mutex[shard]);
+		relationTags[shard][relid] = tags;
 	}
 	bool way_in_any_relations(WayID wayid) {
-		return relationsForWays.find(wayid) != relationsForWays.end();
+		const size_t shard = wayid % mutex.size();
+		return relationsForWays[shard].find(wayid) != relationsForWays[shard].end();
 	}
 	bool node_in_any_relations(NodeID nodeId) {
-		return relationsForNodes.find(nodeId) != relationsForNodes.end();
+		const size_t shard = nodeId % mutex.size();
+		return relationsForNodes[shard].find(nodeId) != relationsForNodes[shard].end();
 	}
 	std::string getRole(uint16_t roleId) const { return relationRoles.getRole(roleId); }
 	const std::vector<std::pair<WayID, uint16_t>>& relations_for_way(WayID wayid) {
-		return relationsForWays[wayid];
+		const size_t shard = wayid % mutex.size();
+		return relationsForWays[shard][wayid];
 	}
 	const std::vector<std::pair<WayID, uint16_t>>& relations_for_node(NodeID nodeId) {
-		return relationsForNodes[nodeId];
+		const size_t shard = nodeId % mutex.size();
+		return relationsForNodes[shard][nodeId];
 	}
 	std::string get_relation_tag(WayID relid, const std::string &key) {
-		auto it = relationTags.find(relid);
-		if (it==relationTags.end()) return "";
+		const size_t shard = relid % mutex.size();
+		auto it = relationTags[shard].find(relid);
+		if (it==relationTags[shard].end()) return "";
 		auto jt = it->second.find(key);
 		if (jt==it->second.end()) return "";
 		return jt->second;
 	}
-	void clear() {
-		std::lock_guard<std::mutex> lock(mutex);
-		relationsForWays.clear();
-		relationTags.clear();
-	}
 };