Merge pull request #660 from cldellow/bug-657

ignore --shard-stores if --compact is present
systemed · Jan 28, 2024 · 1c8ad9d · 1c8ad9d
2 parents 95644d2 + 5b96711
commit 1c8ad9d
Show file tree

Hide file tree

Showing 5 changed files with 18 additions and 4 deletions.
diff --git a/include/node_stores.h b/include/node_stores.h
@@ -59,8 +59,8 @@ class CompactNodeStore : public NodeStore
 	void batchStart() override {}
 
 	// CompactNodeStore has no metadata to know whether or not it contains
-	// a node, so it's not suitable for used in sharded scenarios.
-	bool contains(size_t shard, NodeID id) const override { return true; }
+	// a node, so it's not suitable for use in sharded scenarios
+	bool contains(size_t shard, NodeID id) const override { throw std::runtime_error("CompactNodeStore does not support contains"); }
 	NodeStore& shard(size_t shard) override { return *this; }
 	const NodeStore& shard(size_t shard) const override { return *this; }
 	size_t shards() const override { return 1; }

diff --git a/include/osm_store.h b/include/osm_store.h
@@ -292,6 +292,7 @@ class OSMStore
 	void open(std::string const &osm_store_filename);
 
 	void use_compact_store(bool use) { use_compact_nodes = use; }
+	bool isCompactStore() { return use_compact_nodes; }
 	void enforce_integrity(bool ei) { require_integrity = ei; }
 	bool integrity_enforced() { return require_integrity; }
 

diff --git a/include/pbf_processor.h b/include/pbf_processor.h
@@ -2,6 +2,7 @@
 #ifndef _READ_PBF_H
 #define _READ_PBF_H
 
+#include <atomic>
 #include <string>
 #include <unordered_set>
 #include <vector>
@@ -122,6 +123,7 @@ class PbfProcessor
 
 	OSMStore &osmStore;
 	std::mutex ioMutex;
+	std::atomic<bool> compactWarningIssued;
 };
 
 int ReadPbfBoundingBox(const std::string &inputFile, double &minLon, double &maxLon, 

diff --git a/src/pbf_processor.cpp b/src/pbf_processor.cpp
@@ -21,7 +21,7 @@ std::atomic<uint64_t> blocksProcessed(0), blocksToProcess(0);
 thread_local PbfReader::PbfReader reader;
 
 PbfProcessor::PbfProcessor(OSMStore &osmStore)
-	: osmStore(osmStore)
+	: osmStore(osmStore), compactWarningIssued(false)
 { }
 
 bool PbfProcessor::ReadNodes(OsmLuaProcessing& output, PbfReader::PrimitiveGroup& pg, const PbfReader::PrimitiveBlock& pb, const SignificantTags& nodeKeys)
@@ -31,8 +31,17 @@ bool PbfProcessor::ReadNodes(OsmLuaProcessing& output, PbfReader::PrimitiveGroup
 	TagMap tags;
 
 
+	bool isCompactStore = osmStore.isCompactStore();
+	NodeID lastNodeId = 0;
 	for (auto& node : pg.nodes()) {
 		NodeID nodeId = node.id;
+		if (isCompactStore && lastNodeId != 0 && nodeId != lastNodeId + 1 && !compactWarningIssued.exchange(true)) {
+			std::lock_guard<std::mutex> lock(ioMutex);
+			std::cout << "warning: --compact mode enabled, but PBF has gaps in IDs" << std::endl;
+			std::cout << "         to fix: osmium renumber your-file.osm.pbf -o renumbered.osm.pbf" << std::endl;
+		}
+		lastNodeId = nodeId;
+
 		LatpLon latplon = { int(lat2latp(double(node.lat)/10000000.0)*10000000.0), node.lon };
 
 		tags.reset();

diff --git a/src/tilemaker.cpp b/src/tilemaker.cpp
@@ -189,7 +189,9 @@ int main(const int argc, const char* argv[]) {
 
 	shared_ptr<NodeStore> nodeStore;
 
-	if (options.osm.shardStores) {
+	// CompactNodeStore is a dense datatype; it doesn't make sense to allocate
+	// more than one of them.
+	if (options.osm.shardStores && !options.osm.compact) {
 		nodeStore = std::make_shared<ShardedNodeStore>(createNodeStore);
 	} else {
 		nodeStore = createNodeStore();