From 82a0eb5f03631c759756f7835ea9b0c7f0384bb9 Mon Sep 17 00:00:00 2001 From: Aron Bierbaum Date: Mon, 4 Nov 2013 18:43:39 -0600 Subject: [PATCH 1/2] Add support for PostgreSQL HSTORE --- doc/source/mapping.rst | 3 +++ imposm/db/postgis.py | 13 +++++++++++++ imposm/mapping.py | 13 +++++++++++-- imposm/psqldb.py | 1 + 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/doc/source/mapping.rst b/doc/source/mapping.rst index 6d432c1..4a96bae 100644 --- a/doc/source/mapping.rst +++ b/doc/source/mapping.rst @@ -27,6 +27,9 @@ There are three classes for the base geometries: ``Points``, ``LineStrings`` an ``with_type_field`` Controls if the the *value* of the mapped key/value should be stored in the ``type`` column. Defaults to ``True``. +``use_hstore`` + Controls if all tags should be stored in a ``tags`` hstore column. Defaults to ``False`` + .. _mapping: mapping diff --git a/imposm/db/postgis.py b/imposm/db/postgis.py index 48f8aa6..60c7d88 100644 --- a/imposm/db/postgis.py +++ b/imposm/db/postgis.py @@ -18,6 +18,7 @@ import psycopg2 import psycopg2.extensions +import psycopg2.extras import logging log = logging.getLogger(__name__) @@ -76,6 +77,8 @@ def connection(self): ) self._connection.set_isolation_level( psycopg2.extensions.ISOLATION_LEVEL_READ_COMMITTED) + # Register adapter and typecaster for dict conversion. + psycopg2.extras.register_hstore(self._connection, unicode = True) return self._connection def commit(self): @@ -163,6 +166,12 @@ def _insert_stmt(self, mapping): extra_arg_names = [n for n, t in mapping.fields] extra_args = ', %s' * len(extra_arg_names) extra_arg_names = ', ' + ', '.join('"' + name + '"' for name in extra_arg_names) + + # Add tags argument for hstore. + if mapping.use_hstore: + extra_arg_names += ', tags' + extra_args += ', %s' + return """INSERT INTO "%(tablename)s" (osm_id, geometry %(extra_arg_names)s) VALUES (%%s, ST_Transform(ST_GeomFromWKB(%%s, 4326), %(srid)s) @@ -191,6 +200,10 @@ def create_table(self, mapping): for n, t in mapping.fields: extra_fields += ', "%s" %s ' % (n, t.column_type) + # Add hstore column named tags. + if mapping.use_hstore: + extra_fields += ', "tags" HSTORE ' + if config.imposm_pg_serial_id: serial_column = "id SERIAL PRIMARY KEY," else: diff --git a/imposm/mapping.py b/imposm/mapping.py index 30dd5d8..3e50b39 100644 --- a/imposm/mapping.py +++ b/imposm/mapping.py @@ -87,10 +87,12 @@ class Mapping(object): _insert_stmt = None with_type_field = True - def __init__(self, name, mapping, fields=None, field_filter=None, with_type_field=None): + def __init__(self, name, mapping, fields=None, field_filter=None, with_type_field=None, + use_hstore = False): self.name = name self.mapping = mapping self.fields = fields or tuple(self.fields) + self.use_hstore = use_hstore self.limit_to_polygon = None if with_type_field is not None: # allow subclass to define other default by setting it as class variable @@ -154,12 +156,19 @@ def build_geom(self, osm_elem): raise DropElem(ex) def field_values(self, osm_elem): - return [t.value(osm_elem.tags.get(n), osm_elem) for n, t in self.fields] + values = [t.value(osm_elem.tags.get(n), osm_elem) for n, t in self.fields] + # Add dictionary of all tags which will be converted by psycopg2. + if self.use_hstore: + values.append(osm_elem.tags) + return values def field_dict(self, osm_elem): result = dict((n, t.value(osm_elem.tags.get(n), osm_elem)) for n, t in self.fields) if self.with_type_field: del result['type'] + # Add dictionary of all tags which will be converted by psycopg2. + if self.use_hstore: + result['tags'] = osm_elem.tags result[osm_elem.cls] = osm_elem.type return result diff --git a/imposm/psqldb.py b/imposm/psqldb.py index fdfff42..1b0e45e 100644 --- a/imposm/psqldb.py +++ b/imposm/psqldb.py @@ -24,6 +24,7 @@ createdb -E UTF8 -O ${user} ${dbname} createlang plpgsql ${dbname} ${postgis} +echo "CREATE EXTENSION hstore;" | psql -d ${dbname} echo "ALTER TABLE spatial_ref_sys OWNER TO ${user};" | psql -d ${dbname} echo "ALTER USER ${user} WITH PASSWORD '${password}';" |psql -d ${dbname} echo "host\t${dbname}\t${user}\t127.0.0.1/32\tmd5" >> ${pg_hba} From 818318f7ac299b8b2bf51f5c97bb1f9d192da5a5 Mon Sep 17 00:00:00 2001 From: Aron Bierbaum Date: Thu, 7 Nov 2013 07:58:05 -0600 Subject: [PATCH 2/2] Allow storing all tags --- doc/source/mapping.rst | 8 ++++++++ imposm/mapping.py | 20 +++++++++++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/doc/source/mapping.rst b/doc/source/mapping.rst index 4a96bae..47db918 100644 --- a/doc/source/mapping.rst +++ b/doc/source/mapping.rst @@ -77,6 +77,14 @@ There is a default field for names if you do not supply a field for the `name` c .. autofunction:: set_default_name_type +PostgreSQL hstore +^^^^^^^^^^^^^^^^^ + +By default each mapping only stores the fields that are specified in the `tags` hstore column. If you want to store all OSM tags can call :func:`set_hstore_all_tags`. + +.. autofunction:: set_hstore_all_tags + + Classes ~~~~~~~ diff --git a/imposm/mapping.py b/imposm/mapping.py index 3e50b39..16f2789 100644 --- a/imposm/mapping.py +++ b/imposm/mapping.py @@ -39,9 +39,11 @@ 'FixInvalidPolygons', 'UnionView', 'set_default_name_field', + 'set_hstore_all_tags' ] default_name_field = None +hstore_all_tags = False def set_default_name_type(type, column_name='name'): """ @@ -55,6 +57,17 @@ def set_default_name_type(type, column_name='name'): global default_name_field default_name_field = column_name, type +def set_hstore_all_tags(all_tags): + """ + Set flag that indicates that we should disable the filtering of tags. + + :: + + set_hstore_all_tags(True) + """ + global hstore_all_tags + hstore_all_tags = all_tags + # changed by imposm.app if the projection is epsg:4326 import_srs_is_geographic = False @@ -227,6 +240,10 @@ def for_relations(self, tags): return self._mapping_for_tags(self.polygon_mappings, tags) def _tag_filter(self, filter_tags): + # Disable filtering of tags by returning None. + if hstore_all_tags: + return None + def filter(tags): for k in tags.keys(): if k not in filter_tags: @@ -271,7 +288,8 @@ def rel_filter(tags): tags.clear() return tag_count = len(tags) - _rel_filter(tags) + if _rel_filter is not None: + _rel_filter(tags) if len(tags) < tag_count: # we removed tags... if not set(tags).difference(expected_tags):