diff --git a/doc/source/mapping.rst b/doc/source/mapping.rst index 6d432c1..47db918 100644 --- a/doc/source/mapping.rst +++ b/doc/source/mapping.rst @@ -27,6 +27,9 @@ There are three classes for the base geometries: ``Points``, ``LineStrings`` an ``with_type_field`` Controls if the the *value* of the mapped key/value should be stored in the ``type`` column. Defaults to ``True``. +``use_hstore`` + Controls if all tags should be stored in a ``tags`` hstore column. Defaults to ``False`` + .. _mapping: mapping @@ -74,6 +77,14 @@ There is a default field for names if you do not supply a field for the `name` c .. autofunction:: set_default_name_type +PostgreSQL hstore +^^^^^^^^^^^^^^^^^ + +By default each mapping only stores the fields that are specified in the `tags` hstore column. If you want to store all OSM tags can call :func:`set_hstore_all_tags`. + +.. autofunction:: set_hstore_all_tags + + Classes ~~~~~~~ diff --git a/imposm/db/postgis.py b/imposm/db/postgis.py index 48f8aa6..60c7d88 100644 --- a/imposm/db/postgis.py +++ b/imposm/db/postgis.py @@ -18,6 +18,7 @@ import psycopg2 import psycopg2.extensions +import psycopg2.extras import logging log = logging.getLogger(__name__) @@ -76,6 +77,8 @@ def connection(self): ) self._connection.set_isolation_level( psycopg2.extensions.ISOLATION_LEVEL_READ_COMMITTED) + # Register adapter and typecaster for dict conversion. + psycopg2.extras.register_hstore(self._connection, unicode = True) return self._connection def commit(self): @@ -163,6 +166,12 @@ def _insert_stmt(self, mapping): extra_arg_names = [n for n, t in mapping.fields] extra_args = ', %s' * len(extra_arg_names) extra_arg_names = ', ' + ', '.join('"' + name + '"' for name in extra_arg_names) + + # Add tags argument for hstore. + if mapping.use_hstore: + extra_arg_names += ', tags' + extra_args += ', %s' + return """INSERT INTO "%(tablename)s" (osm_id, geometry %(extra_arg_names)s) VALUES (%%s, ST_Transform(ST_GeomFromWKB(%%s, 4326), %(srid)s) @@ -191,6 +200,10 @@ def create_table(self, mapping): for n, t in mapping.fields: extra_fields += ', "%s" %s ' % (n, t.column_type) + # Add hstore column named tags. + if mapping.use_hstore: + extra_fields += ', "tags" HSTORE ' + if config.imposm_pg_serial_id: serial_column = "id SERIAL PRIMARY KEY," else: diff --git a/imposm/mapping.py b/imposm/mapping.py index 30dd5d8..16f2789 100644 --- a/imposm/mapping.py +++ b/imposm/mapping.py @@ -39,9 +39,11 @@ 'FixInvalidPolygons', 'UnionView', 'set_default_name_field', + 'set_hstore_all_tags' ] default_name_field = None +hstore_all_tags = False def set_default_name_type(type, column_name='name'): """ @@ -55,6 +57,17 @@ def set_default_name_type(type, column_name='name'): global default_name_field default_name_field = column_name, type +def set_hstore_all_tags(all_tags): + """ + Set flag that indicates that we should disable the filtering of tags. + + :: + + set_hstore_all_tags(True) + """ + global hstore_all_tags + hstore_all_tags = all_tags + # changed by imposm.app if the projection is epsg:4326 import_srs_is_geographic = False @@ -87,10 +100,12 @@ class Mapping(object): _insert_stmt = None with_type_field = True - def __init__(self, name, mapping, fields=None, field_filter=None, with_type_field=None): + def __init__(self, name, mapping, fields=None, field_filter=None, with_type_field=None, + use_hstore = False): self.name = name self.mapping = mapping self.fields = fields or tuple(self.fields) + self.use_hstore = use_hstore self.limit_to_polygon = None if with_type_field is not None: # allow subclass to define other default by setting it as class variable @@ -154,12 +169,19 @@ def build_geom(self, osm_elem): raise DropElem(ex) def field_values(self, osm_elem): - return [t.value(osm_elem.tags.get(n), osm_elem) for n, t in self.fields] + values = [t.value(osm_elem.tags.get(n), osm_elem) for n, t in self.fields] + # Add dictionary of all tags which will be converted by psycopg2. + if self.use_hstore: + values.append(osm_elem.tags) + return values def field_dict(self, osm_elem): result = dict((n, t.value(osm_elem.tags.get(n), osm_elem)) for n, t in self.fields) if self.with_type_field: del result['type'] + # Add dictionary of all tags which will be converted by psycopg2. + if self.use_hstore: + result['tags'] = osm_elem.tags result[osm_elem.cls] = osm_elem.type return result @@ -218,6 +240,10 @@ def for_relations(self, tags): return self._mapping_for_tags(self.polygon_mappings, tags) def _tag_filter(self, filter_tags): + # Disable filtering of tags by returning None. + if hstore_all_tags: + return None + def filter(tags): for k in tags.keys(): if k not in filter_tags: @@ -262,7 +288,8 @@ def rel_filter(tags): tags.clear() return tag_count = len(tags) - _rel_filter(tags) + if _rel_filter is not None: + _rel_filter(tags) if len(tags) < tag_count: # we removed tags... if not set(tags).difference(expected_tags): diff --git a/imposm/psqldb.py b/imposm/psqldb.py index fdfff42..1b0e45e 100644 --- a/imposm/psqldb.py +++ b/imposm/psqldb.py @@ -24,6 +24,7 @@ createdb -E UTF8 -O ${user} ${dbname} createlang plpgsql ${dbname} ${postgis} +echo "CREATE EXTENSION hstore;" | psql -d ${dbname} echo "ALTER TABLE spatial_ref_sys OWNER TO ${user};" | psql -d ${dbname} echo "ALTER USER ${user} WITH PASSWORD '${password}';" |psql -d ${dbname} echo "host\t${dbname}\t${user}\t127.0.0.1/32\tmd5" >> ${pg_hba}