Able to fill internal geocoder tables, but get 'too many source rows'…

… error during georeference analysis of countries.

Dockerfile

@@ -220,6 +220,7 @@ ADD ./config/cartodb.nginx.proxy.conf /etc/nginx/sites-enabled/default
 ADD ./config/varnish.vcl /etc/varnish.vcl
 ADD ./geocoder.sh /cartodb/script/geocoder.sh
 ADD ./geocoder_server.sql /cartodb/script/geocoder_server.sql
+ADD ./fill_geocoder_server.sh /cartodb/script/fill_geocoder_server.sh
 ENV PATH /usr/local/rvm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 RUN mkdir -p /cartodb/log && touch /cartodb/log/users_modifications && \
     /opt/varnish/sbin/varnishd -a :6081 -T localhost:6082 -s malloc,256m -f /etc/varnish.vcl && \

README.md

@@ -50,6 +50,9 @@ The chosen hostname should also resolve to an IP adress of the machine where the
 Geocoder
 --------
 
+The external geocoders like heremaps or mapzen have dummy api keys and do not work.
+No attempts haven been made or will be made in this Docker image to get the external geocoders to work.
+
 The internal geocoder is configured, but contains no data inside the image.
 
 To fill the internal geocoder run
@@ -58,4 +61,4 @@ docker exec -ti <carto docker container id> bash -c /cartodb/script/fill_geocode
 ```
 
 This will run the scripts described at https://github.com/CartoDB/data-services/tree/master/geocoder
-It will use at least require 10Gb of diskspace to download the dumps and import them.
+It will use at least require 5.7+7.8Gb of diskspace to download the dumps and import them.

create_dev_user

@@ -33,10 +33,3 @@ rake cartodb:db:set_user_account_type["${SUBDOMAIN}",'[DEDICATED]']
 
 # Set dataservices server
 rake cartodb:db:configure_geocoder_extension_for_non_org_users[$SUBDOMAIN]
-
-USER_DB=`echo "SELECT database_name FROM users WHERE username='dev'" | psql -U postgres -t carto_db_development`
-psql -U postgres $USER_DB < /cartodb/script/geocoder_client.sql
-echo "CREATE EXTENSION cdb_dataservices_client;" | psql -U postgres $USER_DB
-echo "SELECT CDB_Conf_SetConf('user_config', '{"'"is_organization"'": false, "'"entity_name"'": "'"dev"'"}');" | psql -U postgres $USER_DB
-GEOCODER_DB=`echo "SELECT database_name FROM users WHERE username='geocoder'" | psql -A -U postgres -t carto_db_development`
-echo -e "SELECT CDB_Conf_SetConf('geocoder_server_config', '{ \"connection_str\": \"host=localhost port=5432 dbname=${GEOCODER_DB} user=postgres\"}');" | psql -U postgres $USER_DB

geocoder.sh

@@ -1,3 +1,4 @@
+cd /cartodb
 
 rake cartodb:db:create_user --trace SUBDOMAIN="geocoder" \
 	PASSWORD="pass1234" ADMIN_PASSWORD="pass1234" \
@@ -11,6 +12,18 @@ rake cartodb:db:set_user_quota[geocoder,102400]
 echo "--- Allowing unlimited tables creation"
 rake cartodb:db:set_unlimited_table_quota[geocoder]
 
-
-GEOCODER_DB=`echo "SELECT database_name FROM users WHERE username='geocoder'" | psql -U postgres -t carto_db_development`    
+GEOCODER_DB=`echo "SELECT database_name FROM users WHERE username='geocoder'" | psql -U postgres -t carto_db_development`
 psql -U postgres $GEOCODER_DB < /cartodb/script/geocoder_server.sql
+
+# Setup dataservices client
+# dev user
+USER_DB=`echo "SELECT database_name FROM users WHERE username='dev'" | psql -U postgres -t carto_db_development`
+echo "CREATE EXTENSION cdb_dataservices_client;" | psql -U postgres $USER_DB
+echo "SELECT CDB_Conf_SetConf('user_config', '{"'"is_organization"'": false, "'"entity_name"'": "'"dev"'"}');" | psql -U postgres $USER_DB
+echo -e "SELECT CDB_Conf_SetConf('geocoder_server_config', '{ \"connection_str\": \"host=localhost port=5432 dbname=${GEOCODER_DB} user=postgres\"}');" | psql -U postgres $USER_DB
+
+# example organization
+ORGANIZATION_DB=`echo "SELECT database_name FROM users WHERE username='admin4example'" | psql -A -U postgres -t carto_db_development`
+echo "CREATE EXTENSION cdb_dataservices_client;" | psql -U postgres $ORGANIZATION_DB
+echo "SELECT CDB_Conf_SetConf('user_config', '{"'"is_organization"'": true, "'"entity_name"'": "'"example"'"}');" | psql -U postgres $ORGANIZATION_DB
+echo -e "SELECT CDB_Conf_SetConf('geocoder_server_config', '{ \"connection_str\": \"host=localhost port=5432 dbname=${GEOCODER_DB} user=postgres\"}');" | psql -U postgres $ORGANIZATION_DB

geocoder_server.sql

@@ -21,4 +21,11 @@ SELECT CDB_Conf_SetConf(
 SELECT CDB_Conf_SetConf(
     'server_conf',
     '{"environment": "development"}'
-);
+);
+
+SELECT cartodb.cdb_conf_setconf('logger_conf', '{"geocoder_log_path": "/tmp/geocodings.log"}');
+
+-- dummy conf from https://github.com/CartoDB/dataservices-api/blob/master/server/extension/test/sql/00_install_test.sql
+SELECT cartodb.cdb_conf_setconf('heremaps_conf', '{"geocoder": {"app_id": "dummy_id", "app_code": "dummy_code", "geocoder_cost_per_hit": 1}, "isolines": {"app_id": "dummy_id", "app_code": "dummy_code"}}');
+SELECT cartodb.cdb_conf_setconf('mapzen_conf', '{"routing": {"api_key": "routing_dummy_api_key", "monthly_quota": 1500000}, "geocoder": {"api_key": "geocoder_dummy_api_key", "monthly_quota": 1500000}, "matrix": {"api_key": "matrix_dummy_api_key", "monthly_quota": 1500000}}');
+SELECT cartodb.cdb_conf_setconf('data_observatory_conf', '{"connection": {"whitelist": ["ethervoid"], "production": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api", "staging": "host=localhost port=5432 dbname=dataservices_db user=geocoder_api"}, "monthly_quota": 100000}');

setup_organization.sh

@@ -1,6 +1,3 @@
-cd /cartodb
-source /usr/local/rvm/scripts/rvm
-
 ORGANIZATION_NAME="example"
 USERNAME="admin4example"
 EMAIL="[email protected]"
@@ -11,12 +8,3 @@ rake cartodb:db:set_unlimited_table_quota["${USERNAME}"]
 rake cartodb:db:create_new_organization_with_owner ORGANIZATION_NAME="${ORGANIZATION_NAME}" USERNAME="${USERNAME}" ORGANIZATION_SEATS=100 ORGANIZATION_QUOTA=102400 ORGANIZATION_DISPLAY_NAME="${ORGANIZATION_NAME}"
 rake cartodb:db:set_organization_quota[$ORGANIZATION_NAME,5000]
 rake cartodb:db:configure_geocoder_extension_for_organizations[$ORGANIZATION_NAME]
-
-# Set dataservices server
-ORGANIZATION_DB=`echo "SELECT database_name FROM users WHERE username='admin4example'" | psql -A -U postgres -t carto_db_development`
-psql -U postgres $ORGANIZATION_DB < /cartodb/script/geocoder_client.sql
-echo "CREATE EXTENSION cdb_dataservices_client;" | psql -U postgres $ORGANIZATION_DB
-echo "SELECT CDB_Conf_SetConf('user_config', '{"'"is_organization"'": true, "'"entity_name"'": "'"example"'"}');" | psql -U postgres $ORGANIZATION_DB
-GEOCODER_DB=`echo "SELECT database_name FROM users WHERE username='geocoder'" | psql -A -U postgres -t carto_db_development`
-echo -e "SELECT CDB_Conf_SetConf('geocoder_server_config', '{ \"connection_str\": \"host=localhost port=5432 dbname=${GEOCODER_DB} user=postgres\"}');" | psql -U postgres $ORGANIZATION_DB
-