From 6dbf01ee55e2422cf5ec33a49eed1af4a6eec5e1 Mon Sep 17 00:00:00 2001 From: Romain Crestey Date: Thu, 17 Mar 2016 13:27:26 +0100 Subject: [PATCH 1/8] moved osm-related scripts to its own folder --- server/{ => osm}/osmLoader.js | 0 server/{ => osm}/query.json | 0 server/{ => osm}/wayToPointConverter.js | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename server/{ => osm}/osmLoader.js (100%) rename server/{ => osm}/query.json (100%) rename server/{ => osm}/wayToPointConverter.js (100%) diff --git a/server/osmLoader.js b/server/osm/osmLoader.js similarity index 100% rename from server/osmLoader.js rename to server/osm/osmLoader.js diff --git a/server/query.json b/server/osm/query.json similarity index 100% rename from server/query.json rename to server/osm/query.json diff --git a/server/wayToPointConverter.js b/server/osm/wayToPointConverter.js similarity index 100% rename from server/wayToPointConverter.js rename to server/osm/wayToPointConverter.js From 6351e386416b440a3b0988399754c568fbe58b51 Mon Sep 17 00:00:00 2001 From: Romain Crestey Date: Thu, 17 Mar 2016 14:12:04 +0100 Subject: [PATCH 2/8] simplified db management --- server/database/management/connectToDB.js | 2 +- server/database/management/createTables.js | 21 +++++++-------------- server/database/management/dropAllTables.js | 15 ++++++--------- tools/init-database.js | 18 ++++++++---------- 4 files changed, 22 insertions(+), 34 deletions(-) diff --git a/server/database/management/connectToDB.js b/server/database/management/connectToDB.js index 99aa81d..ba606b0 100644 --- a/server/database/management/connectToDB.js +++ b/server/database/management/connectToDB.js @@ -22,7 +22,7 @@ module.exports = function(){ if(err){ console.error("Couldn't connect to db", conString, err); if(attempts >= MAX_ATTEMPTS) - reject(err); + reject("Couldn't connect: " + err); else { // wait twice more to give time and not overwhelm the database with useless attempts to connect console.warn("Retrying in ", 2*time); diff --git a/server/database/management/createTables.js b/server/database/management/createTables.js index 74b3708..21c51a9 100644 --- a/server/database/management/createTables.js +++ b/server/database/management/createTables.js @@ -2,23 +2,16 @@ var fs = require('fs'); -var connectToDB = require('./connectToDB.js'); - var createTableScript = fs.readFileSync( require.resolve('./createTables.sql') ).toString(); +module.exports = function(db){ + console.log('== Creating the database tables =='); -module.exports = function(){ return new Promise(function(resolve, reject){ - connectToDB() - .then(function(db){ - console.log('== Creating the database tables =='); - db.query(createTableScript, function(err, result) { - if(err) reject(err); else resolve(result); - }); - - }) - .catch(function(err){ - console.error('Could not connect', err); + db.query(createTableScript, function(err, result) { + if(err) + reject('Coudn\'t create the tables: ' + err); + else resolve(result); }); - }); + }; }; diff --git a/server/database/management/dropAllTables.js b/server/database/management/dropAllTables.js index 8bd1ffd..0546115 100644 --- a/server/database/management/dropAllTables.js +++ b/server/database/management/dropAllTables.js @@ -2,19 +2,16 @@ var fs = require('fs'); -var connectToDB = require('./connectToDB.js'); - var dropTableScript = fs.readFileSync( require.resolve('./dropAllTables.sql') ).toString(); -module.exports = function(){ +module.exports = function(db){ console.warn('\n\t=====\n\nWARNING! Dropping all tables!\n\n\t=====\n'); return new Promise(function(resolve, reject){ - connectToDB().then(function(db){ - db.query(dropTableScript, function(err, result) { - if(err) reject(err); else resolve(result); - }); - - }).catch(reject); + db.query(dropTableScript, function(err, result) { + if(err) + reject('Coudn\'t drop the tables: ' + err); + else resolve(result); + }); }); }; diff --git a/tools/init-database.js b/tools/init-database.js index e7ddf0c..342a59f 100755 --- a/tools/init-database.js +++ b/tools/init-database.js @@ -9,21 +9,19 @@ var dropAllTables = require('../server/database/management/dropAllTables.js'); var createTables = require('../server/database/management/createTables.js'); connectToDB() -.then(function(){ - return dropAllTables() - .catch(function(err){ - console.error("Couldn't drop tables", err); - process.exit(); - }) - .then(createTables) - .catch(function(err){ - console.error("Couldn't create tables", err); - process.exit(); +.then(function(db){ + return dropAllTables(db) + .then(function(){ + return createTables(db); }) .then(function(){ console.log("Success!"); process.exit(); }) + .catch(function(err){ + console.error("DB error", err); + process.exit(); + }) }) .catch(function(err){ console.error("Couldn't connect to database", err); From 94cb23345488a3dd029ae897d9bae7f3537a7ff8 Mon Sep 17 00:00:00 2001 From: Romain Crestey Date: Thu, 17 Mar 2016 16:30:47 +0100 Subject: [PATCH 3/8] added the update-table capability --- README.md | 4 +++ server/database/management/connectToDB.js | 1 - server/database/management/createTables.js | 4 +-- server/database/management/declarations.js | 22 +++++++++++++-- server/database/management/generateDecl.js | 31 ++++++++++++++++++++++ tools/generate-declarations.js | 26 ------------------ tools/init-database.js | 15 ++++++----- tools/update-tables.js | 25 +++++++++++++++++ 8 files changed, 90 insertions(+), 38 deletions(-) create mode 100644 server/database/management/generateDecl.js delete mode 100755 tools/generate-declarations.js create mode 100644 tools/update-tables.js diff --git a/README.md b/README.md index 85985c9..35b95c4 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,10 @@ In your regular console: node tools/init-database.js ``` +``` +node tools/update-tables.js // if you want to add or modify your tables, it doesnt drop the tables, so you're fine +``` + you can always use `psql` separately to load and dump data: ``` diff --git a/server/database/management/connectToDB.js b/server/database/management/connectToDB.js index ba606b0..fd36404 100644 --- a/server/database/management/connectToDB.js +++ b/server/database/management/connectToDB.js @@ -2,7 +2,6 @@ var pg = require('pg'); var PRIVATE = require('../../../PRIVATE.json'); - var conString = 'postgres://'+ PRIVATE.pg_user + ':' + PRIVATE.pg_pwd + '@localhost:5432/' + PRIVATE.db_name; var MAX_ATTEMPTS = 10; diff --git a/server/database/management/createTables.js b/server/database/management/createTables.js index 21c51a9..b258f8e 100644 --- a/server/database/management/createTables.js +++ b/server/database/management/createTables.js @@ -10,8 +10,8 @@ module.exports = function(db){ return new Promise(function(resolve, reject){ db.query(createTableScript, function(err, result) { if(err) - reject('Coudn\'t create the tables: ' + err); + reject('Coudn\'t create the tables: ' + err.stack); else resolve(result); }); - }; + }); }; diff --git a/server/database/management/declarations.js b/server/database/management/declarations.js index 77f76f6..5bfb233 100644 --- a/server/database/management/declarations.js +++ b/server/database/management/declarations.js @@ -1,4 +1,4 @@ -// autogenerated by sql-generate v1.1.0 on Wed Dec 09 2015 16:38:42 GMT+0000 (UTC) +// autogenerated by sql-generate v1.1.0 on Thu Mar 17 2016 16:22:42 GMT+0100 (CET) var sql = require('sql'); @@ -66,6 +66,23 @@ exports.networks = sql.define({ }); +/** + * SQL definition for public.osmplaces + */ +exports.osmplaces = sql.define({ + name: 'osmplaces', + columns: [ + { name: 'created_at' }, + { name: 'updated_at' }, + { name: 'id' }, + { name: 'tags' }, + { name: 'lat' }, + { name: 'lon' }, + { name: 'geom' } + ] +}); + + /** * SQL definition for public.places */ @@ -120,7 +137,8 @@ exports.raster_columns = sql.define({ { name: 'pixel_types' }, { name: 'nodata_values' }, { name: 'out_db' }, - { name: 'extent' } + { name: 'extent' }, + { name: 'spatial_index' } ] }); diff --git a/server/database/management/generateDecl.js b/server/database/management/generateDecl.js new file mode 100644 index 0000000..8166ca1 --- /dev/null +++ b/server/database/management/generateDecl.js @@ -0,0 +1,31 @@ +#!/usr/bin/env node + +"use strict"; + +var fs = require('fs'); +var path = require('path'); + +var generateSqlDefinition = require('sql-generate'); + +var PRIVATE = require('../../../PRIVATE.json'); +var conString = 'postgres://'+ PRIVATE.pg_user + ':' + PRIVATE.pg_pwd + '@localhost:5432/' + PRIVATE.db_name; + + +module.exports = function(){ + console.log('== Generating definitions =='); + + return new Promise(function(resolve, reject){ + generateSqlDefinition({ dsn: conString }, function(err, definitions) { + if (err) + reject('Coudn\'t generate definitions: ' + err); + + fs.writeFileSync(path.join(__dirname, './declarations.js'), definitions.buffer); + console.log('Definitions generated'); + resolve(); + }); + + process.on('uncaughtException', function(err) { + reject('Coudn\'t generate definitions, caught exception: ', err); + }); + }); +}; diff --git a/tools/generate-declarations.js b/tools/generate-declarations.js deleted file mode 100755 index d80dcab..0000000 --- a/tools/generate-declarations.js +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env node - -"use strict"; - -var fs = require('fs'); -var path = require('path'); - -var generateSqlDefinition = require('sql-generate'); - -var PRIVATE = require('../PRIVATE.json'); - -var conString = 'postgres://'+ PRIVATE.pg_user + ':' + PRIVATE.pg_pwd + '@localhost:5432/' + PRIVATE.db_name; - -generateSqlDefinition({ dsn: conString }, function(err, definitions) { - if (err) { - console.error(err); - return; - } - fs.writeFileSync(path.join(__dirname, '../server/database/management/declarations.js'), definitions.buffer); - console.log('definitions generated'); - process.exit(); -}); - -process.on('uncaughtException', function(err) { - console.error('Caught exception: ', err); -}); diff --git a/tools/init-database.js b/tools/init-database.js index 342a59f..7652d9c 100755 --- a/tools/init-database.js +++ b/tools/init-database.js @@ -7,6 +7,7 @@ require('es6-shim'); var connectToDB = require('../server/database/management/connectToDB.js'); var dropAllTables = require('../server/database/management/dropAllTables.js'); var createTables = require('../server/database/management/createTables.js'); +var generateDeclarations = require('../server/database/management/generateDecl.js'); connectToDB() .then(function(db){ @@ -15,14 +16,14 @@ connectToDB() return createTables(db); }) .then(function(){ - console.log("Success!"); - process.exit(); - }) - .catch(function(err){ - console.error("DB error", err); - process.exit(); - }) + return generateDeclarations(); + }); +}) +.then(function(){ + console.log("Success!"); + process.exit(); }) .catch(function(err){ console.error("Couldn't connect to database", err); + process.exit(); }); diff --git a/tools/update-tables.js b/tools/update-tables.js new file mode 100644 index 0000000..1dccbbc --- /dev/null +++ b/tools/update-tables.js @@ -0,0 +1,25 @@ +#!/usr/bin/env node + +'use strict'; + +require('es6-shim'); + +var connectToDB = require('../server/database/management/connectToDB.js'); +var createTables = require('../server/database/management/createTables.js'); +var generateDeclarations = require('../server/database/management/generateDecl.js'); + +connectToDB() +.then(function(db){ + return createTables(db) + .then(function(){ + return generateDeclarations(); + }); +}) +.then(function(){ + console.log("Success!"); + process.exit(); +}) +.catch(function(err){ + console.error("ERROR ", err); + process.exit(); +}); From 3a99cfcb5202429113b1c8a757b2cd76d02bf914 Mon Sep 17 00:00:00 2001 From: Romain Crestey Date: Thu, 17 Mar 2016 16:31:12 +0100 Subject: [PATCH 4/8] added osmTable to db --- server/database/management/createTables.sql | 28 ++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/server/database/management/createTables.sql b/server/database/management/createTables.sql index 6cbf153..3e14da0 100644 --- a/server/database/management/createTables.sql +++ b/server/database/management/createTables.sql @@ -25,8 +25,11 @@ CREATE TABLE IF NOT EXISTS networks ( color text NOT NULL, url text DEFAULT NULL ) INHERITS(lifecycle); + +DROP TRIGGER IF EXISTS updated_at_networks on networks; CREATE TRIGGER updated_at_networks BEFORE UPDATE ON networks FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); + CREATE TABLE IF NOT EXISTS places ( id SERIAL PRIMARY KEY, name text NOT NULL, @@ -50,6 +53,29 @@ CREATE TABLE IF NOT EXISTS places ( lon real NOT NULL, geom geometry ) INHERITS(lifecycle); + +DROP TRIGGER IF EXISTS updated_at_places on places; CREATE TRIGGER updated_at_places BEFORE UPDATE ON places FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); -CREATE INDEX place_geo_index ON places USING GIST (geom); \ No newline at end of file + +CREATE TABLE IF NOT EXISTS osmPlaces ( + id SERIAL PRIMARY KEY, + tags json NOT NULL, + lat real NOT NULL, + lon real NOT NULL, + geom geometry +) INHERITS(lifecycle); + +DROP TRIGGER IF EXISTS updated_at_osmPlaces on osmPlaces; +CREATE TRIGGER updated_at_osmPlaces BEFORE UPDATE ON osmPlaces FOR EACH ROW EXECUTE PROCEDURE update_updated_at_column(); + + + +-- INDEXES +DROP INDEX IF EXISTS place_geo_index; +CREATE INDEX place_geo_index ON places USING GIST (geom); + +DROP INDEX IF EXISTS osmPlace_geo_index; +CREATE INDEX osmPlace_geo_index ON osmPlaces USING GIST (geom) + + From 6b1e91cb3685a52fd1abe77f7f1e39fe9e5020dd Mon Sep 17 00:00:00 2001 From: Romain Crestey Date: Thu, 17 Mar 2016 18:41:42 +0100 Subject: [PATCH 5/8] fixed dropAllTable to include osmplaces --- server/database/management/dropAllTables.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/server/database/management/dropAllTables.sql b/server/database/management/dropAllTables.sql index a7705f4..30095d2 100644 --- a/server/database/management/dropAllTables.sql +++ b/server/database/management/dropAllTables.sql @@ -1,3 +1,4 @@ -- Destruction -- DROP TABLE IF EXISTS networks CASCADE; DROP TABLE IF EXISTS places CASCADE; +DROP TABLE IF EXISTS osmplaces CASCADE; From b476408f1476487bc712502e87b6d5d950e92737 Mon Sep 17 00:00:00 2001 From: Romain Crestey Date: Thu, 17 Mar 2016 19:37:18 +0100 Subject: [PATCH 6/8] load OSM points to 6element db --- server/database/management/createTables.sql | 1 + server/database/management/declarations.js | 3 +- server/database/models/osmPlaces.js | 255 ++++++++++++++++++++ server/loadOSMtoDb.js | 28 +++ server/osm/osmLoader.js | 13 +- server/osm/query.json | 1 + 6 files changed, 297 insertions(+), 4 deletions(-) create mode 100644 server/database/models/osmPlaces.js create mode 100644 server/loadOSMtoDb.js diff --git a/server/database/management/createTables.sql b/server/database/management/createTables.sql index 3e14da0..a0d7214 100644 --- a/server/database/management/createTables.sql +++ b/server/database/management/createTables.sql @@ -60,6 +60,7 @@ CREATE TRIGGER updated_at_places BEFORE UPDATE ON places FOR EACH ROW EXECUTE PR CREATE TABLE IF NOT EXISTS osmPlaces ( id SERIAL PRIMARY KEY, + osm_id text NOT NULL, tags json NOT NULL, lat real NOT NULL, lon real NOT NULL, diff --git a/server/database/management/declarations.js b/server/database/management/declarations.js index 5bfb233..f28058f 100644 --- a/server/database/management/declarations.js +++ b/server/database/management/declarations.js @@ -1,4 +1,4 @@ -// autogenerated by sql-generate v1.1.0 on Thu Mar 17 2016 16:22:42 GMT+0100 (CET) +// autogenerated by sql-generate v1.1.0 on Thu Mar 17 2016 18:47:55 GMT+0100 (CET) var sql = require('sql'); @@ -75,6 +75,7 @@ exports.osmplaces = sql.define({ { name: 'created_at' }, { name: 'updated_at' }, { name: 'id' }, + { name: 'osm_id' }, { name: 'tags' }, { name: 'lat' }, { name: 'lon' }, diff --git a/server/database/models/osmPlaces.js b/server/database/models/osmPlaces.js new file mode 100644 index 0000000..ef6adf0 --- /dev/null +++ b/server/database/models/osmPlaces.js @@ -0,0 +1,255 @@ +'use strict'; + +var sql = require('sql'); +sql.setDialect('postgres'); +var databaseP = require('../management/databaseClientP'); +var osmPlaces = require('../management/declarations.js').osmplaces; +var networks = require('../management/declarations.js').networks; + + +var jsArrayToPg = function(nodeArray) { + return "ARRAY['" + nodeArray.join("','") + "']"; +} + +module.exports = { + + // ------------- BASICS --------------- + + create: function(osmPlacesData){ + if(!Array.isArray(osmPlacesData)) + osmPlacesData = [osmPlacesData]; + + return databaseP.then(function (db) { + + return Promise.all(osmPlacesData.map(function(data){ + + var query = osmPlaces + .insert(data) + .returning('*') + .toQuery(); + + // console.log('osmPlaces create query', query); + + return new Promise(function (resolve, reject) { + db.query(query, function (err, result) { + if (err) { + console.log("ERROR in createByChunk", query, err.stack); + reject(err); + } + else resolve(result.rows); + }); + }); + })) + }) + .catch(function(err){ + console.error('ERROR in createByChunk', err.stack); + }); + }, + + count: function () { + return databaseP.then(function (db) { + var query = osmPlaces + .select(osmPlaces.count().as('count')) + .toQuery(); + + // console.log('osmPlaces count query', query); + + return new Promise(function (resolve, reject) { + db.query(query, function (err, result) { + if (err) { + console.error("ERROR in count", query, err); + reject(err); + } + else resolve(Number(result.rows[0].count)); + }); + }); + }); + }, + + + // ------------- GETTERS --------------- + + getKNearest: function(coords, k){ + return databaseP.then(function (db) { + + var strDistance = "st_distance_sphere(osmPlaces.geom, st_makepoint(" + coords.lon + ", " + coords.lat + ")) "; + var strDistanceAS = strDistance + "AS distance"; + var query = osmPlaces + .select(osmPlaces.star(), strDistanceAS) + .from(osmPlaces) + .where(strDistance + "< 50000 and type = 'centre'") + .order("distance") + .limit(k) + .toQuery(); + + return new Promise(function (resolve, reject) { + db.query(query, function (err, result) { + if (err) reject(err); + else resolve(result.rows); + }); + }); + }) + .catch(function(err){ + console.log('ERROR in getKNearest', err); + }); + }, + + getByIds: function(placeIds){ + return databaseP.then(function (db) { + + var query = osmPlaces + .select(osmPlaces.star()) + .from(osmPlaces) + .where(osmPlaces.id.in(placeIds)) + .toQuery(); + + return new Promise(function (resolve, reject) { + db.query(query, function (err, result) { + if (err) { + console.log("ERROR in searching place", query); + reject(err); + } + else { + resolve(result.rows); + } + }); + }); + }) + .catch(function(err){ + console.log('ERROR in get Place', err); + }); + }, + + getByOperator: function(operatorName){ + return databaseP.then(function (db) { + + var query = osmPlaces + .select("*") + .from(osmPlaces) + .where(osmPlaces.owner.equals(operatorName)) + .toQuery(); + + return new Promise(function (resolve, reject) { + db.query(query, function (err, result) { + if (err) { + console.log("ERROR in searching place by operatorName", query); + reject(err); + } + else { + resolve(result.rows); + } + }); + }); + }) + .catch(function(err){ + console.log('ERROR in get getByOperator', err); + }); + }, + + + + // ------------- BINS --------------- + + getBins: function(pheromonId){ + return databaseP.then(function (db) { + + var query = osmPlaces + .select(osmPlaces.bins, osmPlaces.owner) + .where(osmPlaces.pheromon_id.equals(pheromonId)) + .toQuery(); + + return new Promise(function (resolve, reject) { + db.query(query, function (err, result) { + if (err) { + console.log("ERROR in searching bins", query); + reject(err); + } + else{ + if(result.rows[0].bins === undefined) resolve (undefined); + else resolve(result.rows[0]); + } + }); + }); + }) + .catch(function(err){ + console.log('ERROR in get Bins', err); + }); + }, + + updateBins: function(pheromonId, bins){ + return databaseP.then(function (db) { + + var query = osmPlaces + .update({'bins': bins}) + .where(osmPlaces.pheromon_id.equals(pheromonId)) + .returning(osmPlaces.bins) + .toQuery(); + + return new Promise(function (resolve, reject) { + db.query(query, function (err, result) { + if (err) reject(err); + else resolve(result.rows[0]); + }); + }); + }) + .catch(function(err){ + console.log('ERROR in update Bins', err); + }); + }, + + updateBin: function(pheromonId, bin){ + + var self = this; + return this.getBins(pheromonId) + .then(function(object){ + + return new Promise(function (resolve, reject) { + var index = object.bins.findIndex(function(elt){ + return elt.id === bin.id; + }) + + if(index === -1) reject('Bin with id=' + bin.id + ' unfound'); + else { + object.bins[index] = bin; + self.updateBins(pheromonId, object.bins) + .then (function(){ + resolve(true); + }) + .catch(function(err){ + reject(err); + }) + } + }); + }) + .catch(function(err){ + console.log('ERROR in getting Bin', err); + }); + }, + + getWithin: function(coords, bbox, categories, limit){ + return databaseP.then(function (db) { + + var strDistance = "st_distance_sphere(osmPlaces.geom, st_makepoint(" + coords.lon + ", " + coords.lat + ")) AS distance"; + var filters = categories[0] === "All" ? "": " AND osmPlaces.objects ?| " + jsArrayToPg(categories); + var query = osmPlaces + .select(osmPlaces.star(),networks.name.as('file'), networks.color.as('color'), strDistance) + .from(osmPlaces.join(networks).on(osmPlaces.network.equals(networks.id))) + .where("osmPlaces.geom && ST_MakeEnvelope(" + bbox.minLon + ", " + bbox.minLat + ", " + bbox.maxLon + ", " + bbox.maxLat + ", 4326)" + filters) + .order("distance") + .limit(limit) + .toQuery(); + + + return new Promise(function (resolve, reject) { + db.query(query, function (err, result) { + if (err) reject(err); + + else resolve(result.rows); + }); + }); + }) + .catch(function(err){ + console.log('ERROR in getWithin', err, err.stack); + }); + } +}; diff --git a/server/loadOSMtoDb.js b/server/loadOSMtoDb.js new file mode 100644 index 0000000..71042bf --- /dev/null +++ b/server/loadOSMtoDb.js @@ -0,0 +1,28 @@ +'use strict'; + +var loader = require('./osm/osmLoader'); +var OsmPlaces = require('./database/models/osmPlaces.js'); + +loader() +.then(function(osmData){ + console.log('Nb of points in OSM', osmData.length); + + var data = osmData.map(function(place){ + return { + osm_id: place.properties.id, + tags: place.properties.tags, + lat: place.geometry.coordinates[1], + lon: place.geometry.coordinates[0] + }; + }); + + return OsmPlaces.create(data); +}) +.then(function(){ + console.log('All OSM Places created'); + process.exit(); +}) +.catch(function(err){ + console.error('ERROR creating OSM Place', err); + process.exit(); +}); \ No newline at end of file diff --git a/server/osm/osmLoader.js b/server/osm/osmLoader.js index 1746cbd..896d61b 100644 --- a/server/osm/osmLoader.js +++ b/server/osm/osmLoader.js @@ -8,10 +8,17 @@ var converter = require('./wayToPointConverter.js'); module.exports = function(bbox){ - var bboxCoords = [bbox.south, bbox.west, bbox.north, bbox.east].join(','); - var bboxString = 'bbox: ' + bboxCoords; + var query; - var query = queryDic.viewport.replace(/bbox/, bboxString); + if (bbox){ + var bboxCoords = [bbox.south, bbox.west, bbox.north, bbox.east].join(','); + var bboxString = 'bbox: ' + bboxCoords; + + query = queryDic.viewport.replace(/bbox/, bboxString); + } + + else + query = queryDic.gironde; var resultP = new Promise(function(resolve, reject){ osm(query, function(error, data){ diff --git a/server/osm/query.json b/server/osm/query.json index d434059..f26fa7d 100644 --- a/server/osm/query.json +++ b/server/osm/query.json @@ -1,4 +1,5 @@ { + "france": "[out:json][timeout:600];area(3601403916)->.gironde;(node['amenity'='recycling']['recycling_type'='centre'](area.gironde);way['amenity'='recycling']['recycling_type'='centre'](area.gironde);node['amenity'='recycling']['recycling_type'='container'](area.gironde););out body;>;out skel qt;", "gironde": "[out:json][timeout:25];area(3600007405)->.gironde;(node['amenity'='recycling']['recycling_type'='centre'](area.gironde);way['amenity'='recycling']['recycling_type'='centre'](area.gironde);node['amenity'='recycling']['recycling_type'='container'](area.gironde););out body;>;out skel qt;", "viewport": "[out:json][timeout:25][bbox];(node['amenity'='recycling']['recycling_type'='centre'];way['amenity'='recycling']['recycling_type'='centre'];node['amenity'='recycling']['recycling_type'='container'];);out body;>;out skel qt;" } \ No newline at end of file From b31312bf463e812bf6a6b3fab2c8c2e2961e76a8 Mon Sep 17 00:00:00 2001 From: Romain Crestey Date: Fri, 18 Mar 2016 11:59:02 +0100 Subject: [PATCH 7/8] fixed various bugs avoiding points to load from osm db --- server/database/management/createTables.sql | 1 + server/database/management/declarations.js | 3 ++- server/database/models/osmPlaces.js | 1 - server/loadOSMtoDb.js | 11 +++++--- server/osm/osmLoader.js | 2 +- server/osm/query.json | 2 +- server/searchPlaces.js | 29 +++++---------------- server/toGeoJson.js | 17 +++++++----- 8 files changed, 30 insertions(+), 36 deletions(-) diff --git a/server/database/management/createTables.sql b/server/database/management/createTables.sql index a0d7214..58d5cc3 100644 --- a/server/database/management/createTables.sql +++ b/server/database/management/createTables.sql @@ -64,6 +64,7 @@ CREATE TABLE IF NOT EXISTS osmPlaces ( tags json NOT NULL, lat real NOT NULL, lon real NOT NULL, + network integer REFERENCES networks (id) NOT NULL, geom geometry ) INHERITS(lifecycle); diff --git a/server/database/management/declarations.js b/server/database/management/declarations.js index f28058f..bd0cc40 100644 --- a/server/database/management/declarations.js +++ b/server/database/management/declarations.js @@ -1,4 +1,4 @@ -// autogenerated by sql-generate v1.1.0 on Thu Mar 17 2016 18:47:55 GMT+0100 (CET) +// autogenerated by sql-generate v1.1.0 on Fri Mar 18 2016 09:36:00 GMT+0100 (CET) var sql = require('sql'); @@ -79,6 +79,7 @@ exports.osmplaces = sql.define({ { name: 'tags' }, { name: 'lat' }, { name: 'lon' }, + { name: 'network' }, { name: 'geom' } ] }); diff --git a/server/database/models/osmPlaces.js b/server/database/models/osmPlaces.js index ef6adf0..d4abf97 100644 --- a/server/database/models/osmPlaces.js +++ b/server/database/models/osmPlaces.js @@ -238,7 +238,6 @@ module.exports = { .order("distance") .limit(limit) .toQuery(); - return new Promise(function (resolve, reject) { db.query(query, function (err, result) { diff --git a/server/loadOSMtoDb.js b/server/loadOSMtoDb.js index 71042bf..56f532c 100644 --- a/server/loadOSMtoDb.js +++ b/server/loadOSMtoDb.js @@ -6,13 +6,18 @@ var OsmPlaces = require('./database/models/osmPlaces.js'); loader() .then(function(osmData){ console.log('Nb of points in OSM', osmData.length); - + var data = osmData.map(function(place){ + var lat = place.geometry.coordinates[1]; + var lon = place.geometry.coordinates[0]; + return { osm_id: place.properties.id, tags: place.properties.tags, - lat: place.geometry.coordinates[1], - lon: place.geometry.coordinates[0] + lat: lat, + lon: lon, + geom: 'POINT(' + lon + ' ' + lat + ')', + network: 5 // Réseau Openstreetmap }; }); diff --git a/server/osm/osmLoader.js b/server/osm/osmLoader.js index 896d61b..414c5f5 100644 --- a/server/osm/osmLoader.js +++ b/server/osm/osmLoader.js @@ -18,7 +18,7 @@ module.exports = function(bbox){ } else - query = queryDic.gironde; + query = queryDic.france; var resultP = new Promise(function(resolve, reject){ osm(query, function(error, data){ diff --git a/server/osm/query.json b/server/osm/query.json index f26fa7d..f811bb4 100644 --- a/server/osm/query.json +++ b/server/osm/query.json @@ -1,5 +1,5 @@ { - "france": "[out:json][timeout:600];area(3601403916)->.gironde;(node['amenity'='recycling']['recycling_type'='centre'](area.gironde);way['amenity'='recycling']['recycling_type'='centre'](area.gironde);node['amenity'='recycling']['recycling_type'='container'](area.gironde););out body;>;out skel qt;", + "france": "[out:json][timeout:600];area(3601403916)->.france;(node['amenity'='recycling']['recycling_type'='centre'](area.france);way['amenity'='recycling']['recycling_type'='centre'](area.france);node['amenity'='recycling']['recycling_type'='container'](area.france););out body;>;out skel qt;", "gironde": "[out:json][timeout:25];area(3600007405)->.gironde;(node['amenity'='recycling']['recycling_type'='centre'](area.gironde);way['amenity'='recycling']['recycling_type'='centre'](area.gironde);node['amenity'='recycling']['recycling_type'='container'](area.gironde););out body;>;out skel qt;", "viewport": "[out:json][timeout:25][bbox];(node['amenity'='recycling']['recycling_type'='centre'];way['amenity'='recycling']['recycling_type'='centre'];node['amenity'='recycling']['recycling_type'='container'];);out body;>;out skel qt;" } \ No newline at end of file diff --git a/server/searchPlaces.js b/server/searchPlaces.js index 76a8c60..f70a89a 100644 --- a/server/searchPlaces.js +++ b/server/searchPlaces.js @@ -1,8 +1,9 @@ "use strict"; -var osmLoader = require('./osmLoader.js'); +// var osmLoader = require('./osmLoader.js'); var toGeoJson = require('./toGeoJson.js'); var Places = require('./database/models/places.js'); +var OsmPlaces = require('./database/models/osmPlaces.js'); var withPlacesMeasurements = require('./withPlacesMeasurements.js'); /* @@ -43,15 +44,8 @@ module.exports = function(req, res){ var dbDataP = Places.getWithin(data.geoloc, data.boundingBox, data.categories, 2000) .then(toGeoJson); - // OSM Search - var bbox = { - north: data.boundingBox.maxLat, - south: data.boundingBox.minLat, - east: data.boundingBox.maxLon, - west: data.boundingBox.minLon - }; - - var osmDataP = osmLoader(bbox); + var osmDataP = OsmPlaces.getWithin(data.geoloc, data.boundingBox, data.categories, 2000) + .then(toGeoJson); Promise.all([dbDataP, osmDataP]) .then(function(results){ @@ -101,19 +95,10 @@ module.exports = function(req, res){ } else if(data.geoloc !== null){ var dbDataP = Places.getKNearest({"lon": data.geoloc.lon, "lat": data.geoloc.lat}, data.nbPlaces, data.categories) - .then(function(results){ - return toGeoJson(results); - }); - - // OSM Search - var bbox = { // raw approx of 50km bounding box around geoloc - north: data.geoloc.lat + 0.5, - south: data.geoloc.lat - 0.5, - east: data.geoloc.lon + 0.5, - west: data.geoloc.lon - 0.5 - }; + .then(toGeoJson); - var osmDataP = osmLoader(bbox); + var osmDataP = OsmPlaces.getKNearest({"lon": data.geoloc.lon, "lat": data.geoloc.lat}, data.nbPlaces, data.categories) + .then(toGeoJson); Promise.all([dbDataP, osmDataP]) .then(function(results){ diff --git a/server/toGeoJson.js b/server/toGeoJson.js index 57fe497..2f7c58a 100644 --- a/server/toGeoJson.js +++ b/server/toGeoJson.js @@ -7,15 +7,18 @@ var dictionnary = require('../data/dictionary.json'); var parseObjects = function(row){ return new Promise(function(resolve){ - hstore.parse(row.objects, function(fromHstore) { + if (row.objects){ + hstore.parse(row.objects, function(fromHstore) { - var objects = {}; - Object.keys(fromHstore).forEach(function(key){ - var fr = dictionnary[key]; - objects[fr]= fromHstore[key]; + var objects = {}; + Object.keys(fromHstore).forEach(function(key){ + var fr = dictionnary[key]; + objects[fr]= fromHstore[key]; + }); + resolve(objects); }); - resolve(objects); - }); + } + else resolve([]); }); } From 455a3ce7585344ae70ee2747f5933571eea7d417 Mon Sep 17 00:00:00 2001 From: Romain Crestey Date: Fri, 18 Mar 2016 15:46:43 +0100 Subject: [PATCH 8/8] removed unused osmplaces methods --- server/database/models/osmPlaces.js | 132 ---------------------------- 1 file changed, 132 deletions(-) diff --git a/server/database/models/osmPlaces.js b/server/database/models/osmPlaces.js index d4abf97..5e72f9b 100644 --- a/server/database/models/osmPlaces.js +++ b/server/database/models/osmPlaces.js @@ -93,138 +93,6 @@ module.exports = { console.log('ERROR in getKNearest', err); }); }, - - getByIds: function(placeIds){ - return databaseP.then(function (db) { - - var query = osmPlaces - .select(osmPlaces.star()) - .from(osmPlaces) - .where(osmPlaces.id.in(placeIds)) - .toQuery(); - - return new Promise(function (resolve, reject) { - db.query(query, function (err, result) { - if (err) { - console.log("ERROR in searching place", query); - reject(err); - } - else { - resolve(result.rows); - } - }); - }); - }) - .catch(function(err){ - console.log('ERROR in get Place', err); - }); - }, - - getByOperator: function(operatorName){ - return databaseP.then(function (db) { - - var query = osmPlaces - .select("*") - .from(osmPlaces) - .where(osmPlaces.owner.equals(operatorName)) - .toQuery(); - - return new Promise(function (resolve, reject) { - db.query(query, function (err, result) { - if (err) { - console.log("ERROR in searching place by operatorName", query); - reject(err); - } - else { - resolve(result.rows); - } - }); - }); - }) - .catch(function(err){ - console.log('ERROR in get getByOperator', err); - }); - }, - - - - // ------------- BINS --------------- - - getBins: function(pheromonId){ - return databaseP.then(function (db) { - - var query = osmPlaces - .select(osmPlaces.bins, osmPlaces.owner) - .where(osmPlaces.pheromon_id.equals(pheromonId)) - .toQuery(); - - return new Promise(function (resolve, reject) { - db.query(query, function (err, result) { - if (err) { - console.log("ERROR in searching bins", query); - reject(err); - } - else{ - if(result.rows[0].bins === undefined) resolve (undefined); - else resolve(result.rows[0]); - } - }); - }); - }) - .catch(function(err){ - console.log('ERROR in get Bins', err); - }); - }, - - updateBins: function(pheromonId, bins){ - return databaseP.then(function (db) { - - var query = osmPlaces - .update({'bins': bins}) - .where(osmPlaces.pheromon_id.equals(pheromonId)) - .returning(osmPlaces.bins) - .toQuery(); - - return new Promise(function (resolve, reject) { - db.query(query, function (err, result) { - if (err) reject(err); - else resolve(result.rows[0]); - }); - }); - }) - .catch(function(err){ - console.log('ERROR in update Bins', err); - }); - }, - - updateBin: function(pheromonId, bin){ - - var self = this; - return this.getBins(pheromonId) - .then(function(object){ - - return new Promise(function (resolve, reject) { - var index = object.bins.findIndex(function(elt){ - return elt.id === bin.id; - }) - - if(index === -1) reject('Bin with id=' + bin.id + ' unfound'); - else { - object.bins[index] = bin; - self.updateBins(pheromonId, object.bins) - .then (function(){ - resolve(true); - }) - .catch(function(err){ - reject(err); - }) - } - }); - }) - .catch(function(err){ - console.log('ERROR in getting Bin', err); - }); - }, getWithin: function(coords, bbox, categories, limit){ return databaseP.then(function (db) {