From 644156de9f6f243e8b988cb04a1ef840039cd58a Mon Sep 17 00:00:00 2001 From: Adam Butcher Date: Wed, 2 Sep 2015 06:47:02 +0100 Subject: [PATCH 1/7] Add a new cache mode 'follow' Rather than specifying 'true' or 'false' for the 'cache' option, specifying 'follow' will uses a changes feed to track external modification to cached documents. Only documents that have been cached during normal write-through operation will be updated. This is useful if you have a system where multiple clients may write directly to the database without going through a single cache. Note: This could be made more optimal for cases where not all changes in the feed are cached as it pulls the full document content in the change feed. Maybe a mode 'follow-lite' could enable a lightweight change feed followed by additional fetch if it is found that a changed document is cached. If, for a particular app however, changes are likely to occur externally to documents in the cache, the existing 'include_docs' mode is probably OK. --- lib/cradle/database/index.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/lib/cradle/database/index.js b/lib/cradle/database/index.js index 9effb77..a870f1e 100644 --- a/lib/cradle/database/index.js +++ b/lib/cradle/database/index.js @@ -6,6 +6,23 @@ var Database = exports.Database = function (name, connection) { this.connection = connection; this.name = encodeURIComponent(name); this.cache = new (cradle.Cache)(connection.options); + + // For any entry already in the cache, update it if it changes + // remotely. + if (connection.options.cache === 'follow') { + var self = this; + this.changes(function (err, list) { + var lastSeq = 0; + if (list && list.length !== 0) + lastSeq = list[list.length - 1]["seq"]; + var feed = self.changes({ since: lastSeq, include_docs: true }); + feed.on('change', function (change) { + var id = change["id"]; + if (id && 'doc' in change && self.cache.has(id)) + self.cache.save(id, change["doc"]); + }); + }); + } }; // A wrapper around `Connection.request`, From e25a05e251ae452d713e1ef9c71a031ec115f414 Mon Sep 17 00:00:00 2001 From: Adam Butcher Date: Wed, 7 Oct 2015 17:07:39 +0100 Subject: [PATCH 2/7] Only start cache feed in 'follow' cache mode when database exists. --- lib/cradle/database/index.js | 46 ++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/lib/cradle/database/index.js b/lib/cradle/database/index.js index a870f1e..c6fc5a6 100644 --- a/lib/cradle/database/index.js +++ b/lib/cradle/database/index.js @@ -6,22 +6,34 @@ var Database = exports.Database = function (name, connection) { this.connection = connection; this.name = encodeURIComponent(name); this.cache = new (cradle.Cache)(connection.options); + this.cacheFeed = null; + var self = this; + this.exists(function(err, result) { + if (result === true) + self.configureCacheFeed(); + }); +}; +Database.prototype.configureCacheFeed = function () { + if (this.cacheFeed) { + this.cacheFeed.stop(); + this.cacheFeed = null; + } // For any entry already in the cache, update it if it changes // remotely. - if (connection.options.cache === 'follow') { - var self = this; - this.changes(function (err, list) { - var lastSeq = 0; - if (list && list.length !== 0) - lastSeq = list[list.length - 1]["seq"]; - var feed = self.changes({ since: lastSeq, include_docs: true }); - feed.on('change', function (change) { - var id = change["id"]; - if (id && 'doc' in change && self.cache.has(id)) - self.cache.save(id, change["doc"]); - }); - }); + if (this.connection.options.cache === 'follow') { + var self = this; + this.changes(function (err, list) { + var lastSeq = 0; + if (list && list.length !== 0) + lastSeq = list[list.length - 1]["seq"]; + self.cacheFeed = self.changes({ since: lastSeq, include_docs: true }); + self.cacheFeed.on('change', function (change) { + var id = change["id"]; + if (id && 'doc' in change && self.cache.has(id)) + self.cache.save(id, change["doc"]); + }); + }); } }; @@ -56,7 +68,11 @@ Database.prototype.info = function (callback) { }; Database.prototype.create = function (callback) { - this.query({ method: 'PUT' }, callback); + var self = this; + this.query({ method: 'PUT' }, function () { + self.configureCacheFeed(); + callback.apply(this, arguments); + }); }; // Destroys a database with 'DELETE' @@ -79,4 +95,4 @@ Database.prototype.destroy = function (callback) { require('./attachments'); require('./changes'); require('./documents'); -require('./views'); \ No newline at end of file +require('./views'); From 2852166231f0d9b16d4b3c92e3deb6da4146e84b Mon Sep 17 00:00:00 2001 From: Adam Butcher Date: Wed, 7 Oct 2015 22:56:19 +0100 Subject: [PATCH 3/7] Database: ctor: Don't unnecessarily check for existence when not in 'follow' cache mode. --- lib/cradle/database/index.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/cradle/database/index.js b/lib/cradle/database/index.js index c6fc5a6..15f193d 100644 --- a/lib/cradle/database/index.js +++ b/lib/cradle/database/index.js @@ -7,11 +7,13 @@ var Database = exports.Database = function (name, connection) { this.name = encodeURIComponent(name); this.cache = new (cradle.Cache)(connection.options); this.cacheFeed = null; - var self = this; - this.exists(function(err, result) { - if (result === true) - self.configureCacheFeed(); - }); + if (connection.options.cache === 'follow') { + var self = this; + this.exists(function(err, result) { + if (result === true) + self.configureCacheFeed(); + }); + } }; Database.prototype.configureCacheFeed = function () { From 19f7ee40b1fee8c0009c57343b419200f5370aeb Mon Sep 17 00:00:00 2001 From: Adam Butcher Date: Wed, 6 Mar 2019 10:10:26 +0000 Subject: [PATCH 4/7] Connection.database: Add options parameter to support disabling usage of the cache for specific databases. --- lib/cradle.js | 4 ++-- lib/cradle/database/index.js | 13 +++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/cradle.js b/lib/cradle.js index 269f1ab..ec8f3b1 100644 --- a/lib/cradle.js +++ b/lib/cradle.js @@ -253,8 +253,8 @@ cradle.Connection.prototype.request = function (options, callback) { // We return an object with database functions, // closing around the `name` argument. // -cradle.Connection.prototype.database = function (name) { - return new cradle.Database(name, this) +cradle.Connection.prototype.database = function (name, opts) { + return new cradle.Database(name, this, opts) }; // diff --git a/lib/cradle/database/index.js b/lib/cradle/database/index.js index 15f193d..667352a 100644 --- a/lib/cradle/database/index.js +++ b/lib/cradle/database/index.js @@ -2,12 +2,17 @@ var querystring = require('querystring'), Args = require('vargs').Constructor, cradle = require('../../cradle'); -var Database = exports.Database = function (name, connection) { +var Database = exports.Database = function (name, connection, opts) { this.connection = connection; this.name = encodeURIComponent(name); - this.cache = new (cradle.Cache)(connection.options); + this.opts = {...connection.options}; + if (opts && opts.disableCache) { + this.opts.cache = false; + this.opts.cacheSize = 0; + } + this.cache = new (cradle.Cache)(this.opts); this.cacheFeed = null; - if (connection.options.cache === 'follow') { + if (this.opts.cache === 'follow') { var self = this; this.exists(function(err, result) { if (result === true) @@ -23,7 +28,7 @@ Database.prototype.configureCacheFeed = function () { } // For any entry already in the cache, update it if it changes // remotely. - if (this.connection.options.cache === 'follow') { + if (this.opts.cache === 'follow') { var self = this; this.changes(function (err, list) { var lastSeq = 0; From dfc64ad85018486ffa42662bec2d6f5dc48643ea Mon Sep 17 00:00:00 2001 From: Adam Butcher Date: Wed, 31 Jul 2024 22:53:40 +0100 Subject: [PATCH 5/7] Database: Don't read entire changes feed to determine last sequence number. --- lib/cradle/database/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cradle/database/index.js b/lib/cradle/database/index.js index 667352a..d40b26f 100644 --- a/lib/cradle/database/index.js +++ b/lib/cradle/database/index.js @@ -30,10 +30,10 @@ Database.prototype.configureCacheFeed = function () { // remotely. if (this.opts.cache === 'follow') { var self = this; - this.changes(function (err, list) { + this.changes({descending: true, limit: 0}, function (err, list) { var lastSeq = 0; - if (list && list.length !== 0) - lastSeq = list[list.length - 1]["seq"]; + if (list && 'last_seq' in list) + lastSeq = list.last_seq; self.cacheFeed = self.changes({ since: lastSeq, include_docs: true }); self.cacheFeed.on('change', function (change) { var id = change["id"]; From 7427e2551730aa577e221f617e1d5a3e4b0feef1 Mon Sep 17 00:00:00 2001 From: Adam Butcher Date: Tue, 11 Jun 2019 08:58:09 +0100 Subject: [PATCH 6/7] core, changes: Support infinite retry for local connections and recover broken changes feed. --- lib/cradle.js | 16 ++++++++++++---- lib/cradle/database/changes.js | 16 +++++++++++++++- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/lib/cradle.js b/lib/cradle.js index ec8f3b1..6c68c34 100644 --- a/lib/cradle.js +++ b/lib/cradle.js @@ -26,7 +26,8 @@ cradle.options = { retries: 0, retryTimeout: 10e3, forceSave: true, - headers: {} + headers: {}, + forceReconnect: false, }; cradle.setup = function (settings) { @@ -155,13 +156,14 @@ cradle.Connection.prototype.rawRequest = function (options, callback) { options.query[k] = String(options.query[k]); } } - options.path += '?' + querystring.stringify(options.query); + options.uri = this._url(options.path + '?' + querystring.stringify(options.query)); + } + else { + options.uri = this._url(options.path); } options.headers['Connection'] = options.headers['Connection'] || 'keep-alive'; options.agent = this.agent; - options.uri = this._url(options.path); - delete options.path; return request(options, callback || function () { }); }; @@ -214,6 +216,12 @@ cradle.Connection.prototype.request = function (options, callback) { return this.rawRequest(options, function _onResponse(err, res, body) { attempts++; if (err) { + if (self.options.forceReconnect && String(err.code).startsWith('ECONN')) { + return setTimeout( + self.rawRequest.bind(self, options, _onResponse), + self.options.retryTimeout + ); + } if (self.options.retries && (!options.method || options.method.toLowerCase() === 'get' || options.body) && String(err.code).indexOf('ECONN') === 0 && attempts <= self.options.retries diff --git a/lib/cradle/database/changes.js b/lib/cradle/database/changes.js index fe0ba8f..999f86c 100644 --- a/lib/cradle/database/changes.js +++ b/lib/cradle/database/changes.js @@ -50,7 +50,21 @@ Database.prototype.changes = function (options, callback) { response.emit.apply(response, ['data'].concat(Array.prototype.slice.call(arguments))); }); - + + var self = this; + // Keep a consistent object for return to the client, even if + // this feed is restarted due to error. + feed.on('error', function (err) { + if (feed.dead && options.follow !== false) { + console.error(self.name, 'ERROR: Cradle changes feed died, restarting', err.message || err); + setTimeout(function() { + console.error(self.name, 'RECOVERY: Restarting feed that died with', err.message || err); + feed.restart(); + feed.emit('recover', err); + }, 1000); + } + }); + if (options.follow !== false) { feed.follow(); } From bfe1c9e07a1a51d63a15d3ccdf0bafd649d4ec2a Mon Sep 17 00:00:00 2001 From: Adam Butcher Date: Wed, 31 Jul 2024 23:05:28 +0100 Subject: [PATCH 7/7] Database: Workaround COUCHDB-1415 by injecting timestamp in POSTed body. --- lib/cradle/database/index.js | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/lib/cradle/database/index.js b/lib/cradle/database/index.js index d40b26f..68c8f4b 100644 --- a/lib/cradle/database/index.js +++ b/lib/cradle/database/index.js @@ -10,6 +10,10 @@ var Database = exports.Database = function (name, connection, opts) { this.opts.cache = false; this.opts.cacheSize = 0; } + + // Workaround https://issues.apache.org/jira/browse/COUCHDB-1415 by default + this.workaroundBug1415 = true; + this.cache = new (cradle.Cache)(this.opts); this.cacheFeed = null; if (this.opts.cache === 'follow') { @@ -47,6 +51,11 @@ Database.prototype.configureCacheFeed = function () { // A wrapper around `Connection.request`, // which prepends the database name. Database.prototype.query = function (options, callback) { + + // XXX: Workaround https://issues.apache.org/jira/browse/COUCHDB-1415 + if (options.body && options.method === 'POST' && this.workaroundBug1415) + options.body.$ts = Date.now(); + options.path = [this.name, options.path].filter(Boolean).join('/'); return this.connection.request(options, callback); }; @@ -88,12 +97,12 @@ Database.prototype.create = function (callback) { Database.prototype.destroy = function (callback) { if (arguments.length > 1) { throw new(Error)("destroy() doesn't take any additional arguments"); - } - + } + this.query({ - method: 'DELETE', - path: '/', - }, callback); + method: 'DELETE', + path: '/', + }, callback); }; //