From ecce008d3141169cf22fb6fe3e66156736d61485 Mon Sep 17 00:00:00 2001 From: Erwan Guyader Date: Wed, 27 Apr 2022 16:01:57 +0200 Subject: [PATCH 1/3] gui/main: Fix MigrationFailedError import Since the creation of the `migrations` module, the `MigrationFailedError` class has been moved to the root of the module out of its `migrations.js` file (which now only contains the migrations themselves). --- gui/main.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gui/main.js b/gui/main.js index 3380cd215..ba61ba8cb 100644 --- a/gui/main.js +++ b/gui/main.js @@ -20,7 +20,7 @@ const { SYNC_DIR_EMPTY_MESSAGE, SYNC_DIR_UNLINKED_MESSAGE } = require('../core/local/errors') -const { MigrationFailedError } = require('../core/migrations/migrations') +const { MigrationFailedError } = require('../core/migrations') const config = require('../core/config') const winRegistry = require('../core/utils/win_registry') From b441aeb390f80f8b861970ed0492d855d8f80d77 Mon Sep 17 00:00:00 2001 From: Erwan Guyader Date: Wed, 27 Apr 2022 16:03:12 +0200 Subject: [PATCH 2/3] dev: Add error to exception's extra attributes Custom error attributes are not displayed by Sentry by default. This prevents us from accessing meaningful data about captured errors so we will now add the full error object to the exception's extra attributes to make sure they're available. --- core/utils/sentry.js | 1 - 1 file changed, 1 deletion(-) diff --git a/core/utils/sentry.js b/core/utils/sentry.js index 84509d2e1..306e15fdf 100644 --- a/core/utils/sentry.js +++ b/core/utils/sentry.js @@ -105,7 +105,6 @@ const handleBunyanMessage = msg => { // Send messages explicitly marked for sentry and all TypeError instances if (msg.sentry || (msg.err && msg.err.name === 'TypeError')) { const extra = _.omit(msg, [ - 'err', 'tags', 'v', 'hostname', From 72d29a5ccabf4b985185000bad44f8a8c3ca099f Mon Sep 17 00:00:00 2001 From: Erwan Guyader Date: Wed, 27 Apr 2022 16:04:46 +0200 Subject: [PATCH 3/3] core/migrations: Limit concurrent network requests Mapping over a large number of documents in a migration that makes network requests for these documents can lead to exhausting available resources. This results in `net::ERR_INSUFFICIENT_RESOURCES` errors and failed migrations. Limiting the concurrency to only 10 documents at a time should prevent these errors. --- core/migrations/migrations.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/core/migrations/migrations.js b/core/migrations/migrations.js index de9116563..b41f40da7 100644 --- a/core/migrations/migrations.js +++ b/core/migrations/migrations.js @@ -4,6 +4,7 @@ */ const path = require('path') +const { Promise } = require('bluebird') const metadata = require('../metadata') const { SCHEMA_INITIAL_VERSION } = require('./constants') @@ -352,8 +353,9 @@ module.exports = ([ docs /*: SavedMetadata[] */, { remote } /*: InjectedDependencies */ ) /*: Promise */ => { - return Promise.all( - docs.map(async doc => { + return Promise.map( + docs, + async doc => { const remoteDir = await remote.remoteCozy.findDir(doc.remote._id) if (remoteDir != null) { @@ -400,7 +402,10 @@ module.exports = ([ } return doc - }) + }, + // XXX: Avoid exhausting network resources by limiting the number of + // concurrent requests. + { concurrency: 10 } ) } }