From b7121a84312e41c1eb5a276647a54472083417c1 Mon Sep 17 00:00:00 2001 From: stefanprobst Date: Wed, 3 Apr 2019 12:44:27 +0200 Subject: [PATCH] Use Loki's elemMatch operator (#13025) --- packages/gatsby/package.json | 2 +- packages/gatsby/src/db/loki/index.js | 2 +- packages/gatsby/src/db/loki/nodes-query.js | 61 +++------------------- yarn.lock | 15 ++++-- 4 files changed, 19 insertions(+), 61 deletions(-) diff --git a/packages/gatsby/package.json b/packages/gatsby/package.json index f706139ceb000..4beec4c72b6bf 100644 --- a/packages/gatsby/package.json +++ b/packages/gatsby/package.json @@ -20,6 +20,7 @@ "@mikaelkristiansson/domready": "^1.0.9", "@pieh/friendly-errors-webpack-plugin": "1.7.0-chalk-2", "@reach/router": "^1.1.1", + "@stefanprobst/lokijs": "^1.5.6-b", "address": "1.0.3", "autoprefixer": "^9.4.3", "babel-core": "7.0.0-bridge.0", @@ -86,7 +87,6 @@ "json-stringify-safe": "^5.0.1", "kebab-hash": "^0.1.2", "lodash": "^4.17.10", - "lokijs": "^1.5.6", "md5": "^2.2.1", "md5-file": "^3.1.1", "mime": "^2.2.0", diff --git a/packages/gatsby/src/db/loki/index.js b/packages/gatsby/src/db/loki/index.js index 37fbea60cb663..1647e7580b6ba 100644 --- a/packages/gatsby/src/db/loki/index.js +++ b/packages/gatsby/src/db/loki/index.js @@ -1,7 +1,7 @@ const _ = require(`lodash`) const fs = require(`fs-extra`) const path = require(`path`) -const loki = require(`lokijs`) +const loki = require(`@stefanprobst/lokijs`) const uuidv4 = require(`uuid/v4`) const customComparators = require(`./custom-comparators`) diff --git a/packages/gatsby/src/db/loki/nodes-query.js b/packages/gatsby/src/db/loki/nodes-query.js index 1f9f22a53064e..c97bd54cc4d35 100644 --- a/packages/gatsby/src/db/loki/nodes-query.js +++ b/packages/gatsby/src/db/loki/nodes-query.js @@ -1,7 +1,6 @@ const _ = require(`lodash`) const prepareRegex = require(`../../utils/prepare-regex`) const { getNodeTypeCollection } = require(`./nodes`) -const sift = require(`sift`) const { emitter } = require(`../../redux`) // Cleared on DELETE_CACHE @@ -14,49 +13,6 @@ emitter.on(`DELETE_CACHE`, () => { } }) -// Takes a raw graphql filter and converts it into a mongo-like args -// object that can be understood by the `sift` library. E.g `eq` -// becomes `$eq` -function siftifyArgs(object) { - const newObject = {} - _.each(object, (v, k) => { - if (_.isPlainObject(v)) { - if (k === `elemMatch`) { - k = `$elemMatch` - } - newObject[k] = siftifyArgs(v) - } else { - // Compile regex first. - if (k === `regex`) { - newObject[`$regex`] = prepareRegex(v) - } else if (k === `glob`) { - const Minimatch = require(`minimatch`).Minimatch - const mm = new Minimatch(v) - newObject[`$regex`] = mm.makeRe() - } else { - newObject[`$${k}`] = v - } - } - }) - return newObject -} - -// filter nodes using the `sift` library. But isn't this a loki query -// file? Yes, but we need to support all functionality provided by -// `run-sift`, and there are some operators that loki can't -// support. Like `elemMatch`, so for those fields, we fall back to -// sift -function runSift(nodes, query) { - if (nodes) { - const siftQuery = { - $elemMatch: siftifyArgs(query), - } - return sift(siftQuery, nodes) - } else { - return null - } -} - // Takes a raw graphql filter and converts it into a mongo-like args // object that can be understood by loki. E.g `eq` becomes // `$eq`. gqlFilter should be the raw graphql filter returned from @@ -96,12 +52,8 @@ function toMongoArgs(gqlFilter, lastFieldType) { _.each(gqlFilter, (v, k) => { if (_.isPlainObject(v)) { if (k === `elemMatch`) { - // loki doesn't support elemMatch, so use sift (see runSift - // comment above) - mongoArgs[`$where`] = obj => { - const result = runSift(obj, v) - return result && result.length > 0 - } + const gqlFieldType = lastFieldType.ofType + mongoArgs[`$elemMatch`] = toMongoArgs(v, gqlFieldType) } else { const gqlFieldType = lastFieldType.getFields()[k].type mongoArgs[k] = toMongoArgs(v, gqlFieldType) @@ -189,7 +141,9 @@ const toDottedFields = (filter, acc = {}, path = []) => { Object.keys(filter).forEach(key => { const value = filter[key] const nextValue = _.isPlainObject(value) && value[Object.keys(value)[0]] - if (_.isPlainObject(nextValue)) { + if (key === `$elemMatch`) { + acc[path.join(`.`)] = { [`$elemMatch`]: toDottedFields(value) } + } else if (_.isPlainObject(nextValue)) { toDottedFields(value, acc, path.concat(key)) } else { acc[path.concat(key).join(`.`)] = value @@ -232,8 +186,7 @@ const convertArgs = (gqlArgs, gqlType) => // Converts graphql Sort args into the form expected by loki, which is // a vector where the first value is a field name, and the second is a -// boolean `isDesc`. Nested fields delimited by `___` are replaced by -// periods. E.g +// boolean `isDesc`. E.g // // { // fields: [ `frontmatter___date`, `id` ], @@ -278,7 +231,7 @@ function ensureFieldIndexes(coll, lokiArgs) { * * @param {Object} args. Object with: * - * {Object} gqlType: built during `./build-node-types.js` + * {Object} gqlType: A GraphQL type * * {Object} queryArgs: The raw graphql query as a js object. E.g `{ * filter: { fields { slug: { eq: "/somepath" } } } }` diff --git a/yarn.lock b/yarn.lock index 92e56f7a4a426..7a91ad3569a94 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2615,6 +2615,11 @@ resolved "https://registry.yarnpkg.com/@sindresorhus/is/-/is-0.7.0.tgz#9a06f4f137ee84d7df0460c1fdb1135ffa6c50fd" integrity sha512-ONhaKPIufzzrlNbqtWFFd+jlnemX6lJAgq9ZeiZtS7I1PIf/la7CW4m83rTXRnVnsMbW2k56pGYu7AUFJD9Pow== +"@stefanprobst/lokijs@^1.5.6-b": + version "1.5.6-b" + resolved "https://registry.yarnpkg.com/@stefanprobst/lokijs/-/lokijs-1.5.6-b.tgz#6a36a86dbe132e702e6b15ffd3ce4139aebfe942" + integrity sha512-MNodHp46og+Sdde/LCxTLrxcD5Dimu21R/Fer2raXMG1XtHSV2+vZnkIV87OPAxuf2NiDj1W5hN7Q2MYUfQQ8w== + "@types/configstore@^2.1.1": version "2.1.1" resolved "https://registry.yarnpkg.com/@types/configstore/-/configstore-2.1.1.tgz#cd1e8553633ad3185c3f2f239ecff5d2643e92b6" @@ -13828,11 +13833,6 @@ loglevel@^1.4.1: resolved "https://registry.yarnpkg.com/loglevel/-/loglevel-1.6.1.tgz#e0fc95133b6ef276cdc8887cdaf24aa6f156f8fa" integrity sha1-4PyVEztu8nbNyIh82vJKpvFW+Po= -lokijs@^1.5.6: - version "1.5.6" - resolved "https://registry.yarnpkg.com/lokijs/-/lokijs-1.5.6.tgz#6de6b8c3ff7a972fd0104169f81e7ddc244c029f" - integrity sha512-xJoDXy8TASTjmXMKr4F8vvNUCu4dqlwY5gmn0g5BajGt1GM3goDCafNiGAh/sfrWgkfWu1J4OfsxWm8yrWweJA== - longest-streak@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/longest-streak/-/longest-streak-1.0.0.tgz#d06597c4d4c31b52ccb1f5d8f8fe7148eafd6965" @@ -18735,6 +18735,11 @@ shell-escape@^0.2.0: resolved "https://registry.yarnpkg.com/shell-escape/-/shell-escape-0.2.0.tgz#68fd025eb0490b4f567a027f0bf22480b5f84133" integrity sha1-aP0CXrBJC09WegJ/C/IkgLX4QTM= +shell-escape@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/shell-escape/-/shell-escape-0.2.0.tgz#68fd025eb0490b4f567a027f0bf22480b5f84133" + integrity sha1-aP0CXrBJC09WegJ/C/IkgLX4QTM= + shell-quote@1.6.1, shell-quote@^1.6.1: version "1.6.1" resolved "https://registry.yarnpkg.com/shell-quote/-/shell-quote-1.6.1.tgz#f4781949cce402697127430ea3b3c5476f481767"