From 63bddf79cecec11901729ed326a3fd15190feb7c Mon Sep 17 00:00:00 2001 From: jgclark Date: Tue, 26 Dec 2023 20:31:47 +0000 Subject: [PATCH] SearchExtensions 1.3.0 --- jgclark.SearchExtensions/CHANGELOG.md | 7 + jgclark.SearchExtensions/README.md | 28 +- .../__tests__/searchHelpers.test.js | 134 ++++---- jgclark.SearchExtensions/plugin.json | 12 +- jgclark.SearchExtensions/src/index.js | 3 +- jgclark.SearchExtensions/src/saveSearch.js | 17 +- jgclark.SearchExtensions/src/searchHelpers.js | 288 +++++++++++------- .../src/searchTriggers.js | 140 +++++++++ 8 files changed, 436 insertions(+), 193 deletions(-) create mode 100644 jgclark.SearchExtensions/src/searchTriggers.js diff --git a/jgclark.SearchExtensions/CHANGELOG.md b/jgclark.SearchExtensions/CHANGELOG.md index d3983a481..bf8ab26d5 100644 --- a/jgclark.SearchExtensions/CHANGELOG.md +++ b/jgclark.SearchExtensions/CHANGELOG.md @@ -2,6 +2,13 @@ (And see the full [README](https://github.com/NotePlan/plugins/tree/main/jgclark.SearchExtensions).) +## [1.3.0] - 2023-12-26 +- Adds ability to **automatically refresh** a saved search when opening its note. To enable this, run "/add trigger" on the saved search note, and select "🔎 Search Extensions: 'refreshSavedSearch'" from the list. To turn this off again, just remove the line starting `triggers: onOpen` from the frontmatter. +- Adds **wildcard operators `*` and `?`** in search terms. These match any number of characters (including none) and just 1 character respectively within a word. For example, `pos*e` matches "possible", "posie" and "pose"; `poli?e` matches "polite" and "police". +- Speeded up searches that have multiple terms (particularly 'must-find' terms) +- Now places the date and time of the search, and the Refresh 'button' under the section heading, not above it. This makes better sense for the auto-refresh (above). +- Now clarified that searches do include the special Archive and Templates folders, unless you exclude them using the 'Folders to exclude' setting. + ## [1.2.4] - 2023-10-04 ### Changes - the /flexiSearch dialog box simplified with a new tooltip help, and better validation checks diff --git a/jgclark.SearchExtensions/README.md b/jgclark.SearchExtensions/README.md index 490289277..f857c51c1 100644 --- a/jgclark.SearchExtensions/README.md +++ b/jgclark.SearchExtensions/README.md @@ -1,8 +1,9 @@ # 🔎 Search Extensions plugin NotePlan can search over your notes, but it is currently not very flexible or easy to use; in particular it's difficult to navigate between the search results and any of the actual notes it shows. This plugin adds some extra power and usability to searching. It: - lets you have keep special notes that lists all open tasks for @colleagueX that you can update in place! -- extends the search syntax +- extends the search syntax to allow much more control, including wildcards - by default the search runs and **saves the results in a note that it opens as a split view** next to where you're working. +- these saved searches can be refreshed automatically when you open the note to consult it. ![demo](qs+refresh-demo.gif) @@ -43,20 +44,24 @@ There are further display options you can set: - the commands to automatically decides the name of the note to save the search results to based on the search term, which avoids the final prompt, by the 'Automatically save?' setting. ### Refreshing Results -Each results note has a ` [🔄 Refresh results for ...]` pseudo-button under the title of the note. Clicking that runs the search again, and replaces the earlier set of results. (Thanks to @dwertheimer for the suggestion, which is a good use of the x-callback mechanism -- see below.) +Each results note has a ` [🔄 Refresh results for ...]` pseudo-button under the title of the note. Clicking that runs the search again, and replaces the earlier set of results. ![refresh results](highlight-refresh-in-search-results.png) This is shown in the demo above. -## Extended search syntax +From v1.3, a saved search can be **automatically refreshed when opening it**. To enable this, run "/add trigger" on the saved search note, and select "🔎 Search Extensions: 'refreshSavedSearch'" from the list. To turn this off again, just remove the line starting `triggers: onOpen` from the frontmatter. +## Extended search syntax - put a `+` and `-` search operator on the front of terms that **must** appear, and **must not** appear, respectively. For example `+must may could -cannot"` has 4 search terms, the first must be present, the last mustn't be present, and the middle two (may, could) can be. - the test for + and - is done per line in notes. If you wish to ignore the whole note that has a term, you can use the ! operator, e.g. `+must_have_me !no_way_jose`. (thanks @dwertheimer for this suggestion) -- the searches ignore case of words (i.e. `SPIRIT` will match `spirit` or `Spirit`) -- the searches are simple ones, matching on whole or partial words (e.g. `wind` matches `Windings` and `unwind`), not using fuzzy matching or regular expressions -- currently, a search term must have at least two alphanumeric characters to be valid -- all notes in the special folders (@Archive, @Templates and @Trash) are ignored. Others can be excluded too using the 'Folders to exclude' setting. If a folder is excluded, then so are its sub-folders. +- TEST: the searches ignore case of words (i.e. `SPIRIT` will match `spirit` or `Spirit`) +- the searches are simple ones, matching on whole or partial words (e.g. `wind` matches `Windings` and `unwind`) +- however from v1.3.0 you can also use two **wildcard** operators: + - `*` in a term means "match any number of characters (including none)" -- e.g. `pos*e` matches "possible", "posie" and "pose". + - `?` in a term means "match any single character" -- e.g. `poli?e` matches "polite" and "police". +- currently, a search term must have at least two alphanumeric characters to be valid. +- all notes in the special Trash folder are ignored. Others can be excluded too using the 'Folders to exclude' setting. If a folder is excluded, then so are its sub-folders. - you can use an empty search term (from v1.1), which might be useful in flexiSearch to find all open tasks. It will warn you first that this might be a lengthy operation. - (from v1.2) to search for an exact multi-word phrases, put it in quotes (e.g. `"Holy Spirit"`) - you can set default search terms in the 'Default Search terms' setting; if set you can still always override them. @@ -67,8 +72,9 @@ To change the default **settings** on **macOS** click the gear button on the 'Se ![search settings](search-settings.png) On **iOS** run the command "/Search: update plugin settings" which provides a multi-step equivalent to the more convenient macOS settings window. + ## Results highlighting -To see **highlighting** of matching terms in Simplified-style output, you'll need to be using a theme that highlights lines using `==this syntax==`. The build-in themes should now include this, but you can [customise an existing theme](https://help.noteplan.co/article/44-customize-themes) by adding something like: +To see **highlighting** of matching terms in Simplified-style output, you'll need to be using a theme that highlights lines using `==this syntax==`. The build-in themes now include this, but you can [customise an existing theme](https://help.noteplan.co/article/44-customize-themes) by adding something like: ```jsonc { @@ -102,6 +108,8 @@ To see **highlighting** of matching terms in Simplified-style output, you'll nee } ``` +Note: I have reported a small layout bug with this highlighting that was introduced about v.3.9.9. + ## Using from x-callback calls It's possible to call these commands from [outside NotePlan using the **x-callback mechanism**](https://help.noteplan.co/article/49-x-callback-url-scheme#runplugin). The URL calls all take the same form: ``` @@ -118,7 +126,7 @@ Notes: |-----|-----------|----------|----------|----------|----------|----------| | /flexiSearch | `noteplan://x-callback-url/runPlugin?pluginID=jgclark.SearchExtensions&command=flexiSearch`
(this takes no args: use this just to display the dialog box)| | | | | | | /quickSearch | `noteplan://x-callback-url/runPlugin?pluginID=jgclark.SearchExtensions&command=quickSearch&` | search term(s) ¶ (separated by commas) | paragraph types to filter by (separated by commas) | noteTypesToInclude either 'project','calendar' or 'both' | | | -| /search | `noteplan://x-callback-url/runPlugin?pluginID=jgclark.SearchExtensions&command=saveSearch&` | search term(s) (separated by commas) | paragraph types to filter by (separated by commas) | | | | +| /search | `noteplan://x-callback-url/runPlugin?pluginID=jgclark.SearchExtensions&command=search&` | search term(s) (separated by commas) | paragraph types to filter by (separated by commas) | | | | | /searchOverCalendar | `noteplan://x-callback-url/runPlugin?pluginID=jgclark.SearchExtensions&command=searchOverCalendar&` | search term(s) (separated by commas) | paragraph types to filter by (separated by commas) | | | | | /searchOverNotes | `noteplan://x-callback-url/runPlugin?pluginID=jgclark.SearchExtensions&command=searchOverNotes&` | search term(s) (separated by commas) | paragraph types to filter by (separated by commas) | | | | | /searchInPeriod | `noteplan://x-callback-url/runPlugin?pluginID=jgclark.SearchExtensions&command=searchInPeriod&` | search term(s) (separated by commas) | start date to search over (YYYYMMDD or YYYY-MM-DD format). If not given, then defaults to 3 months ago. | end date to search over (YYYYMMDD or YYYY-MM-DD format). If not given, then defaults to today. | optional paragraph types to filter by (separated by commas) | optional output destination indicator: 'current', 'newnote', or 'log' | @@ -128,7 +136,7 @@ Notes: ## Support If you find an issue with this plugin, or would like to suggest new features for it, please raise a [Bug or Feature 'Issue'](https://github.com/NotePlan/plugins/issues). -If you would like to support my late-night work extending NotePlan through writing these plugins, you can through +I have spent several weeks of my free time on this plugin. If you would like to support my late-night work extending NotePlan through writing these plugins, you can through [Buy Me A Coffee](https://www.buymeacoffee.com/revjgc) diff --git a/jgclark.SearchExtensions/__tests__/searchHelpers.test.js b/jgclark.SearchExtensions/__tests__/searchHelpers.test.js index 8e12dd434..24b6f2243 100644 --- a/jgclark.SearchExtensions/__tests__/searchHelpers.test.js +++ b/jgclark.SearchExtensions/__tests__/searchHelpers.test.js @@ -372,9 +372,9 @@ describe('searchHelpers.js tests', () => { }) describe('normaliseSearchTerms', () => { - test('empty string', () => { + test('empty string -> empty string', () => { const result = normaliseSearchTerms('') - expect(result).toEqual([]) + expect(result).toEqual(['']) }) test('just spaces', () => { const result = normaliseSearchTerms(' ') @@ -424,74 +424,80 @@ describe('searchHelpers.js tests', () => { const result = normaliseSearchTerms('xxx AND yyy AND zzz') expect(result).toEqual(['+xxx', '+yyy', '+zzz']) }) - test('"1 John", 1Jn (do modify)', () => { - const result = normaliseSearchTerms('"1 John", 1Jn', true) - expect(result).toEqual(['+1', '+John', '1Jn']) - }) test('"1 John", 1Jn (do not modify)', () => { - const result = normaliseSearchTerms('"1 John", 1Jn', false) + const result = normaliseSearchTerms('"1 John" 1Jn') expect(result).toEqual(['1 John', '1Jn']) }) - test('mix of quoted and unquoted terms (do modify)', () => { - const result = normaliseSearchTerms('-term1 "term two" !term3', true) - expect(result).toEqual(['-term1', '+term', '+two', '!term3']) - }) test("mix of quoted and unquoted terms (don't modify)", () => { - const result = normaliseSearchTerms('-term1 "term two" !term3', false) + const result = normaliseSearchTerms('-term1 "term two" !term3') expect(result).toEqual(['-term1', 'term two', '!term3']) }) test("quoted terms with different must/may/cant (don't modify)", () => { - const result = normaliseSearchTerms('-"Bob Smith" "Holy Spirit" !"ice cream cone"', false) + const result = normaliseSearchTerms('-"Bob Smith" "Holy Spirit" !"ice cream cone"') expect(result).toEqual(['-Bob Smith', 'Holy Spirit', '!ice cream cone']) }) - test('terms with apostrophes in quoted terms (do modify)', () => { - const result = normaliseSearchTerms('-term1 "couldn\'t possibly" !term3', true) - expect(result).toEqual(['-term1', "+couldn't", '+possibly', '!term3']) - }) test("terms with apostrophes in quoted terms (don't modify)", () => { - const result = normaliseSearchTerms('-term1 "couldn\'t possibly" !term3', false) + const result = normaliseSearchTerms('-term1 "couldn\'t possibly" !term3') expect(result).toEqual(['-term1', "couldn't possibly", '!term3']) }) - // TODO: This one failing on the apostophe -> [- "can't",+ "can",+ "t", "term2"] - test.skip('terms with apostrophes in unquoted terms', () => { + test('terms with apostrophes in unquoted terms', () => { const result = normaliseSearchTerms("can't term2") expect(result).toEqual(["can't", 'term2']) }) - test('mix of quoted and unquoted terms (do modify)', () => { - const result = normaliseSearchTerms(`bob "xxx",'yyy', "asd'sa" 'bob two' "" hello`, true) - expect(result).toEqual(['bob', 'xxx', 'yyy', "asd'sa", '+bob', '+two', 'hello']) - }) test("mix of quoted and unquoted terms (don't modify)", () => { - const result = normaliseSearchTerms(`bob "xxx",'yyy', "asd'sa" 'bob two' "" hello`, false) - expect(result).toEqual(['bob', 'xxx', 'yyy', "asd'sa", 'bob two', 'hello']) - }) - // TODO: This one failing on "-bob two" -> [- "-bob two", '+bob' '+two'] - test.skip('mix of quoted and unquoted terms and operators (do modify)', () => { - const result = normaliseSearchTerms('+bob "xxx",\'yyy\', !"asd\'sa" -\'bob two\' "" !hello', true) - expect(result).toEqual(['+bob', 'xxx', 'yyy', "!asd'sa", '-bob two', '!hello']) + const result = normaliseSearchTerms(`bob "xxx" 'yyy' "asd'sa" 'bob two' "" hello`) + expect(result).toEqual(['bob', 'xxx', "'yyy'", "asd'sa", "'bob", "two'", 'hello']) }) test("mix of quoted and unquoted terms and operators (don't modify)", () => { - const result = normaliseSearchTerms('+bob "xxx",\'yyy\', !"asd\'sa" -\'bob two\' "" !hello', false) - expect(result).toEqual(['+bob', 'xxx', 'yyy', "!asd'sa", '-bob two', '!hello']) + const result = normaliseSearchTerms('+bob "xxx" \'yyy\' !"asd\'sa" -"bob two" "" !hello') + expect(result).toEqual(['+bob', 'xxx', "'yyy'", "!asd'sa", "-bob two", '!hello']) }) test("test for Greek characters", () => { - const result = normaliseSearchTerms('γιάννης', false) + const result = normaliseSearchTerms('γιάννης') expect(result).toEqual(['γιάννης']) }) - test("test for Greek characters as an @mention", () => { - const result = normaliseSearchTerms('@γιάννης', false) - expect(result).toEqual(['@γιάννης']) + test("mix of terms with ? and * operators (this is just normalising not validating)", () => { + const result = normaliseSearchTerms('spirit* mo? *term mo*blues ?weird') + expect(result).toEqual(['spirit*', 'mo?', '*term', 'mo*blues', '?weird']) + }) + + describe('skipping these tests as removed modifyQuotedTermsToAndedTerms functionality', () => { + test.skip('"1 John", 1Jn (do modify)', () => { + const result = normaliseSearchTerms('"1 John" 1Jn', true) + expect(result).toEqual(['+1', '+John', '1Jn']) + }) + test.skip('mix of quoted and unquoted terms (do modify)', () => { + const result = normaliseSearchTerms('-term1 "term two" !term3', true) + expect(result).toEqual(['-term1', '+term', '+two', '!term3']) + }) + test.skip('terms with apostrophes in quoted terms (do modify)', () => { + const result = normaliseSearchTerms('-term1 "couldn\'t possibly" !term3', true) + expect(result).toEqual(['-term1', "+couldn't", '+possibly', '!term3']) + }) + test.skip('mix of quoted and unquoted terms (do modify)', () => { + const result = normaliseSearchTerms(`bob "xxx" 'yyy' "asd'sa" 'bob two' "" hello`, true) + expect(result).toEqual(['bob', 'xxx', 'yyy', "asd'sa", '+bob', '+two', 'hello']) + }) + test.skip('mix of quoted and unquoted terms and operators (do modify)', () => { + const result = normaliseSearchTerms('+bob "xxx",\'yyy\', !"asd\'sa" -\'bob two\' "" !hello', true) + expect(result).toEqual(['+bob', 'xxx', "'yyy'", "!asd'sa", '-bob', 'two', '!hello']) + }) }) - // TODO: can't mix OR with + }) describe('validateAndTypeSearchTerms', () => { - test('should return empty array from empty input', () => { - const result = validateAndTypeSearchTerms('') - expect(result).toEqual([]) + test('should return empty array from empty input (empty not allowed)', () => { + const result = validateAndTypeSearchTerms('', false) + expect(result).toEqual([]) // and an error + }) + test('should return empty array from empty input (empty allowed)', () => { + const result = validateAndTypeSearchTerms('', true) + expect(result).toEqual([ + { term: '', type: 'must', termRep: '' } + ]) }) test('should return empty array from too many terms', () => { - const result = validateAndTypeSearchTerms('abc def ghi jkl mno pqr stu vwz') + const result = validateAndTypeSearchTerms('abc def ghi jkl mno pqr stu vwz nine ten') expect(result).toEqual([]) }) test('should return empty array from no positive terms', () => { @@ -500,61 +506,50 @@ describe('searchHelpers.js tests', () => { }) test("single term string 'term1'", () => { const result = validateAndTypeSearchTerms('term1') - expect(result).toEqual([{ term: 'term1', type: 'may', termRep: 'term1' }]) + expect(result).toEqual([ + { term: 'term1', type: 'may', termRep: 'term1' } + ]) }) test("single term string 'twitter.com'", () => { const result = validateAndTypeSearchTerms('twitter.com') expect(result).toEqual([{ term: 'twitter.com', type: 'may', termRep: 'twitter.com' }]) }) - // Note: updated to suit better multi-word term handling - test("single quoted term string 'test string'", () => { - const result = validateAndTypeSearchTerms("'test string'") + test("quoted string with apostrophe [shouldn't matter]", () => { + const result = validateAndTypeSearchTerms('"shouldn\'t matter"') expect(result).toEqual([ - // { term: 'test', type: 'must', termRep: '+test' }, - // { term: 'string', type: 'must', termRep: '+string' }, - { term: 'test string', type: 'may', termRep: 'test string' }, + { term: "shouldn't matter", type: 'may', termRep: "shouldn't matter" }, ]) }) - // Note: updated to suit better multi-word term handling test('two term string', () => { const result = validateAndTypeSearchTerms('term1 "term two"') expect(result).toEqual([ { term: 'term1', type: 'may', termRep: 'term1' }, - // { term: 'term', type: 'must', termRep: '+term' }, - // { term: 'two', type: 'must', termRep: '+two' }, { term: 'term two', type: 'may', termRep: 'term two' }, ]) }) - // Note: updated to suit better multi-word term handling - test('three terms with +//-', () => { + test('three terms with [+,-,]', () => { const result = validateAndTypeSearchTerms('+term1 "term two" -term3') expect(result).toEqual([ { term: 'term1', type: 'must', termRep: '+term1' }, - // { term: 'term', type: 'must', termRep: '+term' }, - // { term: 'two', type: 'must', termRep: '+two' }, { term: 'term two', type: 'may', termRep: 'term two' }, { term: 'term3', type: 'not-line', termRep: '-term3' }, ]) }) - // Note: updated to suit better multi-word term handling - test('three terms with +//!', () => { + test('three terms with [+,!,]', () => { const result = validateAndTypeSearchTerms('+term1 "term two" !term3') expect(result).toEqual([ { term: 'term1', type: 'must', termRep: '+term1' }, - // { term: 'term', type: 'must', termRep: '+term' }, - // { term: 'two', type: 'must', termRep: '+two' }, { term: 'term two', type: 'may', termRep: 'term two' }, { term: 'term3', type: 'not-note', termRep: '!term3' }, ]) }) test('+"1 John", 1Jn', () => { - const result = validateAndTypeSearchTerms('+"1 John", 1Jn') + const result = validateAndTypeSearchTerms('+"1 John" 1Jn') expect(result).toEqual([ { term: '1 John', type: 'must', termRep: '+1 John' }, { term: '1Jn', type: 'may', termRep: '1Jn' }, ]) }) - // Note: added to suit better multi-word term handling test("quoted terms with different must/may/cant", () => { const result = validateAndTypeSearchTerms('-"Bob Smith" "Holy Spirit" !"ice cream cone"') expect(result).toEqual([ @@ -562,6 +557,21 @@ describe('searchHelpers.js tests', () => { { term: 'Holy Spirit', type: 'may', termRep: 'Holy Spirit' }, { term: 'ice cream cone', type: 'not-note', termRep: '!ice cream cone' }]) }) + test("mix of terms with valid ? and * operators", () => { + const result = validateAndTypeSearchTerms('spirit* mo?t +term mo*blues we*d') + expect(result).toEqual([ + { term: 'spirit*', type: 'may', termRep: 'spirit*' }, + { term: 'mo?t', type: 'may', termRep: 'mo?t' }, + { term: 'term', type: 'must', termRep: '+term' }, + { term: 'mo*blues', type: 'may', termRep: 'mo*blues' }, + { term: 'we*d', type: 'may', termRep: 'we*d' }]) + }) + test("mix of terms with invalid ? and * operators", () => { + const result = validateAndTypeSearchTerms('*spirit ?moses we*d') + expect(result).toEqual([ + { term: 'we*d', type: 'may', termRep: 'we*d' } + ]) + }) }) // Just a no-result test -- rest too hard to mock up diff --git a/jgclark.SearchExtensions/plugin.json b/jgclark.SearchExtensions/plugin.json index ce89c760f..c5ffaeffa 100644 --- a/jgclark.SearchExtensions/plugin.json +++ b/jgclark.SearchExtensions/plugin.json @@ -8,8 +8,8 @@ "plugin.author": "Jonathan Clark", "plugin.url": "https://github.com/NotePlan/plugins/tree/main/jgclark.SearchExtensions/", "plugin.changelog": "https://github.com/NotePlan/plugins/blob/main/jgclark.SearchExtensions/CHANGELOG.md", - "plugin.version": "1.2.4", - "plugin.lastUpdateInfo": "1.2.4: fix /flexiSearch issues on iOS\n1.2.3: change to allow /quickSearch from x-callback without search term.\n1.2.2: ability to run FlexiSearch without closing the Dashboard and Project list windows from other plugins.\n1.2.1: fix bug in /searchInPeriod. 1.2.0: multi-word search terms.\n1.1: New 'flexiSearch' command. Adds limits to very large search results, to prevent overwhelming the app. Deals with 'twitter.com' case. Lots of other polish.\n1.0: Major new release with more powerful search syntax, a new display style, sync-ing open tasks and more. Please see the README to learn more!", + "plugin.version": "1.3.0", + "plugin.lastUpdateInfo": "1.3.0: adds `*` and `?` wildcard operators. Adds auto-refresh capability. Other improvements. (Please see documentation for details.)\n1.2.4: fix /flexiSearch issues on iOS\n1.2.3: change to allow /quickSearch from x-callback without search term.\n1.2.2: ability to run FlexiSearch without closing the Dashboard and Project list windows from other plugins.\n1.2.1: fix bug in /searchInPeriod. 1.2.0: multi-word search terms.\n1.1: New 'flexiSearch' command. Adds limits to very large search results, to prevent overwhelming the app. Deals with 'twitter.com' case. Lots of other polish.\n1.0: Major new release with more powerful search syntax, a new display style, sync-ing open tasks and more. Please see the README to learn more!", "plugin.dependencies": [], "plugin.script": "script.js", "plugin.isRemote": "false", @@ -139,6 +139,12 @@ "paraTypes" ] }, + { + "name": "refreshSavedSearch", + "hidden": true, + "description": "onOpen", + "jsFunction": "refreshSavedSearch" + }, { "name": "closeDialogWindow", "hidden": true, @@ -219,7 +225,7 @@ { "key": "foldersToExclude", "title": "Folders to exclude", - "description": "Optional list of folders to exclude in these commands. If a folder is listed, then sub-folders are also excluded.\nTo exclude the top-level folder, use '/'.\nNote that @Trash, @Templates and @Archive are always excluded.", + "description": "Optional list of folders to exclude in these commands. If a folder is listed, then sub-folders are also excluded.\nTo exclude the top-level folder, use '/'. (The special Trash folder is always excluded.)", "type": "[string]", "default": [ "Summaries", diff --git a/jgclark.SearchExtensions/src/index.js b/jgclark.SearchExtensions/src/index.js index 478a4e91d..d39c8f81f 100644 --- a/jgclark.SearchExtensions/src/index.js +++ b/jgclark.SearchExtensions/src/index.js @@ -2,7 +2,7 @@ //----------------------------------------------------------------------------- // More advanced searching // Jonathan Clark -// Last updated 30.6.2023 for v1.2.0 +// Last updated 7.12.2023 for v1.3.0 //----------------------------------------------------------------------------- export { @@ -14,6 +14,7 @@ export { searchOverCalendar } from './saveSearch' export { searchPeriod } from './saveSearchPeriod' +export { refreshSavedSearch } from './searchTriggers' export { closeDialogWindow, flexiSearchRequest, diff --git a/jgclark.SearchExtensions/src/saveSearch.js b/jgclark.SearchExtensions/src/saveSearch.js index 55aed8a66..5cf7293ea 100644 --- a/jgclark.SearchExtensions/src/saveSearch.js +++ b/jgclark.SearchExtensions/src/saveSearch.js @@ -3,7 +3,7 @@ // Create list of occurrences of note paragraphs with specified strings, which // can include #hashtags or @mentions, or other arbitrary strings (but not regex). // Jonathan Clark -// Last updated 1.7.2023 for v1.1.0, @jgclark +// Last updated 21.12.2023 for v1.3.0, @jgclark //----------------------------------------------------------------------------- import pluginJson from '../plugin.json' @@ -13,6 +13,7 @@ import { getSearchTermsRep, makeAnySyncs, OPEN_PARA_TYPES, + optimiseOrderOfSearchTerms, resultCounts, type resultOutputTypeV3, runSearchesV2, @@ -93,6 +94,7 @@ export async function searchOverNotes(searchTermsArg?: string, paraTypeFilterArg * Call the main function, searching over all notes, but using a fixed note for results */ export async function quickSearch(searchTermsArg?: string, paraTypeFilterArg?: string, noteTypesToIncludeArg?: string): Promise { + logDebug('quickSearch', `starting with searchTermsArg=${searchTermsArg}, paraTypeFilterArg=${paraTypeFilterArg}, noteTypesToIncludeArg=${noteTypesToIncludeArg}`) await saveSearch( searchTermsArg, noteTypesToIncludeArg ?? 'both', @@ -123,7 +125,7 @@ export async function saveSearch( // get relevant settings const config = await getSearchSettings() const headingMarker = '#'.repeat(config.headingLevel) - logDebug(pluginJson, `arg0 -> searchTermsArg ${typeof searchTermsArg}`) + // logDebug(pluginJson, `arg0 -> searchTermsArg ${typeof searchTermsArg}`) logDebug(pluginJson, `arg0 -> searchTermsArg '${searchTermsArg ?? '(not supplied)'}'`) // work out if we're being called non-interactively (i.e. via x-callback) by seeing whether originatorCommand is not empty @@ -133,6 +135,7 @@ export async function saveSearch( // Get the noteTypes to include const noteTypesToInclude: Array = (noteTypesToIncludeArg === 'both' || noteTypesToIncludeArg === '') ? ['notes', 'calendar'] : [noteTypesToIncludeArg] logDebug(pluginJson, `arg1 -> note types '${noteTypesToInclude.toString()}'`) + logDebug(pluginJson, `arg2 -> originatorCommand = '${originatorCommand}'`) // Get the search terms, either from argument supplied, or by asking user let termsToMatchStr = '' @@ -160,10 +163,11 @@ export async function saveSearch( await showMessage(`These search terms aren't valid. Please see Plugin Console for details.`) return } - logDebug(pluginJson, `arg2 -> originatorCommand = '${originatorCommand}'`) + // Now optimise the order we tackle the search terms + const orderedSearchTerms = optimiseOrderOfSearchTerms(validatedSearchTerms) // If we have a blank search term, then double-check user wants to do this - if (validatedSearchTerms.length === 1 && validatedSearchTerms[0].term === '') { + if (orderedSearchTerms.length === 1 && orderedSearchTerms[0].term === '') { const res = await showMessageYesNo('No search terms specified. Are you sure you want to run a potentially very long search?') if (res === 'No') { logDebug(pluginJson, 'User has cancelled search') @@ -184,7 +188,7 @@ export async function saveSearch( await CommandBar.onAsyncThread() // $FlowFixMe[incompatible-exact] - const resultsProm: resultOutputTypeV3 = runSearchesV2(validatedSearchTerms, noteTypesToInclude, [], config.foldersToExclude, config, paraTypesToInclude) // Note: deliberately no await: this is resolved later + const resultsProm: resultOutputTypeV3 = runSearchesV2(orderedSearchTerms, noteTypesToInclude, [], config.foldersToExclude, config, paraTypesToInclude) // Note: deliberately no await: this is resolved later await CommandBar.onMainThread() @@ -232,7 +236,7 @@ export async function saveSearch( // Do output // logDebug(pluginJson, 'reached do output stage') const searchTermsRepStr = `'${resultSet.searchTermsRepArr.join(' ')}'`.trim() // Note: we normally enclose in [] but here need to use '' otherwise NP Editor renders the link wrongly - const xCallbackURL = createRunPluginCallbackUrl('jgclark.SearchExtensions', originatorCommand, [termsToMatchStr, paraTypeFilterArg ?? '']) + const xCallbackURL = createRunPluginCallbackUrl('jgclark.SearchExtensions', originatorCommand, [termsToMatchStr, paraTypeFilterArg ?? '', noteTypesToInclude.join(',')]) // Note: these params are to the individual functions, not to the underlying saveSearch() function. So they are in a different order. switch (destination) { case 'current': { @@ -314,6 +318,7 @@ export async function saveSearch( break } } + logDebug(pluginJson, `saveSearch() finished.`) } catch (err) { logError(pluginJson, err.message) diff --git a/jgclark.SearchExtensions/src/searchHelpers.js b/jgclark.SearchExtensions/src/searchHelpers.js index 36db8a162..17d84a2d0 100644 --- a/jgclark.SearchExtensions/src/searchHelpers.js +++ b/jgclark.SearchExtensions/src/searchHelpers.js @@ -2,7 +2,7 @@ //----------------------------------------------------------------------------- // Search Extensions helpers // Jonathan Clark -// Last updated 14.7.2023 for v1.2.1, @jgclark +// Last updated 26.12.2023 for v1.3.0, @jgclark //----------------------------------------------------------------------------- import pluginJson from '../plugin.json' @@ -81,6 +81,7 @@ export const SYNCABLE_PARA_TYPES = ['open', 'scheduled', 'checklist', 'checklist export type SearchConfig = { autoSave: boolean, folderToStore: string, + includeSpecialFolders: boolean, foldersToExclude: Array, headingLevel: headingLevelType, defaultSearchTerms: Array, @@ -121,38 +122,30 @@ export async function getSearchSettings(): Promise { /** * Take a simple string as search input and process it to turn into an array of strings ready to validate and type. -* Quoted multi-word search terms (e.g. ["Bob Smith"]) are by default treated as [+Bob +Smith] as I now discover the API doesn't support quoted multi-word search phrases. +* V3: Quoted multi-word search terms (e.g. ["Bob Smith"]) are now left alone (but without the double quotes). The extra parameter 'modifyQuotedTermsToAndedTerms' has now been removed. +* V2: Quoted multi-word search terms (e.g. ["Bob Smith"]) are by default treated as [+Bob +Smith] as I now discover the API doesn't support quoted multi-word search phrases. * @author @jgclark * @tests in jest file * @param {string | Array} searchArg string containing search term(s) or array of search terms -* @param {boolean?} modifyQuotedTermsToAndedTerms? (default true) * @returns {Array} normalised search term(s) */ -export function normaliseSearchTerms( - searchArg: string, - modifyQuotedTermsToAndedTerms?: boolean = true -): Array { - logDebug('normaliseSearchTerms', `starting for [${searchArg}]`) +export function normaliseSearchTerms(searchArg: string): Array { + // logDebug('normaliseSearchTerms', `starting for [${searchArg}]`) let outputArray = [] - // // First deal with edge case of empty searchArg, which is now allowed - // if (searchArg === '') { - // logWarn('normaliseSearchTerms', `Returning special case of single empty search term`) - // return [''] - // } - - // Take a simple string and process it to turn into an array of string, according to one of several schemes: - // if (!searchArg.match(/\w{2,}/)) { - // // this has no words (at least 2 long) -> empty - // logWarn('normaliseSearchTerms', `No valid words found in [${searchArg}]`) - // return [] - // } + // First deal with edge case of empty searchArg, which is now allowed + if (searchArg === '') { + logWarn('normaliseSearchTerms', `Returning special case of single empty search term`) + return [''] + } + + // this has free-floating +/- operators -> error (but single ! is allowed) if (searchArg.match(/\s[\+\-]\s/)) { - // this has free-floating +/- operators -> error (but single ! is allowed) logWarn('normaliseSearchTerms', `Search string not valid: unattached search operators found in [${searchArg}]`) return [] } + // Change older search syntax into newer one // change simple form [x,y,z] style -> array of x,y,z if (searchArg.match(/\w+\s*,\s*\w+/)) { outputArray = searchArg.split(/\s*,\s*/) @@ -168,63 +161,29 @@ export function normaliseSearchTerms( outputArray = searchArg.split(/\sOR\s/) } - // // As we want to modify quoted phrases to +words, Go through terms to find multi-word ones, and change to individual + terms - // else if (modifyQuotedTermsToAndedTerms) { - // const reResults = searchArg.match(/(?:[^!+-])(?:([\"'])(.+?)\1)/g) - // if (reResults) { - // for (const r of reResults) { - // // the match groups are - // // 0/total matches [ "word1 word2"] - // // 1 first of matching pair of quotes - // // 2 phrase in quotes - // // modify searchArg to make +words instead - // const innerTerms = r[2].split(' ') - // for (const t of innerTerms) { - // outputArray.push(`+${t}`) - // } - // } - // } - // // Now need to add terms not in quotes - // // ??? - // } - // else treat as [x y z], with or without quoted phrases. else { + // const searchArgPadded = ' ' + searchArg + ' ' // This Regex attempts to split words: - // - but keeping text in double or single quotes together - // - and prefixed search operators !/+/- + // - but keeping text in double quotes as one term // - and #hashtag/child and @mention(5) possibilities - // - a word now may include any of ./!/#/- - // To make it more manageable we need to add space to front and end - // const RE_WOW = new RegExp(/\s([\-\+\!]?)([\-\+\!]?)(?:([\'"])(.+?)\3)|([\-\+\!]?[\w\.\!\-#@\/\(\)]+)/g) - // Following is attenpt to allow full unicode letter characters (\p{L}) and numbers (\p{N}) rather than ASCII (\w): (info from Dash.) - // const RE_WOW = new RegExp(/\s([\-\+\!]?)([\-\+\!]?)(?:([\'"])(.+?)\3)|([\-\+\!]?[\p{L}\.\!\-#@\/\(\)]+)/gu) - // Following also avoids 'classEscape' compilation errors - // FIXME: but breaks other things - const RE_WOW = new RegExp(/\s([\-\+!]?)([\-\+!]?)(?:(['"])(.+?)\3)|([\-\+!]?[\p{L}\p{N}\.!\-#@\/\(\)]+)/gu) - const searchArgPadded = ' ' + searchArg + ' ' - const reResults = searchArgPadded.matchAll(RE_WOW) + // - a word now may include any of .!+#-*?' + // NB: Allows full unicode letter characters (\p{L}) and numbers (\p{N}) rather than ASCII (\w): (info from Dash.) + // NB: To make the regex easier, add a space to start and end, and switch the order of any [-+!]['"] + const RE_WOW = new RegExp(/(([\p{L}\p{N}\s\-\/@\(\)#*?.+!']*)(?="\s)|([\p{L}\p{N}\-@\/\(\)#.+!'\*\?]*))/gu) + let searchArgPadded = ' ' + searchArg + ' ' + searchArgPadded = searchArgPadded + .replace(/\s-"/, ' "-').replace(/\s\+"/, ' "+').replace(/\s!"/, ' "!') + const reResults = searchArgPadded.match(RE_WOW) if (reResults) { - for (const r of reResults) { - // this concats match groups: - // 1 (optional operator prefix) - // 4 (phrase inside quotes) or - // 5 (word not in quotes) - - if (r[4] && r[4].includes(' ') && modifyQuotedTermsToAndedTerms) { - // if we want to modify quoted phrases to +words, and we have some quoted phrases, - // go through terms to find multi-word ones, and change to individual + terms. - // But if we have a simple quoted ["word"] then strip quotes but don't add + - // TODO: deal with [-"word1 word2"] case -> '-word1', '-word2' I guess. - // TODO: deal with mid-word apostrophe [can't term] case - const innerTerms = r[4].split(' ') - for (const t of innerTerms) { - outputArray.push(`+${t}`) - } - } - else { - // add whichever bit of the term matches - outputArray.push(`${r[1] ?? ''}${r[4] ?? ''}${r[5] ?? ''}`) + logDebug('validateAndTypeSearchTerms', `-> [${String(reResults)}] from [${searchArgPadded}]`) + let carryForward = '' + for (const rr of reResults) { + let r = rr.trim() + // Add term as long as it doesn't start with a * or ? or is empty + if (r !== '') { + // logDebug('r', `[${r}]`) + outputArray.push(r) } } } else { @@ -245,8 +204,7 @@ export function normaliseSearchTerms( * @tests in jest file */ export function validateAndTypeSearchTerms(searchArg: string, allowEmptyOrOnlyNegative: boolean = false): Array { - // TEST: Now change to modifyQuotedTermsToAndedTerms false - const normalisedTerms = normaliseSearchTerms(searchArg, false) + const normalisedTerms = normaliseSearchTerms(searchArg) logDebug('validateAndTypeSearchTerms', `starting with ${String(normalisedTerms.length)} normalised terms: [${String(normalisedTerms)}]`) // Don't allow 0 terms, apart from @@ -260,31 +218,34 @@ export function validateAndTypeSearchTerms(searchArg: string, allowEmptyOrOnlyNe const validatedTerms: Array = [] for (const u of normalisedTerms) { let t = u.trim() - let thisType = '' - const thisRep = t - if (t[0] === '+') { - thisType = 'must' - t = t.slice(1) - } else if (t[0] === '-') { - thisType = 'not-line' - t = t.slice(1) - } else if (t[0] === '!') { - thisType = 'not-note' - t = t.slice(1) + // Only proceed if this doesn't have a wildcard at the start + if (/^[^\*\?]/.test(t)) { + let thisType = '' + const thisRep = t + if (t[0] === '+') { + thisType = 'must' + t = t.slice(1) + } else if (t[0] === '-') { + thisType = 'not-line' + t = t.slice(1) + } else if (t[0] === '!') { + thisType = 'not-note' + t = t.slice(1) + } else { + thisType = 'may' + } + validatedTerms.push({ term: t, type: thisType, termRep: thisRep }) } else { - thisType = 'may' + logDebug('normaliseSearchTerms', `- ignoring invalid search term: [${t}]`) } - validatedTerms.push({ term: t, type: thisType, termRep: thisRep }) } // Stop if we have a silly number of search terms - if (validatedTerms.length > 7) { + if (validatedTerms.length > 9) { logWarn(pluginJson, `Too many search terms given (${validatedTerms.length}); stopping as this might be an error.`) return [] } - // clo(validatedTerms, 'validatedTerms') - // Now check we have a valid set of terms. (If they're not valid, return an empty array.) // Invalid if we don't have any must-have or may-have search terms if (validatedTerms.filter((t) => (t.type === 'may' || t.type === 'must')).length === 0) { @@ -303,6 +264,37 @@ export function validateAndTypeSearchTerms(searchArg: string, allowEmptyOrOnlyNe return validatedTerms } +/** +* Optimise the order to tackle search terms. Assumes these have been normalised and validated already. +* @author @jgclark +* @param {Array} inputTerms +* @returns {Array} output +* TODO: @tests in jest file +*/ +export function optimiseOrderOfSearchTerms(inputTerms: Array): Array { + try { + logDebug('optimiseOrderOfSearchTerms', `starting with ${String(inputTerms.length)} terms`) + // Expand the typedSearchTerm object to include length of terms + const expandedInputTerms = inputTerms.map((i) => { + return { + typeOrder: (i.type === 'must') ? 'aaa' : i.type, // 'must' needs to come first, so make it to 'aaa' in a separate variable in the item + type: i.type, + term: i.term, + termRep: i.termRep, + longestWordLength: i.term.length + } + }) + clo(expandedInputTerms, 'expandedInputTerms = ') + const sortKeys = ['typeOrder', 'longestWordLength'] + logDebug('optimiseOrderOfSearchTerms', `- Will use sortKeys: [${String(sortKeys)}]`) + const sortedTerms: Array = sortListBy(expandedInputTerms, sortKeys) + clo(sortedTerms, 'optimiseOrderOfSearchTerms -> ') + return sortedTerms + } catch (err) { + return [] + } +} + /** * Compute difference of two arrays, by a given property value * from https://stackoverflow.com/a/63745126/3238281 @@ -404,6 +396,8 @@ export function getSearchTermsRep(typedSearchTerms: Array): str * This is where the search logic is applied, using the must/may/not terms. * Returns the subset of results, and can optionally limit the number of results returned to the first 'resultLimit' items. * If fromDateStr and toDateStr are given, then it will filter out results from Project Notes or the Calendar notes from outside that date range (measured at the first date of the Calendar note's period). + * Note: assumes the order of searchTerms has been optimised before now + * * Called by runSearchesV2 * @param {Array} * @param {number} resultLimit (optional; defaults to 500) @@ -428,8 +422,9 @@ export function applySearchOperators( let consolidatedNALs: Array = [] let consolidatedNoteCount = 0 let consolidatedLineCount = 0 - let uniquedFilenames = [] + let uniquedFilenames: Array = [] + // ------------------------------------------------------------ // Write any *first* 'must' search results to consolidated set if (mustResultObjects.length > 0) { const r = mustResultObjects[0] @@ -444,6 +439,19 @@ export function applySearchOperators( consolidatedLineCount = consolidatedNALs.length logDebug('applySearchOperators', `- must: after term 1, ${consolidatedLineCount} results`) + // If no results by now, there's no point finding anything further, so just form up an almost-empty return + if (consolidatedLineCount === 0) { + logInfo('applySearchOperators', `- must: no results found after must term [${r.searchTerm.termRep}] so stopping early.`) + const consolidatedResultsObject: resultOutputTypeV3 = { + searchTermsRepArr: termsResults.map((m) => m.searchTerm.termRep), + resultNoteAndLineArr: [], + resultCount: 0, + resultNoteCount: 0, + fullResultCount: 0 + } + return consolidatedResultsObject + } + // Write any *subsequent* 'must' search results to consolidated set, // having computed the intersection with the consolidated set if (mustResultObjects.length > 1) { @@ -459,7 +467,6 @@ export function applySearchOperators( const intersectionNALArray = noteAndLineIntersection(consolidatedNALs, r.resultNoteAndLineArr) logDebug('applySearchOperators', `- must: intersection of ${r.searchTerm.termRep} -> ${intersectionNALArray.length} results`) consolidatedNALs = intersectionNALArray - // clo(consolidatedNALs, `consolidatedNALs after must[${j}] intersection`) j++ } @@ -468,12 +475,27 @@ export function applySearchOperators( consolidatedNoteCount = numberOfUniqueFilenames(consolidatedNALs) consolidatedLineCount = consolidatedNALs.length // clo(consolidatedNALs, '(after must) consolidatedNALs:') + logDebug('applySearchOperators', `- must: after all ${mustResultObjects.length} terms, ${consolidatedLineCount} results`) + + // If no results by now, there's no point finding anything further, so just form up an almost-empty return + if (consolidatedLineCount === 0) { + logInfo('applySearchOperators', `- must: no results found after must term [${r.searchTerm.termRep}] so stopping early.`) + const consolidatedResultsObject: resultOutputTypeV3 = { + searchTermsRepArr: termsResults.map((m) => m.searchTerm.termRep), + resultNoteAndLineArr: [], + resultCount: 0, + resultNoteCount: 0, + fullResultCount: 0 + } + return consolidatedResultsObject + } } logDebug('applySearchOperators', `Must: at end, ${consolidatedLineCount} results`) } else { logDebug('applySearchOperators', `- must: No results found for must-find search terms`) } + // ------------------------------------------------------------ // Check if we can add the 'may' search results to consolidated set let addedAny = false for (const r of mayResultObjects) { @@ -509,6 +531,7 @@ export function applySearchOperators( } logDebug('applySearchOperators', `May: at end, ${consolidatedLineCount} results from ${consolidatedNoteCount} notes`) + // ------------------------------------------------------------ // Delete any results from the consolidated set that match 'not-...' terms let removedAny = false for (const r of notResultObjects) { @@ -552,6 +575,7 @@ export function applySearchOperators( } logDebug('applySearchOperators', `Not: at end, ${consolidatedLineCount} results from ${consolidatedNoteCount} notes`) + // ------------------------------------------------------------ // If we have date limits, now apply them if (fromDateStr && toDateStr) { logDebug('applySearchOperators', `- Will now filter out Calendar note results outside ${fromDateStr}-${toDateStr} from ${consolidatedLineCount} results`) @@ -565,6 +589,7 @@ export function applySearchOperators( let fullResultCount = consolidatedLineCount + // ------------------------------------------------------------ // Now check to see if we have more than config.resultLimit: if so only use the first amount to return if (resultLimit > 0 && consolidatedLineCount > resultLimit) { // First make a note of the total (to display later) @@ -624,7 +649,7 @@ export function numberOfUniqueFilenames(inArray: Array): number { /** * Run a search over all search terms in 'termsToMatchArr' over the set of notes determined by the parameters. - * V2 of this function + * V3 of this function, which assumes the order of terms in termsToMatchArr has been optimised. * Has an optional 'paraTypesToInclude' parameter of paragraph type(s) to include (e.g. ['open'] to include only open tasks). If not given, then no paragraph types will be excluded. * * @param {Array} termsToMatchArr @@ -655,8 +680,10 @@ export async function runSearchesV2( //------------------------------------------------------------------ // Get results for each search term independently and save + // let lastTermType = '' for (const typedSearchTerm of termsToMatchArr) { - logDebug('runSearchesV2', ` - searching for term [${typedSearchTerm.termRep}] ...`) + let thisTermType = typedSearchTerm.type + logDebug('runSearchesV2', ` - searching for term [${typedSearchTerm.termRep}] type '${thisTermType}':`) const innerStartTime = new Date() // do search for this search term, using configured options @@ -666,21 +693,23 @@ export async function runSearchesV2( termsResults.push(resultObject) resultCount += resultObject.resultCount logDebug('runSearchesV2', ` -> ${resultObject.resultCount} results for '${typedSearchTerm.termRep}' in ${timer(innerStartTime)}`) - } - logDebug('runSearchesV2', `- ${termsToMatchArr.length} searches completed in ${timer(outerStartTime)}s -> ${resultCount} results`) + // If we have no results from previous 'must' term, then return early + if (thisTermType === 'must' && resultCount === 0) { + logInfo('runSearchesV2', `- no results from 'must' term [${typedSearchTerm.termRep}], so not doing further searches.`) + break + } + // TODO: Can we extend the above to check with not as well? + // lastTermType = typedSearchTerm.termType + } - // // If we have no results, then return early - // clo(termsResults, 'resultsProm in top level') - // if (resultCount === 0) { - // return [] - // } + logDebug('runSearchesV2', `- ${termsToMatchArr.length} searches completed in ${timer(outerStartTime)} -> ${resultCount} results`) //------------------------------------------------------------------ // Work out what subset of results to return, taking into the must/may/not terms, and potentially dates too outerStartTime = new Date() const consolidatedResultSet: resultOutputTypeV3 = applySearchOperators(termsResults, config.resultLimit, fromDateStr, toDateStr) - logDebug('runSearchesV2', `- Applied search logic in ${timer(outerStartTime)}s`) + logDebug('runSearchesV2', `- Applied search logic in ${timer(outerStartTime)}`) // For open tasks, add line sync with blockIDs (if we're using 'NotePlan' display style) // clo(consolidatedResultSet, 'after applySearchOperators, consolidatedResultSet =') @@ -731,6 +760,7 @@ export async function runSearchV2( let searchTerm = fullSearchTerm let resultParas: Array = [] let multiWordSearch = false + let wildcardedSearch = false logDebug('runSearchV2', `Starting for [${searchTerm}]`) // V1: get list of matching paragraphs for this string by n.paragraphs.filter @@ -743,13 +773,27 @@ export async function runSearchV2( // we will now just search for the first word in the search term if (searchTerm.includes(" ")) { multiWordSearch = true - searchTerm = searchTerm.split(' ')[0] + const words = searchTerm.split(' ') + // use the longest word not just the first + const longestWord = words.length > 0 ? words.sort((a, b) => b.length - a.length)[0] : '' + searchTerm = longestWord logDebug('runSearchV2', `multi-word: will just use [${searchTerm}] for [${fullSearchTerm}], and then do fuller check on results`) } + // if search term includes * or ? then we need to do further wildcard filtering + // reduce search term to just the part before the wildcard + let beforeWildcardSearchTerm = '' + let wildcardOnwardsSearchTerm = '' + if (searchTerm.includes("*") || searchTerm.includes("?")) { + searchTerm = searchTerm.split(/[\*\?]/, 1)[0] + wildcardOnwardsSearchTerm = fullSearchTerm.slice(searchTerm.length) + wildcardedSearch = true + logDebug('runSearchV2', `wildcard: will now use [${searchTerm}] for [${fullSearchTerm}]`) + } + //------------------------------------------------------- // Finally, the actual Search API Call! - CommandBar.showLoading(true, `Running search for ${fullSearchTerm} ...`) + CommandBar.showLoading(true, `Running search for ${fullSearchTerm} ${fullSearchTerm !== searchTerm ? '(via ' + searchTerm + ') ' : ''}...`) const response = await DataStore.search(searchTerm, noteTypesToInclude, foldersToInclude, foldersToExclude, false) let tempResult: Array = response.slice() // to convert from $ReadOnlyArray to $Array @@ -764,6 +808,16 @@ export async function runSearchV2( logDebug('runSearchV2', `multi-word: after filtering: ${String(tempResult.length)}`) } + // if search term includes * or ? then we need to do further wildcard filtering, but using regex version: + // - replace ? with . + // - replace * with [^\s]*? (i.e. any anything within the same 'word') + if (wildcardedSearch) { + const regexSearchTerm = new RegExp('\\b' + fullSearchTerm.replace(/\?/g, '.').replace(/\*/g, '[^\\s]*?') + '\\b') + logDebug('runSearchV2', `wildcard: before regex filtering with ${String(regexSearchTerm)}: ${String(tempResult.length)}`) + tempResult = tempResult.filter(tr => regexSearchTerm.test(tr.content)) + logDebug('runSearchV2', `wildcard: after filtering: ${String(tempResult.length)}`) + } + if (paraTypesToInclude.length > 0) { CommandBar.showLoading(true, `Now filtering to para types '${String(paraTypesToInclude)}' ...`) // Check each result and add to the resultParas array only if it matches the given paraTypesToInclude @@ -898,23 +952,27 @@ export async function writeSearchResultsToNote( const headingMarker = '#'.repeat(config.headingLevel) const searchTermsRepStr = `'${resultSet.searchTermsRepArr.join(' ')}'`.trim() // Note: we normally enclose in [] but here need to use '' otherwise NP Editor renders the link wrongly logDebug('writeSearchResultsToNote', `Starting with ${resultSet.resultCount} results for [${searchTermsRepStr}] ...`) - const xCallbackLine = (xCallbackURL !== '') ? ` [🔄 Refresh results for ${searchTermsRepStr}](${xCallbackURL})` : '' + const xCallbackText = (xCallbackURL !== '') ? ` [🔄 Refresh results for ${searchTermsRepStr}](${xCallbackURL})` : '' + const timestampAndRefreshLine = `at ${nowLocaleShortDateTime()}${xCallbackText}` // Add each result line to output array - let titleLines = `# ${requestedTitle}\nat ${nowLocaleShortDateTime()}${xCallbackLine}` + // let titleLines = `# ${requestedTitle}\n${timestampAndRefreshLine}` + let titleLines = `# ${requestedTitle}` let headingLine = '' let resultsContent = '' // First check if we have any results if (resultSet.resultCount > 0) { - resultsContent = createFormattedResultLines(resultSet, config).join('\n') + resultsContent = '\n' + createFormattedResultLines(resultSet, config).join('\n') const resultCountsStr = resultCounts(resultSet) headingLine += `${searchTermsRepStr} ${resultCountsStr}` } else { // No results headingLine = `${searchTermsRepStr}` - resultsContent = `(no matches)` + resultsContent = "(no matches)" } + // Prepend the results part with the timestamp+refresh line + resultsContent = `${timestampAndRefreshLine}${resultsContent}` // logDebug('writeSearchResultsToNote', `resultsContent is ${resultsContent.length} bytes`) // Get existing note by start-of-string match on titleToMatch, if that is supplied, or requestedTitle if not. @@ -925,6 +983,13 @@ export async function writeSearchResultsToNote( // Just replace the heading section, to allow for some text to be left between runs logDebug('writeSearchResultsToNote', `- just replacing section '${searchTermsRepStr}' in ${outputNote.filename}`) replaceSection(outputNote, searchTermsRepStr, headingLine, config.headingLevel, resultsContent) + + // Because of a change in where the timestamp is displayed, we potentially need to remove it from line 1 of the note + const line1 = outputNote.paragraphs[1].content + if (line1.startsWith('at ') && line1.includes('Refresh results for ')) { + logDebug('writeSearchResultsToNote', `- removing timestamp from line 1 of ${outputNote.filename}. This should be one-time-only operation.`) + outputNote.removeParagraphAtIndex(1) + } } else { // Replace all note contents @@ -949,7 +1014,7 @@ export async function writeSearchResultsToNote( } /** - * Create nicely-formatted lines to display 'resultSet', using settings from 'config' + * Create nicely-formatted Markdown lines to display 'resultSet', using settings from 'config' * @author @jgclark * @param {resultOutputTypeV2} resultSet * @param {SearchConfig} config @@ -966,11 +1031,12 @@ export function createFormattedResultLines(resultSet: resultOutputTypeV3, config // Take off leading + or ! if necessary const mayOrMustTerms = mayOrMustTermsRep.map((f) => (f.match(/^[\+\!]/)) ? f.slice(1) : f) const notEmptyMayOrMustTerms = mayOrMustTerms.filter((f) => f !== '') - logDebug('createFormattedResultLines', `Starting with ${notEmptyMayOrMustTerms.length} notEmptyMayOrMustTerms (${String(notEmptyMayOrMustTerms)})`) + // logDebug('createFormattedResultLines', `Starting with ${notEmptyMayOrMustTerms.length} notEmptyMayOrMustTerms (${String(notEmptyMayOrMustTerms)}) / simplifyLine? ${String(simplifyLine)} / groupResultsByNote? ${String(config.groupResultsByNote)} / config.resultQuoteLength = ${String(config.resultQuoteLength)}`) // Add each result line to output array let lastFilename: string let nc = 0 for (const rnal of resultSet.resultNoteAndLineArr) { + // clo(rnal, `resultNoteAndLineArr[${nc}]`) if (config.groupResultsByNote) { // Write each line without transformation, grouped by Note, with Note headings inserted accordingly let thisFilename = rnal.noteFilename diff --git a/jgclark.SearchExtensions/src/searchTriggers.js b/jgclark.SearchExtensions/src/searchTriggers.js new file mode 100644 index 000000000..4a2123140 --- /dev/null +++ b/jgclark.SearchExtensions/src/searchTriggers.js @@ -0,0 +1,140 @@ +// @flow +//----------------------------------------------------------------------------- +// Create list of occurrences of note paragraphs with specified strings, which +// can include #hashtags or @mentions, or other arbitrary strings (but not regex). +// Jonathan Clark +// Last updated 8.12.2023 for v1.3.0, @jgclark +//----------------------------------------------------------------------------- + +import pluginJson from '../plugin.json' +import { + quickSearch, + searchOverAll, + searchOverCalendar, + searchOverNotes, + searchOpenTasks, +} from './saveSearch' +import { searchPeriod } from './saveSearchPeriod' +import { clo, logDebug, logInfo, logError, logWarn } from '@helpers/dev' + + +function getUrlParams(query: string): { [key: string]: string } { + const search = /([^&=]+)=?([^&]*)/g + let match + const decode = function (s) { + return decodeURIComponent(s.replace(/\+/g, // Regex for replacing addition symbol with a space + " ")) + } + const urlParams = {} + while (match = search.exec(query)) { + urlParams[decode(match[1])] = decode(match[2]) + console.log(`Found param: ${decode(match[1])} / ${decode(match[2])}`) + } + clo(urlParams) + return urlParams +} + + +/** + * Refresh the saved search results in the note, if the note has a suitable x-callback 'Refresh button' in it. + * Designed to be called by an onOpen trigger. + */ +export async function refreshSavedSearch(): Promise { + try { + if (!(Editor.content && Editor.note)) { + logWarn(pluginJson, `Cannot get Editor details. Please open a note.`) + return + } + const noteReadOnly: CoreNoteFields = Editor.note + + // Check to see if this has been called in the last 5000ms: if so don't proceed, as this could be a double call, which could lead to an infinite loop + const timeSinceLastEdit: number = Date.now() - noteReadOnly.versions[0].date + if (timeSinceLastEdit <= 5000) { + logDebug(pluginJson, `refreshSavedSearch fired, but ignored, as it was called only ${String(timeSinceLastEdit)}ms after the note was last updated`) + return + } + + logDebug(pluginJson, `refreshSavedSearch triggered for '${noteReadOnly.filename}'`) + // Does this note have a Refresh button from the Search Extensions plugin? + const refreshButtonLines = noteReadOnly.paragraphs.filter(p => + /Refresh /.test(p.content) + && /noteplan:\/\/x\-callback\-url\/runPlugin\?pluginID=jgclark\.SearchExtensions&/.test(p.content) + ) + // Only proceed if we have a refresh button + if (refreshButtonLines?.length === 0) { + logDebug(pluginJson, 'Note has no suitable Refresh button') + return + } + + const firstLine = refreshButtonLines[0].content + logDebug(pluginJson, `Note has a suitable Refresh button line: {${firstLine}}`) + + // V2: attempt to reconstruct the parameters to call the plugin's command directly. + const firstUrlInLine = firstLine.match(/noteplan:\/\/[^\s\)]*/)[0] + logDebug(pluginJson, `firstUrlInLine: {${firstUrlInLine}}`) + const params = getUrlParams(firstUrlInLine) + const cmdName = params.command + const arg0 = params.arg0 ?? '' + const arg1 = params.arg1 ?? '' + const arg2 = params.arg2 ?? '' + const arg3 = params.arg3 ?? '' + const arg4 = params.arg4 ?? '' + + await CommandBar.showLoading(true, 'Refreshing search results ...') + await CommandBar.onAsyncThread() + switch (cmdName) { + case "searchOverCalendar": { + // Put up a progress indicator first, though + searchOverCalendar(arg0, arg1) + break + } + case "search": { // -> searchOverAll() + // Put up a progress indicator first, though + searchOverAll(arg0, arg1) + break + } + case "searchOpenTasks": { + // Put up a progress indicator first, though + searchOpenTasks(arg0, arg1) + break + } + case "searchOverNotes": { + // Put up a progress indicator first, though + searchOverNotes(arg0, arg1) + break + } + case "quickSearch": { + // Put up a progress indicator first, though + quickSearch(arg0, arg1, arg2) + break + } + case "searchInPeriod": { // -> searchPeriod() + // Put up a progress indicator first, though + searchPeriod(arg0, arg1, arg2, arg3, arg4) + break + } + } + await CommandBar.onMainThread() + await CommandBar.showLoading(false) + + // V1: use the callback URL from the note directly + // Note: as it triggers a note open, need to stop it creating infinite loops + // const urlMatches = firstLine.match(/\((noteplan:\/\/x\-callback\-url\/runPlugin\?pluginID=jgclark\.SearchExtensions&.*?)\)/) + // noteplan:\/\/[^\s\)]* + // if (urlMatches) { + // const firstURL = urlMatches[1] // first capture group + // logDebug(pluginJson, `First matching URL: {${firstURL}}`) + + // // If we get this far, then we can call this callback to refresh the note + // // Put up a progress indicator first, though + // await CommandBar.showLoading(true, 'Refreshing search results ...') + // await CommandBar.onAsyncThread() + // NotePlan.openURL(firstURL) + // await CommandBar.onMainThread() + // await CommandBar.showLoading(false) + // } + } + catch (error) { + logError(pluginJson, `${error.name}: ${error.message}`) + } +}