From ccf627602d4ed1c1d5ca488f69e310c3395acf83 Mon Sep 17 00:00:00 2001 From: Leo Singer Date: Sun, 26 Nov 2023 21:52:13 -0500 Subject: [PATCH] Support either ElasticSearch or OpenSearch in sandbox OpenSearch has been built for fewer platforms than Elasticsearch. Support either OpenSearch or Elasticsearch depending on the configuration in the Architect project manifest. --- README.md | 4 +++- engines.ts | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ index.ts | 10 ++++++++- install.ts | 59 ++++++++++++++++++++--------------------------------- run.ts | 15 +++++++++++--- 5 files changed, 106 insertions(+), 42 deletions(-) create mode 100644 engines.ts diff --git a/README.md b/README.md index e0ec513..ffb1086 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This is a [plugin](https://arc.codes/docs/en/guides/plugins/overview) for [Architect](https://arc.codes/) that provisions managed [Amazon OpenSearch](https://aws.amazon.com/opensearch-service/) for the application. -When you are using Architect's [sandbox](https://arc.codes/docs/en/reference/cli/sandbox) mode, the plugin [downloads and runs OpenSearch locally](https://opensearch.org/downloads.html#opensearch). +When you are using Architect's [sandbox](https://arc.codes/docs/en/reference/cli/sandbox) mode, the plugin [downloads and runs Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html#elasticsearch-install-packages) or [OpenSearch](https://opensearch.org/downloads.html#opensearch) locally. Pair this pacakge with [@nasa-gcn/architect-functions-search](https://github.com/nasa-gcn/architect-functions-search) to connect to the service from your Node.js Lambda function code. @@ -30,6 +30,8 @@ Pair this pacakge with [@nasa-gcn/architect-functions-search](https://github.com # master nodes are disabled. dedicatedMasterCount 3 dedicatedMasterType t3.small.search + # Use OpenSearch in sandbox mode; default is Elasticsearch. + sandboxEngine opensearch 4. Optionally, create a file called `sandbox-search.json` or `sandbox-search.js` in your project and populate it with sample data to be passed to [`client.bulk()`](https://github.com/opensearch-project/opensearch-js/blob/main/guides/bulk.md). Here are some examples. diff --git a/engines.ts b/engines.ts new file mode 100644 index 0000000..0ebd363 --- /dev/null +++ b/engines.ts @@ -0,0 +1,60 @@ +/*! + * Copyright © 2023 United States Government as represented by the + * Administrator of the National Aeronautics and Space Administration. + * All Rights Reserved. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +export type SandboxEngine = 'elasticsearch' | 'opensearch' + +export const manifest = [ + { + engine: 'elasticsearch', + type: 'Linux', + arch: 'x64', + url: 'https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-8.6.2-linux-x86_64.tar.gz', + }, + { + engine: 'elasticsearch', + type: 'Linux', + arch: 'arm64', + url: 'https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-8.6.2-linux-aarch64.tar.gz', + }, + { + engine: 'elasticsearch', + type: 'Darwin', + arch: 'x64', + url: 'https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-8.6.2-darwin-x86_64.tar.gz', + }, + { + engine: 'elasticsearch', + type: 'Darwin', + arch: 'arm64', + url: 'https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-8.6.2-darwin-aarch64.tar.gz', + }, + { + engine: 'elasticsearch', + type: 'Windows_NT', + arch: 'x64', + url: 'https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-8.6.2-windows-x86_64.zip', + }, + { + engine: 'opensearch', + type: 'Linux', + arch: 'x64', + url: 'https://artifacts.opensearch.org/releases/bundle/opensearch/2.11.0/opensearch-2.11.0-linux-x64.tar.gz', + }, + { + engine: 'opensearch', + type: 'Linux', + arch: 'arm64', + url: 'https://artifacts.opensearch.org/releases/bundle/opensearch/2.11.0/opensearch-2.11.0-linux-arm64.tar.gz', + }, + { + engine: 'opensearch', + type: 'Windows_NT', + arch: 'x64', + url: 'https://artifacts.opensearch.org/releases/bundle/opensearch/2.11.0/opensearch-2.11.0-windows-x64.zip', + }, +] diff --git a/index.ts b/index.ts index 85b6e0e..1498de7 100644 --- a/index.ts +++ b/index.ts @@ -68,8 +68,15 @@ export const deploy = { let local: LocalSearch +function getEngine(name?: string) { + if (name?.toLowerCase() === 'opensearch') return 'opensearch' + else return 'elasticsearch' +} + export const sandbox = { async start({ + // @ts-expect-error: The Architect plugins API has no type definitions. + arc, inventory: { inv: { // @ts-expect-error: The Architect plugins API has no type definitions. @@ -77,7 +84,8 @@ export const sandbox = { }, }, }) { - local = await launch({}) + const engine = getEngine(getConfig(arc).sandboxEngine) + local = await launch({ engine }) await populate(cwd, { node: local.url }) }, async end() { diff --git a/install.ts b/install.ts index 988f9f7..4c5f690 100644 --- a/install.ts +++ b/install.ts @@ -8,40 +8,12 @@ import os from 'os' import { pipeline } from 'stream/promises' -import { join } from 'path' +import { join, posix } from 'path' import fetch from 'make-fetch-happen' import { Extract as unzip } from 'unzip-stream' import { x as untar } from 'tar' import { cache, exists, mkdirP } from './paths.js' - -const version = '2.11.0' - -const types = new Map([ - ['Linux', ['linux', 'tar.gz']], - ['Windows_NT', ['windows', 'zip']], -]) - -const archs = ['x64', 'arm64'] - -function getFilename() { - const os_type = os.type() - const os_arch = os.arch() - - const typeInfo = types.get(os_type) - - if ( - !typeInfo || - !archs.includes(os_arch) || - (os_type === 'Windows_NT' && os_arch === 'arm64') - ) { - throw new Error( - `No OpenSearch binary is available for your OS type (${os_type}) and architecture (${os_arch}). For supported operating systems, see https://opensearch.org/versions/opensearch-2-11-0.html.` - ) - } - - const [type, ext] = typeInfo - return { name: `opensearch-${version}-${type}-${os_arch}`, ext } -} +import { SandboxEngine, manifest } from './engines.js' async function download(url: string) { console.log('Downloading', url, 'to', cache) @@ -50,21 +22,34 @@ async function download(url: string) { return body } -export async function install() { - const { name, ext } = getFilename() - const extractPath = join(cache, name) - const binExt = os.type() === 'Windows_NT' ? '.bat' : '' +export async function install(engine: SandboxEngine) { + const type = os.type() + const arch = os.arch() + const url = manifest.find( + (entry) => + entry.engine === engine && entry.arch === arch && entry.type === type + )?.url + if (!url) { + throw new Error( + `No ${engine} binary is available for your OS type (${type}) and architecture (${arch}).` + ) + } + + const archiveFilename = posix + .basename(new URL(url).pathname) + .replace(/(.tar.gz|.zip)$/, '') + const extractPath = join(cache, archiveFilename) + const binExt = type === 'Windows_NT' ? '.bat' : '' const binPath = join( extractPath, - `opensearch-${version}`, + archiveFilename.split('-').slice(0, 2).join('-'), 'bin', - `opensearch${binExt}` + `${engine}${binExt}` ) const binPathExists = await exists(binPath) if (!binPathExists) { - const url = `https://artifacts.opensearch.org/releases/bundle/opensearch/${version}/${name}.${ext}` const stream = await download(url) let extract diff --git a/run.ts b/run.ts index 075e233..df3ac81 100644 --- a/run.ts +++ b/run.ts @@ -20,6 +20,7 @@ import { } from './processes.js' import { pipeline } from 'stream/promises' import { createReadStream } from 'fs' +import type { SandboxEngine } from './engines.js' export class LocalSearch { private readonly child!: ChildProcess @@ -36,11 +37,17 @@ export class LocalSearch { return Object.assign(this, props) } - static async launch({ port }: { port?: number }) { + static async launch({ + port, + engine, + }: { + port?: number + engine: SandboxEngine + }) { port ??= 9200 const url = `http://localhost:${port}` - const bin = await install() + const bin = await install(engine) let child await mkdirP(temp) @@ -55,7 +62,9 @@ export class LocalSearch { `-Epath.logs=${logsDir}`, `-Ehttp.port=${port}`, '-Ediscovery.type=single-node', - '-Eplugins.security.disabled=true', + engine === 'elasticsearch' + ? '-Expack.security.enabled=false' + : '-Eplugins.security.disabled=true', ] console.log('Spawning', bin, ...args)