From 3bb172fc8131268509a896fd21f1dfd1dd42c613 Mon Sep 17 00:00:00 2001 From: Landry Breuil Date: Fri, 26 Apr 2024 15:21:55 +0200 Subject: [PATCH 1/5] [tools] add support to register pipelines in a secured ES instance --- tools/pipelines/register-es-pipelines.js | 38 ++++++++++++++++++------ 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/tools/pipelines/register-es-pipelines.js b/tools/pipelines/register-es-pipelines.js index 31b0424de2..e22eac1a57 100644 --- a/tools/pipelines/register-es-pipelines.js +++ b/tools/pipelines/register-es-pipelines.js @@ -9,6 +9,8 @@ program .command('register') .description('Register pipelines') .option('--host ', 'ElasticSearch host', 'http://localhost:9200') + .option('--username ', 'ElasticSearch user', '') + .option('--password ', 'ElasticSearch password', '') .option( '--records-index ', 'Name of the index used by GeoNetwork for records', @@ -17,7 +19,10 @@ program .action((options) => { const esUrl = options.host.replace(/\/$/, '') // remove trailing slash if any const recordsIndex = options.recordsIndex - registerPipelines(esUrl, recordsIndex) + const username = options.username + const password = options.password + const authHeader = (options.username && options.password ? 'Basic ' + Buffer.from(username + ":" + password).toString('base64') : ''); + registerPipelines(esUrl, recordsIndex, authHeader) }) program .command('clear') @@ -27,9 +32,14 @@ program 'ElasticSearch host, default is http://localhost:9090', 'http://localhost:9200' ) + .option('--username ', 'ElasticSearch user', '') + .option('--password ', 'ElasticSearch password', '') .action((options) => { const esUrl = options.host.replace(/\/$/, '') // remove trailing slash if any - clearPipelines(esUrl) + const username = options.username + const password = options.password + const authHeader = (options.username && options.password ? 'Basic ' + Buffer.from(username + ":" + password).toString('base64') : ''); + clearPipelines(esUrl, authHeader) }) program.parse(process.argv) @@ -129,13 +139,14 @@ for(int i = ctx.format.length - 1; i >= 0; i--) { ], } -async function registerPipeline(esHost, name, payload) { +async function registerPipeline(esHost, name, payload, authHeader) { console.log(`adding ${name} pipeline...`) await fetch(`${esHost}/_ingest/pipeline/${name}`, { method: 'PUT', body: JSON.stringify(payload), headers: { + 'Authorization': authHeader, 'Content-Type': 'application/json', }, }) @@ -150,11 +161,14 @@ async function registerPipeline(esHost, name, payload) { console.log(`${name} pipeline was successfully registered!`) } -async function clearPipeline(esHost, name) { +async function clearPipeline(esHost, name, authHeader) { console.log(`clearing ${name} pipeline...`) await fetch(`${esHost}/_ingest/pipeline/${name}`, { method: 'DELETE', + headers: { + 'Authorization': authHeader, + }, }) .then((resp) => resp.json()) .then((result) => { @@ -167,13 +181,14 @@ async function clearPipeline(esHost, name) { console.log(`${name} pipeline was successfully cleaned!`) } -async function setDefaultPipeline(esHost, recordsIndex, name) { +async function setDefaultPipeline(esHost, recordsIndex, name, authHeader) { console.log(`setting ${name} as default pipeline...`) await fetch(`${esHost}/${recordsIndex}/_settings`, { method: 'PUT', body: JSON.stringify({ 'index.default_pipeline': name }), headers: { + 'Authorization': authHeader, 'Content-Type': 'application/json', }, }) @@ -188,10 +203,15 @@ async function setDefaultPipeline(esHost, recordsIndex, name) { console.log(`${name} pipeline was successfully set as default!`) } -async function registerPipelines(esHost, recordsIndex) { +async function registerPipelines(esHost, recordsIndex, authHeader) { console.log('querying currently registered pipelines...') - const pipelines = await fetch(`${esHost}/_ingest/pipeline`).then((resp) => + const pipelines = await fetch(`${esHost}/_ingest/pipeline`, { + headers: { + 'Authorization': authHeader, + }, + }) + .then((resp) => resp.json() ) @@ -203,10 +223,10 @@ async function registerPipelines(esHost, recordsIndex) { }) console.log('') - await registerPipeline(esHost, 'geonetwork-ui', GEONETWORK_UI_PIPELINE) + await registerPipeline(esHost, 'geonetwork-ui', GEONETWORK_UI_PIPELINE, authHeader) console.log('') - await setDefaultPipeline(esHost, recordsIndex, 'geonetwork-ui') + await setDefaultPipeline(esHost, recordsIndex, 'geonetwork-ui', authHeader) } async function clearPipelines(esHost) { From ae33e26cfeaa3d5912b8813a03d7422fbc9bdd54 Mon Sep 17 00:00:00 2001 From: Landry Breuil Date: Mon, 29 Apr 2024 08:56:38 +0200 Subject: [PATCH 2/5] Update tools/pipelines/register-es-pipelines.js Co-authored-by: Olivia Guyot --- tools/pipelines/register-es-pipelines.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pipelines/register-es-pipelines.js b/tools/pipelines/register-es-pipelines.js index e22eac1a57..273a9833c1 100644 --- a/tools/pipelines/register-es-pipelines.js +++ b/tools/pipelines/register-es-pipelines.js @@ -21,7 +21,7 @@ program const recordsIndex = options.recordsIndex const username = options.username const password = options.password - const authHeader = (options.username && options.password ? 'Basic ' + Buffer.from(username + ":" + password).toString('base64') : ''); + const authHeader = (options.username && options.password ? 'Basic ' + btoa(username + ":" + password) : ''); registerPipelines(esUrl, recordsIndex, authHeader) }) program From c79d08bf5e718bf09af8776bfd569222df849baa Mon Sep 17 00:00:00 2001 From: Landry Breuil Date: Mon, 29 Apr 2024 08:56:49 +0200 Subject: [PATCH 3/5] Update tools/pipelines/register-es-pipelines.js Co-authored-by: Olivia Guyot --- tools/pipelines/register-es-pipelines.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/pipelines/register-es-pipelines.js b/tools/pipelines/register-es-pipelines.js index 273a9833c1..7cabc7dc7c 100644 --- a/tools/pipelines/register-es-pipelines.js +++ b/tools/pipelines/register-es-pipelines.js @@ -38,7 +38,7 @@ program const esUrl = options.host.replace(/\/$/, '') // remove trailing slash if any const username = options.username const password = options.password - const authHeader = (options.username && options.password ? 'Basic ' + Buffer.from(username + ":" + password).toString('base64') : ''); + const authHeader = (options.username && options.password ? 'Basic ' + btoa(username + ":" + password) : ''); clearPipelines(esUrl, authHeader) }) From b904a1f5c6f0c37683b94d73117c58ce86355855 Mon Sep 17 00:00:00 2001 From: Landry Breuil Date: Mon, 29 Apr 2024 09:02:24 +0200 Subject: [PATCH 4/5] document --username & --password options to register-es-pipelines --- docs/guide/deploy.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/guide/deploy.md b/docs/guide/deploy.md index f504838b24..3b00de61cb 100644 --- a/docs/guide/deploy.md +++ b/docs/guide/deploy.md @@ -178,7 +178,9 @@ Then run the following script with the appropriate options: node tools/pipelines/register-es-pipelines.js register --host=http://localhost:9090 ``` -The `--host` option is used to point to the ElasticSearch instance. Additionally, the `--records-index` option can be used if the index containing the metadata records is not called `gn-records`. +The `--host` option is used to point to the ElasticSearch instance. +If ElasticSearch is secured, `--username` and `--password` can be used to pass HTTP Authentication. +Additionally, the `--records-index` option can be used if the index containing the metadata records is not called `gn-records`. ### Option B: Running a docker image From 765c5b17033b70a9e9a99ded7dc9f5cb8f6014f2 Mon Sep 17 00:00:00 2001 From: Olivia Guyot Date: Mon, 29 Apr 2024 13:10:39 +0200 Subject: [PATCH 5/5] chore: npm run format:write --- tools/pipelines/register-es-pipelines.js | 30 +++++++++++++++--------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/tools/pipelines/register-es-pipelines.js b/tools/pipelines/register-es-pipelines.js index 7cabc7dc7c..5acb18ccfd 100644 --- a/tools/pipelines/register-es-pipelines.js +++ b/tools/pipelines/register-es-pipelines.js @@ -21,7 +21,10 @@ program const recordsIndex = options.recordsIndex const username = options.username const password = options.password - const authHeader = (options.username && options.password ? 'Basic ' + btoa(username + ":" + password) : ''); + const authHeader = + options.username && options.password + ? 'Basic ' + btoa(username + ':' + password) + : '' registerPipelines(esUrl, recordsIndex, authHeader) }) program @@ -38,7 +41,10 @@ program const esUrl = options.host.replace(/\/$/, '') // remove trailing slash if any const username = options.username const password = options.password - const authHeader = (options.username && options.password ? 'Basic ' + btoa(username + ":" + password) : ''); + const authHeader = + options.username && options.password + ? 'Basic ' + btoa(username + ':' + password) + : '' clearPipelines(esUrl, authHeader) }) @@ -146,7 +152,7 @@ async function registerPipeline(esHost, name, payload, authHeader) { method: 'PUT', body: JSON.stringify(payload), headers: { - 'Authorization': authHeader, + Authorization: authHeader, 'Content-Type': 'application/json', }, }) @@ -167,7 +173,7 @@ async function clearPipeline(esHost, name, authHeader) { await fetch(`${esHost}/_ingest/pipeline/${name}`, { method: 'DELETE', headers: { - 'Authorization': authHeader, + Authorization: authHeader, }, }) .then((resp) => resp.json()) @@ -188,7 +194,7 @@ async function setDefaultPipeline(esHost, recordsIndex, name, authHeader) { method: 'PUT', body: JSON.stringify({ 'index.default_pipeline': name }), headers: { - 'Authorization': authHeader, + Authorization: authHeader, 'Content-Type': 'application/json', }, }) @@ -208,12 +214,9 @@ async function registerPipelines(esHost, recordsIndex, authHeader) { const pipelines = await fetch(`${esHost}/_ingest/pipeline`, { headers: { - 'Authorization': authHeader, + Authorization: authHeader, }, - }) - .then((resp) => - resp.json() - ) + }).then((resp) => resp.json()) const names = Object.keys(pipelines) names.forEach((name) => { @@ -223,7 +226,12 @@ async function registerPipelines(esHost, recordsIndex, authHeader) { }) console.log('') - await registerPipeline(esHost, 'geonetwork-ui', GEONETWORK_UI_PIPELINE, authHeader) + await registerPipeline( + esHost, + 'geonetwork-ui', + GEONETWORK_UI_PIPELINE, + authHeader + ) console.log('') await setDefaultPipeline(esHost, recordsIndex, 'geonetwork-ui', authHeader)