From a0ba4eaca4e2abdceeedb78f999ba59fe19b1e39 Mon Sep 17 00:00:00 2001 From: brenault Date: Tue, 21 Feb 2023 18:06:08 +0100 Subject: [PATCH] devs on import confiance --- frontend/src/views/app-datasets-manager.js | 115 +------ frontend/src/views/app-project-manager.js | 105 +++++- server/router.js | 7 +- server/routes/datasets.js | 179 +--------- server/routes/minio_plugin.js | 2 +- server/routes/tasks.js | 377 ++++++++++++++++++--- 6 files changed, 432 insertions(+), 353 deletions(-) diff --git a/frontend/src/views/app-datasets-manager.js b/frontend/src/views/app-datasets-manager.js index 26d7792..cfdc784 100644 --- a/frontend/src/views/app-datasets-manager.js +++ b/frontend/src/views/app-datasets-manager.js @@ -31,10 +31,6 @@ import { importDataset, createDatasetFrom, importFromKafka, - projsFromDP, - selectionsFromDP, - id_listFromDP, - minio_urisFromDP, deleteDataset } from '../actions/media'; @@ -51,9 +47,7 @@ class AppDatasetsManager extends connect(store)(TemplatePage) { datasetIdx: { type: Number }, pathOrURL: { type: String }, nbSelectedRows: { type: Number }, - items: { type: Array }, - dialog_projsel_template: { type: Array }, - dialog_selsel_template: { type: Array } + items: { type: Array } }; } @@ -64,7 +58,6 @@ class AppDatasetsManager extends connect(store)(TemplatePage) { this.pathOrURL = "undetermined"; this.default_path = ""; this.nbSelectedRows = 0; - this.dialog_projsel_template = null; // ELISE this.similarityLevel = 0;//similarity in % @@ -289,83 +282,6 @@ class AppDatasetsManager extends connect(store)(TemplatePage) { .catch(error => this.errorPopup(error.message)); } - /** - * Fired when clic on import from DP - */ - async onImportFromDP() { - store.dispatch(projsFromDP()).then((projs) => { - console.log("projsFromDP", projs); - //project selector - this.project_list = projs; - if (Object.entries(projs).length > 0) { - this.fillProjectList(projs); - } else { this.errorPopup("No available project !") } - }).catch(error => this.errorPopup(error.message)); - } - - fillProjectList(projs) { - this.dialog_projsel_template = [] - for (const [key, _] of Object.entries(projs)) { - this.dialog_projsel_template.push(html`${key}`); - }; - const projSelectorElem = this.shadowRoot.getElementById('dialog-project-selector'); - console.log("SelectorElement", projSelectorElem) - projSelectorElem.open = true; - } - - fillSelectionList(sels) { - this.dialog_selsel_template = [] - for (const [_, value] of Object.entries(sels)) { - this.dialog_selsel_template.push(html` - ${value.name} - id: ${value.id}, nb_samples: ${value.nbSamples}`); - - }; - const selSelectorElem = this.shadowRoot.getElementById('dialog-selection-selector'); - selSelectorElem.open = true; - } - - onProjectSelected(selected) { - const proj = this.project_list[Object.keys(this.project_list)[selected.detail.index]] - this.project_name = proj.name; - console.log(" selected project: ", this.project_name); - - store.dispatch(selectionsFromDP(this.project_name)).then((sels) => { - console.log("selectionsFromDP", sels); - //selection selector - this.selection_list = sels; - - if (this.selection_list.length > 0) { - this.fillSelectionList(this.selection_list); - } else { this.errorPopup("No available selection !")} - }).catch(error => this.errorPopup(error.message)); - } - - onSelectionSelected(selected) { - const sel = this.selection_list[Object.keys(this.selection_list)[selected.detail.index]] - console.log(" selected selection full:", sel); - console.log(" selected selection:", sel.id); - store.dispatch(id_listFromDP(this.project_name, sel)).then((newtask) => { - console.log("id_listFromDP", newtask); - this.datasetIdx = this.datasets.length;//select the newly created dataset - this.onActivate(); - // update local copy of Redux - this.tasks = getState('application').tasks; - this.taskIdx = getState('application').tasks.findIndex((t) => t.name === getState('application').taskName); - }).catch(error => { - this.errorPopup(error.message) - }); - - /*** - //tmp test... maintenant on crée une task, pas juste un dataset, alors faut gerer différemment (voire tout deplacer dans tasks.js ?) - store.dispatch(importTaskFromKafka(task)).then((newtask) => { - // update local copy of Redux - this.tasks = getState('application').tasks; - this.taskIdx = getState('application').tasks.findIndex((t) => t.name === getState('application').taskName); - }).catch((error) => this.errorPopup(error.message)); - **/ - } - /** * Fired when user wants to delete a dataset. */ @@ -695,14 +611,6 @@ class AppDatasetsManager extends connect(store)(TemplatePage) { @click="${this.onImportFromKafka}"> Import from Kafka - - Import from DebiAI - `; } @@ -850,27 +758,6 @@ class AppDatasetsManager extends connect(store)(TemplatePage) { `; } - - get dialogProjectSelector() { - return html` - - - ${this.dialog_projsel_template} - - - `; - } - - get dialogSelectionSelector() { - return html` - - - ${this.dialog_selsel_template} - - - `; - } - } customElements.define('app-datasets-manager', AppDatasetsManager); diff --git a/frontend/src/views/app-project-manager.js b/frontend/src/views/app-project-manager.js index 5b6249e..0345518 100644 --- a/frontend/src/views/app-project-manager.js +++ b/frontend/src/views/app-project-manager.js @@ -18,6 +18,7 @@ import '@material/mwc-tab'; import '@material/mwc-tab-bar'; import '@material/mwc-textfield'; import '@material/mwc-select'; +import '@material/mwc-list/mwc-list'; import '@material/mwc-list/mwc-list-item'; import '@material/mwc-switch'; import '@trystan2k/fleshy-jsoneditor/fleshy-jsoneditor.js'; @@ -34,7 +35,12 @@ import { putTask, getTasks } from '../actions/application'; -import { getDatasets } from '../actions/media'; +import { + getDatasets, + projsFromDP, + selectionsFromDP, + id_listFromDP +} from '../actions/media'; class AppProjectManager extends connect(store)(TemplatePage) { @@ -45,6 +51,8 @@ class AppProjectManager extends connect(store)(TemplatePage) { creatingTask: { type: Boolean }, importExportText: { type: String }, pathOrURL: { type: String }, + dialog_projsel_template: { type: Array }, + dialog_selsel_template: { type: Array } }; } @@ -65,6 +73,7 @@ class AppProjectManager extends connect(store)(TemplatePage) { this.pathOrURL = "undetermined"; this.default_path = ""; this.datasetIds = []; + this.dialog_projsel_template = null; } onActivate() { @@ -112,6 +121,74 @@ class AppProjectManager extends connect(store)(TemplatePage) { browserElem.open = true; } + /** + * Fired when clic on import from DP + */ + onImportFromDP() { + store.dispatch(projsFromDP()).then((projs) => { + console.log("projsFromDP", projs); + //project selector + this.project_list = projs; + if (Object.entries(projs).length > 0) { + this.fillProjectList(projs); + } else { this.errorPopup("No available project !") } + }).catch(error => this.errorPopup(error.message)); + } + + fillProjectList(projs) { + this.dialog_projsel_template = [] + for (const [key, val] of Object.entries(projs)) { + this.dialog_projsel_template.push(html` + ${key} + nb_samples: ${val.nbSamples}, nb_sel: ${val.nbSelections}`); + }; + const projSelectorElem = this.shadowRoot.getElementById('dialog-project-selector'); + console.log("SelectorElement", projSelectorElem) + projSelectorElem.open = true; + } + + fillSelectionList(sels) { + this.dialog_selsel_template = [] + for (const [_, value] of Object.entries(sels)) { + this.dialog_selsel_template.push(html` + ${value.name} + id: ${value.id}, nb_samples: ${value.nbSamples}`); + + }; + const selSelectorElem = this.shadowRoot.getElementById('dialog-selection-selector'); + selSelectorElem.open = true; + } + + onProjectSelected(selected) { + const proj = this.project_list[Object.keys(this.project_list)[selected.detail.index]] + this.project_name = proj.name; + console.log(" selected project: ", this.project_name); + + store.dispatch(selectionsFromDP(this.project_name)).then((sels) => { + console.log("selectionsFromDP", sels); + //selection selector + this.selection_list = sels; + + if (this.selection_list.length > 0) { + this.fillSelectionList(this.selection_list); + } else { this.errorPopup("No available selection !")} + }).catch(error => this.errorPopup(error.message)); + } + + onSelectionSelected(selected) { + const sel = this.selection_list[Object.keys(this.selection_list)[selected.detail.index]] + console.log(" selected selection full:", sel); + console.log(" selected selection:", sel.id); + store.dispatch(id_listFromDP(this.project_name, sel)).then(() => { + store.dispatch(getTasks()).then(() => { + this.onActivate(); + }) + }).catch(error => { + this.errorPopup(error.message); + store.dispatch(getTasks()); + }); + } + /** * Fired when exporting a project */ @@ -338,6 +415,11 @@ class AppProjectManager extends connect(store)(TemplatePage) { type="button" title="Import annotations from json files" @click="${this.onImport}">Import from files + Import from Confiance DB `; @@ -482,6 +564,27 @@ class AppProjectManager extends connect(store)(TemplatePage) { + ${this.dialogProjectSelector} + ${this.dialogSelectionSelector} + `; + } + get dialogProjectSelector() { + return html` + + + ${this.dialog_projsel_template} + + + `; + } + + get dialogSelectionSelector() { + return html` + + + ${this.dialog_selsel_template} + + `; } } diff --git a/server/router.js b/server/router.js index 484f773..e4aca76 100644 --- a/server/router.js +++ b/server/router.js @@ -18,10 +18,6 @@ const { get_datasets, post_datasets, post_dataset_from, import_dataset_from_kafka, - projects_from_dataprovider, - selections_from_dataprovider, - id_list_from_dataprovider, - minio_uris_from_dataprovider, get_dataset, delete_dataset, get_data, @@ -41,6 +37,9 @@ const { get_tasks, put_task, delete_task, import_tasks, + projects_from_dataprovider, + selections_from_dataprovider, + id_list_from_dataprovider, export_tasks } = require('./routes/tasks'); const { snapshot_project } = require('./routes/project'); const { get_results, diff --git a/server/routes/datasets.js b/server/routes/datasets.js index b6cae15..de1dd8b 100644 --- a/server/routes/datasets.js +++ b/server/routes/datasets.js @@ -7,9 +7,6 @@ const { checkAdmin } = require('./users'); const populator = require('../helpers/data-populator'); const { elise_remove_image } = require('../routes/elise_plugin.js');// ELISE const { getSelectionFromKafka } = require('./kafka_plugin'); -const { downloadFilesFromMinio } = require('./minio_plugin').default; -const { createTasksFromDPImport } = require('./tasks'); -const fetch = require("node-fetch"); /** * @api {get} /datasets Get list of datasets @@ -110,177 +107,6 @@ async function post_dataset_from(req, res) { }); } -/** - * @api {post} /datasets/projects_from_dataprovider get projects list from confiance dp - * @apiName GetProjsFromDP - * @apiGroup Dataset - * @apiPermission admin - * - * @apiParam {RestDataset} body - * - * @apiSuccessExample Success-Response: - * HTTP/1.1 201 OK - * - * @apiErrorExample Error-Response: - * HTTP/1.1 404 Error in DP import - * HTTP/1.1 401 Unauthorized - */ -async function projects_from_dataprovider(req, res) { - return await checkAdmin(req, async () => { - console.log('##### Importing project list from dataprovider'); - return await get_dp("/debiai/projects").then(res => res); - }) - .then(projs=>{return res.status(200).send(projs);}) - .catch(err => {return res.status(400).send(err);}) -} - -/** - * @api {post} /datasets/selections_from_dataprovider get projects list from confiance dp - * @apiName GetSelectionsFromDP - * @apiGroup Dataset - * @apiPermission admin - * - * @apiParam {RestDataset} body - * - * @apiSuccessExample Success-Response: - * HTTP/1.1 201 OK - * - * @apiErrorExample Error-Response: - * HTTP/1.1 404 Error in DP import - * HTTP/1.1 401 Unauthorized - */ -async function selections_from_dataprovider(req, res) { - return await checkAdmin(req, async () => { - console.log('##### Importing selections from dataprovider'); - //console.log('##### BR req.params:', req.params); - // params_id ? - return await get_dp("/debiai/projects/"+req.params.project_name+"/selections").then(res => res); - }) - .then((selections)=>{return res.status(200).send(selections);}) - .catch(err => {return res.status(400).send(err);}) -} - -/** - * @api {post} /datasets/id_list_from_dataprovider get id list from confiance dp - * @apiName GetIdListFromDP - * @apiGroup Dataset - * @apiPermission admin - * - * @apiParam {RestDataset} body - * - * @apiSuccessExample Success-Response: - * HTTP/1.1 201 OK - * - * @apiErrorExample Error-Response: - * HTTP/1.1 404 Error in DP import - * HTTP/1.1 401 Unauthorized - */ -async function id_list_from_dataprovider(req, res) { - return await checkAdmin(req, async () => { - console.log('##### Importing id list from dataprovider'); - //console.log('##### BR req.params:', req.params); - return await get_dp("/debiai/projects/"+req.params.project_name+"/selections/"+req.params.sel_id+"/selected-data-id-list").then(res => res); - }) - .then(async (selections)=>{ - task = await process_selection(res, req.params.project_name, req.params.sel_name, selections) - .then(res => {return res}) - .catch(err => {throw err}) - //console.log("Created Task", task); - return res.status(200).send(task); - }) - .catch(err => { - console.log("ERREUR in id_list_from_dataprovider:", err); - return res.status(400).json({ message: err }); - }) -} - -//// remaniement, ce n'est plus une API --nom temporaire, j'ai pas mieux en stock -// get infos on "selections" from data-provider (dp_info) -// extract minio paths from dp_info -// get images from minio -// create Dataset -// extract annotations from dp_info -// create Task -async function process_selection(res, project_name, sel_name, selections) { - console.log('##### Importing Minio uris from dataprovider'); - dp_res = await get_dp_minio_uris(project_name, selections).then(res => {return res}); - - bucket_names = [] - bucket_paths = [] - filenames = [] - if (!dp_res || dp_res.length == 0) { throw "ERREUR dp_res empty"; } - Object.keys(dp_res).forEach(key => { - console.log("dp_res[", key, "]", dp_res[key].storage); - //console.log("dp_res[", key, "].annotations", dp_res[key].annotations); - if(!dp_res[key].storage) { throw "No storage info in selection, import aborted";} - if(!dp_res[key].storage.minio) { throw "No minio storage info in selection, import aborted";} - if(!dp_res[key].storage.minio.bucket_name) { throw "No minio bucket_name info in selection, import aborted";} - if(!dp_res[key].storage.minio.basepath) { throw "No minio basepath info in selection, import aborted";} - bname = dp_res[key].storage.minio.bucket_name; - bpath = dp_res[key].storage.minio.basepath; - if (bpath.startsWith(bname)) { - //slice parce que le bucket est mis en prefixe du path !! - bpath = bpath.slice(bname.length); - } - bucket_names.push(bname); - bucket_paths.push(bpath); - filenames.push(dp_res[key].storage.filename); - }); - console.log(" - buckets:", bucket_names); - console.log(" - paths:", bucket_paths); - console.log(" - filenames:", filenames); - - console.log('# 1) Create a new dataset'); - console.log('# 1.1) get/set members'); - sel_name = sel_name.replace(/\s/g, '_'); - let dataset = {}; - dataset.date = Date.now(); - dataset.path = 'importedFromDebiai/' + sel_name; - dataset.id = project_name.replace(/\s/g, '_') + "_" + sel_name; - //dataset.data_type = kafkaSelection.data_type ? kafkaSelection.data_type : dataset.data_type = 'image';// ... TODO : set to 'remote_image' when minio will work without local copy - dataset.data_type = 'image' // TODO: gérer autres cas... - console.log("dataset=",dataset); - console.log('# 1.2) getPathFromIds'); - dataset.urlList = await downloadFilesFromMinio(filenames, workspace, sel_name, bucket_names[0], bucket_paths[0]).catch((e) => { - console.error('Error in Minio import\n'+e); - throw 'Error in Minio import\n'+e; - }) - console.log('# 1.3) getImagesFromPath'); - const newDataset = await getOrcreateDataset(dataset, workspace); - if (newDataset) { - return await createTasksFromDPImport(dp_res, newDataset) - .then(task => {return task}) - .catch(err => { - console.log("Error while creating task", err); - throw 'Error while creating task:' + err; - }); - } else throw 'Error while creating dataset'; -} - -async function get_dp(url) { - const dp_host = await db.get(dbkeys.keyForCliOptions).then((options) => { return options.dataProvider }); - return await fetch(dp_host + url, { method: 'get', headers: { 'Content-Type': 'application/json' } }) - .then(res => { - if (res.statusText == 'OK') { return res.json().then(data => Promise.resolve(data)); } - else { return res.status(200).json({ message: 'Response error: '+res });} - }) - .catch(err => { return Promise.reject(err);}); -} - -async function get_dp_minio_uris(project_name, ids) { - const dp_host = await db.get(dbkeys.keyForCliOptions).then((options) => { return options.dataProvider }); - return await fetch(dp_host+"/project/"+project_name+"/data", { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(ids) - }) - .then(res => { - if (res.statusText == 'OK') {return res.json().then(data => Promise.resolve(data))} - else return res.status(200).json({ message: 'Response error: '+res }); - }) - .catch(err => { return Promise.reject(err);}); -} - /** * @api {post} /datasets/import_from_kafka Import a dataset from kafka+minio * @apiName GetDatasetsFromKafka @@ -654,7 +480,7 @@ const getDatasetFromId = (db, id) => { * @param {String} dataset_id * @param {String} type */ -const getAllDataFromDataset = (dataset_id) => { +function getAllDataFromDataset(dataset_id) { const dataList = []; const stream = utils.iterateOnDB(db, dbkeys.keyForData(dataset_id), true, false) return new Promise((resolve) => { @@ -692,9 +518,6 @@ module.exports = { post_datasets, post_dataset_from, import_dataset_from_kafka, - projects_from_dataprovider, - selections_from_dataprovider, - id_list_from_dataprovider, get_dataset, delete_dataset, get_data, diff --git a/server/routes/minio_plugin.js b/server/routes/minio_plugin.js index e6c9485..033a32c 100644 --- a/server/routes/minio_plugin.js +++ b/server/routes/minio_plugin.js @@ -38,7 +38,7 @@ const downloadFilesFromMinio = async (listIds,workspace,selection_name, bucket_n // console.log("Minio config:", minio_config); const minioClient = new Client(minio_config); - console.log("Minio :", minioClient); + //console.log("Minio :", minioClient); // check if bucket exists/can be accessed diff --git a/server/routes/tasks.js b/server/routes/tasks.js index 905c7e6..eaf7a91 100644 --- a/server/routes/tasks.js +++ b/server/routes/tasks.js @@ -10,6 +10,7 @@ const { getOrcreateSpec } = require('./specs'); const { getOrcreateDataset, getAllDataFromDataset, getAllPathsFromDataset, getDataDetails } = require('./datasets'); +const { downloadFilesFromMinio } = require('./minio_plugin').default; const { createJob } = require('./jobs'); const fetch = require("node-fetch"); @@ -254,115 +255,379 @@ async function import_tasks(req, res) { } +/**********************************************************************************/ +/***************************** DATA PROVIDER IMPORT ****************************/ +/**********************************************************************************/ + +async function get_dp(url) { + const dp_host = await db.get(dbkeys.keyForCliOptions).then((options) => { return options.dataProvider }); + return await fetch(dp_host + url, { method: 'get', headers: { 'Content-Type': 'application/json' } }) + .then(res => { + if (res.statusText == 'OK') { return res.json().then(data => Promise.resolve(data)); } + else { return res.status(200).json({ message: 'Response error: '+res });} + }) + .catch(err => { return Promise.reject(err);}); +} + +async function get_dp_minio_uris(project_name, ids) { + const dp_host = await db.get(dbkeys.keyForCliOptions).then((options) => { return options.dataProvider }); + return await fetch(dp_host+"/project/"+project_name+"/data", { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(ids) + }) + .then(res => { + if (res.statusText == 'OK') { + return res.json().then(data => { + if(data == {}) { return Promise.reject() } else { return Promise.resolve(data) } + }) + } else { return res.status(200).json({ message: 'Response error: '+res }) }; + }) + .catch(err => { return Promise.reject(err);}); +} + +/** + * @api {post} /datasets/projects_from_dataprovider get projects list from confiance dp + * @apiName GetProjsFromDP + * @apiGroup Dataset + * @apiPermission admin + * + * @apiParam {RestDataset} body + * + * @apiSuccessExample Success-Response: + * HTTP/1.1 201 OK + * + * @apiErrorExample Error-Response: + * HTTP/1.1 404 Error in DP import + * HTTP/1.1 401 Unauthorized + */ +async function projects_from_dataprovider(req, res) { + return await checkAdmin(req, async () => { + console.log('##### Importing project list from dataprovider'); + return await get_dp("/debiai/projects").then(res => res); + }) + .then(projs=>{return res.status(200).send(projs);}) + .catch(err => {return res.status(400).send(err);}) +} + +/** + * @api {post} /datasets/selections_from_dataprovider get projects list from confiance dp + * @apiName GetSelectionsFromDP + * @apiGroup Dataset + * @apiPermission admin + * + * @apiParam {RestDataset} body + * + * @apiSuccessExample Success-Response: + * HTTP/1.1 201 OK + * + * @apiErrorExample Error-Response: + * HTTP/1.1 404 Error in DP import + * HTTP/1.1 401 Unauthorized + */ +async function selections_from_dataprovider(req, res) { + return await checkAdmin(req, async () => { + console.log('##### Importing selections from dataprovider'); + //console.log('##### BR req.params:', req.params); + return await get_dp("/debiai/projects/"+req.params.project_name+"/selections").then(res => res); + }) + .then((selections)=>{return res.status(200).send(selections);}) + .catch(err => {return res.status(400).send(err);}) +} + +/** + * @api {post} /datasets/id_list_from_dataprovider get id list from confiance dp + * @apiName GetIdListFromDP + * @apiGroup Dataset + * @apiPermission admin + * + * @apiParam {RestDataset} body + * + * @apiSuccessExample Success-Response: + * HTTP/1.1 201 OK + * + * @apiErrorExample Error-Response: + * HTTP/1.1 404 Error in DP import + * HTTP/1.1 401 Unauthorized + */ +//TODO renommer, elle fait tout maintenent : creation Dataset + Task +async function id_list_from_dataprovider(req, res) { + return await checkAdmin(req, async () => { + console.log('##### Importing id list from dataprovider'); + //console.log('##### BR req.params:', req.params); + return await get_dp("/debiai/projects/"+req.params.project_name+"/selections/"+req.params.sel_id+"/selected-data-id-list").then(res => res); + }) + .then(async (selections)=>{ + task = await process_selection(req.params.project_name, req.params.sel_name, selections) + .then(res => {return res}) + .catch(err => {throw err}) + console.log("Created Task", task); + return res.status(200).send(task); + }) + .catch(err => { + console.log("ERREUR in id_list_from_dataprovider:", err); + return res.status(400).json({ message: err }); + }) +} + +//// remaniement, ce n'est plus une API --nom temporaire, j'ai pas mieux en stock +// get infos on "selections" from data-provider (dp_info) +// extract minio paths from dp_info +// get images from minio +// create Dataset +// extract annotations from dp_info +// create Task +async function process_selection(project_name, sel_name, selections) { + console.log('##### Importing Minio uris from dataprovider'); + dp_res = await get_dp_minio_uris(project_name, selections) + .then(res => {return res}) + .catch(err=>{ throw "Error in get_dp_minio_uris "+ err}); + console.log("dp_res FULL", dp_res); + bucket_names = [] + bucket_paths = [] + filenames = [] + if (!dp_res || dp_res.length == 0) { throw "ERROR empty data for selection"; } + Object.keys(dp_res).forEach(key => { + console.log("dp_res[", key, "]", dp_res[key].storage); + //console.log("dp_res[", key, "].annotations", dp_res[key].annotations); + if(!dp_res[key].storage) { throw "No storage info in selection, import aborted";} + if(!dp_res[key].storage.minio) { throw "No minio storage info in selection, import aborted";} + if(!dp_res[key].storage.minio.bucket_name && !dp_res[key].storage.minio.bucket) { throw "No minio bucket[_name] info in selection, import aborted";} + if(!dp_res[key].storage.minio.basepath) { throw "No minio basepath info in selection, import aborted";} + bname = ""; + if(dp_res[key].storage.minio.bucket) { + bname = dp_res[key].storage.minio.bucket; + } else { + bname = dp_res[key].storage.minio.bucket_name; + } + bpath = dp_res[key].storage.minio.basepath; + if (bpath.startsWith(bname)) { + //slice parce que le bucket est mis en prefixe du path !! + bpath = bpath.slice(bname.length); + } + bucket_names.push(bname); + bucket_paths.push(bpath); + filenames.push(dp_res[key].storage.filename); + }); + console.log(" - buckets:", bucket_names); + console.log(" - paths:", bucket_paths); + console.log(" - filenames:", filenames); + + console.log('# 1) Create a new dataset'); + console.log('# 1.1) get/set members'); + sel_name = sel_name.replace(/\s/g, '_'); + let dataset = {}; + dataset.date = Date.now(); + dataset.path = 'importedFromDebiai/' + sel_name; + dataset.id = project_name.replace(/\s/g, '_') + "_" + sel_name; + dataset.data_type = 'image' // TODO: gérer autres cas... + console.log("dataset=",dataset); + console.log('# 1.2) getPathFromIds'); + dataset.urlList = await downloadFilesFromMinio(filenames, workspace, sel_name, bucket_names[0], bucket_paths[0]).catch((e) => { + console.error('Error in Minio import\n'+e); + throw 'Error in Minio import\n'+e; + }) + console.log('# 1.3) getImagesFromPath'); + const newDataset = await getOrcreateDataset(dataset, workspace); + if (newDataset) { + const task = await createTasksFromDPImport(dp_res, newDataset) + .catch(err => { + console.log("Error while creating task", err); + throw 'Error while creating task:' + err; + }); + //console.log("Task", task); + return task; + } else throw 'Error while creating dataset'; +} + async function createTasksFromDPImport(dp_res, dataset) { let dp_info = [] //TODO gerer le fait qu'on recupère pleins de fois les infos de task... // que faire si différentes ? - const task_types = new Set() - + const task_types = new Set(); + const class_list = new Set(); + const class_defs = {}; + console.log("DSQFS", dp_res); Object.keys(dp_res).forEach(key => { Object.keys(dp_res[key].annotations).forEach(ann_key => { - console.log("ZZZZ", key, ann_key); - //console.log("ZQQ", dp_res[key]); - console.log("ZZAA", dp_res[key]['annotations'][ann_key]); - - let task_type = dp_res[key]['annotations'][ann_key].type; + // TMP we take [0] because it's a list for several annotator, we take the first (???) + console.log("dsqd", dp_res[key].annotations[ann_key]); + let ann = dp_res[key].annotations[ann_key]; + if(Array.isArray(ann)) { + ann = ann[0]; //case of different annotators, we take first(last?) one only for now + } + console.log("ZAEER", ann); + let task_type = String(ann.type).toLowerCase(); //tmp if (!task_type) { + console.log("ERROR annotation type not defined, default to classification") task_type = 'classification'; } if (task_type === "classification") { + options = {}; + options[ann_key] = ann.value; + cl_ann = { + category: task_type, + options: options, + //actorId etc. + } dp_info.push({ - id: key, //ou plutot dp_res[key].storage. ?? imageid ? - name: ann_key, //TODO checker si on a besoin de ça, et ce qu'il y a dedans au juste... + //id: key, //ou plutot dp_res[key].storage. ?? imageid ? + id: dp_res[key].storage.filename, + //name: ann_key, //-> blurred, object_detection_vdp, ... type: task_type, - anns: dp_res[key]['annotations'][ann_key].value, - //actorId etc. + anns: [cl_ann] }); + class_list.add(ann_key); + class_defs[ann_key] = typeof ann.value; } else if (task_type === "object_detection") { - ann_type = dp_res[key]['annotations'][ann_key].geometry['type']; - if (ann_type == "rectangle") { - task_type = "rectangle"; + const geom_types = new Set(); + for(it of ann.items) { + class_list.add(it['category']); + console.log("AZAZA", it, it['geometry']); + class_defs[it['category']] = it['geometry'].type; //keep it because we need to differentiate polygon/mpolygon + // mapping geometry -> plugin + geom = it['geometry'].type + if(geom === "mpolygon") geom = "polygon" + geom_types.add(geom); + } + if(geom_types.size>1) { + console.log("WARNING: Several types of object geometry, this is not implemented yet") } + task_type = geom_types.values().next().value; //TODO dp_info.push({ - id: key, //ou plutot dp_res[key].storage. ?? imageid ? - name: ann_key, + //id: key, //ou plutot dp_res[key].storage. ?? imageid ? + id: dp_res[key].storage.filename, + //name: ann_key, type: task_type, - anns: dp_res[key]['annotations'][ann_key].items, + anns: ann.items, //... }) - } - task_types.add(task_type) + } //TODO else ... autres types d'annotations + task_types.add(task_type); }); }); - console.log("dataset", dataset); - console.log("dataset id", dataset['id']); - //tmp - classif_label_value = { - category: [ - { - name: 'classification', color: "black", properties: [ - { name: 'checkbox example', type: 'checkbox', default: false }, - { name: 'dropdown example', type: 'dropdown', enum: ['something', 'something else', 'anything else'], default: 'something' }, - { name: 'textfield example', type: 'textfield', default: 'some text' } - ] - } - ], - default: 'classification' - } - + //console.log("dataset", dataset); + //console.log("dataset id", dataset['id']); + const importedTasks = []; for (let task_type of task_types) { + let task; const plugin_name = task_type; const task_name = dataset['id']; // + "_task"; - const task = { - name : task_name, - spec: { - plugin_name, - label_schema: classif_label_value, // defaultLabelValues(plugin_name), - settings: {}, //TMP defaultSettings(plugin_name), - data_type: dataset.type - }, - dataset: dataset - }; + //cas classif + if (task_type === "classification") { + let props = []; + for (let classe of class_list) { + if(class_defs[classe] === 'boolean') { + props.push({name: classe, type: 'checkbox', default: false}) + } else if(class_defs[classe] === 'string') { + props.push({name: classe, type: 'textfield', default: ""}) + } //... else ??? + } + classif_label_value = { + category: [ + { + name: task_type, + color: "black", + properties: props + } + ], + default: 'classification' + } + + task = { + name : task_name, + spec: { + plugin_name: plugin_name, + label_schema: classif_label_value, // defaultLabelValues(plugin_name), + settings: {}, //TMP defaultSettings(plugin_name), + data_type: dataset.type + }, + dataset: dataset + }; + } else { //cas non classif + let cats = []; + for (let classe of class_list) { + cats.push({ + name: classe, + color: "black", + properties: [] + }); + } + console.log("DEDE", cats); + + objdetect_label_value = { + category: cats, + default: cats[0].name + } + + task = { + name : task_name, + spec: { + plugin_name: plugin_name, + label_schema: objdetect_label_value, // defaultLabelValues(plugin_name), + settings: {}, //TMP defaultSettings(plugin_name), + data_type: dataset.type + }, + dataset: dataset + }; + } const spec = await getOrcreateSpec(task.spec); + + // managing task with same name + let newTaskName = task.name; try { - await db.get(dbkeys.keyForTask(task_name)); - return res.status(400).json({message: 'Taskname already existing'}); - } catch(e) {}//catch === everything is ok (Taskname not in use) + // check if task already exist and search for another name + let cpt = 1; + do { + const task = await db.get(dbkeys.keyForTask(newTaskName)); + newTaskName = `${task.name}-${cpt}`; + cpt += 1; + } while (true) + } catch (err) { + // task with this updated name does not exist, use this name + task.name = newTaskName; + } + //console.log("AAAAAAAAA", task); + console.log('# 2) Push the new task + dataset'); // Task does not exist create it const newTask = {name: task_name, dataset_id: dataset.id, spec_id: spec.id} - await db.put(dbkeys.keyForTask(newTask.name), newTask); + const bm = new batchManager.BatchManager(db); + await bm.add({ type: 'put', key: dbkeys.keyForTask(newTask.name), value: newTask }) //why ??? + //await db.put(dbkeys.keyForTask(newTask.name), newTask); // Generate first job list await generateJobResultAndLabelsLists(newTask); // Send back the created task const taskDetail = await getTaskDetails(newTask.name); console.log('Task created', taskDetail.name) - - for (dp in dp_info) { - if (dp.type === task_type) { + for (dp of dp_info) { + if (dp.type == task_type) { // Create Labels const newLabels = { task_name: newTask.name, data_id: dp.id, annotations: dp.anns }; - + console.log("Label to add", newLabels, newLabels.annotations); await bm.add({ type: 'put', key: dbkeys.keyForLabels(newTask.name, newLabels.data_id), value: newLabels }); + /** TMP On ne gere pas encore le statut (to_validate, to_annotate, ...) if (ann.data.status) {//if existing, get status back resultData = await db.get(dbkeys.keyForResult(newTask.name, newLabels.data_id));//get the status for this data resultData.status = ann.data.status;//add the status await bm.add({ type: 'put', key: dbkeys.keyForResult(newTask.name, newLabels.data_id), value: resultData }); } + **/ } } await bm.flush(); - importedTasks.push(newTaskName); + importedTasks.push(newTask); } + return importedTasks; } @@ -848,7 +1113,6 @@ const getTaskDetails = async (taskName) => { * @param {Object} task */ async function generateJobResultAndLabelsLists(task) { - console.log("TTTT", getAllDataFromDataset) const dataIdList = await getAllDataFromDataset(task.dataset_id); const bm = new batchManager.BatchManager(db); const bar = new cliProgress.SingleBar({ @@ -962,6 +1226,9 @@ module.exports = { put_task, delete_task, import_tasks, + projects_from_dataprovider, + selections_from_dataprovider, + id_list_from_dataprovider, export_tasks, getAllTasksDetails, createTasksFromDPImport