Skip to content

Commit

Permalink
export Confiance - WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
@BertrandRenault
BertrandRenault committed Mar 2, 2023
1 parent a0ba4ea commit aad9c65
Show file tree
Hide file tree
Showing 12 changed files with 381 additions and 106 deletions.
2 changes: 1 addition & 1 deletion cli/pixano-cli.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ export function cli(argv) {
.option('minio-endpoint', 'Minio endpoint', 'minio-storage.apps.confianceai-public.irtsysx.fr')
.option('minio-accessKey', 'Minio accessKey', "developer")
.option('minio-secretKey', 'Minio secretKey', "password")
.option('data-provider', 'Data Provider url', 'http://localhost:3011') // 'http://192.168.102.130:3010')
.option('data-provider', 'Data Provider url', 'http://localhost:3010') // 'http://192.168.102.130:3010')
//.option('data-provider', 'Data Provider url', ' http://debiai-data-provider-os:3011') //version interne Kubernetes, ecrasé par YAML
.example('pixano /path/to/workspace','The most common way to use Pixano:')
.example('pixano --workspace /path/to/workspace --port 5001','Run on a specific port:')
Expand Down
8 changes: 0 additions & 8 deletions config/minio.json
View file

This file was deleted.

9 changes: 0 additions & 9 deletions config/minio_old.json
View file

This file was deleted.

8 changes: 8 additions & 0 deletions frontend/src/actions/media.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,14 @@ export const importFromKafka = () => (dispatch) => {
return GET('/api/v1/datasets/import_from_kafka', dispatch);
}

/**
* partial (by image) export to Data Provider (Confiance).
*/
export const partialExporttoDP = (task_name, media_id) => (dispatch) => {
//console.log("partialExporttoDP");
return GET('/api/v1/dataprovider/partial_export_to_dataprovider/'+task_name+'/'+media_id, dispatch);
}

/**
* get project list from Data Provider (Confiance).
*/
Expand Down
2 changes: 1 addition & 1 deletion frontend/src/views/app-dashboard-admin.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -705,7 +705,7 @@ class AppDashboardAdmin extends TemplatePage {
return html`
<div id="overview" class="section">
<h1 class="display-4" style="margin: auto;">Select a task: </h1>
<mwc-select label='Task' @selected=${(e) => {
<mwc-select label='Task' style='width: 400px' @selected=${(e) => {
if (tasks[e.detail.index] && tasks[e.detail.index].name !== taskName) {
store.dispatch(updateTaskName(tasks[e.detail.index].name));
this.refreshGrid();
Expand Down
2 changes: 1 addition & 1 deletion frontend/src/views/app-datasets-manager.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -571,7 +571,7 @@ class AppDatasetsManager extends connect(store)(TemplatePage) {
return html`
<div id="overview" class="section">
<h1 class="display-4" style="margin: auto;">Select a dataset: </h1>
<mwc-select label='Dataset' @selected=${(e) => {
<mwc-select label='Dataset' style='width: 400px' @selected=${(e) => {
if (this.datasets[e.detail.index] && this.datasets[e.detail.index].id !== datasetId) {
this.datasetIdx = e.detail.index;
this.refreshGrid();
Expand Down
16 changes: 15 additions & 1 deletion frontend/src/views/app-label.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ import '@material/mwc-icon-button';
import '@material/mwc-snackbar';

import { AppExplore } from './app-explore';
import { partialExporttoDP } from '../actions/media';


class AppLabel extends AppExplore {
static get properties() {
Expand Down Expand Up @@ -114,6 +116,18 @@ class AppLabel extends AppExplore {
console.log('_submissionHelper');
await store.dispatch(putJob(objective));
await store.dispatch(putLabels());

//Confiance DP export
//TODO: test si on est dans Confiance (mais pour le moment on va dire que oui)
//TODO: export sur Submit only (+validate/Reject)? sur skip too ?
if(objective !== 'skip') {
const taskName = getState().application.taskName;
const media_id = getState('media').info.id;
await store.dispatch(partialExporttoDP(taskName, media_id))
.then(ret => { console.log("Export OK"+ ret); })
.catch(err => { console.log("ERROR Export", err); this.errorPopup("EXPORT ERROR\n" + err.message); })
}

} // Job has either been reassigned to someone else or is dead.
catch (err) { console.log('err1', err); this.errorPopup(err.message); }

Expand Down Expand Up @@ -152,7 +166,7 @@ class AppLabel extends AppExplore {
}

/**
* Submit job.
* Skip job.
*/
skip() {
this._submissionHelper('skip');
Expand Down
21 changes: 18 additions & 3 deletions frontend/src/views/app-project-manager.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,16 @@ class AppProjectManager extends connect(store)(TemplatePage) {
browserElem.open = true;
}

/**
* Fired when exporting a project to Confiance DB
*/
onExportToDP() {
const browserElem = this.shadowRoot.getElementById('dialog-import-export-path');
this.importExportText = 'export';
browserElem.mode = 'export';
browserElem.open = true;
}

/**
* Fired when importing a project
*/
Expand Down Expand Up @@ -407,10 +417,14 @@ class AppProjectManager extends connect(store)(TemplatePage) {
type="button"
title="Copy database with annotation and their status into an archive"
@click="${() => snapshotProject()}">Snapshot</mwc-button>
<mwc-button outlined
<mwc-button outlined
type="button"
title="Export annotations only to json files"
@click="${this.onExport}">Export</mwc-button>
<mwc-button outlined
type="button"
title="Export annotations to Confiance DataBase"
@click="${this.onExportToDP}">Export to Confiance DB</mwc-button>
<mwc-button outlined
type="button"
title="Import annotations from json files"
Expand All @@ -430,20 +444,21 @@ class AppProjectManager extends connect(store)(TemplatePage) {
const taskName = t ? t.name : '';
const datasetId = t ? t.dataset.id : '';
const pluginName = t ? t.spec.plugin_name : '';
//Note: style removed from "${this.tasks.map((t) => html`<mwc-tab label="Task ${t.name}" style="max-width: 200px;"></mwc-tab>`)}"
return html`
<form class="section">
${this.taskHeader}
<div>
<mwc-tab-bar @MDCTabBar:activated=${this.onTabChanged} activeindex="${this.taskIdx}">
${this.tasks.map((t) => html`<mwc-tab label="Task ${t.name}" style="max-width: 200px;"></mwc-tab>`)}
<mwc-button outlined
<mwc-button outlined
id="add-task"
class="add-task ${this.tasks.length ? 'multi' : 'single'}"
style=${this.creatingTask ? 'display: none;' : 'display: flex;'}
type="button"
icon="add"
title="Add new annotation task"
@click="${this.onAddTask}">New task</mwc-button>
${this.tasks.map((t) => html`<mwc-tab label="Task ${t.name}"></mwc-tab>`)}
</mwc-tab-bar>
<div style="${t != undefined && this.tasks.length ? 'display: block;' : 'display: none;'}">
<div class="form-group" style="display: flex; flex-wrap: wrap;">
Expand Down
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
"fs": "0.0.1-security",
"git-rev-sync": "^3.0.2",
"glob": "^7.1.6",
"google-palette": "^1.1.0",
"image-thumbnail": "1.0.14",
"jsonwebtoken": "^8.5.1",
"kafkajs": "^1.15.0",
Expand Down
4 changes: 4 additions & 0 deletions server/router.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ const { get_tasks,
projects_from_dataprovider,
selections_from_dataprovider,
id_list_from_dataprovider,
partial_export_to_dataprovider,
export_tasks_to_dataprovider,
export_tasks } = require('./routes/tasks');
const { snapshot_project } = require('./routes/project');
const { get_results,
Expand Down Expand Up @@ -77,6 +79,8 @@ router.delete('/users/:username', middleware.checkToken, delete_user);
router.get('/profile', middleware.checkToken, get_profile);

//for Confiance data provider
router.get('/dataprovider/export_tasks_to_dataprovider/:task_name', middleware.checkToken, export_tasks_to_dataprovider);
router.get('/dataprovider/partial_export_to_dataprovider/:task_name/:media_id', middleware.checkToken, partial_export_to_dataprovider);
router.get('/dataprovider/projects_from_dataprovider', middleware.checkToken, projects_from_dataprovider);
router.get('/dataprovider/selections_from_dataprovider/:project_name', middleware.checkToken, selections_from_dataprovider);
router.get('/dataprovider/id_list_from_dataprovider/:project_name/:sel_id/:sel_name', middleware.checkToken, id_list_from_dataprovider);
Expand Down
169 changes: 102 additions & 67 deletions server/routes/minio_plugin.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,21 @@ function waitFor(conditionFunction) {//rajouter un timeout

/**
* Download files form minio
* @param ["id1","id2",...]: a list of ids to search for in the bucket
* @param minio_files {path0_full: [{bucket: bucket name, path: path0, file: file_id }, ...], ..., pathN: [{...}, ...]}: a struct of bucket paths and ids to search for
* pathX_full is the full "key", aka bucket_name/path, used to group files for seeking
* @param workspace: Pixano's current workspace (images will be copied inside of it)
* @return ["url1","url2",...]: returns the list of the corresponding URLs
* @doc https://docs.min.io/docs/javascript-client-api-reference
*/
const downloadFilesFromMinio = async (listIds,workspace,selection_name, bucket_name, bucket_path) => {
console.log("downloadFilesFromMinio (",selection_name,") - nbSamples", listIds.length);
const downloadFilesFromMinio = async (minio_files, workspace, selection_name) => {
//TODO : wrong, need to iterate and sum in each array
let num_ids = 0;
for (const k in minio_files) {
num_ids += minio_files[k].length;
}
console.log("downloadFilesFromMinio (",selection_name,") - nbSamples", num_ids);

const pixano_local_save_image_directory = workspace+'/minio_saved_images/'+'importedFromDebiai/'+selection_name+'/';//... TODO
const pixano_local_save_image_directory = workspace+'/minio_saved_images/importedFromDebiai/'+selection_name+'/';//... TODO
var listOfURLs = [];

// Instantiate the minio client with the endpoint
Expand All @@ -35,83 +41,112 @@ const downloadFilesFromMinio = async (listIds,workspace,selection_name, bucket_n
accessKey: options.minioAccessKey,
secretKey: options.minioSecretKey
}});
// console.log("Minio config:", minio_config);
//console.log("Minio config:", minio_config);

const minioClient = new Client(minio_config);

//console.log("Minio :", minioClient);
/** test: list availble buckets
minioClient.listBuckets(function(err, buckets) {
if (err) return console.log(err);
console.log('available buckets :', buckets);
})
*/


// check if bucket exists/can be accessed
// var bucket_name = CONFIG.bucket_name;
// if (project_name==='Valeo') bucket_name = 'pixanovaleousecase';// special case : different bucket

var exists = await minioClient.bucketExists(bucket_name).catch((e) => {throw "Minio: Bucket does not exist\n"+e;});
if (!exists) throw "Minio: Bucket "+bucket_name+" does not exist";
console.log(`Bucket ${bucket_name} exists.`);
// check if bucket(s) exists/can be accessed
const unique_buckets = new Set()
for (const k in minio_files) {
for(const f of minio_files[k]) {
unique_buckets.add(f['bucket']);
}
}
console.log('Buckets', unique_buckets);
for (const bucket_name of unique_buckets) {
var exists = await minioClient.bucketExists(bucket_name).catch((e) => {throw "Minio: Bucket does not exist\n"+e;});
if (!exists) throw "Minio: Bucket "+bucket_name+" does not exist";
console.log(`Bucket ${bucket_name} exists.`);
}

// Extract the list of image from the bucket
var data = [];
var doneData = 0;

console.log(`Seek in ${bucket_name} path: ${bucket_path}`);
const bar = new cliProgress.SingleBar({
format: 'Image retrieval from Minio | {bar} | {percentage}% || {value}/{total} jobs'
});
bar.start(listIds.length, 0);
var stream = minioClient.listObjects(bucket_name, bucket_path, true);
stream.on('error', function (e) { throw(e); });
stream.on('data', function (obj) { data.push(obj); });
stream.on('end', function () {
if (data.length===0) throw "Minio: no data found in bucket "+bucket_name;
//for (var i=0; i<data.length; i++) {//search for urls that correspond to the input list and get them
data.forEach(obj => {//search for urls that correspond to the input list and get them
//const obj = data[i];
if ('name' in obj) {
// console.log("name in obj");
var corresponding = false;
// FORMAT of sample_ids :
// "sample_ids": [
// {
// "dataset": "not_labeled",
// "subject": "c34",
// "relative_path": "not_labeled/c34/",
// "url": "",
// "type": "image",
// "id": "191003-2237_2953236_ - C101_OK.jpgImage"
// }, ... ]
var sample;
for (var i=0; i<listIds.length ; i++) {
sample = listIds[i];
if (!sample) throw(e);
// console.log("sample=",sample);
// console.log("sample.id.replace('.jpgImage', '.jpg')=",sample.id.replace('.jpgImage', '.jpg'));
if (obj.name.includes(sample.replace('.jpgImage', '.jpg'))) {
//console.log(`obj.name ${obj.name} includes sample ${sample}`);
corresponding=true;
break;
}
}
if (corresponding) {
console.log("corresponding sample=",sample);
//Download image in current directory//... TODO : use web links when available
minioClient.fGetObject(bucket_name, obj.name, pixano_local_save_image_directory + obj.name, function (e) {
if (e) {
console.log("ERERERER", e);
throw "Error while importing from Minio: " + e;
bar.start(num_ids, 0);
for(const k in minio_files) {
//assume (by construction) unique bucket / path for each key in minio_files
const bucket_name = minio_files[k][0]['bucket'];
const bucket_path = minio_files[k][0]['path'];
const listIds = []
var data = [];
var doneData = 0;

for( const f of minio_files[k]) {
listIds.push(f['file']);
}
console.log(`Seek in ${bucket_name} path: ${bucket_path}`);
var stream = minioClient.listObjects(bucket_name, bucket_path, true);
stream.on('error', function (e) { throw(e); });
stream.on('data', function (obj) { data.push(obj); });
stream.on('end', function () {
if (data.length===0) throw "Minio: no data found in bucket "+bucket_name;
//for (var i=0; i<data.length; i++) {//search for urls that correspond to the input list and get them
data.forEach(obj => {//search for urls that correspond to the input list and get them
//const obj = data[i];
if ('name' in obj) {
// console.log("name in obj");
var corresponding = false;
// FORMAT of sample_ids :
// "sample_ids": [
// {
// "dataset": "not_labeled",
// "subject": "c34",
// "relative_path": "not_labeled/c34/",
// "url": "",
// "type": "image",
// "id": "191003-2237_2953236_ - C101_OK.jpgImage"
// }, ... ]
var sample;
for (var i=0; i<listIds.length ; i++) {
sample = listIds[i];
if (!sample) throw(e);
// console.log("sample=",sample);
// console.log("sample.id.replace('.jpgImage', '.jpg')=",sample.id.replace('.jpgImage', '.jpg'));
if (obj.name.includes(sample.replace('.jpgImage', '.jpg'))) {
//console.log(`obj.name ${obj.name} includes sample ${sample}`);
corresponding=true;
break;
}
//console.info('append:',{url: pixano_local_save_image_directory + obj.name, id: sample});
listOfURLs.push({url: pixano_local_save_image_directory + obj.name, id: sample});
doneData++;
});
bar.increment();
}
if (corresponding) {
console.log("corresponding sample=",sample);
//Download image in current directory//... TODO : use web links when available

//minioClient.fGetObject(bucket_name, obj.name, pixano_local_save_image_directory + obj.name, function (e) {
//TMP On applati l'arborescence Minio. Ce n'est pas solution optimale dans tout les cas
// mais c'est le + simple dans le cadre d'une selection DP
// -> on vire les répertoires du path, on ne garde que le filename
const obj_flat_name = obj.name.split('/').pop();
minioClient.fGetObject(bucket_name, obj.name, pixano_local_save_image_directory + obj_flat_name, function (e) {
if (e) {
console.log("ERERERER", e);
throw "Error while importing from Minio: " + e;
}
//console.info('append:',{url: pixano_local_save_image_directory + obj.name, id: sample});
//////TMP applati listOfURLs.push({url: pixano_local_save_image_directory + obj.name, id: sample});
listOfURLs.push({url: pixano_local_save_image_directory + obj_flat_name, id: sample});
doneData++;
});
bar.increment();
} else doneData++;
} else doneData++;
} else doneData++;
});//throw errors further
});//throw errors further
});//throw errors further
console.log("waitFor",doneData,data.length);
await waitFor(() => { console.log("test",doneData,data.length); if (data.length>0) return(doneData === data.length); });
console.log("waitFor",doneData,data.length);
await waitFor(() => { console.log("test",doneData,data.length); if (data.length>0) return(doneData === data.length); });
}
//console.log("listOfURLs=",listOfURLs);
console.info("Minio: got "+listOfURLs.length+" images over "+listIds.length+" in the input list");
console.info("Minio: got "+listOfURLs.length+" images over "+num_ids+" in the input list");
bar.stop();
if (listOfURLs.length===0) throw "Minio: no corresponding data found in bucket "+bucket_name;

Expand Down
Loading

0 comments on commit aad9c65

Please sign in to comment.