[Feature] Progress Tracking

.vscode/settings.json

@@ -13,5 +13,8 @@
     "npm-debug.log.*": true,
     "test/**/__snapshots__": true,
     "yarn.lock": true
-  }
+  },
+  "[javascript]": {
+    "editor.tabSize": 2
+  },
 }

app/actions/cpacpy.js

@@ -0,0 +1,261 @@
+export const CPACPY_INIT = 'CPACPY_INIT'
+export const CPACPY_INIT_DONE = 'CPACPY_INIT_DONE'
+
+export const CPACPY_SCHEDULER_CALL = 'CPACPY_SCHEDULER_CALL'
+
+export const CPACPY_SCHEDULER_POLLING = 'CPACPY_SCHEDULER_POLLING'
+export const CPACPY_SCHEDULER_POLLING_CANCEL = 'CPACPY_SCHEDULER_POLLING_CANCEL'
+
+export const CPACPY_SCHEDULER_SCHEDULER = 'CPACPY_SCHEDULER_SCHEDULER'
+
+export const CPACPY_SCHEDULER_CONNECT = 'CPACPY_SCHEDULER_CONNECT'
+export const CPACPY_SCHEDULER_CONNECT_CANCEL = 'CPACPY_SCHEDULER_CONNECT_CANCEL'
+export const CPACPY_SCHEDULER_CONNECT_MESSAGE = 'CPACPY_SCHEDULER_CONNECT_MESSAGE'
+export const CPACPY_SCHEDULER_CONNECT_SEND = 'CPACPY_SCHEDULER_CONNECT_SEND'
+export const CPACPY_SCHEDULER_CONNECT_SEND_CALLBACK = 'CPACPY_SCHEDULER_CONNECT_SEND_CALLBACK'
+
+export const CPACPY_SCHEDULER_INFO = 'CPACPY_SCHEDULER_INFO'
+export const CPACPY_SCHEDULER_DETECT = 'CPACPY_SCHEDULER_DETECT'
+
+export const CPACPY_SCHEDULER_ONLINE = 'CPACPY_SCHEDULER_ONLINE'
+export const CPACPY_SCHEDULER_OFFLINE = 'CPACPY_SCHEDULER_OFFLINE'
+
+export const CPACPY_SCHEDULER_ADDNEW = 'CPACPY_SCHEDULER_ADDNEW'
+export const CPACPY_SCHEDULER_TEST_TEMP_CONNECTION = 'CPACPY_SCHEDULER_TEST_TEMP_CONNECTION'
+export const CPACPY_SCHEDULER_TEST_TEMP_CONNECTION_SUCCESS = 'CPACPY_SCHEDULER_TEST_TEMP_CONNECTION_SUCCESS'
+export const CPACPY_SCHEDULER_TEST_TEMP_CONNECTION_FAILED = 'CPACPY_SCHEDULER_TEST_TEMP_CONNECTION_FAILED'
+export const CPACPY_SCHEDULER_DELETE = 'CPACPY_SCHEDULER_DELETE'
+
+export const CPACPY_CONFIG_LOAD = 'CPACPY_CONFIG_LOAD'
+export const CPACPY_CONFIG_LOAD_SUCCESS = 'CPACPY_CONFIG_LOAD_SUCCESS'
+export const CPACPY_CONFIG_LOAD_ERROR = 'CPACPY_CONFIG_LOAD_ERROR'
+export const CPACPY_CONFIG_SAVE = 'CPACPY_CONFIG_SAVE'
+export const CPACPY_CONFIG_SAVE_SUCCESS = 'CPACPY_CONFIG_SAVE_SUCCESS'
+export const CPACPY_CONFIG_SAVE_ERROR = 'CPACPY_CONFIG_SAVE_ERROR'
+export const CPACPY_CONFIG_CLEAR = 'CPACPY_CONFIG_CLEAR'
+export const CPACPY_CONFIG_CLEAR_SUCCESS = 'CPACPY_CONFIG_CLEAR_SUCCESS'
+export const CPACPY_CONFIG_CLEAR_ERROR = 'CPACPY_CONFIG_CLEAR_ERROR'
+
+export function init() {
+  return {
+    type: CPACPY_INIT,
+  }
+}
+export function selectCurrentScheduler(scheduler) {
+  return {
+    type: CPACPY_SCHEDULER_SCHEDULER,
+    scheduler,
+  }
+}
+
+export function detect(scheduler, authKey='', poll=true, current=false) {
+  return {
+    type: CPACPY_SCHEDULER_DETECT,
+    scheduler, authKey, poll, current,
+  }
+}
+
+export function poll(scheduler, current=false) {
+  return {
+    type: CPACPY_SCHEDULER_POLLING,
+    scheduler, current,
+  }
+}
+
+export function pollCancel(scheduler) {
+  return {
+    type: CPACPY_SCHEDULER_POLLING_CANCEL,
+    scheduler,
+  }
+}
+
+export function watch(scheduler) {
+  return {
+    type: CPACPY_SCHEDULER_CONNECT,
+    scheduler,
+  }
+}
+
+export function watchCancel(scheduler) {
+  return {
+    type: CPACPY_SCHEDULER_CONNECT_CANCEL,
+    scheduler,
+  }
+}
+
+export function connectSendCallback(scheduler, callbackId, callbackAction) {
+  return {
+    type: CPACPY_SCHEDULER_CONNECT_SEND_CALLBACK,
+    scheduler,
+    callbackId,
+    callbackAction
+  }
+}
+
+export function connectSendCallbackCall(scheduler, callbackType, data) {
+  return {
+    type: callbackType,
+    scheduler,
+    data
+  }
+}
+
+export function connectSendWatch(scheduler, schedule, { action, error }) {
+  return {
+    type: CPACPY_SCHEDULER_CONNECT_SEND,
+    scheduler,
+    action,
+    error,
+    message: {
+      type: 'watch',
+      schedule,
+      watchers: ['Spawn', 'Start', 'End', 'Status', 'Result', 'Log']
+    }
+  }
+}
+
+export function call(scheduler, method='GET', endpoint, data, { success, error }, headers = {}) {
+  return {
+    type: CPACPY_SCHEDULER_CALL,
+    scheduler,
+    method,
+    endpoint,
+    data,
+    response: {
+      success,
+      error,
+    },
+    headers,
+  }
+}
+
+export function callSuccess(scheduler, type, data) {
+  return {
+    type,
+    scheduler,
+    data
+  }
+}
+
+export function callError(scheduler, type, exception) {
+  return {
+    type,
+    scheduler,
+    exception
+  }
+}
+
+export function schedule(scheduler, data, { success, error }) {
+  return call(
+    scheduler,
+    'POST',
+    '/schedule',
+    data,
+    {
+      success,
+      error,
+    }
+  )
+}
+
+export function scheduleDataSettings(scheduler, dataSettings, { success, error }){
+  return schedule(
+    scheduler,
+    {
+      type: 'data_settings',
+      data_settings: dataSettings,
+    },
+    { success, error }
+  )
+}
+
+export function schedulePipeline(scheduler, dataConfig, pipeline, profile, { success, error }){
+  return schedule(
+    scheduler,
+    {
+      type: 'pipeline',
+      data_config: dataConfig,
+      pipeline,
+      profile,
+    },
+    { success, error }
+  )
+}
+
+export function fetchResults(scheduler, schedule, result, { success, error }, { start, end } = { start: 0, end: null }) {
+  return call(
+    scheduler,
+    'GET',
+    `/schedule/${schedule}/result/${result}`,
+    null,
+    {
+      success: (body, headers) => {
+        let name = null
+        if (headers.get('content-disposition')) {
+          const disposition = headers.get('content-disposition')
+          if (disposition && disposition.indexOf('attachment') !== -1) {
+            const filenameRegex = /filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/;
+            const matches = filenameRegex.exec(disposition);
+            if (matches != null && matches[1]) {
+              name = matches[1].replace(/['"]/g, '');
+            }
+          }
+        }
+        return success(body, name)
+      },
+      error
+    },
+    {
+      Range: `bytes=${start}-${end ? end : ''}`
+    }
+  )
+}
+
+export function info(scheduler, info) {
+  return {
+    type: CPACPY_SCHEDULER_INFO,
+    scheduler, info
+  }
+}
+
+export function online(scheduler) {
+  return {
+    type: CPACPY_SCHEDULER_ONLINE,
+    scheduler
+  }
+}
+
+export function offline(scheduler) {
+  return {
+    type: CPACPY_SCHEDULER_OFFLINE,
+    scheduler
+  }
+}
+
+export function addNew(name, ip, port, authKey) {
+  return {
+    type: CPACPY_SCHEDULER_ADDNEW,
+    payload: {newName: name, newIP: ip, newPort: port, newAuthKey: authKey},
+  }
+}
+
+export function test(name, ip, port, authKey) {
+  return {
+    type: CPACPY_SCHEDULER_TEST_TEMP_CONNECTION,
+    name, ip, port, authKey
+  }
+}
+
+export function testFailed(error) {
+  return {
+    type: CPACPY_SCHEDULER_TEST_TEMP_CONNECTION_FAILED,
+    error
+  }
+}
+
+export function deleteScheduler(id) {
+  return  {
+    type: CPACPY_SCHEDULER_DELETE,
+    id,
+  }
+}

app/actions/dataset.js

@@ -0,0 +1,72 @@
+export const DATASET_CONFIG_LOAD = 'DATASET_CONFIG_LOAD'
+export const DATASET_CONFIG_LOAD_SUCCESS = 'DATASET_CONFIG_LOAD_SUCCESS'
+export const DATASET_CONFIG_LOAD_ERROR = 'DATASET_CONFIG_LOAD_ERROR'
+export const DATASET_CONFIG_SAVE = 'DATASET_CONFIG_SAVE'
+export const DATASET_CONFIG_SAVE_SUCCESS = 'DATASET_CONFIG_SAVE_SUCCESS'
+export const DATASET_CONFIG_SAVE_ERROR = 'DATASET_CONFIG_SAVE_ERROR'
+export const DATASET_CONFIG_CLEAR = 'DATASET_CONFIG_CLEAR'
+export const DATASET_CONFIG_CLEAR_SUCCESS = 'DATASET_CONFIG_CLEAR_SUCCESS'
+export const DATASET_CONFIG_CLEAR_ERROR = 'DATASET_CONFIG_CLEAR_ERROR'
+
+export const DATASET_SETTINGS_CREATE = 'DATASET_SETTINGS_CREATE'
+export const DATASET_SETTINGS_UPDATE = 'DATASET_SETTINGS_UPDATE'
+
+export const DATASET_GENERATE_DATA_CONFIG = 'DATASET_GENERATE_DATA_CONFIG'
+export const DATASET_GENERATE_DATA_CONFIG_SUCCESS = 'DATASET_GENERATE_DATA_CONFIG_SUCCESS'
+export const DATASET_GENERATE_DATA_CONFIG_ERROR = 'DATASET_GENERATE_DATA_CONFIG_ERROR'
+
+export const DATASET_GENERATE_DATA_CONFIG_SCHEDULED = 'DATASET_GENERATE_DATA_CONFIG_SCHEDULED'
+export const DATASET_GENERATE_DATA_CONFIG_FINISHED = 'DATASET_GENERATE_DATA_CONFIG_FINISHED'
+export const DATASET_GENERATE_DATA_CONFIG_FETCHED = 'DATASET_GENERATE_DATA_CONFIG_FETCHED'
+
+export const DATASET_GENERATE_DATA_CONFIG_URL = 'DATASET_GENERATE_DATA_CONFIG_URL'
+
+export const DATASET_SELECT_SCHEDULER = 'DATASET_SELECT_SCHEDULER'
+
+export function createDataSettings(name, configuration) {
+  return {
+    type: DATASET_SETTINGS_CREATE,
+    dataset: { name, configuration }
+  }
+}
+
+export function updateDataSettings(id, name, configuration) {
+  return {
+    type: DATASET_SETTINGS_UPDATE,
+    dataset: { id, name, configuration }
+  }
+}
+
+export function generateDataConfig(scheduler, dataset) {
+  return {
+    type: DATASET_GENERATE_DATA_CONFIG,
+    scheduler, dataset
+  }
+}
+
+export function importDataConfig(dataset, config) {
+  return {
+    type: DATASET_GENERATE_DATA_CONFIG_SUCCESS,
+    dataset, config
+  }
+}
+
+export function generateDataConfigUrlFetch(dataset, url) {
+  return {
+    type : DATASET_GENERATE_DATA_CONFIG_URL,
+    dataset, url
+  }
+}
+
+export function updateDatasetError(dataset, exception) {
+  return {
+    type: DATASET_GENERATE_DATA_CONFIG_ERROR,
+    dataset, exception
+  }
+}
+
+export function datasetSaveAll() {
+  return {
+    type: DATASET_CONFIG_SAVE,
+  }
+}

app/actions/execution.js

@@ -0,0 +1,36 @@
+import uuid from 'uuid/v4'
+
+export const EXECUTION_CONFIG_LOAD = 'EXECUTION_CONFIG_LOAD'
+export const EXECUTION_CONFIG_LOAD_SUCCESS = 'EXECUTION_CONFIG_LOAD_SUCCESS'
+export const EXECUTION_CONFIG_LOAD_ERROR = 'EXECUTION_CONFIG_LOAD_ERROR'
+export const EXECUTION_CONFIG_SAVE = 'EXECUTION_CONFIG_SAVE'
+export const EXECUTION_CONFIG_SAVE_SUCCESS = 'EXECUTION_CONFIG_SAVE_SUCCESS'
+export const EXECUTION_CONFIG_SAVE_ERROR = 'EXECUTION_CONFIG_SAVE_ERROR'
+export const EXECUTION_CONFIG_CLEAR = 'EXECUTION_CONFIG_CLEAR'
+export const EXECUTION_CONFIG_CLEAR_SUCCESS = 'EXECUTION_CONFIG_CLEAR_SUCCESS'
+export const EXECUTION_CONFIG_CLEAR_ERROR = 'EXECUTION_CONFIG_CLEAR_ERROR'
+
+export const EXECUTION_PREPROCESS_DATASET = 'EXECUTION_PREPROCESS_DATASET'
+export const EXECUTION_PREPROCESS_DATASET_SUCCESS = 'EXECUTION_PREPROCESS_DATASET_SUCCESS'
+export const EXECUTION_PREPROCESS_DATASET_ERROR = 'EXECUTION_PREPROCESS_DATASET_ERROR'
+export const EXECUTION_PREPROCESS_DATASET_SCHEDULED = 'EXECUTION_PREPROCESS_DATASET_SCHEDULED'
+export const EXECUTION_PREPROCESS_DATASET_PROCESSING_SCHEDULED = 'EXECUTION_PREPROCESS_DATASET_PROCESSING_SCHEDULED'
+export const EXECUTION_PREPROCESS_DATASET_PROCESSING_STATUS = 'EXECUTION_PREPROCESS_DATASET_PROCESSING_STATUS'
+export const EXECUTION_PREPROCESS_DATASET_PROCESSING_RESULT = 'EXECUTION_PREPROCESS_DATASET_PROCESSING_RESULT'
+export const EXECUTION_PREPROCESS_DATASET_PROCESSING_NODELOG = 'EXECUTION_PREPROCESS_DATASET_PROCESSING_NODELOG'
+export const EXECUTION_PREPROCESS_DATASET_PROCESSING_CRASH = 'EXECUTION_PREPROCESS_DATASET_PROCESSING_CRASH'
+export const EXECUTION_PREPROCESS_DATASET_PROCESSING_CRASH_ERROR = 'EXECUTION_PREPROCESS_DATASET_PROCESSING_CRASH_ERROR'
+export const EXECUTION_PREPROCESS_DATASET_PROCESSING_LOG = 'EXECUTION_PREPROCESS_DATASET_PROCESSING_LOG'
+export const EXECUTION_PREPROCESS_DATASET_PROCESSING_LOG_ERROR = 'EXECUTION_PREPROCESS_DATASET_PROCESSING_LOG_ERROR'
+export const EXECUTION_PREPROCESS_DATASET_PROCESSING_FINISHED = 'EXECUTION_PREPROCESS_DATASET_PROCESSING_FINISHED'
+export const EXECUTION_PREPROCESS_DATASET_PROCESSING_ERROR = 'EXECUTION_PREPROCESS_DATASET_PROCESSING_ERROR'
+export const EXECUTION_PREPROCESS_DATASET_FINISHED = 'EXECUTION_PREPROCESS_DATASET_FINISHED'
+export const EXECUTION_PREPROCESS_DATASET_FETCHED = 'EXECUTION_PREPROCESS_DATASET_FETCHED'
+
+export function preprocessDataset(execution, scheduler, dataset, pipeline, note=null) {
+  // const execution = uuid()
+  return {
+    type: EXECUTION_PREPROCESS_DATASET,
+    execution, scheduler, dataset, pipeline, note
+  }
+}