Added wail-ui/constants/(uiStrings,acronyms).js and refactored 40 files

fixes #51 relaxed minimum width for WAIL started to address #77 refactored large list rendering components to not use arrow functions refactored components to use bound methods for setState callbacks
N0taN3rd · May 11, 2017 · a3936ad · a3936ad
1 parent 2d26cb5
commit a3936ad
Show file tree

Hide file tree

Showing 117 changed files with 2,462 additions and 1,290 deletions.
diff --git a/package.json b/package.json
@@ -58,39 +58,39 @@
     "bytewise": "^1.1.0",
     "checksum": "^0.1.1",
     "cheerio": "^0.22.0",
-    "chokidar": "^1.6.1",
-    "codemirror": "^5.25.0",
+    "chokidar": "^1.7.0",
+    "codemirror": "^5.25.2",
     "del": "2.2.2",
     "drag-drop": "^2.13.2",
     "dropbox": "^2.5.2",
-    "electron": "^1.6.6",
+    "electron": "^1.6.9",
     "electron-oauth-twitter": "^1.1.0",
     "electron-settings": "1.1.1",
     "electron-window": "^0.8.1",
     "eventemitter3": "^2.0.3",
-    "extract-zip": "^1.6.0",
+    "extract-zip": "^1.6.5",
     "file-exists": "^3.0.2",
     "filenamify": "^2.0.0",
     "filenamify-url": "^1.0.0",
     "find-process": "^1.0.5",
     "flexboxgrid": "^6.3.1",
-    "fs-extra": "2.1.2",
-    "fuse.js": "^2.6.2",
+    "fs-extra": "3.0.1",
+    "fuse.js": "^2.7.3",
     "get-port": "^3.1.0",
     "graceful-fs": "^4.1.11",
     "history": "^4.6.1",
     "http-mitm-proxy": "^0.5.2",
     "immutable": "^3.8.1",
     "is-running": "^2.1.0",
     "joinable": "^1.4.1",
-    "js-search": "^1.4.0",
+    "js-search": "^1.4.2",
     "js-yaml": "^3.8.3",
     "key-path-helpers": "^0.4.0",
     "keymirror": "^0.1.1",
     "klaw": "^1.3.1",
     "lodash": "^4.17.4",
     "material-design-icons-iconfont": "^3.0.3",
-    "material-ui": "^0.17.3",
+    "material-ui": "^0.18.0",
     "mime-types": "^2.1.15",
     "mkdirp": "^0.5.1",
     "moment": "^2.18.1",
@@ -108,28 +108,28 @@
     "pretty-bytes": "^4.0.2",
     "pretty-ms": "^2.1.0",
     "pretty-seconds": "^0.2.1",
-    "prop-types": "^15.5.8",
+    "prop-types": "^15.5.9",
     "ps-tree": "^1.1.0",
     "react": "15.5.4",
     "react-addons-shallow-compare": "^15.5.2",
     "react-addons-update": "^15.5.2",
     "react-dom": "^15.5.4",
     "react-fittext": "0.0.7",
     "react-flex": "^2.2.8",
-    "react-notification-system": "^0.2.13",
+    "react-notification-system": "^0.2.14",
     "react-redux": "^5.0.4",
     "react-router": "^4.1.1",
     "react-router-dom": "^4.1.1",
     "react-router-redux": "^5.0.0-alpha.6",
-    "react-swipeable-views": "^0.12.0",
+    "react-swipeable-views": "^0.12.1",
     "react-tap-event-plugin": "^2.0.1",
     "react-textfit": "^0.1.9",
     "recompose": "^0.23.1",
     "redux": "^3.6.0",
-    "redux-batched-actions": "^0.1.5",
+    "redux-batched-actions": "^0.2.0",
     "redux-electron-ipc": "^1.1.9",
-    "redux-form": "^6.6.3",
-    "redux-form-material-ui": "^4.1.3",
+    "redux-form": "^6.7.0",
+    "redux-form-material-ui": "^4.2.0",
     "redux-ignore": "^1.2.4",
     "redux-immutable": "^4.0.0",
     "redux-observable": "^0.14.1",
@@ -138,9 +138,9 @@
     "redux-undo": "^0.6.1",
     "request": "^2.81.0",
     "request-progress": "^3.0.0",
-    "request-promise": "^4.2.0",
+    "request-promise": "^4.2.1",
     "roboto-fontface": "^0.7.0",
-    "rxjs": "^5.3.0",
+    "rxjs": "^5.4.0",
     "serialize-error": "^2.1.0",
     "shelljs": "0.7.7",
     "sliced": "^1.0.1",
@@ -154,22 +154,22 @@
     "through2": "^2.0.3",
     "tldjs": "^1.7.0",
     "twit": "^2.2.5",
-    "twix": "^1.1.4",
+    "twix": "^1.1.5",
     "unzip": "^0.1.11",
     "url-type": "^0.2.1",
     "uuid": "^3.0.1",
     "valid-url": "^1.0.9",
     "validator": "^7.0.0",
     "warning": "^3.0.0",
-    "yamljs": "^0.2.9"
+    "yamljs": "^0.2.10"
   },
   "devDependencies": {
     "autoprefixer": "^6.7.7",
     "ava": "^0.19.1",
     "babel-cli": "^6.24.1",
     "babel-core": "^6.24.1",
-    "babel-eslint": "^7.2.2",
-    "babel-loader": "^6.4.1",
+    "babel-eslint": "^7.2.3",
+    "babel-loader": "^7.0.0",
     "babel-plugin-add-module-exports": "0.2.1",
     "babel-plugin-espower": "^2.3.2",
     "babel-plugin-react-html-attrs": "2.0.0",
@@ -192,12 +192,12 @@
     "babel-plugin-transform-object-rest-spread": "^6.23.0",
     "babel-plugin-transform-react-constant-elements": "^6.23.0",
     "babel-plugin-transform-react-inline-elements": "^6.22.0",
-    "babel-plugin-transform-react-remove-prop-types": "^0.4.0",
+    "babel-plugin-transform-react-remove-prop-types": "^0.4.5",
     "babel-plugin-transform-remove-console": "^6.8.1",
     "babel-plugin-transform-runtime": "^6.23.0",
     "babel-plugin-webpack-loaders": "^0.9.0",
     "babel-polyfill": "^6.23.0",
-    "babel-preset-env": "1.3.3",
+    "babel-preset-env": "1.4.0",
     "babel-preset-es2017-node7": "^0.5.2",
     "babel-preset-node7": "^1.5.0",
     "babel-preset-react": "^6.24.1",
@@ -211,19 +211,19 @@
     "cron-parser": "^2.4.0",
     "cross-env": "^4.0.0",
     "cross-zip": "^2.1.5",
-    "css-loader": "^0.28.0",
-    "debug": "^2.6.3",
+    "css-loader": "^0.28.1",
+    "debug": "^2.6.4",
     "depcheck": "^0.6.7",
     "devtron": "^1.4.0",
     "electron-chromedriver": "^1.6.0",
     "electron-debug": "^1.1.0",
     "electron-devtools-installer": "^2.2.0",
     "electron-installer-dmg": "^0.2.1",
-    "electron-packager": "^8.6.0",
+    "electron-packager": "^8.7.0",
     "electron-rebuild": "^1.5.7",
     "enzyme": "^2.8.2",
     "eslint": "^3.19.0",
-    "eslint-plugin-react": "^6.10.3",
+    "eslint-plugin-react": "^7.0.0",
     "extract-text-webpack-plugin": "^2.1.0",
     "fbjs-scripts": "^0.7.1",
     "file-loader": "^0.11.1",
@@ -240,16 +240,15 @@
     "request-debug": "^0.2.0",
     "run-sequence": "^1.2.2",
     "snazzy": "^7.0.0",
-    "spectron": "^3.6.1",
-    "standard": "^10.0.1",
+    "spectron": "^3.6.2",
+    "standard": "^10.0.2",
     "standard-format": "^2.2.4",
     "strip-loader": "^0.1.2",
-    "style-loader": "0.16.1",
-    "uglify-js": "git://github.com/mishoo/UglifyJS2.git#harmony",
+    "style-loader": "0.17.0",
     "url-loader": "^0.5.8",
-    "webpack": "2.3.3",
-    "webpack-dev-middleware": "1.10.1",
-    "webpack-dev-server": "2.4.2",
+    "webpack": "2.5.1",
+    "webpack-dev-middleware": "1.10.2",
+    "webpack-dev-server": "2.4.5",
     "webpack-externals-plugin": "1.0.0",
     "webpack-hot-middleware": "2.18.0",
     "webpack-middleware": "1.5.1",

diff --git a/wail-core/fs/index.js b/wail-core/fs/index.js
@@ -1,6 +1,6 @@
 import chokidar from 'chokidar'
 import walk from 'klaw'
-import fsep from './fsePromise'
+import fs from 'fs-extra'
 import directorySize from './directorySize'
 import pathExists from './pathExists'
 import pathIs from './pathIs'
@@ -12,4 +12,4 @@ const base = {
   FSWatcher: chokidar.FSWatcher
 }
 
-export default completeAssign(base, fsep, directorySize, pathExists, pathIs)
+export default completeAssign(base, fs, directorySize, pathExists, pathIs)
diff --git a/wail-core/managers/appManager.js b/wail-core/managers/appManager.js
@@ -121,10 +121,8 @@ export default class AppManager2 {
 
     this.winConf = {
       width: this.w,
-      minWidth: this.w,
       // maxWidth: this.w,
       height: this.h,
-      minHeight: this.h,
       // maxHeight: this.h,
       title: 'Web Archiving Integration Layer',
       show: false,

diff --git a/wail-core/managers/archiveMan.js b/wail-core/managers/archiveMan.js
@@ -213,7 +213,7 @@ export default class ArchiveManager extends EventEmitter {
       console.log('no exist')
       return await this._handleColDirNoExistence(noExist)
     }
-    let {exist, empty, doNotExist} = this._getAllColsWithExistCheck()
+    let {exist, empty, doNotExist} = await this._getAllColsWithExistCheck()
     let colSeeds = await this._colSeeds.wemFindAll('Could not retrieve the seeds from the database')
     if (empty) {
       if (colSeeds.length > 0) {

diff --git a/wail-core/managers/heritrix/crawlManager.js b/wail-core/managers/heritrix/crawlManager.js
@@ -24,7 +24,7 @@ const pathMan = remote.getGlobal('pathMan')
 export default class CrawlManager {
   constructor () {
     this.db = new Db({
-      filename: pathMan.join(settings.get('wailCore.db'), 'crawls.db'),
+      filename: path.join(settings.get('wailCore.db'), 'crawls.db'),
       autoload: true
     })
     this.csMonitor = new CrawlStatsMonitor()
@@ -63,7 +63,7 @@ export default class CrawlManager {
       // so babel will make this Math.max.apply(Math,array)
       let latestLaunch = Math.max(...files.filter(item => this.launchId.test(item)))
       let warcPath = path.join(jobPath, `${latestLaunch}`, 'warcs', '*.warc')
-      ipc.send('add-warcs-to-col', { forCol, warcs: warcPath })
+      ipc.send('add-warcs-to-col', {forCol, warcs: warcPath})
     })
   }
 
@@ -76,7 +76,7 @@ export default class CrawlManager {
           console.log(docs)
           let pDocs = docs
           if (pDocs.length > 0) {
-            pDocs = _.orderBy(pDocs.map(r => new CrawlInfo(r)), [ 'jobId' ], [ 'desc' ])
+            pDocs = _.orderBy(pDocs.map(r => new CrawlInfo(r)), ['jobId'], ['desc'])
           }
           resolve(pDocs)
         }
@@ -85,7 +85,7 @@ export default class CrawlManager {
   }
 
   crawlStarted (jobId) {
-    this.db.update({ jobId }, { $set: { running: true } }, { returnUpdatedDocs: true }, (error, numUpdated, updated) => {
+    this.db.update({jobId}, {$set: {running: true}}, {returnUpdatedDocs: true}, (error, numUpdated, updated) => {
       if (error) {
         console.error('error inserting document', error)
         ipc.send('managers-error', {
@@ -118,7 +118,7 @@ export default class CrawlManager {
         }
       }
     }
-    this.db.update({ jobId: update.jobId }, theUpdate, { returnUpdatedDocs: true }, (error, numUpdated, updated) => {
+    this.db.update({jobId: update.jobId}, theUpdate, {returnUpdatedDocs: true}, (error, numUpdated, updated) => {
       if (error) {
         console.error('error updating document', update, error)
         ipc.send('managers-error', {
@@ -134,7 +134,7 @@ export default class CrawlManager {
           col: updated.forCol,
           warcs: update.stats.warcs,
           lastUpdated: update.stats.timestamp,
-          seed: { url: updated.urls, jobId: update.jobId }
+          seed: {url: updated.urls, jobId: update.jobId}
         })
       }
     })
@@ -143,7 +143,7 @@ export default class CrawlManager {
   areCrawlsRunning () {
     console.log('checking if crawls are running')
     return new Promise((resolve, reject) => {
-      this.db.count({ running: true }, (err, runningCount) => {
+      this.db.count({running: true}, (err, runningCount) => {
         if (err) {
           console.error('error finding if crawls are running')
           reject(err)
@@ -156,13 +156,82 @@ export default class CrawlManager {
     })
   }
 
+  _insert (data) {
+    return new Promise((resolve, reject) => {
+      this.db.insert(data, (err, doc) => {
+        if (err) {
+          reject(err)
+        } else {
+          resolve(doc)
+        }
+      })
+    })
+  }
+
+  makeCrawlConf2 (options) {
+    let {urls, forCol, jobId, depth} = options
+    return fs.readFile(settings.get('heritrix.jobConf'), 'utf8').then(data => {
+      let doc = cheerio.load(data, {xmlMode: true})
+      if (!jobId) {
+        jobId = new Date().getTime()
+      }
+      let urlConf = doc('bean[id="longerOverrides"]').find('prop[key="seeds.textSource.value"]')
+      let urlText
+      if (Array.isArray(urls)) {
+        urlText = `${os.EOL}${urls.join(os.EOL)}${os.EOL}`
+      } else {
+        urlText = `${os.EOL}${urls}${os.EOL}`
+      }
+      urlConf.text(urlText)
+      let maxHops = doc('bean[class="org.archive.modules.deciderules.TooManyHopsDecideRule"]').find('property[name="maxHops"]')
+      maxHops.attr('value', `${depth}`)
+      let confPath = path.join(settings.get('heritrix.jobsDir'), `${jobId}`)
+      let cfp = path.join(confPath, 'crawler-beans.cxml')
+      return fs.ensureDir(confPath).then(() =>
+        fs.writeFile(cfp, doc.xml(), 'utf8').then(() => {
+          let crawlInfo = {
+            depth,
+            jobId,
+            latestRun: {
+              ending: false,
+              started: false,
+              ended: true,
+              timestamp: jobId,
+              discovered: 0,
+              queued: 0,
+              downloaded: 0
+            },
+            path: path.join(settings.get('heritrix.jobsDir'), `${jobId}`),
+            confP: cfp,
+            urls: urls,
+            running: false,
+            forCol
+          }
+          // {"url":"http://cs.odu.edu","jobIds":[1473098189935],"mementos":1,"added":"2016-09-05T13:56:29-04:00","lastUpdated":"2016-09-16T00:12:16-04:00"}
+          let lastUpdated = moment(jobId).format()
+          ipc.send('made-heritrix-jobconf', {
+            forMain: {
+              crawlInfo,
+              jobId
+            },
+            forArchives: {
+              forCol,
+              lastUpdated,
+              seed: {forCol, url: urls, jobIds: [jobId], lastUpdated, added: lastUpdated, mementos: 0}
+            }
+          })
+          return this._insert({_id: `${jobId}`, ...crawlInfo}).then(() => crawlInfo)
+        }))
+    })
+  }
+
   /**
    * Creates a new crawler-beans.cxml(heritrix crawl config)
    * @param {Object} options
    * @returns {Promise|Promise<Object>}
    */
   makeCrawlConf (options) {
-    let { urls, forCol, jobId, depth } = options
+    let {urls, forCol, jobId, depth} = options
     return new Promise((resolve, reject) =>
       readFile(settings.get('heritrix.jobConf'))
         .then(data => {
@@ -216,11 +285,11 @@ export default class CrawlManager {
                 forArchives: {
                   forCol,
                   lastUpdated,
-                  seed: { forCol, url: urls, jobIds: [ jobId ], lastUpdated, added: lastUpdated, mementos: 0 }
+                  seed: {forCol, url: urls, jobIds: [jobId], lastUpdated, added: lastUpdated, mementos: 0}
                 }
               })
 
-              this.db.insert({ _id: `${jobId}`, ...crawlInfo }, (iError, doc) => {
+              this.db.insert({_id: `${jobId}`, ...crawlInfo}, (iError, doc) => {
                 if (iError) {
                   console.error(iError)
                   reject(iError)
@@ -241,7 +310,7 @@ export default class CrawlManager {
   }
 
   removeCrawl (jobId) {
-    this.db.remove({ _id: `${jobId}` }, (err, numRemoved) => {
+    this.db.remove({_id: `${jobId}`}, (err, numRemoved) => {
       if (err) {
         console.error(`Could not remove crawl for ${jobId}`, err)
         ipc.send('managers-error', {

diff --git a/wail-core/util/collectionsUtils.js b/wail-core/util/collectionsUtils.js
@@ -1,5 +1,5 @@
 import Promise from 'bluebird'
-import fs from 'fs-extra'
+import fs from '../fs'
 import path from 'path'
 import through2 from 'through2'
 import prettyBytes from 'pretty-bytes'