Merge pull request #294 from pmoleri/async-io

Async IO

app/core/contents.js

@@ -2,23 +2,19 @@
 'use strict';
 
 const path = require('path');
-const fs = require('fs');
-const glob = require('glob');
+const fs = require('fs-extra');
 const _ = require('underscore');
 const _s = require('underscore.string');
 const yaml = require('js-yaml');
 const utils = require('./utils');
 const contentProcessors = require('../functions/contentProcessors');
 
-function handler (activePageSlug, config) {
+async function handler (activePageSlug, config) {
  activePageSlug = activePageSlug || '';
  const baseSlug = activePageSlug.split(/[\\/]/).slice(0, -1).join('/');
- const page_sort_meta = config.page_sort_meta || '';
- const category_sort = config.category_sort || false;
  const contentDir = utils.normalizeDir(path.normalize(config.content_dir));
 
- const files = glob.sync(contentDir + '**/*');
- const content_dir = path.normalize(contentDir);
+ const files = await utils.promiseGlob(contentDir + '**/*');
  const filesProcessed = [];
 
  filesProcessed.push({
@@ -32,105 +28,115 @@ function handler (activePageSlug, config) {
  files: []
  });
 
- files.forEach(filePath => {
+ const results = await Promise.all(
+ files.map(filePath => processFile(config, activePageSlug, contentDir, filePath))
+ );
+
+ for (const result of results) {
+ if (result && result.is_directory) {
+ filesProcessed.push(result);
+ } else if (result && !result.is_directory) {
+ const dirSlug = path.dirname(result.slug);
+ const parent = _.find(filesProcessed, item => item.slug === dirSlug);
+ parent.files.push(result);
+ }
+ }
 
- const shortPath = path.normalize(filePath).replace(content_dir, '').trim();
- const fileSlug = shortPath.split('\\').join('/');
- let stat = fs.lstatSync(filePath);
+ const sortedFiles = _.sortBy(filesProcessed, cat => cat.sort);
+ sortedFiles.forEach(category => {
+ category.files = _.sortBy(category.files, file => file.sort);
+ });
 
- if (stat.isSymbolicLink()) {
- stat = fs.lstatSync(fs.readlinkSync(filePath));
- }
+ return sortedFiles;
+}
 
- if (stat.isDirectory()) {
+async function processFile (config, activePageSlug, contentDir, filePath) {
+ const content_dir = path.normalize(contentDir);
+ const page_sort_meta = config.page_sort_meta || '';
+ const category_sort = config.category_sort || false;
+ const shortPath = path.normalize(filePath).replace(content_dir, '').trim();
+ const fileSlug = shortPath.split('\\').join('/');
+ const stat = await fs.stat(filePath);
+
+ if (stat.isDirectory()) {
+
+ let sort = 0;
+ // ignore directories that has an ignore file under it
+ const ignoreFile = contentDir + shortPath + '/ignore';
 
- let sort = 0;
- // ignore directories that has an ignore file under it
- const ignoreFile = contentDir + shortPath + '/ignore';
+ const ignoreExists = await fs.lstat(ignoreFile).then(stat => stat.isFile(), () => {});
+ if (ignoreExists) {
+ return true;
+ }
 
- if (fs.existsSync(ignoreFile) && fs.lstatSync(ignoreFile).isFile()) {
- return true;
+ let dirMetadata = {};
+ try {
+ const metaFile = await fs.readFile(path.join(contentDir, shortPath, 'meta'));
+ dirMetadata = contentProcessors.cleanObjectStrings(yaml.safeLoad(metaFile.toString('utf-8')));
+ } catch (e) {
+ if (config.debug) {
+ console.log('No meta file for', contentDir + shortPath);
  }
+ }
 
-  let dirMetadata = {};
+ if (category_sort && !dirMetadata.sort) {
  try {
- const metaFile = fs.readFileSync(contentDir + shortPath + '/meta');
- dirMetadata = contentProcessors.cleanObjectStrings(yaml.safeLoad(metaFile.toString('utf-8')));
+ const sortFile = await fs.readFile(path.join(contentDir, shortPath, 'sort'));
+ sort = parseInt(sortFile.toString('utf-8'), 10);
  } catch (e) {
  if (config.debug) {
- console.log('No meta file for', contentDir + shortPath);
- }
- }
-
- if (category_sort && !dirMetadata.sort) {
- try {
- const sortFile = fs.readFileSync(contentDir + shortPath + '/sort');
- sort = parseInt(sortFile.toString('utf-8'), 10);
- } catch (e) {
- if (config.debug) {
- console.log('No sort file for', contentDir + shortPath);
- }
+ console.log('No sort file for', contentDir + shortPath);
  }
  }
-
- filesProcessed.push({
- slug: shortPath,
- title: dirMetadata.title || _s.titleize(_s.humanize(path.basename(shortPath))),
- show_on_home: dirMetadata.show_on_home ? (dirMetadata.show_on_home === 'true') : config.show_on_home_default,
- is_index: false,
- is_directory: true,
- active: activePageSlug.startsWith('/' + fileSlug),
- class: 'category-' + contentProcessors.cleanString(shortPath),
- sort: dirMetadata.sort || sort,
- files: []
- });
-
  }
 
- if (stat.isFile() && path.extname(shortPath) === '.md') {
- try {
-
- const file = fs.readFileSync(filePath);
- let slug = fileSlug;
- let pageSort = 0;
-
- if (fileSlug.indexOf('index.md') > -1) {
- slug = slug.replace('index.md', '');
- }
-
- slug = slug.replace('.md', '').trim();
+ return {
+ slug: fileSlug,
+ title: dirMetadata.title || _s.titleize(_s.humanize(path.basename(shortPath))),
+ show_on_home: dirMetadata.show_on_home ? (dirMetadata.show_on_home === 'true') : config.show_on_home_default,
+ is_index: false,
+ is_directory: true,
+ active: activePageSlug.startsWith('/' + fileSlug),
+ class: 'category-' + contentProcessors.cleanString(fileSlug),
+ sort: dirMetadata.sort || sort,
+ files: []
+ };
+
+ }
+
+ if (stat.isFile() && path.extname(shortPath) === '.md') {
+ try {
+ const file = await fs.readFile(filePath);
+ let slug = fileSlug;
+ let pageSort = 0;
+
+ if (fileSlug.indexOf('index.md') > -1) {
+ slug = slug.replace('index.md', '');
+ }
 
- const dir = path.dirname(shortPath);
- const meta = contentProcessors.processMeta(file.toString('utf-8'));
+ slug = slug.replace('.md', '').trim();
 
- if (page_sort_meta && meta[page_sort_meta]) {
- pageSort = parseInt(meta[page_sort_meta], 10);
- }
+ const meta = contentProcessors.processMeta(file.toString('utf-8'));
 
- const val = _.find(filesProcessed, item => item.slug === dir);
- val.files.push({
- slug: slug,
- title: meta.title ? meta.title : contentProcessors.slugToTitle(slug),
- show_on_home: meta.show_on_home ? (meta.show_on_home === 'true') : config.show_on_home_default,
- is_directory: false,
- active: (activePageSlug.trim() === '/' + slug),
- sort: pageSort
- });
+ if (page_sort_meta && meta[page_sort_meta]) {
+ pageSort = parseInt(meta[page_sort_meta], 10);
+ }
 
- } catch (e) {
- if (config.debug) {
- console.log(e);
- }
+ return {
+ slug: slug,
+ title: meta.title ? meta.title : contentProcessors.slugToTitle(slug),
+ show_on_home: meta.show_on_home ? (meta.show_on_home === 'true') : config.show_on_home_default,
+ is_directory: false,
+ active: (activePageSlug.trim() === '/' + slug),
+ sort: pageSort
+ };
+
+ } catch (e) {
+ if (config.debug) {
+ console.log(e);
  }
  }
- });
-
- const sortedFiles = _.sortBy(filesProcessed, cat => cat.sort);
- sortedFiles.forEach(category => {
- category.files = _.sortBy(category.files, file => file.sort);
- });
-
- return sortedFiles;
+ }
 }
 
 exports.default = handler;

app/core/page.js

@@ -2,17 +2,17 @@
 'use strict';
 
 const path = require('path');
-const fs = require('fs');
+const fs = require('fs-extra');
 const utils = require('./utils');
 const _s = require('underscore.string');
 const marked = require('marked');
 const contentProcessors = require('../functions/contentProcessors');
 
-function handler (filePath, config) {
+async function handler (filePath, config) {
  const contentDir = utils.normalizeDir(path.normalize(config.content_dir));
 
  try {
- const file = fs.readFileSync(filePath);
+ const file = await fs.readFile(filePath);
  let slug = utils.getSlug(filePath, contentDir);
 
  if (slug.indexOf('index.md') > -1) {

app/core/search.js

@@ -2,7 +2,6 @@
 'use strict';
 
 const path = require('path');
-const glob = require('glob');
 const contentProcessors = require('../functions/contentProcessors');
 const utils = require('./utils');
 const pageHandler = require('./page');
@@ -30,13 +29,15 @@ function getStemmers (config) {
  return stemmers;
 }
 
-function handler (query, config) {
+async function handler (query, config) {
  const contentDir = utils.normalizeDir(path.normalize(config.content_dir));
- const documents = glob
-  .sync(contentDir + '**/*.md')
- .map(filePath => contentProcessors.extractDocument(
+ const rawDocuments = await utils.promiseGlob(contentDir + '**/*.md');
+ const potentialDocuments = await Promise.all(
+ rawDocuments.map(filePath => contentProcessors.extractDocument(
  contentDir, filePath, config.debug
  ))
+ );
+ const documents = potentialDocuments
  .filter(doc => doc !== null);
 
  const lunrInstance = getLunr(config);
@@ -48,17 +49,21 @@ function handler (query, config) {
  documents.forEach((doc) => this.add(doc), this);
  });
 
- const results = idx.search(query);
- const searchResults = [];
+ const results = idx.search(query);
 
- results.forEach(result => {
- const p = pageHandler(contentDir + result.ref, config);
- p.excerpt = p.excerpt.replace(new RegExp('(' + query + ')', 'gim'), '<span class="search-query">$1</span>');
- searchResults.push(p);
- });
+ const searchResults = await Promise.all(
+ results.map(result => processSearchResult(contentDir, config, query, result))
+ );
 
  return searchResults;
 }
 
+async function processSearchResult (contentDir, config, query, result) {
+ const page = await pageHandler(contentDir + result.ref, config);
+ page.excerpt = page.excerpt.replace(new RegExp('(' + query + ')', 'gim'), '<span class="search-query">$1</span>');
+
+ return page;
+}
+
 exports.default = handler;
 module.exports = exports.default;

app/core/utils.js

@@ -1,24 +1,30 @@
 
 'use strict';
 
-const fs = require('fs');
+const glob = require('glob');
+const fs = require('fs-extra');
+const util = require('util');
 const moment = require('moment');
 
+const promiseGlob = util.promisify(glob);
+
 const normalizeDir = (dir) => dir.replace(/\\/g, '/');
 const getSlug = (filePath, contentDir) => normalizeDir(filePath).replace(normalizeDir(contentDir), '').trim();
 
-function getLastModified (config, meta, file_path) {
+async function getLastModified (config, meta, file_path) {
  if (typeof meta.modified !== 'undefined') {
  return moment(meta.modified).format(config.datetime_format);
  } else {
- return moment(fs.lstatSync(file_path).mtime).format(config.datetime_format);
+ const { mtime } = await fs.lstat(file_path);
+ return moment(mtime).format(config.datetime_format);
  }
 }
 
 exports.default = {
  normalizeDir,
  getLastModified,
- getSlug
+ getSlug,
+ promiseGlob
 };
 
 module.exports = exports.default;

app/functions/build_nested_pages.js

@@ -1,14 +1,12 @@
 
 'use strict';
 
-var path = require('path');
-
 function build_nested_pages (pages) {
  var result = [];
  var i = pages.length;
 
  while (i--) {
- if (pages[i].slug.split(path.sep).length > 1) {
+ if (pages[i].slug.split('/').length > 1) {
  var parent = find_by_slug(pages, pages[i]);
  parent.files.unshift(pages[i]);
  } else {
@@ -21,7 +19,7 @@ function build_nested_pages (pages) {
 
 function find_by_slug (pages, page) {
  return pages.find(function (element) {
- return element.slug === page.slug.split(path.sep).slice(0, -1).join(path.sep);
+ return element.slug === page.slug.split('/').slice(0, -1).join('/');
  });
 }
 

app/functions/contentProcessors.js

@@ -2,7 +2,7 @@
 'use strict';
 
 const path = require('path');
-const fs = require('fs');
+const fs = require('fs-extra');
 const _s = require('underscore.string');
 const yaml = require('js-yaml');
 
@@ -112,9 +112,9 @@ function processVars (markdownContent, config) {
  return markdownContent;
 }
 
-function extractDocument (contentDir, filePath, debug) {
+async function extractDocument (contentDir, filePath, debug) {
  try {
- const file = fs.readFileSync(filePath);
+ const file = await fs.readFile(filePath);
  const meta = processMeta(file.toString('utf-8'));
 
  const id = filePath.replace(contentDir, '').trim();

app/index.js

@@ -31,8 +31,7 @@ function initialize (config) {
  var oauth2 = require('./middleware/oauth2.js');
  var route_login = require('./routes/login.route.js') (config);
  var route_login_page = require('./routes/login_page.route.js') (config);
- var route_logout = require('./routes/logout.route.js')
- (config);
+ var route_logout = require('./routes/logout.route.js') (config);
  var route_page_edit = require('./routes/page.edit.route.js') (config);
  var route_page_delete = require('./routes/page.delete.route.js') (config);
  var route_page_create = require('./routes/page.create.route.js') (config);
@@ -73,7 +72,7 @@ function initialize (config) {
  router.use(express.static(path.join(config.theme_dir, config.theme_name, 'public')));
  }
  router.use(config.image_url, express.static(path.normalize(config.content_dir + config.image_url)));
- router.use('/translations', express.static(path.normalize(__dirname + '/translations')));
+ router.use('/translations', express.static(path.normalize(path.join(__dirname, 'translations'))));
 
  // HTTP Authentication
  if (config.authentication === true || config.authentication_for_edit || config.authentication_for_read) {