Feature/env variables

README.md

@@ -37,6 +37,12 @@ You find an example in [`test/template-escape`](test/template-escape).
 If you want to use for example `$(_name)` as both an external reference and a normal reference,
 then you add a `\` for the latter resulting in `$(\_name)` for the latter.
 
+If your YARRRML document contains sensitive information such as database credentials,
+you can use dollar variables with curly braces and specify a variable in a `.env` file.
+For example, use `${DB_PASSWORD}` in the YARRRML document and add a `.env` file with content `DB_PASSWORD=mySecretPassword`.
+During generation of an RML document, the variables will be replaced.
+If a variable is not found in the current environment, the yarrrml-parser will emit a warning.
+
 #### yarrrml-generator
 
 If you want to generate YARRRML rules from an RML document, you do the following: `yarrrml-generator -i rules.rml.ttl`.

bin/parser.js

@@ -17,6 +17,9 @@ const watch = require('../lib/watcher.js');
 const glob = require('glob');
 const Logger = require('../lib/logger');
 
+// load environment variables from .env into process.env
+require('dotenv').config();
+
 namespaces.ql = 'http://semweb.mmlab.be/ns/ql#';
 
 pkginfo(module, 'version');

lib/abstract-generator.js

@@ -48,8 +48,35 @@ class AbstractGenerator {
 
       let json;
 
+      // replace found variables with the values from .env
+      let populatedYarrrml = yarrrml;
+      let reVars = /.*?(\${.*?}).*?/gm;
+      let usedEnvVariables = yarrrml.matchAll(reVars);
+      let nonFoundVars = [];
+      for(let match of usedEnvVariables) {
+        let found = match[1];
+        // extract var name, e.g. DB_HOST from ${DB_HOST} so we can compare with env variables
+        let varName = found.substring(2, (found.length - 1) );
+
+        if(varName in process.env) {
+          /* split-join solution because 'replaceAll' might not exist and 'replace' needs a global flag
+           * which is problematic because then a regex has to be used instead of a string
+           * and then our ${} variables are not recognized
+           * see Stackoverlow: https://stackoverflow.com/a/542305
+           */
+          populatedYarrrml = populatedYarrrml.split(found).join(process.env[varName]);
+
+        } else {
+          nonFoundVars.push(varName);
+        }
+      }
+
+      if(nonFoundVars.length > 0) {
+        Logger.warn(`No value set for the following used environment variables: ${nonFoundVars}`);
+      }
+
       try {
-        json = YAML.parse(yarrrml);
+        json = YAML.parse(populatedYarrrml);
       } catch (e) {
         e.code = 'INVALID_YAML';
         e.file = file

lib/rml-generator.test.js

@@ -75,6 +75,48 @@ describe('YARRRML to RML', function () {
     work('template-escape/mapping.yml', 'template-escape/mapping.rml.ttl', done, {includeMetadata: false});
   });
 
+
+
+  describe('environment variables', function() {
+    let envCache;
+
+    beforeEach(() => {
+      // cache current environment to reset after tests
+      envCache = Object.assign({}, process.env);
+    });
+
+    afterEach(() => {
+      // reset environment
+      process.env = envCache;
+    });
+
+    it('works for expanded environment variables', function(done) {
+      process.env.DB_USER = "dbUser";
+      process.env.DB_PASSWORD = "dbPassword";
+      process.env.DB_HOST = "myHost";
+      process.env.DB_PORT = "myPort";
+      process.env.DB = "myDB";
+      work('env-variables/env-replaced/mapping.yml', 'env-variables/env-replaced/mapping.rml.ttl', done, {includeMetadata: true});
+    });
+
+    it('has warning if no environment variable was replaced', () => {
+      const y2r = new convertYAMLtoRML();
+
+      y2r.convert(fs.readFileSync(path.resolve(__dirname, '../test/env-variables/env-not-defined-warning-all/mapping.yml'), 'utf8'));
+      assert.strictEqual(y2r.getLogger().has('warning'), true);
+      assert.strictEqual(y2r.getLogger().getAll().length, 1);
+    });
+
+    it('has warning if only a few environment variables were not replaced', () => {
+      process.env.DB_HOST = "myHost";
+      process.env.DB_PORT = "myPort";
+      const y2r = new convertYAMLtoRML();
+      y2r.convert(fs.readFileSync(path.resolve(__dirname, '../test/env-variables/env-not-defined-warning/mapping.yml'), 'utf8'));
+      assert.strictEqual(y2r.getLogger().has('warning'), true);
+      assert.strictEqual(y2r.getLogger().getAll().length, 1);
+    });
+  })
+
   describe('between our worlds rules', function () {
     it('anime', function (done) {
       work('betweenourworlds/anime/mapping.yarrrml', 'betweenourworlds/anime/mapping.rml.ttl', done, {includeMetadata: false});