Support for JSON path, added masking types ref

dandraka · May 30, 2024 · 1dda19b · 1dda19b
1 parent 291d922
commit 1dda19b
Show file tree

Hide file tree

Showing 19 changed files with 830 additions and 79 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -13,7 +13,8 @@
             "args": ["../Zoro.Tests/data/debugconfig.xml"],
             "cwd": "${workspaceFolder}/Zoro",
             "console": "internalConsole",
-            "stopAtEntry": false
+            "stopAtEntry": false,
+            "requireExactSource": true
         },
         {
             "name": ".NET Core Attach",

diff --git a/MaskingTypes.md b/MaskingTypes.md
diff --git a/README.md b/README.md
@@ -46,7 +46,25 @@ var masker = new Zoro.Processor.DataMasking(config);
 masker.Mask();
 ```
 
-**Sample config file:**
+## Documentation:
+
+### Masking types reference
+
+Please see the [anonymization and masking types reference doc](https://github.com/dandraka/Zoro/blob/master/MaskingTypes.md).
+
+### Developer documentation
+
+Please see the [generated docs](https://github.com/dandraka/Zoro/blob/master/docs/Zoro.Processor.md).
+
+### Notes on usage
+
+- If using a database to write data (DataDestination=Database), the names of parameters in SqlCommand ($field) must match the names of FieldMasks.
+- If input is a JSON file (DataSource=JsonFile) and one or more FieldMasks are type List (FieldMask.MaskType=List), one 1 Replacement entry is allowed, which has to have an empty Selector (Selector="").
+- If input is a JSON file (DataSource=JsonFile), FieldMasks that perform a database query (FieldMask.MaskType=Query) are not allowed.
+
+## Examples:
+
+**Sample config file for CSV source and destination**
 
 ```
 <?xml version="1.0"?>
@@ -87,7 +105,7 @@ ID;Name;BankAccount
 4;botaahjazlvojknub.qi;****************
 ```
 
-**A more complete sample of a config file is the following:**
+**Sample config file for DB source and destination using lists and queries**
 
 ```
 <?xml version="1.0"?>
@@ -137,15 +155,77 @@ ID;Name;BankAccount
 </MaskConfig>
 ```
 
-#### Notes on usage
+**Sample config file for JSON source and destination using an Expression and a List**
 
-- If using a database to write data (DataDestination=Database), the number and names of parameters in SqlCommand ($field) must match the number and names of FieldMasks.
-- If input is a JSON file (DataSource=JsonFile) and one or more FieldMasks are type List (FieldMask.MaskType=List), one 1 Replacement entry is allowed, which has to have an empty Selector (Selector="").
-- If input is a JSON file (DataSource=JsonFile), FieldMasks that perform a database query (FieldMask.MaskType=Query) are not allowed.
+```
+<?xml version="1.0"?>
+<MaskConfig xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <FieldMasks>
+    <FieldMask>
+      <FieldName>name</FieldName>
+      <MaskType>Expression</MaskType>
+      <Expression>Customer {{$.id}}</Expression>
+    </FieldMask>
+    <FieldMask>
+      <FieldName>salary</FieldName>
+      <MaskType>Similar</MaskType>
+    </FieldMask>  
+    <FieldMask>
+      <FieldName>spouse</FieldName>
+      <MaskType>List</MaskType>
+		<ListOfPossibleReplacements>
+			<Replacement Selector="" List="Eleni Koufaki,Athina Lefkogianaki,Mihaela Papadomanolaki" />
+		</ListOfPossibleReplacements>
+	</FieldMask>	  
+  </FieldMasks>
+  <InputFile>%TestInstanceDir%\data2.json</InputFile>
+  <OutputFile>%TestInstanceDir%\maskedata2.json</OutputFile>
+  <DataSource>JsonFile</DataSource>
+  <DataDestination>JsonFile</DataDestination>
+</MaskConfig>
+```
 
-### Developer documentation
+The above config file can process for example the following JSON:
 
-Please see the [generated docs](https://github.com/dandraka/Zoro/blob/master/docs/Zoro.Processor.md).
+```
+{
+    "employees": [
+        {
+            "id": "1",
+            "name": "Aleksander Singh",
+            "salary": 105000,
+            "spouse": "Ingrid Díaz"
+        },
+        {
+            "id": "2",
+            "name": "Alicja Bakshi",
+            "salary": 142500,
+            "spouse": "Ellinore Alvarez"
+        }
+    ]
+}
+```
+
+and the result will be something like the following:
+
+```
+{
+    "employees": [
+        {
+            "id": "1",
+            "name": "Customer-1",
+            "salary": 902473,
+            "spouse": "Mihaela Papadomanolaki"
+        },
+        {
+            "id": "2",
+            "name": "Customer-2",
+            "salary": 046795,
+            "spouse": "Eleni Koufaki"
+        }
+    ]
+}
+```
 
 ### Note:
 

diff --git a/Zoro.Processor/Dandraka.Zoro.nuspec b/Zoro.Processor/Dandraka.Zoro.nuspec
@@ -2,7 +2,7 @@
 <package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
   <metadata>
     <id>Dandraka.Zoro</id>
-    <version>2.3.0</version>
+    <version>2.3.1</version>
     <authors>Jim (Dimitrios) Andrakakis</authors>
     <requireLicenseAcceptance>true</requireLicenseAcceptance>
     <icon>icon.png</icon>
@@ -14,7 +14,8 @@
       You can copy the tools directory to use the tool as a standalone program for Windows and Linux, both 64bit.</description>
     <releaseNotes>
     Added new masking type 'Expression'.
-		Added full JSON support. 
+    Added full JSON support. 
+    Added masking types reference documentation.    		
 		Added developer docs. 
 		Addressed SQL Data Provider Security Feature Bypass Vulnerability CVE-2024-0056 on Microsoft.Data.SqlClient and System.Data.SqlClient
 	</releaseNotes>
@@ -23,7 +24,8 @@
     <dependencies>
       <group targetFramework=".netstandard2.1">
         <dependency id="GenericParsing" version="1.2.2" />
-        <dependency id="System.Data.SqlClient" version="4.8.6" />        
+        <dependency id="System.Data.SqlClient" version="4.8.6" />   
+        <dependency id="System.Data.OleDb" version="8.0.0" />   
       </group>
     </dependencies>
   </metadata>

diff --git a/Zoro.Processor/DataMasking.cs b/Zoro.Processor/DataMasking.cs
@@ -30,10 +30,31 @@ public class DataMasking
         /// </summary>
         /// <param name="config"></param>
         public DataMasking(MaskConfig config)
-        {
-            this.config = config;
+        {            
+            this.config = config;            
         }
 
+        private static DbProviderFactory GetDbFactory(string connType)
+        {
+            switch(connType)
+            {
+                case "System.Data.SqlClient":
+                    DbProviderFactories.RegisterFactory(connType, typeof(System.Data.SqlClient.SqlClientFactory));
+                    break;
+                case "System.Data.OleDb":
+                    DbProviderFactories.RegisterFactory(connType, typeof(System.Data.OleDb.OleDbFactory));
+                    break; 
+                /*                    
+                case "System.Data.Odbc":
+                    DbProviderFactories.RegisterFactory(connType, typeof(System.Data.Odbc.OdbcFactory));
+                    break;
+                */                                      
+                default:
+                    throw new NotSupportedException($"Connection type {connType} is not yet supported");
+            }            
+            return DbProviderFactories.GetFactory(connType);
+        }        
+
         /// <summary>
         /// Performs masking.
         /// </summary>
@@ -46,7 +67,7 @@ public void Mask()
                 // flat types
                 case DataSource.CsvFile:
                 case DataSource.Database:
-                    AnonymizeFlatData((DataTable)dt);                    
+                    AnonymizeFlatData((DataTable)dt);
                     break;
                 // nested types
                 case DataSource.JsonFile:
@@ -106,7 +127,7 @@ private void AnonymizeJSONProperty(JProperty c)
                         c.Value = s;
                         break;
                 }
-            }            
+            }
 
             if (c.HasValues)
             {
@@ -258,14 +279,14 @@ private string GetMaskedString(string data, FieldMask fieldMask, DataRow row, JP
         }
 
         private string GetExpressionString(DataRow row, string expression, JProperty jsonNode)
-        {            
+        {
             var r = fieldsRegEx.Match(expression);
 
-            foreach(Group rxGroup in r.Groups)
+            foreach (Group rxGroup in r.Groups)
             {
                 string fieldName = rxGroup.Value.Replace("{{", "").Replace("}}", "").ToLower();
 
-                switch(this.config.DataSource)
+                switch (this.config.DataSource)
                 {
                     case DataSource.CsvFile:
                     case DataSource.Database:
@@ -278,10 +299,14 @@ private string GetExpressionString(DataRow row, string expression, JProperty jso
                         break;
                     case DataSource.JsonFile:
                         // field name is supposed to be a JsonPath
-                        var jsonPathResult = jsonNode.Root.SelectToken(fieldName);
+                        var jsonPathResult = jsonNode.Parent.SelectToken(fieldName); //jsonNode.Root.SelectToken(fieldName);
                         if (jsonPathResult == null || jsonPathResult.Value<object>() == null)
                         {
-                            throw new FieldNotFoundException(fieldName);
+                            jsonPathResult = jsonNode.Root.SelectToken(fieldName);
+                            if (jsonPathResult == null || jsonPathResult.Value<object>() == null)
+                            {
+                                throw new FieldNotFoundException(fieldName);
+                            }
                         }
                         fieldValue = jsonPathResult.Value<string>();
                         expression = expression.Replace(rxGroup.Value, fieldValue);
@@ -361,7 +386,7 @@ private string GetStringFromList(DataRow row, List<Replacement> listOfReplacemen
             if (replacementStr == null)
             {
                 // warning, nothing found
-                throw new DataException($"No match could be located for data row\r\n{string.Join(",", row.ItemArray)}");
+                throw new DataNotFoundException(row);
             }
             var list = replacementStr.Split(',').Select(x => x.Trim()).ToList();
             string str = list[rnd.Next(0, list.Count - 1)];
@@ -499,12 +524,12 @@ private JContainer ReadDataFromJSONFile()
             {
                 jsonObj = JObject.Parse(json);
             }
-            catch(JsonReaderException e)
+            catch (JsonReaderException e)
             {
                 e.ToString();
                 jsonObj = JArray.Parse(json);
             }
-            
+
             return jsonObj;
         }
 
@@ -559,7 +584,7 @@ private void SaveDataToDb(DataTable dt)
             int numParams = config.SqlCommand.Count(c => c == this.DbParamChar);
             if (numParams != config.FieldMasks.Count)
             {
-                throw new ArgumentException($"Sql Command parameter mismatch: '{config.SqlCommand}' does not contain the same number of parameters $field ({numParams}) as the number of FieldMasks ({config.FieldMasks.Count})");
+                throw new ArgumentException($"Sql Command parameter mismatch: '{config.SqlCommand}' does not contain the same number of parameters {this.DbParamChar}field ({numParams}) as the number of FieldMasks ({config.FieldMasks.Count})");
             }
 
             bool wasOpen = EnsureOpenDbConnection();
@@ -591,12 +616,12 @@ private void SaveDataToDb(DataTable dt)
         /// <summary>Creates and opens a db connection, if needed.</summary>
         /// <returns>True if the connection was already open, false otherwise.</returns>
         private bool EnsureOpenDbConnection()
-        {
+        {            
             bool wasOpen = false;
             if (config.GetConnection() == null)
             {
-                DbProviderFactory factory = DbProviderFactories.GetFactory(config.ConnectionType);
-                config.SetConnection(factory.CreateConnection());
+                var dbFactory = GetDbFactory(config.ConnectionType);
+                config.SetConnection(dbFactory.CreateConnection());
                 config.GetConnection().ConnectionString = config.ConnectionString;
                 config.GetConnection().Open();
             }

diff --git a/Zoro.Processor/DataNotFoundException.cs b/Zoro.Processor/DataNotFoundException.cs
@@ -0,0 +1,34 @@
+using System;
+using System.Data;
+
+namespace Dandraka.Zoro.Processor
+{
+    /// <summary>
+    /// Exception raised when a field is not found.
+    /// </summary>
+    public class DataNotFoundException : Exception
+    {
+        /// <summary>
+        /// Creates a FieldNotFoundException.
+        /// </summary>
+        public DataNotFoundException()
+        {
+        }
+
+        /// <summary>
+        /// Creates a FieldNotFoundException specifying the field name.
+        /// </summary>
+        public DataNotFoundException(DataRow row)
+            : base($"No match could be located for data row\r\n{string.Join(",", row.ItemArray)}")
+        {
+        }
+
+        /// <summary>
+        /// Creates a FieldNotFoundException specifying the field name and an inner exception.
+        /// </summary>
+        public DataNotFoundException(DataRow row, Exception inner)
+            : base($"No match could be located for data row\r\n{string.Join(",", row.ItemArray)}", inner)
+        {
+        }
+    }
+}
diff --git a/Zoro.Processor/FieldNotFoundException.cs b/Zoro.Processor/FieldNotFoundException.cs
@@ -1,30 +1,33 @@
 using System;
 
-/// <summary>
-/// Exception raised when a field is not found.
-/// </summary>
-public class FieldNotFoundException : Exception
+namespace Dandraka.Zoro.Processor
 {
     /// <summary>
-    /// Creates a FieldNotFoundException.
+    /// Exception raised when a field is not found.
     /// </summary>
-    public FieldNotFoundException()
+    public class FieldNotFoundException : Exception
     {
-    }
+        /// <summary>
+        /// Creates a FieldNotFoundException.
+        /// </summary>
+        public FieldNotFoundException()
+        {
+        }
 
-    /// <summary>
-    /// Creates a FieldNotFoundException specifying the field name.
-    /// </summary>
-    public FieldNotFoundException(string field)
-        : base($"Field {field} was not found.")
-    {
-    }
+        /// <summary>
+        /// Creates a FieldNotFoundException specifying the field name.
+        /// </summary>
+        public FieldNotFoundException(string field)
+            : base($"Field or JsonPath {field} was not found.")
+        {
+        }
 
-    /// <summary>
-    /// Creates a FieldNotFoundException specifying the field name and an inner exception.
-    /// </summary>
-    public FieldNotFoundException(string field, Exception inner)
-        : base($"Field {field} was not found.", inner)
-    {
+        /// <summary>
+        /// Creates a FieldNotFoundException specifying the field name and an inner exception.
+        /// </summary>
+        public FieldNotFoundException(string field, Exception inner)
+            : base($"Field or JsonPath {field} was not found.", inner)
+        {
+        }
     }
 }