Trimmed Binary & Output Properties (#8)

* Modified JsonResolver to use a mixture of techniques to include or exclude certain properties * Found that `PublishTrimmed` breaks the program as the JSON serializer uses reflection * Cleaned up some code * Added `--output` argument with error handling * Compressing each executable individually
joernio · Nov 28, 2023 · 0ecfee0 · 0ecfee0
1 parent 42419d2
commit 0ecfee0
Show file tree

Hide file tree

Showing 5 changed files with 101 additions and 51 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -58,6 +58,14 @@ jobs:
           chmod +x dotnetastgen-linux-arm
           chmod +x dotnetastgen-macos
           chmod +x dotnetastgen-macos-arm
+      - name: Compress all executables
+        run: |
+          zip dotnetastgen-linux.zip dotnetastgen-linux
+          zip dotnetastgen-linux-arm.zip  dotnetastgen-linux-arm
+          zip dotnetastgen-macos.zip  dotnetastgen-macos
+          zip dotnetastgen-macos-arm.zip  dotnetastgen-macos-arm
+          zip dotnetastgen-win.zip dotnetastgen-win.exe
+          zip dotnetastgen-win-arm.zip dotnetastgen-win-arm.exe
       - name: Set next release version
         id: taggerFinal
         uses: anothrNick/[email protected]
@@ -69,9 +77,9 @@ jobs:
         with:
           tag_name: ${{ steps.taggerDryRun.outputs.new_tag }}
           files: |
-            dotnetastgen-linux
-            dotnetastgen-linux-arm
-            dotnetastgen-macos
-            dotnetastgen-macos-arm
-            dotnetastgen-win.exe
-            dotnetastgen-win-arm.exe
+            dotnetastgen-linux.zip
+            dotnetastgen-linux-arm.zip
+            dotnetastgen-macos.zip
+            dotnetastgen-macos-arm.zip
+            dotnetastgen-win.exe.zip
+            dotnetastgen-win-arm.exe.zip
diff --git a/DotNetAstGen/DotNetAstGen.csproj b/DotNetAstGen/DotNetAstGen.csproj
@@ -16,4 +16,8 @@
       <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     </ItemGroup>
 
+    <ItemGroup>
+      <Folder Include="release\" />
+    </ItemGroup>
+
 </Project>
diff --git a/DotNetAstGen/Program.cs b/DotNetAstGen/Program.cs
@@ -1,4 +1,4 @@
-using System.Reflection;
+using System.Diagnostics;
 using System.Text;
 using CommandLine;
 using DotNetAstGen.Utils;
@@ -34,62 +34,83 @@ public static void Main(string[] args)
                     _logger = LoggerFactory.CreateLogger<Program>();
                     _logger.LogDebug("Show verbose output.");
 
-                    _RunAstGet(options.InputFilePath);
+                    _RunAstGet(options.InputFilePath, new DirectoryInfo(options.OutputDirectory));
                 });
         }
 
-        private static void _RunAstGet(string inputPath)
+        private static void _RunAstGet(string inputPath, DirectoryInfo rootOutputPath)
         {
+            if (!rootOutputPath.Exists)
+            {
+                rootOutputPath.Create();
+            }
+
             if (Directory.Exists(inputPath))
             {
                 _logger?.LogInformation("Parsing directory {dirName}", inputPath);
-                foreach (FileInfo fileInfo in new DirectoryInfo(inputPath).EnumerateFiles("*.cs"))
+                var rootDirectory = new DirectoryInfo(inputPath);
+                foreach (var inputFile in new DirectoryInfo(inputPath).EnumerateFiles("*.cs"))
                 {
-                    _AstForFile(fileInfo);
+                    _AstForFile(rootDirectory, rootOutputPath, inputFile);
                 }
             }
             else if (File.Exists(inputPath))
             {
                 _logger?.LogInformation("Parsing file {fileName}", inputPath);
-                _AstForFile(new FileInfo(inputPath));
+                var file = new FileInfo(inputPath);
+                Debug.Assert(file.Directory != null, "Given file has a null parent directory!");
+                _AstForFile(file.Directory, rootOutputPath, file);
             }
             else
             {
                 _logger?.LogError("The path {inputPath} does not exist!", inputPath);
                 Environment.Exit(1);
             }
-            _logger?.LogInformation("Parsing successful!");
+
+            _logger?.LogInformation("AST generation complete");
         }
 
-        private static void _AstForFile(FileInfo filePath)
+        private static void _AstForFile(FileSystemInfo rootInputPath, FileSystemInfo rootOutputPath, FileInfo filePath)
         {
             var fullPath = filePath.FullName;
             _logger?.LogDebug("Parsing file: {filePath}", fullPath);
-            using var streamReader = new StreamReader(fullPath, Encoding.UTF8);
-            var programText = streamReader.ReadToEnd();
-            var tree = CSharpSyntaxTree.ParseText(programText);
-            _logger?.LogDebug("Successfully parsed: {filePath}", fullPath);
-            var root = tree.GetCompilationUnitRoot();
-            var rootType = root.GetType();
-            IList<PropertyInfo> props = new List<PropertyInfo>(rootType.GetProperties());
-            var jsonString = JsonConvert.SerializeObject(root, Formatting.Indented, new JsonSerializerSettings
+            try
             {
-                ReferenceLoopHandling = ReferenceLoopHandling.Ignore,
-                ContractResolver = new IgnorePropertiesResolver() // Comment this to see the unfiltered parser output
-            });
-            var outputName = Path.Combine(filePath.DirectoryName ?? "./", $"{Path.GetFileNameWithoutExtension(fullPath)}.json");
-            _logger?.LogDebug("Writing AST to {astJsonPath}", outputName);
-            File.WriteAllText(outputName, jsonString);
+                using var streamReader = new StreamReader(fullPath, Encoding.UTF8);
+                var programText = streamReader.ReadToEnd();
+                var tree = CSharpSyntaxTree.ParseText(programText);
+                _logger?.LogDebug("Successfully parsed: {filePath}", fullPath);
+                var root = tree.GetCompilationUnitRoot();
+                var jsonString = JsonConvert.SerializeObject(root, Formatting.Indented, new JsonSerializerSettings
+                {
+                    ReferenceLoopHandling = ReferenceLoopHandling.Ignore,
+                    ContractResolver =
+                        new IgnorePropertiesResolver() // Comment this to see the unfiltered parser output
+                });
+                var outputName = Path.Combine(filePath.DirectoryName ?? "./",
+                        $"{Path.GetFileNameWithoutExtension(fullPath)}.json")
+                    .Replace(rootInputPath.FullName, rootOutputPath.FullName);
+
+                File.WriteAllText(outputName, jsonString);
+                _logger?.LogInformation("Successfully wrote AST to '{astJsonPath}'", outputName);
+            }
+            catch (Exception e)
+            {
+                _logger?.LogError("Error encountered while parsing '{filePath}': {errorMsg}", fullPath, e.Message);
+            }
         }
     }
 
 
     internal class Options
     {
         [Option('v', "verbose", Required = false, HelpText = "Enable verbose output.")]
-        public bool Verbose { get; set; }
+        public bool Verbose { get; set; } = false;
 
         [Option('i', "input", Required = true, HelpText = "Input file or directory.")]
-        public string InputFilePath { get; set; }
+        public string InputFilePath { get; set; } = "";
+
+        [Option('o', "input", Required = false, HelpText = "Output directory. (default `./.ast`)")]
+        public string OutputDirectory { get; set; } = ".ast";
     }
 }
diff --git a/DotNetAstGen/Utils/JsonResolver.cs b/DotNetAstGen/Utils/JsonResolver.cs
@@ -1,34 +1,54 @@
 using System.Reflection;
+using System.Text.RegularExpressions;
+using Microsoft.Extensions.Logging;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Serialization;
 
 namespace DotNetAstGen.Utils
 {
-    class IgnorePropertiesResolver : DefaultContractResolver
+    internal class IgnorePropertiesResolver : DefaultContractResolver
     {
-        private readonly string[] _propsToIgnore =
+        private static readonly ILogger? Logger = Program.LoggerFactory?.CreateLogger("IgnorePropertiesResolver");
+
+
+        private readonly HashSet<string> _propsToAllow = new(new[]
         {
-            "Parent", "ParentTrivia", "ContainsSkippedText", "SyntaxTree", "SpanStart", "IsMissing",
-            "IsStructuredTrivia", "HasStructuredTrivia", "ContainsDiagnostics", "ContainsDirectives",
-            "HasLeadingTrivia", "HasTrailingTrivia", "ContainsAnnotations", "Span", "FullSpan", "LeadingTrivia",
-            "TrailingTrivia"
-        };
+            "Value", "Externs", "Usings", "Name", "Identifier", "Left", "Right", "Members", "ConstraintClauses",
+            "Alias", "NamespaceOrType", "Arguments", "Expression", "Declaration", "ElementType", "Initializer", "Else",
+            "Condition", "Statement", "Statements", "Variables", "WhenNotNull", "AllowsAnyExpression", "Expressions",
+            "Modifiers", "ReturnType", "IsUnboundGenericName", "Default", "IsConst", "Parameters", "Types",
+            "ExplicitInterfaceSpecifier", "Text", "Length", "Location"
+        });
 
-        private readonly HashSet<string> _ignoreProps;
+        private readonly List<string> _regexToAllow = new(new[]
+        {
+            ".*Token$", ".*Keyword$", ".*Lists?$", ".*Body$", "(Span)?Start"
+        });
+
+        private readonly List<string> _regexToIgnore = new(new[]
+        {
+            ".*(Semicolon|Brace|Bracket|EndOfFile|Paren|Dot)Token$"
+        });
 
-        public IgnorePropertiesResolver()
+        private bool MatchesAllow(string input)
         {
-            _ignoreProps = new HashSet<string>(this._propsToIgnore);
+            return _regexToAllow.Any(regex => Regex.IsMatch(input, regex));
+        }
+
+        private bool MatchesIgnore(string input)
+        {
+            return _regexToIgnore.Any(regex => Regex.IsMatch(input, regex));
         }
 
         protected override JsonProperty CreateProperty(MemberInfo member, MemberSerialization memberSerialization)
         {
             var property = base.CreateProperty(member, memberSerialization);
-            if (_ignoreProps.Contains(property.PropertyName ?? ""))
-            {
-                property.ShouldSerialize = _ => false;
-            }
-
+            var propertyName = property.PropertyName ?? "";
+            var shouldSerialize = propertyName != "" &&
+                                  (_propsToAllow.Contains(propertyName) || MatchesAllow(propertyName)) &&
+                                  !MatchesIgnore(propertyName);
+            Logger?.LogDebug(shouldSerialize ? $"Allowing {propertyName}" : $"Ignoring {propertyName}");
+            property.ShouldSerialize = _ => shouldSerialize;
             return property;
         }
     }

diff --git a/DotNetAstGen/Utils/MapSerializer.cs b/DotNetAstGen/Utils/MapSerializer.cs
@@ -1,19 +1,16 @@
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.CSharp;
 using Microsoft.CodeAnalysis.CSharp.Syntax;
-using Newtonsoft.Json.Linq;
-using System.Data;
-using System.Runtime.InteropServices;
-using DotNetAstGen.Utils;
 using Microsoft.Extensions.Logging;
+using Newtonsoft.Json.Linq;
 
 namespace DotNetAstGen.Utils
 {
     public class MapSerializer
 
     {
-        private static ILogger? _logger = Program.LoggerFactory?.CreateLogger("MapSerializer");
-        
+        private static readonly ILogger? Logger = Program.LoggerFactory?.CreateLogger("MapSerializer");
+
         public static JArray ProcessSyntaxTokenList(SyntaxTokenList list)
         {
             JArray syntaxTokenList = new();