From 0ecfee0bc49b022ee683b52a93a4502727874601 Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Tue, 28 Nov 2023 11:42:32 +0200 Subject: [PATCH] Trimmed Binary & Output Properties (#8) * Modified JsonResolver to use a mixture of techniques to include or exclude certain properties * Found that `PublishTrimmed` breaks the program as the JSON serializer uses reflection * Cleaned up some code * Added `--output` argument with error handling * Compressing each executable individually --- .github/workflows/release.yml | 20 ++++++--- DotNetAstGen/DotNetAstGen.csproj | 4 ++ DotNetAstGen/Program.cs | 69 +++++++++++++++++++---------- DotNetAstGen/Utils/JsonResolver.cs | 50 ++++++++++++++------- DotNetAstGen/Utils/MapSerializer.cs | 9 ++-- 5 files changed, 101 insertions(+), 51 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a3a94cb..f3c699d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -58,6 +58,14 @@ jobs: chmod +x dotnetastgen-linux-arm chmod +x dotnetastgen-macos chmod +x dotnetastgen-macos-arm + - name: Compress all executables + run: | + zip dotnetastgen-linux.zip dotnetastgen-linux + zip dotnetastgen-linux-arm.zip dotnetastgen-linux-arm + zip dotnetastgen-macos.zip dotnetastgen-macos + zip dotnetastgen-macos-arm.zip dotnetastgen-macos-arm + zip dotnetastgen-win.zip dotnetastgen-win.exe + zip dotnetastgen-win-arm.zip dotnetastgen-win-arm.exe - name: Set next release version id: taggerFinal uses: anothrNick/github-tag-action@1.61.0 @@ -69,9 +77,9 @@ jobs: with: tag_name: ${{ steps.taggerDryRun.outputs.new_tag }} files: | - dotnetastgen-linux - dotnetastgen-linux-arm - dotnetastgen-macos - dotnetastgen-macos-arm - dotnetastgen-win.exe - dotnetastgen-win-arm.exe \ No newline at end of file + dotnetastgen-linux.zip + dotnetastgen-linux-arm.zip + dotnetastgen-macos.zip + dotnetastgen-macos-arm.zip + dotnetastgen-win.exe.zip + dotnetastgen-win-arm.exe.zip \ No newline at end of file diff --git a/DotNetAstGen/DotNetAstGen.csproj b/DotNetAstGen/DotNetAstGen.csproj index 4b2c629..02d47c7 100644 --- a/DotNetAstGen/DotNetAstGen.csproj +++ b/DotNetAstGen/DotNetAstGen.csproj @@ -16,4 +16,8 @@ + + + + diff --git a/DotNetAstGen/Program.cs b/DotNetAstGen/Program.cs index 19b80b3..977fe56 100644 --- a/DotNetAstGen/Program.cs +++ b/DotNetAstGen/Program.cs @@ -1,4 +1,4 @@ -using System.Reflection; +using System.Diagnostics; using System.Text; using CommandLine; using DotNetAstGen.Utils; @@ -34,52 +34,70 @@ public static void Main(string[] args) _logger = LoggerFactory.CreateLogger(); _logger.LogDebug("Show verbose output."); - _RunAstGet(options.InputFilePath); + _RunAstGet(options.InputFilePath, new DirectoryInfo(options.OutputDirectory)); }); } - private static void _RunAstGet(string inputPath) + private static void _RunAstGet(string inputPath, DirectoryInfo rootOutputPath) { + if (!rootOutputPath.Exists) + { + rootOutputPath.Create(); + } + if (Directory.Exists(inputPath)) { _logger?.LogInformation("Parsing directory {dirName}", inputPath); - foreach (FileInfo fileInfo in new DirectoryInfo(inputPath).EnumerateFiles("*.cs")) + var rootDirectory = new DirectoryInfo(inputPath); + foreach (var inputFile in new DirectoryInfo(inputPath).EnumerateFiles("*.cs")) { - _AstForFile(fileInfo); + _AstForFile(rootDirectory, rootOutputPath, inputFile); } } else if (File.Exists(inputPath)) { _logger?.LogInformation("Parsing file {fileName}", inputPath); - _AstForFile(new FileInfo(inputPath)); + var file = new FileInfo(inputPath); + Debug.Assert(file.Directory != null, "Given file has a null parent directory!"); + _AstForFile(file.Directory, rootOutputPath, file); } else { _logger?.LogError("The path {inputPath} does not exist!", inputPath); Environment.Exit(1); } - _logger?.LogInformation("Parsing successful!"); + + _logger?.LogInformation("AST generation complete"); } - private static void _AstForFile(FileInfo filePath) + private static void _AstForFile(FileSystemInfo rootInputPath, FileSystemInfo rootOutputPath, FileInfo filePath) { var fullPath = filePath.FullName; _logger?.LogDebug("Parsing file: {filePath}", fullPath); - using var streamReader = new StreamReader(fullPath, Encoding.UTF8); - var programText = streamReader.ReadToEnd(); - var tree = CSharpSyntaxTree.ParseText(programText); - _logger?.LogDebug("Successfully parsed: {filePath}", fullPath); - var root = tree.GetCompilationUnitRoot(); - var rootType = root.GetType(); - IList props = new List(rootType.GetProperties()); - var jsonString = JsonConvert.SerializeObject(root, Formatting.Indented, new JsonSerializerSettings + try { - ReferenceLoopHandling = ReferenceLoopHandling.Ignore, - ContractResolver = new IgnorePropertiesResolver() // Comment this to see the unfiltered parser output - }); - var outputName = Path.Combine(filePath.DirectoryName ?? "./", $"{Path.GetFileNameWithoutExtension(fullPath)}.json"); - _logger?.LogDebug("Writing AST to {astJsonPath}", outputName); - File.WriteAllText(outputName, jsonString); + using var streamReader = new StreamReader(fullPath, Encoding.UTF8); + var programText = streamReader.ReadToEnd(); + var tree = CSharpSyntaxTree.ParseText(programText); + _logger?.LogDebug("Successfully parsed: {filePath}", fullPath); + var root = tree.GetCompilationUnitRoot(); + var jsonString = JsonConvert.SerializeObject(root, Formatting.Indented, new JsonSerializerSettings + { + ReferenceLoopHandling = ReferenceLoopHandling.Ignore, + ContractResolver = + new IgnorePropertiesResolver() // Comment this to see the unfiltered parser output + }); + var outputName = Path.Combine(filePath.DirectoryName ?? "./", + $"{Path.GetFileNameWithoutExtension(fullPath)}.json") + .Replace(rootInputPath.FullName, rootOutputPath.FullName); + + File.WriteAllText(outputName, jsonString); + _logger?.LogInformation("Successfully wrote AST to '{astJsonPath}'", outputName); + } + catch (Exception e) + { + _logger?.LogError("Error encountered while parsing '{filePath}': {errorMsg}", fullPath, e.Message); + } } } @@ -87,9 +105,12 @@ private static void _AstForFile(FileInfo filePath) internal class Options { [Option('v', "verbose", Required = false, HelpText = "Enable verbose output.")] - public bool Verbose { get; set; } + public bool Verbose { get; set; } = false; [Option('i', "input", Required = true, HelpText = "Input file or directory.")] - public string InputFilePath { get; set; } + public string InputFilePath { get; set; } = ""; + + [Option('o', "input", Required = false, HelpText = "Output directory. (default `./.ast`)")] + public string OutputDirectory { get; set; } = ".ast"; } } \ No newline at end of file diff --git a/DotNetAstGen/Utils/JsonResolver.cs b/DotNetAstGen/Utils/JsonResolver.cs index da5e224..36c1e8f 100644 --- a/DotNetAstGen/Utils/JsonResolver.cs +++ b/DotNetAstGen/Utils/JsonResolver.cs @@ -1,34 +1,54 @@ using System.Reflection; +using System.Text.RegularExpressions; +using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Newtonsoft.Json.Serialization; namespace DotNetAstGen.Utils { - class IgnorePropertiesResolver : DefaultContractResolver + internal class IgnorePropertiesResolver : DefaultContractResolver { - private readonly string[] _propsToIgnore = + private static readonly ILogger? Logger = Program.LoggerFactory?.CreateLogger("IgnorePropertiesResolver"); + + + private readonly HashSet _propsToAllow = new(new[] { - "Parent", "ParentTrivia", "ContainsSkippedText", "SyntaxTree", "SpanStart", "IsMissing", - "IsStructuredTrivia", "HasStructuredTrivia", "ContainsDiagnostics", "ContainsDirectives", - "HasLeadingTrivia", "HasTrailingTrivia", "ContainsAnnotations", "Span", "FullSpan", "LeadingTrivia", - "TrailingTrivia" - }; + "Value", "Externs", "Usings", "Name", "Identifier", "Left", "Right", "Members", "ConstraintClauses", + "Alias", "NamespaceOrType", "Arguments", "Expression", "Declaration", "ElementType", "Initializer", "Else", + "Condition", "Statement", "Statements", "Variables", "WhenNotNull", "AllowsAnyExpression", "Expressions", + "Modifiers", "ReturnType", "IsUnboundGenericName", "Default", "IsConst", "Parameters", "Types", + "ExplicitInterfaceSpecifier", "Text", "Length", "Location" + }); - private readonly HashSet _ignoreProps; + private readonly List _regexToAllow = new(new[] + { + ".*Token$", ".*Keyword$", ".*Lists?$", ".*Body$", "(Span)?Start" + }); + + private readonly List _regexToIgnore = new(new[] + { + ".*(Semicolon|Brace|Bracket|EndOfFile|Paren|Dot)Token$" + }); - public IgnorePropertiesResolver() + private bool MatchesAllow(string input) { - _ignoreProps = new HashSet(this._propsToIgnore); + return _regexToAllow.Any(regex => Regex.IsMatch(input, regex)); + } + + private bool MatchesIgnore(string input) + { + return _regexToIgnore.Any(regex => Regex.IsMatch(input, regex)); } protected override JsonProperty CreateProperty(MemberInfo member, MemberSerialization memberSerialization) { var property = base.CreateProperty(member, memberSerialization); - if (_ignoreProps.Contains(property.PropertyName ?? "")) - { - property.ShouldSerialize = _ => false; - } - + var propertyName = property.PropertyName ?? ""; + var shouldSerialize = propertyName != "" && + (_propsToAllow.Contains(propertyName) || MatchesAllow(propertyName)) && + !MatchesIgnore(propertyName); + Logger?.LogDebug(shouldSerialize ? $"Allowing {propertyName}" : $"Ignoring {propertyName}"); + property.ShouldSerialize = _ => shouldSerialize; return property; } } diff --git a/DotNetAstGen/Utils/MapSerializer.cs b/DotNetAstGen/Utils/MapSerializer.cs index 25d580a..550148a 100644 --- a/DotNetAstGen/Utils/MapSerializer.cs +++ b/DotNetAstGen/Utils/MapSerializer.cs @@ -1,19 +1,16 @@ using Microsoft.CodeAnalysis; using Microsoft.CodeAnalysis.CSharp; using Microsoft.CodeAnalysis.CSharp.Syntax; -using Newtonsoft.Json.Linq; -using System.Data; -using System.Runtime.InteropServices; -using DotNetAstGen.Utils; using Microsoft.Extensions.Logging; +using Newtonsoft.Json.Linq; namespace DotNetAstGen.Utils { public class MapSerializer { - private static ILogger? _logger = Program.LoggerFactory?.CreateLogger("MapSerializer"); - + private static readonly ILogger? Logger = Program.LoggerFactory?.CreateLogger("MapSerializer"); + public static JArray ProcessSyntaxTokenList(SyntaxTokenList list) { JArray syntaxTokenList = new();