From d718e88a2581af7aeb7ec4fe75f079e38822a099 Mon Sep 17 00:00:00 2001 From: David Baker Effendi Date: Wed, 29 Nov 2023 10:47:21 +0200 Subject: [PATCH] Kind & Location Information (#9) * Made sure artifact is correctly versioned and this can be viewed by `--version` * Extended property resolver to add meta data class containing type and location info on serialization * Removed unused code --- .github/workflows/release.yml | 25 ++-- .gitignore | 3 +- DotNetAstGen/DotNetAstGen.csproj | 2 + DotNetAstGen/Program.cs | 9 +- DotNetAstGen/SyntaxMetaDataProvider.cs | 76 ++++++++++++ ...ver.cs => SyntaxNodePropertiesResolver.cs} | 28 +++-- DotNetAstGen/Utils/MapSerializer.cs | 111 ------------------ DotNetAstGen/Utils/Utilities.cs | 99 ---------------- build-with-version.sh | 7 ++ publish-release.sh | 28 +++++ 10 files changed, 151 insertions(+), 237 deletions(-) create mode 100644 DotNetAstGen/SyntaxMetaDataProvider.cs rename DotNetAstGen/{Utils/JsonResolver.cs => SyntaxNodePropertiesResolver.cs} (68%) delete mode 100644 DotNetAstGen/Utils/MapSerializer.cs delete mode 100644 DotNetAstGen/Utils/Utilities.cs create mode 100755 build-with-version.sh create mode 100755 publish-release.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9154094..f7574b3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,10 +19,6 @@ jobs: uses: actions/setup-dotnet@v3 with: dotnet-version: '7.x' - - name: Install dependencies - run: dotnet restore - - name: Build - run: dotnet build - name: Get next release version (dry run) id: taggerDryRun uses: anothrNick/github-tag-action@1.61.0 @@ -32,18 +28,23 @@ jobs: DRY_RUN: true - name: echo new tag run: | - echo "The next tag version will be: ${{ steps.taggerDryRun.outputs.new_tag }}" + export NEXT_VERSION=${{ steps.taggerDryRun.outputs.new_tag }} + echo "The next tag version will be: $NEXT_VERSION" - name: echo tag run: | - echo "The current tag is: ${{ steps.taggerDryRun.outputs.tag }}" + echo "The current tag is: ${{ steps.taggerDryRun.outputs.tag }}" + - name: Install dependencies + run: dotnet restore + - name: Compile + run: ./build-with-version $NEXT_VERSION - name: Build run: | - dotnet publish -c Release -r linux-x64 -o ./release/linux DotNetAstGen/DotNetAstGen.csproj - dotnet publish -c Release -r linux-arm -o ./release/linux-arm DotNetAstGen/DotNetAstGen.csproj - dotnet publish -c Release -r osx-x64 -o ./release/macos DotNetAstGen/DotNetAstGen.csproj - dotnet publish -c Release -r osx-arm64 -o ./release/macos-arm DotNetAstGen/DotNetAstGen.csproj - dotnet publish -c Release -r win-x64 -o ./release/win DotNetAstGen/DotNetAstGen.csproj - dotnet publish -c Release -r win-arm -o ./release/win-arm DotNetAstGen/DotNetAstGen.csproj + ./publish-release.sh linux x64 + ./publish-release.sh linux arm + ./publish-release.sh osx x64 + ./publish-release.sh osx arm64 + ./publish-release.sh win x64 + ./publish-release.sh win arm - name: Rename run: | mv ./release/linux/DotNetAstGen dotnetastgen-linux diff --git a/.gitignore b/.gitignore index bf5454a..bade137 100644 --- a/.gitignore +++ b/.gitignore @@ -477,4 +477,5 @@ $RECYCLE.BIN/ *.lnk .idea -**/*.json \ No newline at end of file +**/*.json +.ast diff --git a/DotNetAstGen/DotNetAstGen.csproj b/DotNetAstGen/DotNetAstGen.csproj index 02d47c7..47cf4d1 100644 --- a/DotNetAstGen/DotNetAstGen.csproj +++ b/DotNetAstGen/DotNetAstGen.csproj @@ -1,6 +1,8 @@ + 0.0.1-local + $(NEXT_VERSION) Exe net7.0 enable diff --git a/DotNetAstGen/Program.cs b/DotNetAstGen/Program.cs index 977fe56..f7b949c 100644 --- a/DotNetAstGen/Program.cs +++ b/DotNetAstGen/Program.cs @@ -1,7 +1,6 @@ using System.Diagnostics; using System.Text; using CommandLine; -using DotNetAstGen.Utils; using Microsoft.CodeAnalysis.CSharp; using Microsoft.Extensions.Logging; using Newtonsoft.Json; @@ -25,7 +24,7 @@ public static void Main(string[] args) .AddConsole() .AddDebug(); - if (options.Verbose) + if (options.Debug) { builder.SetMinimumLevel(LogLevel.Debug); } @@ -85,7 +84,7 @@ private static void _AstForFile(FileSystemInfo rootInputPath, FileSystemInfo roo { ReferenceLoopHandling = ReferenceLoopHandling.Ignore, ContractResolver = - new IgnorePropertiesResolver() // Comment this to see the unfiltered parser output + new SyntaxNodePropertiesResolver() // Comment this to see the unfiltered parser output }); var outputName = Path.Combine(filePath.DirectoryName ?? "./", $"{Path.GetFileNameWithoutExtension(fullPath)}.json") @@ -104,8 +103,8 @@ private static void _AstForFile(FileSystemInfo rootInputPath, FileSystemInfo roo internal class Options { - [Option('v', "verbose", Required = false, HelpText = "Enable verbose output.")] - public bool Verbose { get; set; } = false; + [Option('d', "debug", Required = false, HelpText = "Enable verbose output.")] + public bool Debug { get; set; } = false; [Option('i', "input", Required = true, HelpText = "Input file or directory.")] public string InputFilePath { get; set; } = ""; diff --git a/DotNetAstGen/SyntaxMetaDataProvider.cs b/DotNetAstGen/SyntaxMetaDataProvider.cs new file mode 100644 index 0000000..0900533 --- /dev/null +++ b/DotNetAstGen/SyntaxMetaDataProvider.cs @@ -0,0 +1,76 @@ +using Newtonsoft.Json.Serialization; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp; +using Newtonsoft.Json; + +namespace DotNetAstGen +{ + public class SyntaxMetaDataProvider : IValueProvider + { + public object GetValue(object target) + { + return target.GetType().IsAssignableTo(typeof(SyntaxNode)) + ? GetNodeMetadata((SyntaxNode)target) + : new SyntaxMetaData(); + } + + private static SyntaxMetaData GetNodeMetadata(SyntaxNode node) + { + var span = node.SyntaxTree.GetLineSpan(node.Span); + return new SyntaxMetaData( + $"ast.{node.Kind()}", + span.StartLinePosition.Line, + span.EndLinePosition.Line, + span.StartLinePosition.Character, + span.EndLinePosition.Character + ); + } + + public void SetValue(object target, object value) + { + // ignore + } + + public static JsonProperty CreateMetaDataProperty() + { + return new JsonProperty + { + PropertyName = "MetaData", + PropertyType = typeof(SyntaxMetaData), + DeclaringType = typeof(SyntaxNode), + ValueProvider = new SyntaxMetaDataProvider(), + AttributeProvider = null, + Readable = true, + Writable = false, + ShouldSerialize = _ => true + }; + } + } + + public class SyntaxMetaData + { + public SyntaxMetaData() + { + } + + public SyntaxMetaData(string kind, int lineStart, int lineEnd, int columnStart, int columnEnd) + { + Kind = kind; + LineStart = lineStart; + LineEnd = lineEnd; + ColumnStart = columnStart; + ColumnEnd = columnEnd; + } + + public string Kind { get; set; } = "ast.None"; + public int LineStart { get; set; } = -1; + public int LineEnd { get; set; } = -1; + public int ColumnStart { get; set; } = -1; + public int ColumnEnd { get; set; } = -1; + + public override string ToString() + { + return JsonConvert.SerializeObject(this); + } + } +} \ No newline at end of file diff --git a/DotNetAstGen/Utils/JsonResolver.cs b/DotNetAstGen/SyntaxNodePropertiesResolver.cs similarity index 68% rename from DotNetAstGen/Utils/JsonResolver.cs rename to DotNetAstGen/SyntaxNodePropertiesResolver.cs index 36c1e8f..c599cef 100644 --- a/DotNetAstGen/Utils/JsonResolver.cs +++ b/DotNetAstGen/SyntaxNodePropertiesResolver.cs @@ -1,33 +1,34 @@ using System.Reflection; using System.Text.RegularExpressions; +using Microsoft.CodeAnalysis; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Newtonsoft.Json.Serialization; -namespace DotNetAstGen.Utils +namespace DotNetAstGen { - internal class IgnorePropertiesResolver : DefaultContractResolver + internal class SyntaxNodePropertiesResolver : DefaultContractResolver { - private static readonly ILogger? Logger = Program.LoggerFactory?.CreateLogger("IgnorePropertiesResolver"); - - + private static readonly ILogger? Logger = Program.LoggerFactory?.CreateLogger("SyntaxNodePropertiesResolver"); + private readonly HashSet _propsToAllow = new(new[] { - "Value", "Externs", "Usings", "Name", "Identifier", "Left", "Right", "Members", "ConstraintClauses", + "Value", "Usings", "Name", "Identifier", "Left", "Right", "Members", "ConstraintClauses", "Alias", "NamespaceOrType", "Arguments", "Expression", "Declaration", "ElementType", "Initializer", "Else", "Condition", "Statement", "Statements", "Variables", "WhenNotNull", "AllowsAnyExpression", "Expressions", "Modifiers", "ReturnType", "IsUnboundGenericName", "Default", "IsConst", "Parameters", "Types", - "ExplicitInterfaceSpecifier", "Text", "Length", "Location" + "ExplicitInterfaceSpecifier", "MetaData", "Kind" }); private readonly List _regexToAllow = new(new[] { - ".*Token$", ".*Keyword$", ".*Lists?$", ".*Body$", "(Span)?Start" + ".*Token$", ".*Keyword$", ".*Lists?$", ".*Body$", "(Line|Column)(Start|End)" }); private readonly List _regexToIgnore = new(new[] { - ".*(Semicolon|Brace|Bracket|EndOfFile|Paren|Dot)Token$" + ".*(Semicolon|Brace|Bracket|EndOfFile|Paren|Dot)Token$", "AttributeLists", + "(Unsafe|Global|Static|Using)Keyword" }); private bool MatchesAllow(string input) @@ -40,6 +41,15 @@ private bool MatchesIgnore(string input) return _regexToIgnore.Any(regex => Regex.IsMatch(input, regex)); } + protected override IList CreateProperties(Type type, MemberSerialization memberSerialization) + { + var properties = base.CreateProperties(type, memberSerialization); + var isSyntaxNode = type.IsAssignableTo(typeof(SyntaxNode)); + if (!isSyntaxNode) return properties; + properties.Add(SyntaxMetaDataProvider.CreateMetaDataProperty()); + return properties; + } + protected override JsonProperty CreateProperty(MemberInfo member, MemberSerialization memberSerialization) { var property = base.CreateProperty(member, memberSerialization); diff --git a/DotNetAstGen/Utils/MapSerializer.cs b/DotNetAstGen/Utils/MapSerializer.cs deleted file mode 100644 index 550148a..0000000 --- a/DotNetAstGen/Utils/MapSerializer.cs +++ /dev/null @@ -1,111 +0,0 @@ -using Microsoft.CodeAnalysis; -using Microsoft.CodeAnalysis.CSharp; -using Microsoft.CodeAnalysis.CSharp.Syntax; -using Microsoft.Extensions.Logging; -using Newtonsoft.Json.Linq; - -namespace DotNetAstGen.Utils -{ - public class MapSerializer - - { - private static readonly ILogger? Logger = Program.LoggerFactory?.CreateLogger("MapSerializer"); - - public static JArray ProcessSyntaxTokenList(SyntaxTokenList list) - { - JArray syntaxTokenList = new(); - foreach (SyntaxToken token in list) - { - syntaxTokenList.Add( - ProcessSyntaxToken(token) - ); - } - - Console.WriteLine(syntaxTokenList); - return syntaxTokenList; - } - - public static JObject ProcessSyntaxToken(SyntaxToken token) - { - return Utilities.AddNodeMetadataToMap(token); - } - - public static void ProcessSyntaxNodeList(dynamic list) - { - if (list == null) - { - return; - } - - foreach (SyntaxNode node in list) - { - foreach (var key in node.GetType().GetProperties()) - { - var valueType = key.GetValue(node)?.GetType() ?? null; - var value = key.GetValue(node); - var genericType = (valueType?.IsGenericType ?? false) ? valueType?.GetGenericArguments()[0] : null; - - if (key?.GetValue(node)?.GetType() == typeof(SyntaxToken)) - { - Console.WriteLine(ProcessSyntaxToken((SyntaxToken)key.GetValue(node))); - // Console.WriteLine(key?.GetValue(node)?.GetType()); - } - else if (key?.GetValue(node)?.GetType() == typeof(SyntaxNode)) - { - Console.WriteLine("SyntaxNode inside SyntaxNode"); - } - - - if (genericType?.IsSubclassOf(typeof(SyntaxNode)) ?? false) - { - ProcessSyntaxNodeList(value); - } - } - } - } - - public static void Helper(dynamic node, JObject map) - { - var keys = node.GetType().GetProperties(); - foreach (var key in keys) - { - var valueType = key.GetValue(node)?.GetType() ?? null; - var value = key.GetValue(node); - if (valueType?.IsPrimitive ?? true) - { - Console.WriteLine("Primitive type"); - continue; - } - - var genericType = (valueType?.IsGenericType ?? false) ? valueType?.GetGenericArguments()[0] : null; - if ( - genericType?.IsSubclassOf(typeof(CSharpSyntaxNode)) ?? false - ) - { - // List of syntax nodes - Console.WriteLine(value.GetType()); - ProcessSyntaxNodeList(value); - } - - if ( - valueType == typeof(SyntaxToken) - ) - { - Console.WriteLine($"{key}: SyntaxToken type"); - } - } - } - - - public static List CreateList(params T[] elements) - { - return new List(elements); - } - - public static void SerializeToMap(CompilationUnitSyntax root, Dictionary map) - { - JObject customMap = JObject.Parse("{}"); - Helper(root, customMap); - } - } -} \ No newline at end of file diff --git a/DotNetAstGen/Utils/Utilities.cs b/DotNetAstGen/Utils/Utilities.cs deleted file mode 100644 index a0cbf95..0000000 --- a/DotNetAstGen/Utils/Utilities.cs +++ /dev/null @@ -1,99 +0,0 @@ -using System.Text.RegularExpressions; -using System.Text.Json; -using Microsoft.CodeAnalysis; -using Microsoft.CodeAnalysis.CSharp; -using Microsoft.Extensions.Logging; -using Newtonsoft.Json.Linq; -using Newtonsoft.Json; - -namespace DotNetAstGen.Utils -{ - public static class Utilities - { - - private static ILogger? _logger = Program.LoggerFactory?.CreateLogger("Utilities"); - - public static (string kind, int line_start, int line_end, int col_start, int col_end) GetNodeMetadata( - SyntaxNode node) - { - FileLinePositionSpan span = node.SyntaxTree.GetLineSpan(node.Span); - return ( - $"ast.{node.Kind()}", - span.StartLinePosition.Line, - span.EndLinePosition.Line, - span.StartLinePosition.Character, - span.EndLinePosition.Character - ); - } - - - public static (string kind, int line_start, int line_end, int col_start, int col_end) GetNodeMetadata( - SyntaxToken node) - { - try - { - FileLinePositionSpan span = node.SyntaxTree.GetLineSpan(node.Span); - return ( - $"ast.{node.Kind()}", - span.StartLinePosition.Line, - span.EndLinePosition.Line, - span.StartLinePosition.Character, - span.EndLinePosition.Character - ); - } - catch - { - return ( - $"ast.{node.Kind()}", - -1, - -1, - -1, - -1 - ); - } - } - - public static JObject AddNodeMetadataToMap(SyntaxNode node) - { - JObject map = JObject.FromObject(node, new Newtonsoft.Json.JsonSerializer - { - ReferenceLoopHandling = ReferenceLoopHandling.Ignore - }); - var (kind, lineStart, lineEnd, colStart, colEnd) = GetNodeMetadata(node); - if (kind == "ast.None") - { - return map; - } - - map.Add("kind", kind); - map.Add("line_start", lineStart); - map.Add("line_end", lineEnd); - map.Add("col_start", colStart); - map.Add("col_end", colEnd); - - return map; - } - - public static JObject AddNodeMetadataToMap(SyntaxToken node) - { - JObject map = JObject.FromObject(node, new Newtonsoft.Json.JsonSerializer - { - ReferenceLoopHandling = ReferenceLoopHandling.Ignore - }); - - var (kind, lineStart, lineEnd, colStart, colEnd) = Utilities.GetNodeMetadata(node); - if (kind == "ast.None") - { - return map; - } - - map.Add("kind", kind); - map.Add("line_start", lineStart); - map.Add("line_end", lineEnd); - map.Add("col_start", colStart); - map.Add("col_end", colEnd); - - return map; - } - } -} \ No newline at end of file diff --git a/build-with-version.sh b/build-with-version.sh new file mode 100755 index 0000000..a091eae --- /dev/null +++ b/build-with-version.sh @@ -0,0 +1,7 @@ +#!/bin/bash +LATEST_TAG=`git describe --tags --abbrev=0 | cut -c2-` +VERSION=${1:-$LATEST_TAG} + +echo "Running build with assigned version: $VERSION"; + +dotnet build -c Release /p:Version=$VERSION /p:AssemblyVersion=$VERSION diff --git a/publish-release.sh b/publish-release.sh new file mode 100755 index 0000000..3182ce7 --- /dev/null +++ b/publish-release.sh @@ -0,0 +1,28 @@ +#!/bin/bash +OS=$1 +ARCH=$2 +LATEST_TAG=`git describe --tags --abbrev=0 | cut -c2-` + +echo "OS: $OS"; +echo "Architecture: $ARCH"; + +if [[ -z "${NEXT_VERSION}" ]]; then + echo "Version not set, defaulting to latest tag version $LATEST_TAG"; + export NEXT_VERSION=$LATEST_TAG +fi + +RELEASE_DIR="./release" +OUTPUT_TARGET="" +if [[ $OS == "osx" ]]; then + OUTPUT_TARGET="macos" +else + OUTPUT_TARGET=$OS +fi +if [[ $ARCH == arm* ]]; then + OUTPUT_TARGET="$OUTPUT_TARGET-arm" +fi + +OUTPUT_PATH="$RELEASE_DIR/$OUTPUT_TARGET" +TARGET="$OS-$ARCH" + +dotnet publish -c Release -r $TARGET -o $OUTPUT_PATH