Compiler multi threading support (#2240)

* Extend examples, fix Dictionary bugs

* Make sure EqualityComparer..cctors are sorted first

* Extended example works

* Support communication via WiFi

* Add implementation of Number.ToString() to library, for debugging purposes

This code from the System library implements the number formatting.
Following the code makes it a lot easier to debug EE errors (such as
missing sign extensions)

* Start adding File I/O support

* File system support mostly working

But seems to require proper try/catch/finally support,
otherwise files are not closed.

* Basic support for exception handling added

* Use common constants

* Write as ASCII

* Remove wrong warnings

* Convert List to Dictionary

By first only checking the name, method comparison is
significantly faster (up to 10 times)

Gives more clean internal structure

* Fix date pattern

* Fix a potential crash when humidity is 0

See comment in code

* Support replacing static constructors

... in classes that are not fully replaced.
Not really used now, since must wait for UnitsNET 5.0 to
retry the full support again. The current reflection hack is
just to complex.

* A bit documentation update

* Debugger help improved, allow selecting thread for debug

* File is already in Compiler directory

* New compiler command group: Exec

Currently only one new command: --stop

* Extended WeatherStation works

* Build the compiler in a separate pipeline

.github/workflows/arduino.yml

@@ -0,0 +1,28 @@
+# This is a basic workflow to help you get started with Actions
+
+name: Arduino_Compiler_CI
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push or pull request events
+  push:
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+  pull_request:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: windows-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      - name: Checkout
+        uses: actions/checkout@master
+
+      - run: |
+          eng/ArduinoCsCI.cmd %USERPROFILE% Debug
+

azure-pipelines.yml

@@ -68,6 +68,10 @@ stages:
         sourceFolder: $(Build.SourcesDirectory)/artifacts/packages/$(BuildConfiguration)/Shipping
         targetFolder: $(Build.ArtifactStagingDirectory)/Packages
 
+    - task: PublishTestResults@2
+      inputs:
+        mergeTestResults: true
+
     - publish: $(Build.ArtifactStagingDirectory)/Packages
       displayName: Publish Build Artifacts
       artifact: BuildPackages
@@ -91,28 +95,6 @@ stages:
       displayName: 'Execute markdown-link-check'
       condition: eq(variables['build.reason'], 'PullRequest')
 
-  # - job: Windows_ArduinoIntegration
-  #   displayName: Arduino Integration Tests
-  #   timeoutInMinutes: 120
-  #   pool:
-  #     vmImage: windows-2022
-
-  #   strategy:
-  #     matrix:
-  #       Build_Release:
-  #         BuildConfiguration: Release
-  #       Build_Debug:
-  #         BuildConfiguration: Debug
-
-  #   steps:
-  #   - script: build.cmd -ci 
-  #       -configuration $(BuildConfiguration)
-  #       -prepareMachine
-  #     displayName: Build Iot
-
-  #   - script: eng\ArduinoCsCI.cmd $(UserProfile) $(BuildConfiguration)
-  #     displayName: Build and run Arduino Integration Tests
-
   - job: Linux
     displayName: Linux Build
     container: LinuxContainer

eng/ArduinoCsCI.cmd

@@ -5,7 +5,7 @@ REM Second argument is either "Debug" or "Release"
 if %1!==! goto :usage
 
 REM Defines the revision to check out in the ExtendedConfigurableFirmata repo
-set FIRMATA_SIMULATOR_CHECKOUT_REVISION=e2cfb5223aeb71e3a0756d67619db6c238b6acb5
+set FIRMATA_SIMULATOR_CHECKOUT_REVISION=4a3b895c062c8e48685b9018d642d2c5ea84c354
 set RUN_COMPILER_TESTS=FALSE
 
 choco install -y --no-progress arduino-cli
@@ -16,13 +16,18 @@ arduino-cli config init
 arduino-cli config add board_manager.additional_urls https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_dev_index.json
 arduino-cli core update-index
 
+REM directly execute PS, we can ignore any test errors.
+powershell -ExecutionPolicy ByPass -command "%~dp0common\Build.ps1" -restore -build -ci -configuration %2 -preparemachine
+
 set ArduinoRootDir=%1\Documents\Arduino
 set acspath=%~dp0\..\tools\ArduinoCsCompiler\Frontend\bin\%2\net6.0\acs.exe
 
 git clone https://github.com/firmata/ConfigurableFirmata %ArduinoRootDir%\libraries\ConfigurableFirmata
 git clone https://github.com/pgrawehr/ExtendedConfigurableFirmata %ArduinoRootDir%\ExtendedConfigurableFirmata
 arduino-cli core install esp32:esp32
 
+git fetch --all
+git branch --list
 REM Check whether any compiler files have changed - if so, enable the (long running) compiler tests
 git diff --name-status origin/main | find /C /I "tools/ArduinoCsCompiler"
 REM Find returns 1 when the string was NOT found, we want to set the variable to true when it does find something

src/devices/Arduino/ArduinoBoard.cs

@@ -329,6 +329,27 @@ public void AddCommandHandler<T>(T newCommandHandler)
             }
         }
 
+        /// <summary>
+        /// Unregisters the given command handler
+        /// </summary>
+        /// <typeparam name="T">A type derived from <see cref="ExtendedCommandHandler"/></typeparam>
+        /// <param name="commandHandler">The instance</param>
+        /// <remarks>This is intended mostly for unit test scenarios, where the command handlers are recreated. It does not
+        /// remove the modes supported by the handler</remarks>
+        public void RemoveCommandHandler<T>(T commandHandler)
+            where T : ExtendedCommandHandler
+        {
+            _commandHandlersLock.EnterWriteLock();
+            try
+            {
+                _extendedCommandHandlers.Remove(commandHandler);
+            }
+            finally
+            {
+                _commandHandlersLock.ExitWriteLock();
+            }
+        }
+
         /// <summary>
         /// Gets the command handler with the provided type. An exact type match is performed.
         /// </summary>

src/devices/Arduino/FirmataDevice.cs

@@ -30,7 +30,7 @@ internal sealed class FirmataDevice : IDisposable
         private const byte FIRMATA_PROTOCOL_MAJOR_VERSION = 2;
         private const byte FIRMATA_PROTOCOL_MINOR_VERSION = 5; // 2.5 works, but 2.6 is recommended
         private const int FIRMATA_INIT_TIMEOUT_SECONDS = 2;
-        internal static readonly TimeSpan DefaultReplyTimeout = TimeSpan.FromMilliseconds(2000);
+        internal static readonly TimeSpan DefaultReplyTimeout = TimeSpan.FromMilliseconds(3000);
 
         private byte _firmwareVersionMajor;
         private byte _firmwareVersionMinor;
@@ -1209,7 +1209,7 @@ internal void DisableAnalogReporting(int pinNumber, int analogChannel)
                     pwmCommandSequence.WriteByte((byte)0);
                     pwmCommandSequence.WriteByte((byte)FirmataCommand.END_SYSEX);
                     SendCommand(pwmCommandSequence);
-                }
+        }
             }
         }
 

src/devices/CharacterLcd/LcdInterface.I2c4Bit.cs

@@ -24,11 +24,13 @@ private sealed class I2c4Bit : LcdInterface
 
             private readonly I2cDevice _i2cDevice;
             private bool _backlightOn;
+            private int _backlightFlag;
 
             public I2c4Bit(I2cDevice i2cDevice)
             {
                 _i2cDevice = i2cDevice ?? throw new ArgumentNullException(nameof(i2cDevice));
                 _backlightOn = true;
+                _backlightFlag = LCD_BACKLIGHT;
                 InitDisplay();
             }
 
@@ -44,26 +46,12 @@ public override bool BacklightOn
                 set
                 {
                     _backlightOn = value;
+                    _backlightFlag = value ? LCD_BACKLIGHT : 0;
                     // Need to send a command to make this happen immediately.
                     SendCommandAndWait(0);
                 }
             }
 
-            private byte BacklightFlag
-            {
-                get
-                {
-                    if (BacklightOn)
-                    {
-                        return LCD_BACKLIGHT;
-                    }
-                    else
-                    {
-                        return 0;
-                    }
-                }
-            }
-
             private void InitDisplay()
             {
                 // This sequence (copied from a python example) completely resets the display (if it was
@@ -89,8 +77,8 @@ public override void SendCommand(byte command)
 
             private void Write4Bits(byte command)
             {
-                _i2cDevice.WriteByte((byte)(command | ENABLE | BacklightFlag));
-                _i2cDevice.WriteByte((byte)((command & ~ENABLE) | BacklightFlag));
+                _i2cDevice.WriteByte((byte)(command | ENABLE | _backlightFlag));
+                _i2cDevice.WriteByte((byte)((command & ~ENABLE) | _backlightFlag));
             }
 
             public override void SendCommands(ReadOnlySpan<byte> commands)

tools/ArduinoCsCompiler/ArduinoCsCompiler.csproj

@@ -29,4 +29,10 @@
     <ProjectReference Include="..\..\src\System.Device.Gpio\System.Device.Gpio.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
+      <_Parameter1>$(AssemblyName).Tests</_Parameter1>
+    </AssemblyAttribute>
+  </ItemGroup>
+
 </Project>

tools/ArduinoCsCompiler/ArduinoImplementationAttribute.cs

@@ -107,6 +107,15 @@ public bool MergeGenericImplementations
             set;
         }
 
+        /// <summary>
+        /// Set to true for methods that are only called by the runtime (e.g. thread start callbacks)
+        /// </summary>
+        public bool InternalCall
+        {
+            get;
+            set;
+        }
+
         /// <summary>
         /// Computes a hash code for a string that stays consistent over different architectures and between program runs.
         /// </summary>

tools/ArduinoCsCompiler/ArduinoMethodDeclaration.cs

@@ -40,13 +40,13 @@ public ArduinoMethodDeclaration(int token, EquatableMethod methodBase, ArduinoMe
             Name = $"{MethodBase.MethodSignature()} (Token 0x{Token:X})";
         }
 
-        public ArduinoMethodDeclaration(int token, EquatableMethod methodBase, ArduinoMethodDeclaration? requestedBy, IlCode code)
+        public ArduinoMethodDeclaration(int token, EquatableMethod methodBase, ArduinoMethodDeclaration? requestedBy, IlCode code, MethodFlags extraFlags)
         {
             Index = -1;
             MethodBase = methodBase;
             RequestedBy = requestedBy;
             Code = code;
-            Flags = MethodFlags.None;
+            Flags = extraFlags;
             Token = token;
 
             var attribs = methodBase.GetCustomAttributes(typeof(ArduinoImplementationAttribute)).Cast<ArduinoImplementationAttribute>().ToList();

tools/ArduinoCsCompiler/ClassDeclaration.cs

@@ -7,7 +7,7 @@
 #pragma warning disable CS1591
 namespace ArduinoCsCompiler
 {
-    public class ClassDeclaration
+    public class ClassDeclaration : IEquatable<ClassDeclaration>
     {
         private readonly List<ClassMember> _members;
         private readonly List<Type> _interfaces;
@@ -70,6 +70,57 @@ public bool SuppressInit
             }
         }
 
+        public bool Equals(ClassDeclaration? other)
+        {
+            if (ReferenceEquals(null, other))
+            {
+                return false;
+            }
+
+            if (ReferenceEquals(this, other))
+            {
+                return true;
+            }
+
+            // Here, Type and MiniType must be distinct.
+            return NewToken == other.NewToken && Name == other.Name;
+        }
+
+        public override bool Equals(object? obj)
+        {
+            if (ReferenceEquals(null, obj))
+            {
+                return false;
+            }
+
+            if (ReferenceEquals(this, obj))
+            {
+                return true;
+            }
+
+            if (obj.GetType() != GetType())
+            {
+                return false;
+            }
+
+            return Equals((ClassDeclaration)obj);
+        }
+
+        public override int GetHashCode()
+        {
+            return NewToken;
+        }
+
+        public static bool operator ==(ClassDeclaration? left, ClassDeclaration? right)
+        {
+            return Equals(left, right);
+        }
+
+        public static bool operator !=(ClassDeclaration? left, ClassDeclaration? right)
+        {
+            return !Equals(left, right);
+        }
+
         public void AddClassMember(ClassMember member)
         {
             if (ReadOnly)

tools/ArduinoCsCompiler/CompilerCommandHandler.cs

@@ -196,7 +196,7 @@ private bool ParseReply(byte[] data, ExecutorCommand expectedCommand, ref Comman
                     }
                     else if (data[2] == (byte)ExecutorCommand.ConditionalBreakpointHit || data[2] == (byte)ExecutorCommand.Variables)
                     {
-                        _compiler.OnCompilerCallback(data[3] | (data[4] << 7), MethodState.Debugging, data);
+                        _compiler.OnCompilerCallback(data[4] | (data[5] << 7), MethodState.Debugging, data);
                     }
                 }
                 else
@@ -344,7 +344,7 @@ public void AddMethodDeclarations(IList<FirmataCommandSequence> sequences, int d
         {
             FirmataIlCommandSequence sequence = new FirmataIlCommandSequence(ExecutorCommand.DeclareMethod);
             sequence.SendInt32(declarationToken);
-            sequence.WriteByte((byte)methodFlags);
+            sequence.SendUInt14((ushort)methodFlags);
             sequence.WriteByte(maxStack);
             sequence.WriteByte(argCount);
             sequence.SendInt32((int)nativeMethod);

tools/ArduinoCsCompiler/CompilerSettings.cs

@@ -19,6 +19,7 @@ public CompilerSettings()
             AdditionalSuppressions = new List<string>();
             LaunchProgramFromFlash = false;
             MaxMemoryUsage = 256 * 1024;
+            UsePreviewFeatures = false;
         }
 
         /// <summary>
@@ -93,6 +94,8 @@ public bool ForceFlashWrite
             set;
         }
 
+        public bool UsePreviewFeatures { get; set; }
+
         object ICloneable.Clone()
         {
             return MemberwiseClone();