Merge pull request #24 from davewalker5/BSR-42-AircraftDuplications

Modify the record locking code to avoid unnecessary aircraft record duplications

README.md

@@ -49,6 +49,7 @@
 | ApplicationSettings | TimeToRecent | --recent | -r | Threshold, in ms, after the most recent message at which an aircraft is considered "recent" (see states, below) |
 | ApplicationSettings | TimeToStale | --stale | -s | Threshold, in ms, after the most recent message at which an aircraft is considered "stale" (see states, below) |
 | ApplicationSettings | TimeToRemoval | --remove | -x | Threshold, in ms, after the most recent message at which an aircraft is removed from tracking (see states, below) |
+| ApplicationSettings | TimeToLock | --lock | -k | Threshold, in ms, after which an active aircraft record is locked, having received no updates |
 | ApplicationSettings | LogFile | --log-file | -l | Path and name of the log file |
 | ApplicationSettings | MinimumLogLevel | --log-level | -ll | Minimum message severity to log (Debug, Info, Warning or Error) |
 | ApplicationSettings | EnableSqlWriter | --enable-sql-writer | -w | Set to true to enable the SQL writer or false to disable it |
@@ -121,6 +122,7 @@
     - Recent
     - Stale
     - Removed
+    - Locked
 - The states have the following meanings:
 
 | State | Meaning |
@@ -129,6 +131,7 @@
 | Recent | Messages are not being received from the aircraft but they have been received recently |
 | Stale | Messages have not been received from the aircraft for some time and it will shortly be removed from the tracking list |
 | Removed | The aircraft has been removed from the tracking list |
+| Locked | The aircraft's database record has been locked against further updates. New sightings result in a new record |
 
 - Changes in state are communicated to AircraftTracker subscribers via the "aircraft updated" event (see above), with the state as a property of the tracking object
 
@@ -179,7 +182,9 @@ dotnet ef database update -s ../BaseStationReader.Terminal/BaseStationReader.Ter
 - The record for a given address should only be updated while the aircraft in question remains in range
 - Once it passes out of range, or when a new tracking session is started, if the address is seen again it should result in a new tracking record
 - This is achieved using the "Locked" flag on tracking records (see the screenshot, above):
-    - When an aircraft moves out of range and is removed from the tracking collection, its associated record is marked as "Locked"
+    - When an aircraft moves out of range and is removed from the tracking collection, a notional "lock timer" starts
+    - If it's seen again within the timout, the record remains unlocked to avoid duplication of aircraft records for the same flight
+    - Once the timeout is reached, the record is locked and any further updates for that ICAO address result in a new record
     - When the QueuedWriter starts, it immediately queues updates to mark all records that are not currently locked as locked, before accepting any other updates into the queue
 - Records marked as "Locked" are not considered candidates for further updates
 

sql/SummariseByAddress.sql

@@ -0,0 +1,6 @@
+PRAGMA journal_mode=WAL;
+
+SELECT Address, COUNT(1) AS 'Number'
+FROM AIRCRAFT
+GROUP BY Address
+ORDER BY COUNT(1) DESC;

src/BaseStationReader.Data/BaseStationReader.Data.csproj

@@ -5,7 +5,7 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <PackageId>BaseStationReader.Data</PackageId>
-    <PackageVersion>1.18.0.0</PackageVersion>
+    <PackageVersion>1.19.0.0</PackageVersion>
     <Authors>Dave Walker</Authors>
     <Copyright>Copyright (c) Dave Walker 2023</Copyright>
     <Owners>Dave Walker</Owners>
@@ -17,7 +17,7 @@
     <PackageProjectUrl>https://github.com/davewalker5/ADS-B-BaseStationReader</PackageProjectUrl>
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
     <PackageRequireLicenseAcceptance>false</PackageRequireLicenseAcceptance>
-    <ReleaseVersion>1.18.0.0</ReleaseVersion>
+    <ReleaseVersion>1.19.0.0</ReleaseVersion>
   </PropertyGroup>
 
   <ItemGroup>

src/BaseStationReader.Entities/BaseStationReader.Entities.csproj

@@ -5,7 +5,7 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <PackageId>BaseStationReader.Entities</PackageId>
-    <PackageVersion>1.18.0.0</PackageVersion>
+    <PackageVersion>1.19.0.0</PackageVersion>
     <Authors>Dave Walker</Authors>
     <Copyright>Copyright (c) Dave Walker 2023</Copyright>
     <Owners>Dave Walker</Owners>
@@ -17,7 +17,7 @@
     <PackageProjectUrl>https://github.com/davewalker5/ADS-B-BaseStationReader</PackageProjectUrl>
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
     <PackageRequireLicenseAcceptance>false</PackageRequireLicenseAcceptance>
-    <ReleaseVersion>1.18.0.0</ReleaseVersion>
+    <ReleaseVersion>1.19.0.0</ReleaseVersion>
   </PropertyGroup>
 
   <ItemGroup>

src/BaseStationReader.Entities/Config/ApplicationSettings.cs

@@ -14,6 +14,7 @@ public class ApplicationSettings
         public int TimeToRecent { get; set; }
         public int TimeToStale { get; set; }
         public int TimeToRemoval { get; set; }
+        public int TimeToLock { get; set; }
         public string LogFile { get; set; } = "";
         public bool EnableSqlWriter { get; set; }
         public int WriterInterval { get; set; }

src/BaseStationReader.Entities/Config/CommandLineOptionType.cs

@@ -10,6 +10,7 @@ public enum CommandLineOptionType
         TimeToRecent,
         TimeToStale,
         TimeToRemoval,
+        TimeToLock,
         LogFile,
         EnableSqlWriter,
         WriterInterval,

src/BaseStationReader.Entities/Interfaces/IAircraftLockManager.cs

@@ -0,0 +1,9 @@
+using BaseStationReader.Entities.Tracking;
+
+namespace BaseStationReader.Logic
+{
+    public interface IAircraftLockManager
+    {
+        Task<Aircraft?> GetActiveAircraft(string address);
+    }
+}

src/BaseStationReader.Logic/AircraftLockManager.cs

@@ -0,0 +1,40 @@
+using BaseStationReader.Entities.Interfaces;
+using BaseStationReader.Entities.Tracking;
+
+namespace BaseStationReader.Logic
+{
+    public class AircraftLockManager : IAircraftLockManager
+    {
+        private readonly IAircraftWriter _writer;
+        private readonly int _timeToLock;
+
+        public AircraftLockManager(IAircraftWriter writer, int timeToLockMs)
+        {
+            _writer = writer;
+            _timeToLock = timeToLockMs;
+        }
+
+        /// <summary>
+        /// Get the active aircraft with the specified address
+        /// </summary>
+        /// <param name="address"></param>
+        /// <returns></returns>
+        public async Task<Aircraft?> GetActiveAircraft(string address)
+        {
+            // Get the aircraft. This method is guaranteed to return the most recent record for a given aircraft
+            // address
+            Aircraft? aircraft = await _writer.GetAsync(x => x.Address == address);
+
+            // If the last seen date has exceeded the time to lock timeout, this record should no longer be active
+            if ((aircraft != null) && ((DateTime.Now - aircraft.LastSeen).TotalMilliseconds >= _timeToLock))
+            {
+                // Timeout has been exceeded, so lock the record and return null
+                aircraft.Locked = true;
+                await _writer.WriteAsync(aircraft);
+                aircraft = null;
+            }
+
+            return aircraft;
+        }
+    }
+}

src/BaseStationReader.Logic/AircraftWriter.cs

@@ -21,7 +21,7 @@ public AircraftWriter(BaseStationReaderDbContext context)
         }
 
         /// <summary>
-        /// Get the first aircraft matching the specified criteria
+        /// Get the most recent aircraft matching the specified criteria
         /// </summary>
         /// <param name="predicate"></param>
         /// <returns></returns>
@@ -34,12 +34,12 @@ public async Task<Aircraft> GetAsync(Expression<Func<Aircraft, bool>> predicate)
         }
 
         /// <summary>
-        /// List all aircraft matching the specified criteria
+        /// List all aircraft matching the specified criteria, most recent first
         /// </summary>
         /// <param name="predicate"></param>
         /// <returns></returns>
         public async Task<List<Aircraft>> ListAsync(Expression<Func<Aircraft, bool>> predicate)
-            => await _context.Aircraft.Where(predicate).ToListAsync();
+            => await _context.Aircraft.Where(predicate).OrderByDescending(x => x.LastSeen).ToListAsync();
 
         /// <summary>
         /// Write an aircraft to the database, either creating a new record or updating an existing one

src/BaseStationReader.Logic/BaseStationReader.Logic.csproj

@@ -5,7 +5,7 @@
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
     <PackageId>BaseStationReader.Logic</PackageId>
-    <PackageVersion>1.18.0.0</PackageVersion>
+    <PackageVersion>1.19.0.0</PackageVersion>
     <Authors>Dave Walker</Authors>
     <Copyright>Copyright (c) Dave Walker 2023</Copyright>
     <Owners>Dave Walker</Owners>
@@ -17,7 +17,7 @@
     <PackageProjectUrl>https://github.com/davewalker5/ADS-B-BaseStationReader</PackageProjectUrl>
     <PackageLicenseExpression>MIT</PackageLicenseExpression>
     <PackageRequireLicenseAcceptance>false</PackageRequireLicenseAcceptance>
-    <ReleaseVersion>1.18.0.0</ReleaseVersion>
+    <ReleaseVersion>1.19.0.0</ReleaseVersion>
   </PropertyGroup>
 
   <ItemGroup>

src/BaseStationReader.Logic/QueuedWriter.cs

@@ -11,6 +11,7 @@ public class QueuedWriter : IQueuedWriter
     {
         private readonly IAircraftWriter _aircraftWriter;
         private readonly IPositionWriter _positionWriter;
+        private readonly IAircraftLockManager _locker;
         private readonly ConcurrentQueue<object> _queue = new ConcurrentQueue<object>();
         private readonly ITrackerLogger _logger;
         private readonly ITrackerTimer _timer;
@@ -21,12 +22,14 @@ public class QueuedWriter : IQueuedWriter
         public QueuedWriter(
             IAircraftWriter aircraftWriter,
             IPositionWriter positionWriter,
+            IAircraftLockManager locker,
             ITrackerLogger logger,
             ITrackerTimer timer,
             int batchSize)
         {
             _aircraftWriter = aircraftWriter;
             _positionWriter = positionWriter;
+            _locker = locker;
             _logger = logger;
             _timer = timer;
             _timer.Tick += OnTimer;
@@ -133,18 +136,24 @@ private async Task WriteDequeuedObject(object queued)
             AircraftPosition? position = null;
             if (aircraft != null)
             {
-                // See if this is an existing aircraft for which the record hasn't been locked, to get the ID for update
-                aircraft.Id = await MatchAircraftAddress(aircraft.Address, false);
+                // Get the active aircraft with the specified address, if there is one, so it can be updated
+                var activeAircraft = await _locker.GetActiveAircraft(aircraft.Address);
+                if (activeAircraft != null)
+                {
+                    aircraft.Id = activeAircraft.Id;
+                }
             }
             else
             {
-                // Not an aircraft so it must be a position. If the aircraft Id isn't specified, try to find a match
-                // for the address. Note that we allow matches to locked aircraft as the final position update may
-                // come after the aircraft record has been locked
+                // Not an aircraft so it must be a position - match this to an active aircraft
                 position = queued as AircraftPosition;
                 if (position != null)
                 {
-                    position.AircraftId = await MatchAircraftAddress(position.Address, true);
+                    var activeAircraft = await _locker.GetActiveAircraft(position.Address);
+                    if (activeAircraft != null)
+                    {
+                        position.AircraftId = activeAircraft.Id;
+                    }
                 }
             }
 
@@ -169,27 +178,5 @@ private async Task WriteDequeuedObject(object queued)
                 _logger.LogException(ex);
             }
         }
-
-        /// <summary>
-        /// Find an existing aircraft with the specified address and return it's Id, if found, or 0 if not found
-        /// </summary>
-        /// <param name="address"></param>
-        /// <param name="matchLockedAircraft"></param>
-        /// <returns></returns>
-        private async Task<int> MatchAircraftAddress(string address, bool matchLockedAircraft)
-        {
-            int matchingId = 0;
-
-            if (!string.IsNullOrEmpty(address))
-            {
-                var existing = await _aircraftWriter.GetAsync(x => (x.Address == address) && (matchLockedAircraft || !x.Locked));
-                if (existing != null)
-                {
-                    matchingId = existing.Id;
-                }
-            }
-
-            return matchingId;
-        }
     }
 }

src/BaseStationReader.Terminal/BaseStationReader.Terminal.csproj

@@ -3,9 +3,9 @@
   <PropertyGroup>
     <OutputType>Exe</OutputType>
     <TargetFramework>net7.0</TargetFramework>
-    <ReleaseVersion>1.18.0.0</ReleaseVersion>
-    <FileVersion>1.18.0.0</FileVersion>
-    <ProductVersion>1.18.0</ProductVersion>
+    <ReleaseVersion>1.19.0.0</ReleaseVersion>
+    <FileVersion>1.19.0.0</FileVersion>
+    <ProductVersion>1.19.0</ProductVersion>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
   </PropertyGroup>

src/BaseStationReader.Terminal/Logic/TrackerSettingsBuilder.cs

@@ -26,7 +26,8 @@ public class TrackerSettingsBuilder : ITrackerSettingsBuilder
             parser.Add(CommandLineOptionType.ApplicationTimeout, false, "--app-timeout", "-a", "Timeout (ms) after which the application will quit of no messages are recieved", 1, 1);
             parser.Add(CommandLineOptionType.TimeToRecent, false, "--recent", "-r", "Time (ms) to 'recent' staleness", 1, 1);
             parser.Add(CommandLineOptionType.TimeToStale, false, "--stale", "-s", "Time (ms) to 'stale' staleness", 1, 1);
-            parser.Add(CommandLineOptionType.TimeToRemoval, false, "--remove", "-x", "Time (ms) removal of stale records", 1, 1);
+            parser.Add(CommandLineOptionType.TimeToRemoval, false, "--remove", "-x", "Time (ms) to removal of stale records", 1, 1);
+            parser.Add(CommandLineOptionType.TimeToLock, false, "--lock", "-k", "Time (ms) to locking of active database records", 1, 1);
             parser.Add(CommandLineOptionType.LogFile, false, "--log-file", "-l", "Log file path and name", 1, 1);
             parser.Add(CommandLineOptionType.MinimumLogLevel, false, "--log-level", "-ll", "Minimum logging level (Debug, Info, Warning or Error)", 1, 1);
             parser.Add(CommandLineOptionType.EnableSqlWriter, false, "--enable-sql-writer", "-w", "Log file path and name", 1, 1);
@@ -58,6 +59,9 @@ public class TrackerSettingsBuilder : ITrackerSettingsBuilder
             values = parser.GetValues(CommandLineOptionType.TimeToRemoval);
             if (values != null) settings!.TimeToRemoval = int.Parse(values[0]);
 
+            values = parser.GetValues(CommandLineOptionType.TimeToLock);
+            if (values != null) settings!.TimeToLock = int.Parse(values[0]);
+
             values = parser.GetValues(CommandLineOptionType.LogFile);
             if (values != null) settings!.LogFile = values[0];
 

src/BaseStationReader.Terminal/Program.cs

@@ -104,8 +104,9 @@ private static async Task ShowTrackingTable(LiveDisplayContext ctx)
                 BaseStationReaderDbContext context = new BaseStationReaderDbContextFactory().CreateDbContext(Array.Empty<string>());
                 var aircraftWriter = new AircraftWriter(context);
                 var positionWriter = new PositionWriter(context);
+                var aircraftLocker = new AircraftLockManager(aircraftWriter, _settings.TimeToLock);
                 var writerTimer = new TrackerTimer(_settings.WriterInterval);
-                _writer = new QueuedWriter(aircraftWriter, positionWriter, _logger!, writerTimer, _settings.WriterBatchSize);
+                _writer = new QueuedWriter(aircraftWriter, positionWriter, aircraftLocker, _logger!, writerTimer, _settings.WriterBatchSize);
                 _writer.BatchWritten += OnBatchWritten;
                 _writer.Start();
             }
@@ -197,15 +198,6 @@ private static void OnAircraftRemoved(object? sender, AircraftNotificationEventA
             // Update the timestamp used to implement the application timeout
             _lastUpdate = DateTime.Now;
 
-            // Lock the aircraft record - if we see it again, a new record will be created
-            e.Aircraft.Locked = true;
-            if (_settings!.EnableSqlWriter)
-            {
-#pragma warning disable CS8602
-                _writer.Push(e.Aircraft);
-#pragma warning restore CS8602
-            }
-
             // Remove the aircraft from the index
             var rowNumber = _tableManager!.RemoveAircraft(e.Aircraft);
             _logger!.LogMessage(Severity.Info, $"Removed aircraft {e.Aircraft.Address} at row {rowNumber}");

src/BaseStationReader.Terminal/appsettings.desktop.json

@@ -7,6 +7,7 @@
     "TimeToRecent": 60000,
     "TimeToStale": 120000,
     "TimeToRemoval": 180000,
+    "TimeToLock": 900000,
     "LogFile": "C:\\MyApps\\AircraftTracker.log",
     "MinimumLogLevel": "Info",
     "EnableSqlWriter": false,

src/BaseStationReader.Terminal/appsettings.json

@@ -7,6 +7,7 @@
     "TimeToRecent": 60000,
     "TimeToStale": 120000,
     "TimeToRemoval": 180000,
+    "TimeToLock":  900000,
     "LogFile": "C:\\MyApps\\AircraftTracker.log",
     "MinimumLogLevel": "Info",
     "EnableSqlWriter": false,

src/BaseStationReader.Terminal/appsettings.pi.json

@@ -7,6 +7,7 @@
     "TimeToRecent": 60000,
     "TimeToStale": 120000,
     "TimeToRemoval": 180000,
+    "TimeToLock": 900000,
     "LogFile": "/home/dave/MyApps/AircraftTracker.log",
     "MinimumLogLevel": "Info",
     "EnableSqlWriter": false,