diff --git a/src/Aardvark.Algodat.Tests/Program.cs b/src/Aardvark.Algodat.Tests/Program.cs index d6516326..c17ee39b 100644 --- a/src/Aardvark.Algodat.Tests/Program.cs +++ b/src/Aardvark.Algodat.Tests/Program.cs @@ -34,16 +34,22 @@ namespace Aardvark.Geometry.Tests { public class Program { - internal static Task CreateStore(string filename, string storeDir, double minDist) + #region CreateStore + + internal static Task CreateStore(string filename, string storeDir, double minDist) => CreateStore(filename, new DirectoryInfo(storeDir), minDist); - internal static Task CreateStore(string filename, DirectoryInfo storeDir, double minDist) + internal static Task CreateStore(IEnumerable chunks, string storeDir, double minDist, int? splitLimit = null) + => CreateStore(chunks, new DirectoryInfo(storeDir), minDist, splitLimit); + + internal static Task CreateStore(string filename, DirectoryInfo storeDir, double minDist) { if (!storeDir.Exists) storeDir.Create(); var key = Path.GetFileName(filename); using var store = new SimpleDiskStore(Path.Combine(storeDir.FullName, "data.uds")).ToPointCloudStore(); + var config = ImportConfig.Default .WithStorage(store) .WithKey(key) @@ -53,6 +59,7 @@ internal static Task CreateStore(string filename, DirectoryInfo storeDir, double .WithNormalizePointDensityGlobal(true) //.WithMaxChunkPointCount(32 * 1024 * 1024) //.WithOctreeSplitLimit(8192*4) + //.WithEnabledPartIndices(false) ; Report.BeginTimed($"importing {filename}"); @@ -61,9 +68,42 @@ internal static Task CreateStore(string filename, DirectoryInfo storeDir, double File.WriteAllText(Path.Combine(storeDir.FullName, "key.txt"), key); - return Task.CompletedTask; + return Task.FromResult(key); + } + + internal static Task CreateStore(IEnumerable chunks, DirectoryInfo storeDir, double minDist, int? splitLimit = null) + { + if (!storeDir.Exists) storeDir.Create(); + + var filename = storeDir.FullName; + var key = Path.GetFileName(filename); + + using var store = new SimpleDiskStore(Path.Combine(storeDir.FullName, "data.uds")).ToPointCloudStore(); + + var config = ImportConfig.Default + .WithStorage(store) + .WithKey(key) + .WithVerbose(true) + .WithMaxDegreeOfParallelism(0) + .WithMinDist(minDist) + .WithNormalizePointDensityGlobal(true) + //.WithMaxChunkPointCount(32 * 1024 * 1024) + //.WithEnabledPartIndices(false) + ; + + if (splitLimit.HasValue) config = config.WithOctreeSplitLimit(splitLimit.Value); + + Report.BeginTimed($"importing {filename}"); + var pcl = PointCloud.Import(chunks, config); + Report.EndTimed(); + + File.WriteAllText(Path.Combine(storeDir.FullName, "key.txt"), key); + + return Task.FromResult(key); } + #endregion + internal static void PerfTestJuly2019() { var filename = @"T:\Vgm\Data\2017-10-20_09-44-27_1mm_shade_norm_5pp - Cloud.pts"; @@ -2689,22 +2729,51 @@ static Task Parts_Test_20231006() return Task.CompletedTask; } + static async Task Parts_Test_20231006_Merge() + { + V3d[] randomPoints(int n, Box3d bb) + { + var size = bb.Size; + var ps = new V3d[n]; + for (var i = 0; i < n; i++) + { + ps[i] = bb.Min + new V3d(Random.Shared.NextDouble(), Random.Shared.NextDouble(), Random.Shared.NextDouble()) * size; + } + + return ps; + } + + var c0 = new Chunk(positions: randomPoints(10, Box3d.Unit ), null, null, null, null, partIndices: 0, null, null); + var c1 = new Chunk(positions: randomPoints(10, Box3d.Unit + V3d.IOO), null, null, null, null, partIndices: 1, null, null); + + var storeDir = @"W:\Datasets\Vgm\Stores\test_partindex"; + var key = await CreateStore( + new [] { c0, c1 }, + storeDir, + minDist: 0.01, + splitLimit: 10 + ); + + using var store = new SimpleDiskStore(Path.Combine(storeDir, "data.uds")).ToPointCloudStore(); + var ps = store.GetPointSet(key); + var root = ps.Root.Value; + } public static async Task Main(string[] _) { - await Task.Delay(0); // avoid warnings if main contains no await + //await Task.Delay(0); // avoid warnings if main contains no await + await CreateStore( + @"W:\Datasets\Vgm\Data\structured_pointclouds\lowergetikum 20230321.e57", + @"W:\Datasets\Vgm\Stores\lowergetikum 20230321.e57_0.01", + minDist: 0.01 + ); + //await Parts_Test_20231006_Merge(); //await Parts_Test_20231006(); - Test_Import_Regression(); - - //await CreateStore( - // @"T:\Kindergarten.pts", - // @"W:\Aardworx\pointshare2\testdata\2023-08-01_kindergarten.store", - // minDist: 0.0 - // ); + //Test_Import_Regression(); //await Ranges_Test_20230802(); diff --git a/src/Aardvark.Data.E57/Aardvark.Data.E57.csproj b/src/Aardvark.Data.E57/Aardvark.Data.E57.csproj index b8396a73..c06f399c 100644 --- a/src/Aardvark.Data.E57/Aardvark.Data.E57.csproj +++ b/src/Aardvark.Data.E57/Aardvark.Data.E57.csproj @@ -3,6 +3,7 @@ netstandard2.0 10 + false ..\..\bin\Debug diff --git a/src/Aardvark.Data.E57/ImportE57.cs b/src/Aardvark.Data.E57/ImportE57.cs index 2afd4035..ef44ba82 100644 --- a/src/Aardvark.Data.E57/ImportE57.cs +++ b/src/Aardvark.Data.E57/ImportE57.cs @@ -195,7 +195,7 @@ public static IEnumerable Chunks(this Stream stream, long streamLengthInB var yieldedRecordCount = 0L; var partIndex = config.PartIndexOffset; - foreach (var data3d in header.E57Root.Data3D) + foreach (var data3d in header.E57Root.Data3D.Take(2)) { foreach (var (Positions, Properties) in data3d.StreamPointsFull(config.MaxChunkPointCount, config.Verbose, exclude)) { diff --git a/src/Aardvark.Geometry.PointSet/Aardvark.Geometry.PointSet.csproj b/src/Aardvark.Geometry.PointSet/Aardvark.Geometry.PointSet.csproj index 1d2b8143..84a33b8d 100644 --- a/src/Aardvark.Geometry.PointSet/Aardvark.Geometry.PointSet.csproj +++ b/src/Aardvark.Geometry.PointSet/Aardvark.Geometry.PointSet.csproj @@ -8,6 +8,7 @@ ..\..\bin\Release\netstandard2.0\Aardvark.Geometry.PointSet.xml true Aardvark.Geometry.Points + false ..\..\bin\Debug diff --git a/src/Aardvark.Geometry.PointSet/Octrees/IPointCloudNodeExtensions.cs b/src/Aardvark.Geometry.PointSet/Octrees/IPointCloudNodeExtensions.cs index 4311ccf9..e2c572c7 100644 --- a/src/Aardvark.Geometry.PointSet/Octrees/IPointCloudNodeExtensions.cs +++ b/src/Aardvark.Geometry.PointSet/Octrees/IPointCloudNodeExtensions.cs @@ -839,6 +839,7 @@ IEnumerable QueryRec(IPointCloudNode n) public static void CheckDerivedAttributes(this IPointCloudNode self) { var isTmpNode = self.Has(PointSetNode.TemporaryImportNode); + if (self.HasPositions) { if (!self.HasKdTree && !isTmpNode) throw new InvalidOperationException( @@ -863,12 +864,34 @@ public static void CheckDerivedAttributes(this IPointCloudNode self) // "Missing PointDistanceStandardDeviation. Invariant 4a03c3f5-a625-4124-91f6-6f79fd1b5d0e." // ); } + if (!self.HasMaxTreeDepth) throw new InvalidOperationException( "Missing MaxTreeDepth. Invariant c7c3c337-5404-4773-aae3-01d213e575b0." ); if (!self.HasMinTreeDepth) throw new InvalidOperationException( "Missing MinTreeDepth. Invariant 2df9fb7b-684a-4103-8f14-07785607d2f4." ); + + if (self.HasPartIndexRange && !self.HasPartIndices) throw new InvalidOperationException( + "Missing part indices. Invariant fc4ff983-151f-4956-b882-54d649245049." + ); + + if (self.HasPartIndices) + { + if (!self.HasPartIndexRange) throw new InvalidOperationException( + "Missing PartIndexRange. Invariant 90571267-b593-4458-83d6-5448dd9c5257." + ); + + if (self.Has(Durable.Octree.PerPointPartIndex1b)) throw new InvalidOperationException( + "PerPointPartIndex1b should be PerPointPartIndex1bReference. Invariant 85879fd8-7ee8-45a0-b221-67f826bc42df." + ); + if (self.Has(Durable.Octree.PerPointPartIndex1s)) throw new InvalidOperationException( + "PerPointPartIndex1s should be PerPointPartIndex1sReference. Invariant 598f615b-289a-4bfe-b5cf-b61e4bd380d5." + ); + if (self.Has(Durable.Octree.PerPointPartIndex1i)) throw new InvalidOperationException( + "PerPointPartIndex1i should be PerPointPartIndex1iReference. Invariant d5391b75-bbc1-4c83-b963-50fd0f409931." + ); + } } /// diff --git a/src/Aardvark.Geometry.PointSet/Octrees/InMemoryPointSet.cs b/src/Aardvark.Geometry.PointSet/Octrees/InMemoryPointSet.cs index 8b6cafae..5b91ff02 100644 --- a/src/Aardvark.Geometry.PointSet/Octrees/InMemoryPointSet.cs +++ b/src/Aardvark.Geometry.PointSet/Octrees/InMemoryPointSet.cs @@ -25,18 +25,6 @@ namespace Aardvark.Geometry.Points { public class InMemoryPointSet { - /// - /// The following attribute arrays will be stored stand-alone and referenced via id. - /// - public static Dictionary StoreAsReference { get; } = new Dictionary() - { - { Octree.PositionsLocal3f, Octree.PositionsLocal3fReference }, - { Octree.Colors4b, Octree.Colors4bReference }, - { Octree.Normals3f, Octree.Normals3fReference }, - { Octree.Intensities1i, Octree.Intensities1iReference }, - { Octree.Classifications1b, Octree.Classifications1bReference } - }; - private readonly ImmutableDictionary m_data; private readonly int m_splitLimit; private readonly Node m_root; @@ -166,145 +154,137 @@ public Node(InMemoryPointSet octree, Cell cell) _centerX = c.X; _centerY = c.Y; _centerZ = c.Z; } + + /// + /// The following arrays will be stored as separate entries in the store and referenced via id. + /// + private static readonly Dictionary StoreAsReference = new() + { + { Octree.PositionsLocal3f , Octree.PositionsLocal3fReference }, + { Octree.Colors4b , Octree.Colors4bReference }, + { Octree.Normals3f , Octree.Normals3fReference }, + { Octree.Intensities1i , Octree.Intensities1iReference }, + { Octree.Classifications1b , Octree.Classifications1bReference }, + { Octree.PerPointPartIndex1b, Octree.PerPointPartIndex1bReference }, + { Octree.PerPointPartIndex1s, Octree.PerPointPartIndex1sReference }, + { Octree.PerPointPartIndex1i, Octree.PerPointPartIndex1iReference }, + }; + + internal PointSetNode ToPointSetNode(Storage storage, bool isTemporaryImportNode) { - var center = new V3d(_centerX, _centerY, _centerZ); - V3f[]? localPositions = null; + // final result data (which we will use to create a PointSetNode) + var resultData = ImmutableDictionary.Empty + .Add(Octree.NodeId, Guid.NewGuid()) + .Add(Octree.Cell, _cell) + ; + + if (isTemporaryImportNode) resultData = resultData.Add(PointSetNode.TemporaryImportNode, 0); + + // keep track whether we really convert all existing data entries to a PointSetNode + // (so we can fail if something is left over at the end, something we did not expect or handle correctly) + var notHandled = new HashSet(_octree.m_data.Keys); - var attributes = ImmutableDictionary.Empty; + var center = new V3d(_centerX, _centerY, _centerZ); - if (_ia != null) + if (_subnodes == null) { - var count = _ia.Count; + // leaf node ... - // compute positions attribute (relative to center) - var allPs = _octree.m_ps; - localPositions = new V3f[count]; - for (var i = 0; i < count; i++) localPositions[i] = (V3f)(allPs[_ia[i]] - center); // relative to center - attributes = attributes.Add(Octree.PositionsLocal3f, localPositions); + #region positions - // create all other attributes ... - foreach (var kv in _octree.m_data) + // transform global float64 positions to local float32 positions (relative to cell center) to save space + var psGlobal = _octree.m_ps; + V3f[] psLocal; + if (_ia != null) { - if (kv.Key == Octree.PositionsGlobal3d || - kv.Key == Octree.PerCellPartIndex1i || - kv.Key == Octree.PerCellPartIndex1ui || - kv.Key == Octree.PerPointPartIndex1b || - kv.Key == Octree.PerPointPartIndex1s || - kv.Key == Octree.PerPointPartIndex1i || - kv.Key == Octree.PartIndexRange - ) - continue; - var subset = kv.Value.Subset(_ia); - attributes = attributes.Add(kv.Key, subset); + psLocal = new V3f[_ia.Count]; + for (var i = 0; i < _ia.Count; i++) psLocal[i] = (V3f)(psGlobal[_ia[i]] - center); // relative to center + } + else + { + psLocal = new V3f[psGlobal.Count]; + for (var i = 0; i < psGlobal.Count; i++) psLocal[i] = (V3f)(psGlobal[i] - center); // relative to center } - } - // subcells ... - var subcells = _subnodes?.Map(x => x?.ToPointSetNode(storage, isTemporaryImportNode)); - var subcellIds = subcells?.Map(x => x?.Id); - var isLeaf = _subnodes == null; + var bbExactLocal = new Box3f(psLocal); -#if DEBUG - if (_subnodes != null) - { - if (localPositions != null) - throw new InvalidOperationException("Invariant d98ea55b-760c-4564-8076-ce9cf7d293a0."); + resultData = resultData + .Add(Octree.PositionsLocal3f, psLocal) + .Add(Octree.PointCountCell, psLocal.Length) + .Add(Octree.PointCountTreeLeafs, (long)psLocal.Length) + .Add(Octree.BoundingBoxExactLocal, bbExactLocal) + .Add(Octree.BoundingBoxExactGlobal, (Box3d)bbExactLocal + center) + ; - for (var i = 0; i < 8; i++) + #endregion + + #region per-point attribute arrays + + var handledPerPointAttributeArrays = new List(); + foreach (var k in notHandled) { - var sn = _subnodes[i]; if (sn == null) continue; - if (sn._cell.Exponent != this._cell.Exponent - 1) + if (StoreAsReference.TryGetValue(k, out var refdef)) { - throw new InvalidOperationException("Invariant 2c33afb4-683b-4f71-9e1f-36ec4a79fba1."); - } - } - } -#endif - var pointCountTreeLeafs = subcells != null - ? subcells.Sum(n => n != null ? n.PointCountTree : 0) - : localPositions!.Length - ; + var xs = (Array)_octree.m_data[k]; + if (_ia != null) xs = xs.Subset(_ia); - var data = ImmutableDictionary.Empty - .Add(Octree.NodeId, Guid.NewGuid()) - .Add(Octree.Cell, _cell) - .Add(Octree.PointCountTreeLeafs, pointCountTreeLeafs) - ; + // store separately and reference by id ... + var id = Guid.NewGuid(); + storage.Add(id, xs); + resultData = resultData.Add(refdef, id); - if (isTemporaryImportNode) - { - data = data.Add(PointSetNode.TemporaryImportNode, 0); - } + handledPerPointAttributeArrays.Add(k); + } + } + foreach (var k in handledPerPointAttributeArrays) notHandled.Remove(k); - if (attributes.TryGetValue(Octree.PositionsLocal3f, out var psObj)) - { - var ps = (V3f[])psObj; - var bbExactLocal = new Box3f(ps); + #endregion - data = data - .Add(Octree.PointCountCell, ps.Length) - .Add(Octree.BoundingBoxExactLocal, bbExactLocal) - .Add(Octree.BoundingBoxExactGlobal, (Box3d)bbExactLocal + center) - ; - } - else - { - data = data - .Add(Octree.PointCountCell, 0) - ; - } + #region other well-known entries - // part indices ... - { void copy(Def def) { if (_octree.m_data.TryGetValue(def, out var o)) { - data = data.Add(def, o); + resultData = resultData.Add(def, o); + notHandled.Remove(def); } } + copy(Octree.PartIndexRange); copy(Octree.PerCellPartIndex1i); copy(Octree.PerCellPartIndex1ui); - copy(Octree.PartIndexRange); - if ( - _octree.m_data.TryGetValue(Octree.PerPointPartIndex1b, out object? qs) || - _octree.m_data.TryGetValue(Octree.PerPointPartIndex1s, out qs) || - _octree.m_data.TryGetValue(Octree.PerPointPartIndex1i, out qs) - ) - { - qs = _ia != null ? PartIndexUtils.Subset(qs, _ia) : qs; - if (qs != null) data = data.Add(PartIndexUtils.GetDurableDefForPartIndices(qs), qs); - } - } + #endregion - // save attribute arrays to store ... - foreach (var kv in attributes) - { - if (StoreAsReference.TryGetValue(kv.Key, out var refdef)) - { - // store separately and reference by id ... - var id = Guid.NewGuid(); - storage.Add(id, (Array)kv.Value); - data = data.Add(refdef, id); - } - else - { - // store inline (inside node) ... - data = data.Add(kv.Key, kv.Value); - } - } + if (notHandled.Count > 0) throw new Exception( + $"Unhandled entries {string.Join(", ", notHandled.Select(x => x.ToString()))}. " + + $"Invariant 42656f92-f5ac-43ca-a1b7-c7ec95fe9cb3." + ); - if (isLeaf) // leaf - { - var result = new PointSetNode(data, storage, writeToStore: true); - if (storage.GetPointCloudNode(result.Id) == null) throw new InvalidOperationException("Invariant d1022027-2dbf-4b11-9b40-4829436f5789."); + var result = new PointSetNode(resultData, storage, writeToStore: true); + if (storage.GetPointCloudNode(result.Id) == null) throw new InvalidOperationException("Invariant 9e863bc5-e9f4-4d39-bd53-3d81e12af6b1."); return result; } else { + // inner node ... + + var subcells = _subnodes?.Map(x => x?.ToPointSetNode(storage, isTemporaryImportNode)); + var subcellIds = subcells?.Map(x => x?.Id); + + var pointCountTreeLeafs = subcells.Sum(n => n != null ? n.PointCountTree : 0); + var bbExactGlobal = new Box3d(subcells.Where(x => x != null).Select(x => x!.BoundingBoxExactGlobal)); + var bbExactLocal = (Box3f)(bbExactGlobal - center); + + resultData = resultData + .Add(Octree.PointCountTreeLeafs, pointCountTreeLeafs) + .Add(Octree.BoundingBoxExactLocal, bbExactLocal) + .Add(Octree.BoundingBoxExactGlobal, bbExactGlobal) + .Add(Octree.SubnodesGuids, subcellIds.Map(x => x ?? Guid.Empty)) + ; + for (var i = 0; i < 8; i++) { var x = subcellIds![i]; @@ -314,15 +294,76 @@ void copy(Def def) if (storage.GetPointCloudNode(id) == null) throw new InvalidOperationException("Invariant 01830b8b-3c0e-4a8b-a1bd-bfd1b1be1844."); } } - var bbExactGlobal = new Box3d(subcells.Where(x => x != null).Select(x => x!.BoundingBoxExactGlobal)); - data = data - .Add(Octree.BoundingBoxExactGlobal, bbExactGlobal) - .Add(Octree.SubnodesGuids, subcellIds.Map(x => x ?? Guid.Empty)) - ; - var result = new PointSetNode(data, storage, writeToStore: true); + + var result = new PointSetNode(resultData, storage, writeToStore: true); if (storage.GetPointCloudNode(result.Id) == null) throw new InvalidOperationException("Invariant 7b09eccb-b6a0-4b99-be7a-eeff53b6a98b."); return result; } + + // stuff + //var attributes = ImmutableDictionary.Empty; + + /* + * EXPLAINER: + * + * If _ia exists, then this means that a subset of all per-point attribute arrays (positions, colors, ...) has to be taken (according to the indices stored in _ia) + * otherwise these arrays can be used as they are + * + */ + + + + //if (_ia != null) + //{ + // var count = _ia.Count; + + // // create all other attributes ... + // foreach (var kv in _octree.m_data) + // { + // if (kv.Key == Octree.PositionsGlobal3d || + // kv.Key == Octree.PerCellPartIndex1i || + // kv.Key == Octree.PerCellPartIndex1ui || + // kv.Key == Octree.PerPointPartIndex1b || + // kv.Key == Octree.PerPointPartIndex1s || + // kv.Key == Octree.PerPointPartIndex1i || + // kv.Key == Octree.PartIndexRange + // ) + // continue; + // var subset = kv.Value.Subset(_ia); + // attributes = attributes.Add(kv.Key, subset); + // } + //} + //else + //{ + + //} + + + //// part indices ... + //{ + // void copy(Def def) + // { + // if (_octree.m_data.TryGetValue(def, out var o)) + // { + // data = data.Add(def, o); + // } + // } + + // copy(Octree.PerCellPartIndex1i); + // copy(Octree.PerCellPartIndex1ui); + // copy(Octree.PartIndexRange); + + // //if ( + // // _octree.m_data.TryGetValue(Octree.PerPointPartIndex1b, out object? qs) || + // // _octree.m_data.TryGetValue(Octree.PerPointPartIndex1s, out qs) || + // // _octree.m_data.TryGetValue(Octree.PerPointPartIndex1i, out qs) + // // ) + // //{ + // // qs = _ia != null ? PartIndexUtils.Subset(qs, _ia) : qs; + // // if (qs != null) data = data.Add(PartIndexUtils.GetDurableDefForPartIndices(qs), qs); + // //} + //} + } public Node Insert(int index) diff --git a/src/Aardvark.Geometry.PointSet/Octrees/Merge.cs b/src/Aardvark.Geometry.PointSet/Octrees/Merge.cs index 168747c3..d3b1fc90 100644 --- a/src/Aardvark.Geometry.PointSet/Octrees/Merge.cs +++ b/src/Aardvark.Geometry.PointSet/Octrees/Merge.cs @@ -378,11 +378,27 @@ public static (IPointCloudNode, bool) Merge(this IPointCloudNode a, IPointCloudN if (qs != null) { if (qsRange == null) throw new Exception("Invariant 7c01f554-d833-42cc-ab1d-9dbaa61bef45."); - var def = PartIndexUtils.GetDurableDefForPartIndices(qs); - data = data - .Add(def, qs) - .Add(Durable.Octree.PartIndexRange, qsRange) - ; + data = data.Add(Durable.Octree.PartIndexRange, qsRange); + + if (qs is Array qsArray) + { + // store separately and reference by id ... + var id = Guid.NewGuid(); + storage.Add(id, qsArray); + data = qs switch + { + byte[] => data.Add(Durable.Octree.PerPointPartIndex1bReference, id), + short[] => data.Add(Durable.Octree.PerPointPartIndex1sReference, id), + int[] => data.Add(Durable.Octree.PerPointPartIndex1iReference, id), + _ => throw new Exception($"Unexpected type {qs.GetType()}. Invariant cc05e74c-8cac-4d32-9972-0ea53e6e0911."), + }; + } + else + { + var def = PartIndexUtils.GetDurableDefForPartIndices(qs); + data = data.Add(def, qs); + } + } var result = new PointSetNode(data, config.Storage, writeToStore: true); diff --git a/src/Aardvark.Geometry.PointSet/Octrees/PointSet.cs b/src/Aardvark.Geometry.PointSet/Octrees/PointSet.cs index 13581cbd..d32ec5a1 100644 --- a/src/Aardvark.Geometry.PointSet/Octrees/PointSet.cs +++ b/src/Aardvark.Geometry.PointSet/Octrees/PointSet.cs @@ -16,12 +16,10 @@ You should have received a copy of the GNU Affero General Public License using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; -using System.Linq; using System.Text.Json; using System.Text.Json.Nodes; using System.Text.Json.Serialization; using System.Threading; -using static Aardvark.Base.MultimethodTest; namespace Aardvark.Geometry.Points { diff --git a/src/Aardvark.Geometry.PointSet/Octrees/PointSetNode.cs b/src/Aardvark.Geometry.PointSet/Octrees/PointSetNode.cs index 7facce95..64ecbc17 100644 --- a/src/Aardvark.Geometry.PointSet/Octrees/PointSetNode.cs +++ b/src/Aardvark.Geometry.PointSet/Octrees/PointSetNode.cs @@ -116,9 +116,9 @@ public PointSetNode( var bb = new Box3d(PositionsAbsolute); if (!Cell.BoundingBox.Contains(bb)) throw new Exception($"Invariant 71e949d7-21b1-4150-bfe3-337db52c4c7d."); - // Invariant: cell of bounding box of global positions MUST EQUAL node cell - var bbCell = new Cell(bb); - if (Cell != bbCell) throw new InvalidOperationException($"Invariant 7564f5b1-1b70-45cf-bfbe-5c955b90d1fe."); + //// Invariant: cell of bounding box of global positions MUST EQUAL node cell + //var bbCell = new Cell(bb); + //if (Cell != bbCell) throw new InvalidOperationException($"Invariant 7564f5b1-1b70-45cf-bfbe-5c955b90d1fe."); } #endif @@ -153,7 +153,7 @@ public PointSetNode( #endregion -#region Per-point attributes + #region Per-point attributes var psId = PositionsId; var csId = ColorsId; @@ -186,9 +186,9 @@ Func throwWhenNull(Func f) where T : notnull if (isId != null) PersistentRefs[Durable.Octree.Intensities1iReference ] = new PersistentRef(isId.Value, throwWhenNull(storage.GetIntArray ), storage.TryGetIntArrayFromCache ); if (ksId != null) PersistentRefs[Durable.Octree.Classifications1bReference] = new PersistentRef(ksId.Value, throwWhenNull(storage.GetByteArray), storage.TryGetByteArrayFromCache); -#endregion + #endregion -#region Durable.Octree.BoundingBoxExactLocal, Durable.Octree.BoundingBoxExactGlobal + #region Durable.Octree.BoundingBoxExactLocal, Durable.Octree.BoundingBoxExactGlobal if (HasPositions && (!HasBoundingBoxExactLocal || !HasBoundingBoxExactGlobal)) { @@ -218,18 +218,18 @@ Func throwWhenNull(Func f) where T : notnull } } -#endregion + #endregion -#region Durable.Octree.PointCountCell + #region Durable.Octree.PointCountCell if (!Has(Durable.Octree.PointCountCell)) { Data = Data.Add(Durable.Octree.PointCountCell, HasPositions ? Positions.Value!.Length : 0); } -#endregion + #endregion -#region Durable.Octree.PointCountTreeLeafs + #region Durable.Octree.PointCountTreeLeafs if (!Has(Durable.Octree.PointCountTreeLeafs)) { @@ -243,9 +243,9 @@ Func throwWhenNull(Func f) where T : notnull } } -#endregion + #endregion -#region Centroid* + #region Centroid* if (HasPositions && (!HasCentroidLocal || !HasCentroidLocalStdDev)) { @@ -268,9 +268,10 @@ Func throwWhenNull(Func f) where T : notnull } } } -#endregion -#region TreeDepth* + #endregion + + #region TreeDepth* if (!HasMinTreeDepth || !HasMaxTreeDepth) { @@ -316,9 +317,9 @@ Func throwWhenNull(Func f) where T : notnull } } -#endregion + #endregion -#region KdTree + #region KdTree if (HasPositions && !HasKdTree && !IsTemporaryImportNode) { @@ -419,34 +420,6 @@ PointRkdTreeF LoadKdTreeObsolete(string key) ); } - //(bool, PointRkdTreeF?) TryLoadKdTree(string key) - //{ - // var (ok, value) = Storage.TryGetPointRkdTreeFData(key); - // if (ok == false) return (false, default); - // var ps = Positions.Value!; - // return (true, new PointRkdTreeF( - // 3, ps.Length, ps, - // (xs, i) => xs[(int)i], (v, i) => (float)v[i], - // (a, b) => Vec.Distance(a, b), (i, a, b) => b - a, - // (a, b, c) => Vec.DistanceToLine(a, b, c), Fun.Lerp, 1e-6f, - // value - // )); - //} - - //(bool, PointRkdTreeF?) TryLoadKdTreeObsolete(string key) - //{ - // var (ok, value) = Storage.TryGetPointRkdTreeFDataFromD(key); - // if (ok == false) return (false, default); - // var ps = Positions.Value!; - // return (true, new PointRkdTreeF( - // 3, ps.Length, ps, - // (xs, i) => xs[(int)i], (v, i) => (float)v[i], - // (a, b) => Vec.Distance(a, b), (i, a, b) => b - a, - // (a, b, c) => Vec.DistanceToLine(a, b, c), Fun.Lerp, 1e-6f, - // value - // )); - //} - bool TryLoadKdTreeOut(string key, [NotNullWhen(true)] out PointRkdTreeF? result) { var (ok, value) = Storage.TryGetPointRkdTreeFData(key); @@ -922,8 +895,11 @@ public PersistentRef? Classifications Data.ContainsKey(Durable.Octree.PerCellPartIndex1i ) || Data.ContainsKey(Durable.Octree.PerCellPartIndex1ui) || Data.ContainsKey(Durable.Octree.PerPointPartIndex1b) || + Data.ContainsKey(Durable.Octree.PerPointPartIndex1bReference) || Data.ContainsKey(Durable.Octree.PerPointPartIndex1s) || - Data.ContainsKey(Durable.Octree.PerPointPartIndex1i) + Data.ContainsKey(Durable.Octree.PerPointPartIndex1sReference) || + Data.ContainsKey(Durable.Octree.PerPointPartIndex1i) || + Data.ContainsKey(Durable.Octree.PerPointPartIndex1iReference) ; /// @@ -937,12 +913,24 @@ public object? PartIndices if (Data.TryGetValue(Durable.Octree.PerCellPartIndex1i, out var pcpi)) return pcpi; else if (Data.TryGetValue(Durable.Octree.PerCellPartIndex1ui, out var pcpui)) return pcpui; else if (Data.TryGetValue(Durable.Octree.PerPointPartIndex1b, out var pppi1b)) return pppi1b; + else if (Data.TryGetValue(Durable.Octree.PerPointPartIndex1bReference, out var key1b)) return Storage.GetByteArray((Guid)key1b); else if (Data.TryGetValue(Durable.Octree.PerPointPartIndex1s, out var pppi1s)) return pppi1s; + else if (Data.TryGetValue(Durable.Octree.PerPointPartIndex1sReference, out var key1s)) return Storage.GetInt16Array((Guid)key1s); else if (Data.TryGetValue(Durable.Octree.PerPointPartIndex1i, out var pppi1i)) return pppi1i; + else if (Data.TryGetValue(Durable.Octree.PerPointPartIndex1iReference, out var key1i)) return Storage.GetIntArray((Guid)key1i); else return null; } } + [JsonIgnore] + public Guid? PerPointPartIndex1bId => Data.TryGetValue(Durable.Octree.PerPointPartIndex1bReference, out var id) ? (Guid?)id : null; + + [JsonIgnore] + public Guid? PerPointPartIndex1sId => Data.TryGetValue(Durable.Octree.PerPointPartIndex1sReference, out var id) ? (Guid?)id : null; + + [JsonIgnore] + public Guid? PerPointPartIndex1iId => Data.TryGetValue(Durable.Octree.PerPointPartIndex1iReference, out var id) ? (Guid?)id : null; + /// /// Get per-point part indices as an int array (regardless of internal representation). /// Returns false if node has no part indices. @@ -984,7 +972,7 @@ public bool TryGetPartIndices([NotNullWhen(true)] out int[]? result) [Obsolete("Use custom attributes instead.")] public PersistentRef? Velocities => null; -#endregion + #endregion #region KdTree @@ -1278,7 +1266,7 @@ private static Guid ComputeAndStoreKdTree(Storage storage, V3f[] ps) return kdId; } -#endregion + #endregion #region Durable codec diff --git a/src/Aardvark.Geometry.PointSet/Utils/StorageExtensions.cs b/src/Aardvark.Geometry.PointSet/Utils/StorageExtensions.cs index 50a90de2..874671a6 100644 --- a/src/Aardvark.Geometry.PointSet/Utils/StorageExtensions.cs +++ b/src/Aardvark.Geometry.PointSet/Utils/StorageExtensions.cs @@ -25,6 +25,7 @@ You should have received a copy of the GNU Affero General Public License using System.Text; using System.Text.Json.Nodes; using Uncodium.SimpleStore; +using static Aardvark.Base.IL.Constant; namespace Aardvark.Geometry.Points { @@ -80,6 +81,18 @@ public static int[] BufferToIntArray(byte[] buffer) #endregion + #region int16[] + + /// int16[] -> byte[] + public static byte[] Int16ArrayToBuffer(short[] data) + => ArrayToBuffer(data, sizeof(short), (bw, x) => bw.Write(x)); + + /// byte[] -> int16[] + public static short[] BufferToInt16Array(byte[] buffer) + => BufferToArray(buffer, sizeof(short), br => br.ReadInt16()); + + #endregion + #region V2f[] /// V2f[] -> byte[] @@ -292,14 +305,15 @@ public static void Add(this Storage storage, string key, Array data) { switch (data) { - case null: throw new ArgumentNullException(nameof(data)); - case byte[] xs: storage.f_add(key, xs, () => xs); break; - case int[] xs: storage.f_add(key, xs, () => Codec.IntArrayToBuffer(xs)); break; - case V2f[] xs: storage.f_add(key, xs, () => Codec.V2fArrayToBuffer(xs)); break; - case V2d[] xs: storage.f_add(key, xs, () => Codec.V2dArrayToBuffer(xs)); break; - case V3f[] xs: storage.f_add(key, xs, () => Codec.V3fArrayToBuffer(xs)); break; - case V3d[] xs: storage.f_add(key, xs, () => Codec.V3dArrayToBuffer(xs)); break; - case C4b[] xs: storage.f_add(key, xs, () => Codec.C4bArrayToBuffer(xs)); break; + case null : throw new ArgumentNullException(nameof(data)); + case byte[] xs: storage.f_add(key, xs, () => xs); break; + case int[] xs: storage.f_add(key, xs, () => Codec.IntArrayToBuffer(xs)); break; + case short[] xs: storage.f_add(key, xs, () => Codec.Int16ArrayToBuffer(xs)); break; + case V2f[] xs: storage.f_add(key, xs, () => Codec.V2fArrayToBuffer(xs)); break; + case V2d[] xs: storage.f_add(key, xs, () => Codec.V2dArrayToBuffer(xs)); break; + case V3f[] xs: storage.f_add(key, xs, () => Codec.V3fArrayToBuffer(xs)); break; + case V3d[] xs: storage.f_add(key, xs, () => Codec.V3dArrayToBuffer(xs)); break; + case C4b[] xs: storage.f_add(key, xs, () => Codec.C4bArrayToBuffer(xs)); break; default: throw new Exception($"Type {data.GetType()} not supported."); } } @@ -444,6 +458,10 @@ public static void Add(this Storage storage, string key, IList data) public static void Add(this Storage storage, string key, int[] data) => storage.f_add(key, data, () => Codec.IntArrayToBuffer(data)); + /// + public static int[]? GetIntArray(this Storage storage, Guid key) + => GetIntArray(storage, key.ToString()); + /// public static int[]? GetIntArray(this Storage storage, string key) { @@ -476,6 +494,51 @@ public static (bool, int[]?) TryGetIntArray(this Storage storage, string key) #endregion + #region int16[] + + /// + public static void Add(this Storage storage, Guid key, short[] data) => Add(storage, key.ToString(), data); + + /// + public static void Add(this Storage storage, string key, short[] data) + => storage.f_add(key, data, () => Codec.Int16ArrayToBuffer(data)); + + /// + public static short[]? GetInt16Array(this Storage storage, Guid key) + => GetInt16Array(storage, key.ToString()); + + /// + public static short[]? GetInt16Array(this Storage storage, string key) + { + if (storage.HasCache && storage.Cache.TryGetValue(key, out object o)) return (short[])o; + + var buffer = storage.f_get(key); + if (buffer == null) return null; + var data = Codec.BufferToInt16Array(buffer); + + if (storage.HasCache) + storage.Cache.Add(key, data, buffer.Length, onRemove: default); + + return data; + } + + /// + public static (bool, short[]?) TryGetInt16Array(this Storage storage, string key) + { + if (storage.HasCache && storage.Cache.TryGetValue(key, out object o)) + { + return (true, (short[])o); + } + else + { + return (false, default); + } + } + + public static bool TryGetInt16ArrayFromCache(this Storage storage, string key, [NotNullWhen(true)] out short[]? result) => TryGetFromCache(storage, key, out result); + + #endregion + #region C4b[] /// diff --git a/src/Apps/Viewer/Program.fs b/src/Apps/Viewer/Program.fs index 6bda0503..35118d70 100644 --- a/src/Apps/Viewer/Program.fs +++ b/src/Apps/Viewer/Program.fs @@ -14,7 +14,7 @@ let main args = //view @"C:\Users\sm\Downloads\C_30DN2.LAZ.store" [File.readAllText @"C:\Users\sm\Downloads\C_30DN2.LAZ.key"] (Args.parse [||]) //view @"C:\Users\sm\Downloads\test.store" ["128330b1-8761-4a07-b160-76bcd7e2f70a"; "ab2f6f76-7eae-47c9-82d1-ad28b816abb9"] (Args.parse [||]) - let store = @"C:\bla\store\lowergetikum\data.bin" + let store = @"W:\Datasets\Vgm\Stores\lowergetikum 20230321.e57_0.01_2\data.uds" let key = Path.combine [System.IO.Path.GetDirectoryName store;"key.txt"] |> File.readAllText view store [key] (Args.parse [||])