Merge branch 'parts' into 20231126_experiment

RELEASE_NOTES.md

@@ -1,3 +1,9 @@
+### 5.3.0-prerelease008
+- fix part index handling in JoinNonOverlappingTrees (prerelease, for testing only)
+
+### 5.3.0-prerelease007
+- structured point clouds query (prerelease, for testing only)
+
 ### 5.3.0-prerelease006
 - structured point clouds delete (prerelease, for testing only)
 

src/Aardvark.Algodat.Tests/Program.cs

@@ -2763,18 +2763,18 @@ public static async Task Main(string[] _)
         {
             //await Task.Delay(0); // avoid warnings if main contains no await
 
-            //await CreateStore(
-            //    @"W:\Datasets\Vgm\Data\structured_pointclouds\lowergetikum 20230321.e57",
-            //    @"W:\Datasets\Vgm\Stores\lowergetikum 20230321.e57_0.001",
-            //    minDist: 0.001
-            //    );
-
             await CreateStore(
-                @"W:\Datasets\Vgm\Data\E57\erdgeschoss.e57",
-                @"W:\Datasets\Vgm\Stores\erdgeschoss.e57_0.0025",
-                minDist: 0.0025
+                @"W:\Datasets\Vgm\Data\structured_pointclouds\lowergetikum 20230321.e57",
+                @"W:\Datasets\Vgm\Stores\lowergetikum 20230321.e57_0.01",
+                minDist: 0.01
                 );
 
+            //await CreateStore(
+            //    @"W:\Datasets\Vgm\Data\E57\erdgeschoss.e57",
+            //    @"W:\Datasets\Vgm\Stores\erdgeschoss.e57_0.0025",
+            //    minDist: 0.0025
+            //    );
+
             //await Parts_Test_20231006_Merge();
 
             //await Parts_Test_20231006();

src/Aardvark.Algodat.Tests/QueryTests.cs

@@ -20,6 +20,7 @@ You should have received a copy of the GNU Affero General Public License
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
+using static Aardvark.Base.MultimethodTest;
 
 namespace Aardvark.Geometry.Tests
 {
@@ -118,7 +119,6 @@ public void CanQueryPointsNearDirectedRay3d()
             foreach (var x in pointset.QueryPointsNearRay(new Ray3d(new V3d(0.5, -1.0, 0.5), V3d.OIO), 0.2, 1.0, 2.0)) count1 += x.Positions.Count;
             foreach (var x in pointset.QueryPointsNearRay(new Ray3d(new V3d(0.5, -1.0, 0.5), V3d.OIO), 0.2, 1.5, 2.0)) count2 += x.Positions.Count;
             foreach (var x in pointset.QueryPointsNearRay(new Ray3d(new V3d(0.5, 0.75, 0.5), V3d.OIO), 0.2, 0.0, 1.0)) count3 += x.Positions.Count;
-
             Assert.IsTrue(count1 > count2);
             Assert.IsTrue(count2 > count3);
         }
@@ -1178,5 +1178,27 @@ public void EnumerateCells_Kernel_2()
         #endregion
 
 
+        [Test]
+        public void CanQueryPointsWithAttributes()
+        {
+            var r = new Random(0);
+            var ps = new V3d[50000];
+            for (var i = 0; i < 50000; i++) ps[i] = new V3d(r.NextDouble(), r.NextDouble(), r.NextDouble());
+            var config = ImportConfig.Default
+                .WithStorage(PointCloud.CreateInMemoryStore(cache: default))
+                .WithKey("test")
+                .WithOctreeSplitLimit(50000)
+                ;
+            var chunk = new Chunk(ps);
+            var pis = new int[50000];
+            for (var i = 0; i < 50000; i++) pis[i] = r.Next(4);
+            var pir = new Range1i(pis);
+            chunk = chunk.WithPartIndices(pis, pir);
+            var pointset = PointCloud.Chunks(chunk, config);
+
+            var q = pointset.QueryPointsNearPoint(V3d.Zero, 1.0, 5);
+            Assert.IsTrue(q.PartIndices != null);
+            Assert.IsTrue(q.PartIndices.All(x => x >= 0 && x <= 3));
+        }
     }
 }

src/Aardvark.Algodat.sln

@@ -79,6 +79,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Unofficial.laszip.netstanda
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "import", "import\import.csproj", "{000B3CDA-C67B-4F6A-B6F8-69A1B5052343}"
 EndProject
+Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "ScratchFSharp", "Apps\ScratchFSharp\ScratchFSharp.fsproj", "{E5B3A633-A8F4-4D56-BB64-4A885E22B3B2}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -193,6 +195,10 @@ Global
 		{000B3CDA-C67B-4F6A-B6F8-69A1B5052343}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{000B3CDA-C67B-4F6A-B6F8-69A1B5052343}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{000B3CDA-C67B-4F6A-B6F8-69A1B5052343}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E5B3A633-A8F4-4D56-BB64-4A885E22B3B2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E5B3A633-A8F4-4D56-BB64-4A885E22B3B2}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E5B3A633-A8F4-4D56-BB64-4A885E22B3B2}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E5B3A633-A8F4-4D56-BB64-4A885E22B3B2}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -205,9 +211,10 @@ Global
 		{85F64ED4-AAA6-4EEF-BC97-ED5E65839FD7} = {6CA9BC7A-0123-43E9-A26B-5FF0B9E01B08}
 		{5CA8E4F4-E30B-4FC8-A510-A0FFD8A9880B} = {6CA9BC7A-0123-43E9-A26B-5FF0B9E01B08}
 		{000B3CDA-C67B-4F6A-B6F8-69A1B5052343} = {6CA9BC7A-0123-43E9-A26B-5FF0B9E01B08}
+		{E5B3A633-A8F4-4D56-BB64-4A885E22B3B2} = {6CA9BC7A-0123-43E9-A26B-5FF0B9E01B08}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
-		SolutionGuid = {E996B5E6-49B2-4479-9DDE-455A40509F43}
 		EnterpriseLibraryConfigurationToolBinariesPathV6 = packages\EnterpriseLibrary.TransientFaultHandling.6.0.1304.0\lib\portable-net45+win+wp8;packages\EnterpriseLibrary.TransientFaultHandling.Data.6.0.1304.1\lib\NET45
+		SolutionGuid = {E996B5E6-49B2-4479-9DDE-455A40509F43}
 	EndGlobalSection
 EndGlobal

src/Aardvark.Geometry.PointSet/Octrees/InMemoryPointSet.cs

@@ -17,6 +17,7 @@ You should have received a copy of the GNU Affero General Public License
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Diagnostics;
+using System.IO;
 using System.Linq;
 using static Aardvark.Data.Durable;
 
@@ -84,7 +85,7 @@ private InMemoryPointSet(ImmutableDictionary<Def, object> data, Cell cell, int o
             foreach (var kv in data)
             {
                 if (kv.Key == Octree.PerCellPartIndex1i || kv.Key == Octree.PerCellPartIndex1ui) continue;
-                //if (kv.Key == Octree.PartIndexRange) { m_hasPartIndices = true; break; }
+                if (kv.Key == Octree.PartIndexRange) continue;
                 if (kv.Value is not Array) throw new ArgumentException($"Entry {kv.Key} must be array.");
             }
 

src/Aardvark.Geometry.PointSet/Octrees/Merge.cs

@@ -639,12 +639,13 @@ private static IPointCloudNode JoinNonOverlappingTrees(Cell rootCell, IPointClou
             // PRE: we further assume, that both trees are non-empty
             if (a.PointCountTree == 0 && b.PointCountTree == 0) throw new InvalidOperationException();
 
-            // PRE:
+            // PRE: assume that part indices are available in both trees or in no tree (but not in one or the other)
             if (a.HasPartIndexRange != b.HasPartIndexRange) throw new Exception("Invariant b3feedfd-927d-4436-9eb9-350d377ab852.");
 
             #endregion
 
             #region Case reduction
+
             // REDUCE CASES:
             // if one tree ('a' or 'b') is centered at origin, then ensure that 'a' is centered
             // (by swapping 'a' and 'b' if necessary)
@@ -785,6 +786,13 @@ private static IPointCloudNode JoinNonOverlappingTrees(Cell rootCell, IPointClou
                     .Add(Durable.Octree.PointCountTreeLeafs, a.PointCountTree + b.PointCountTree)
                     .Add(Durable.Octree.SubnodesGuids, subcells.Map(x => x?.Id ?? Guid.Empty))
                     ;
+
+                if (a.HasPartIndexRange)
+                {
+                    var mergedPartIndexRange = PartIndexUtils.MergeRanges(a.PartIndexRange, b.PartIndexRange) ?? throw new Exception("Invariant d4ed616f-a348-4303-8e64-651d669cb7bc.");
+                    data = data.Add(Durable.Octree.PartIndexRange, mergedPartIndexRange);
+                }
+
                 var result = new PointSetNode(data, config.Storage, writeToStore: false).CollapseLeafNodes(config).Item1;
 
 #if DEBUG

src/Aardvark.Geometry.PointSet/Octrees/PointsNearObject.cs

@@ -55,7 +55,7 @@ public class PointsNearObject<T>
         public static readonly PointsNearObject<T> Empty = new(
             default!, 0.0, 
             Array.Empty<V3d>(), Array.Empty<C4b>(), Array.Empty<V3f>(), 
-            Array.Empty<int>(), Array.Empty<byte>(),
+            Array.Empty<int>(), Array.Empty<int>(), Array.Empty<byte>(),
             Array.Empty<double>()
             );
 
@@ -76,15 +76,18 @@ public class PointsNearObject<T>
 
         /// <summary></summary>
         public int[]? Intensities { get; }
-
+
+        /// <summary></summary>
+        public int[]? PartIndices { get; }
+
         /// <summary></summary>
         public byte[]? Classifications { get; }
 
         /// <summary></summary>
         public double[]? Distances { get; }
 
         /// <summary></summary>
-        public PointsNearObject(T obj, double maxDistance, V3d[] positions, C4b[]? colors, V3f[]? normals, int[]? intensities, byte[]? classifications, double[]? distances)
+        public PointsNearObject(T obj, double maxDistance, V3d[] positions, C4b[]? colors, V3f[]? normals, int[]? intensities, int[]? partIndices, byte[]? classifications, double[]? distances)
         {
             if (maxDistance < 0.0) throw new ArgumentOutOfRangeException(nameof(maxDistance), $"Parameter 'maxDistance' must not be less than 0.0, but is {maxDistance}.");
 
@@ -94,6 +97,7 @@ public PointsNearObject(T obj, double maxDistance, V3d[] positions, C4b[]? color
             Colors = colors;
             Normals = normals;
             Intensities = intensities;
+            PartIndices = partIndices;
             Classifications = classifications;
             Distances = distances;
         }
@@ -121,6 +125,7 @@ public PointsNearObject<T> Merge(PointsNearObject<T> other, int maxCount)
                 Colors?.Append(other.Colors),
                 Normals.Append(other.Normals),
                 Intensities.Append(other.Intensities),
+                PartIndices.Append(other.PartIndices),
                 Classifications.Append(other.Classifications),
                 Distances.Append(other.Distances)
                 );
@@ -153,6 +158,7 @@ public PointsNearObject<T> Merge(PointsNearObject<T> other, int maxCount)
             Colors?.Length > 0 ? Colors.Reordered(ia) : Colors,
             Normals?.Length > 0 ? Normals.Reordered(ia) : Normals,
             Intensities?.Length > 0 ? Intensities.Reordered(ia) : Intensities,
+            PartIndices?.Length > 0 ? PartIndices.Reordered(ia) : PartIndices,
             Classifications?.Length > 0 ? Classifications.Reordered(ia) : Classifications,
             Distances!.Reordered(ia)
             );
@@ -165,13 +171,13 @@ public PointsNearObject<T> Take(int count)
             if (count >= Count) return this;
             var ds = Distances!.Take(count);
             return new PointsNearObject<T>(Object, ds.Max(),
-                Positions.Take(count), Colors?.Take(count), Normals?.Take(count), Intensities?.Take(count), Classifications!.Take(count), ds
+                Positions.Take(count), Colors?.Take(count), Normals?.Take(count), Intensities?.Take(count), PartIndices?.Take(count), Classifications!.Take(count), ds
                 );
         }
 
         /// <summary>
         /// </summary>
         public PointsNearObject<U> WithObject<U>(U other)
-            => new(other, MaxDistance, Positions, Colors, Normals, Intensities, Classifications, Distances);
+            => new(other, MaxDistance, Positions, Colors, Normals, Intensities, PartIndices, Classifications, Distances);
     }
 }

src/Aardvark.Geometry.PointSet/Queries/QueriesV3d.cs

@@ -75,8 +75,10 @@ public static PointsNearObject<V3d> QueryPointsNearPoint(
                     var cs = node.Colors?.Value?.Subset(ia);
                     var ns = node.Normals?.Value?.Subset(ia);
                     var js = node.Intensities?.Value?.Subset(ia);
+                    node.TryGetPartIndices(out var pis);
+                    pis = pis?.Subset(ia);
                     var ks = node.Classifications?.Value?.Subset(ia);
-                    var chunk = new PointsNearObject<V3d>(query, maxDistanceToPoint, ps, cs, ns, js, ks, ds);
+                    var chunk = new PointsNearObject<V3d>(query, maxDistanceToPoint, ps, cs, ns, js, pis, ks, ds);
                     return chunk;
                 }
                 else

src/Apps/ScratchFSharp/Program.fs

@@ -0,0 +1,90 @@
+namespace ScratchFSharp
+
+open System
+open System.IO
+open Aardvark.Base
+open Aardvark.Data.Points
+open Aardvark.Geometry.Points
+
+module Bla = 
+    [<EntryPoint>]
+    let main a =
+        //let b = Box3d.Unit
+        //let o = V3d.III*0.5
+        //let d = V3d.IOO
+        //let r = Ray3d(o+999.0*d,-d)
+        //let rf = FastRay3d(o,d)
+        //let mutable t = 0.0
+        //let mutable tMin = 0.0
+        //let mutable tMax = 0.0
+        //let i = b.Intersects(r,&t)
+        //let i2 = rf.Intersects(b,&tMin,&tMax)
+        //Log.line "%A %A %A %A %A" i2 i t tMin tMax
+
+
+
+        let storePath = @"C:\bla\store\teststore\data.uds"
+        let key = "3d9654b1-c37a-4d41-ba83-393d58a9bdce"
+
+        let bounds = Box3d(V3d.III*2.0, V3d.III*20.0)
+        let rnd = RandomSystem()
+
+        let rec genPoints loc =
+            let mutable phi = 0.0
+            let mutable theta = 0.0
+            let dPhi = 1.0 * Constant.RadiansPerDegree
+            let dTheta = 1.1 * Constant.RadiansPerDegree
+            let ps = System.Collections.Generic.List()
+            while phi < Constant.PiTimesTwo do  
+                theta <- 0.0
+                while theta < Constant.PiTimesTwo  do 
+                    let d = V2d(phi,theta).CartesianFromSpherical()
+                    let r = Ray3d(loc+999.0*d,-d)
+                    let mutable t = 0.0
+                    if bounds.Intersects(r,&t) then 
+                        let p = r.GetPointOnRay(t)
+                        ps.Add p |> ignore
+                    theta <- theta+dTheta
+                phi <- phi+dPhi
+            ps |> CSharpList.toArray
+
+        let locations = Array.init 5 (fun _ -> rnd.UniformV3d(Box3d.FromCenterAndSize(bounds.Center,bounds.Size*0.8)))
+
+        let chunks = 
+            locations |> Array.mapi (fun i loc -> 
+                let pts = genPoints loc
+                Chunk(pts,null,null,null,null,i,Range1i(i),bounds)
+            ) 
+
+        let store = 
+            new Uncodium.SimpleStore.SimpleDiskStore(storePath,System.Action<_>(fun ss -> ss |> Array.iter (printfn "%A")))
+
+        let add = 
+            Action<string,obj,Func<byte[]>>(fun (name : string) (value : obj) (create : Func<byte[]>) -> store.Add(name, create.Invoke()))
+        let get s = store.Get s
+        let getSlice k o i = store.GetSlice(k,o,i)
+        let remove k = store.Remove k
+        let dispose() = store.Dispose()
+        let flush() = store.Flush()
+        let dict = LruDictionary(1<<<30)
+        let storage = 
+            new Storage(
+                add, 
+                get, 
+                getSlice, 
+                remove, 
+                dispose, 
+                flush, 
+                dict
+            )
+
+        let config = 
+            ImportConfig.Default
+                .WithStorage(storage)
+                .WithKey(key)
+                .WithOctreeSplitLimit(8192)
+
+        let pc = PointCloud.Chunks(chunks,config)
+        File.WriteAllText(Path.Combine(Path.GetDirectoryName(storePath),"key.txt"),key)
+        Log.line "done %A %A" pc.PointCount pc.Id
+        0

src/Apps/ScratchFSharp/ScratchFSharp.fsproj

@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+  </PropertyGroup>
+  <ItemGroup>
+    <Compile Include="Program.fs" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\Aardvark.Data.E57\Aardvark.Data.E57.csproj" />
+    <ProjectReference Include="..\..\Aardvark.Data.Points.Base\Aardvark.Data.Points.Base.csproj" />
+    <ProjectReference Include="..\..\Aardvark.Rendering.PointSet\Aardvark.Rendering.PointSet.fsproj" />
+  </ItemGroup>
+  <Import Project="..\..\..\.paket\Paket.Restore.targets" />
+</Project>

src/Apps/ScratchFSharp/paket.references

@@ -0,0 +1,3 @@
+Aardvark.Build
+Aardvark.Application.Utilities
+FSharp.Core