From fdc08006f5ba46b8edbda28c28290506285c8acc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E9=9B=A8?= Date: Tue, 11 Apr 2023 23:05:55 +0800 Subject: [PATCH] v1.7.0 release --- .gitignore | 8 +- .../giscat/vector/pojo/PojoConstant.java | 26 ++ .../converter/FoolStyleFeatureConverter.java | 255 ++++++++++++++++++ .../FoolStyleFeatureConverterTest.java | 82 ++++++ .../vector/pojo/converter/MakePbfTest1.java | 69 +++++ .../giscat/vector/pojo/converter/Test1.java | 83 ++---- .../giscat-vector-rocksrtree/pom.xml | 4 + .../giscat/vector/rocksrtree/Branch.java | 5 +- .../giscat/vector/rocksrtree/RTree.java | 44 ++- .../giscat/vector/rocksrtree/RectNd.java | 20 +- .../vector/rocksrtree/RocksRtreePb.java | 1 - .../giscat/vector/rocksrtree/TreeBuilder.java | 139 +++++++--- .../vector/rocksrtree/TreeTransaction.java | 104 ++++++- .../giscat/vector/rocksrtreetest/Test.java | 32 ++- pom.xml | 3 + 15 files changed, 732 insertions(+), 143 deletions(-) create mode 100644 giscat-vector/giscat-vector-pojo/src/main/java/org/wowtools/giscat/vector/pojo/PojoConstant.java create mode 100644 giscat-vector/giscat-vector-pojo/src/main/java/org/wowtools/giscat/vector/pojo/converter/FoolStyleFeatureConverter.java create mode 100644 giscat-vector/giscat-vector-pojo/src/test/java/org/wowtools/giscat/vector/pojo/converter/FoolStyleFeatureConverterTest.java create mode 100644 giscat-vector/giscat-vector-pojo/src/test/java/org/wowtools/giscat/vector/pojo/converter/MakePbfTest1.java diff --git a/.gitignore b/.gitignore index d90bedc..b2bac12 100644 --- a/.gitignore +++ b/.gitignore @@ -2,28 +2,26 @@ target logs +*.log -# windows # Thumbs.db -# Mac # .DS_Store -# eclipse # .settings .project .classpath .log *.class -# idea # .idea *.iml -# Package Files # *.war *.ear /target /public jmhResult.json +/D_tmp1rocksrtree/ +/arthas-output/ diff --git a/giscat-vector/giscat-vector-pojo/src/main/java/org/wowtools/giscat/vector/pojo/PojoConstant.java b/giscat-vector/giscat-vector-pojo/src/main/java/org/wowtools/giscat/vector/pojo/PojoConstant.java new file mode 100644 index 0000000..8d9d4b0 --- /dev/null +++ b/giscat-vector/giscat-vector-pojo/src/main/java/org/wowtools/giscat/vector/pojo/PojoConstant.java @@ -0,0 +1,26 @@ +/* + * + * * Copyright (c) 2022- "giscat (https://github.com/codingmiao/giscat)" + * * + * * 本项目采用自定义版权协议，在不同行业使用时有不同约束，详情参阅： + * * + * * https://github.com/codingmiao/giscat/blob/main/LICENSE + * + */ + +package org.wowtools.giscat.vector.pojo; + +import org.locationtech.jts.geom.GeometryFactory; + +/** + * 常量 + * @author liuyu + * @date 2023/4/6 + */ +public class PojoConstant { + + /** + * 通用的geometryFactory + */ + public static final GeometryFactory geometryFactory = new GeometryFactory(); +} diff --git a/giscat-vector/giscat-vector-pojo/src/main/java/org/wowtools/giscat/vector/pojo/converter/FoolStyleFeatureConverter.java b/giscat-vector/giscat-vector-pojo/src/main/java/org/wowtools/giscat/vector/pojo/converter/FoolStyleFeatureConverter.java new file mode 100644 index 0000000..c9a35f9 --- /dev/null +++ b/giscat-vector/giscat-vector-pojo/src/main/java/org/wowtools/giscat/vector/pojo/converter/FoolStyleFeatureConverter.java @@ -0,0 +1,255 @@ +/* + * + * * Copyright (c) 2022- "giscat (https://github.com/codingmiao/giscat)" + * * + * * 本项目采用自定义版权协议，在不同行业使用时有不同约束，详情参阅： + * * + * * https://github.com/codingmiao/giscat/blob/main/LICENSE + * + */ + +package org.wowtools.giscat.vector.pojo.converter; + + +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.LineString; +import org.locationtech.jts.geom.Point; +import org.locationtech.jts.geom.Polygon; + +import java.util.Iterator; +import java.util.List; + +import static org.wowtools.giscat.vector.pojo.PojoConstant.geometryFactory; + +/** + * 傻瓜式的对象转换工具，例如把一个坐标list转为一条线 + * + * @author liuyu + * @date 2023/4/6 + */ +public class FoolStyleFeatureConverter { + + /** + * 将 x y转为point对象 + * + * @param x x + * @param y y + * @return Point + */ + public static Point xy2Point(double x, double y) { + return geometryFactory.createPoint(new Coordinate(x, y)); + } + + /** + * 将list转为point + * + * @param list list + * @return Point + */ + public static Point list2Point(List list) { + return geometryFactory.createPoint(new Coordinate(list.get(0), list.get(1))); + } + + /** + * 将array转为point + * + * @param array array + * @return Point + */ + public static Point array2Point(double[] array) { + return geometryFactory.createPoint(new Coordinate(array[0], array[1])); + } + + /** + * 将string转为point，例如str2Point("10,2",",")转为POINT(10 2) + * + * @param str str + * @param regex 分隔符 + * @return Point + */ + public static Point str2Point(String str, String regex) { + String[] strs = str.trim().split(regex); + return geometryFactory.createPoint(new Coordinate(Double.parseDouble(strs[0]), Double.parseDouble(strs[1]))); + } + + /** + * 将list转为线 + * + * @param list list 例如[1,2,3,4]转为LINESTRING(1 2,3 4) + * @return LineString + */ + public static LineString list2Line(List list) { + Coordinate[] coords = new Coordinate[list.size() / 2]; + int i = 0; + Iterator iterator = list.iterator(); + while (iterator.hasNext()) { + double x = iterator.next(); + double y = iterator.next(); + coords[i] = new Coordinate(x, y); + i++; + } + return geometryFactory.createLineString(coords); + } + + /** + * 将array转为线 + * + * @param array list 例如[1,2,3,4]转为LINESTRING(1 2,3 4) + * @return LineString + */ + public static LineString array2Line(double[] array) { + Coordinate[] coords = new Coordinate[array.length / 2]; + for (int i = 0; i < array.length; i += 2) { + double x = array[i]; + double y = array[i + 1]; + coords[i / 2] = new Coordinate(x, y); + } + return geometryFactory.createLineString(coords); + } + + /** + * 将list转为线 + * + * @param list list 例如[[1,2],[3,4]]转为LINESTRING(1 2,3 4) + * @return LineString + */ + public static LineString lists2Line(List list) { + Coordinate[] coords = new Coordinate[list.size()]; + int i = 0; + for (double[] doubles : list) { + double[] ds = doubles; + coords[i] = new Coordinate(ds[0], ds[1]); + i++; + } + return geometryFactory.createLineString(coords); + } + + /** + * 将array转为线 + * + * @param array array 例如[[1,2],[3,4]]转为LINESTRING(1 2,3 4) + * @return LineString + */ + public static LineString arrays2Line(double[][] array) { + Coordinate[] coords = new Coordinate[array.length]; + for (int i = 0; i < array.length; i++) { + double[] ds = array[i]; + coords[i] = new Coordinate(ds[0], ds[1]); + } + return geometryFactory.createLineString(coords); + } + + /** + * 将string转为线，例如str2Line("10,2;15,3",",",";")转为LINESTRING(10 2,15 3) + * + * @param str str + * @param xyRegex xy坐标间的分隔符 + * @param pointRegex 点之间的分隔符 + * @return LineString + */ + public static LineString str2Line(String str, String xyRegex, String pointRegex) { + String[] pointArray = str.trim().split(pointRegex); + Coordinate[] coords = new Coordinate[pointArray.length]; + for (int i = 0; i < pointArray.length; i++) { + String[] xy = pointArray[i].trim().split(xyRegex); + coords[i] = new Coordinate(Double.parseDouble(xy[0]), Double.parseDouble(xy[1])); + } + return geometryFactory.createLineString(coords); + } + + + /** + * 将list转为面 + * + * @param list list 例如[1,2,3,4,6,6]转为POLYGON ((1 2, 3 4, 6 6, 1 2)) + * @return Polygon + */ + public static Polygon list2Polygon(List list) { + double[] array = new double[list.size()]; + int i = 0; + for (Double v : list) { + array[i] = v; + i++; + } + return array2Polygon(array); + } + + /** + * 将array转为面 + * + * @param array array 例如[1,2,3,4,6,6]转为POLYGON ((1 2, 3 4, 6 6, 1 2)) + * @return Polygon + */ + public static Polygon array2Polygon(double[] array) { + boolean endSame = array[array.length - 1] == array[1] && array[array.length - 2] == array[0]; + + Coordinate[] coords = new Coordinate[endSame ? array.length / 2 : array.length / 2 + 1]; + for (int i = 0; i < array.length; i += 2) { + double x = array[i]; + double y = array[i + 1]; + coords[i / 2] = new Coordinate(x, y); + } + if (!endSame) { + coords[coords.length - 1] = coords[0].copy(); + } + return geometryFactory.createPolygon(coords); + } + + /** + * 将list转为面 + * + * @param list list 例如[[1,2],[3,4],[6,6]]转为POLYGON ((1 2, 3 4, 6 6, 1 2)) + * @return Polygon + */ + public static Polygon lists2Polygon(List list) { + double[][] array = new double[list.size()][]; + int i = 0; + for (double[] v : list) { + array[i] = v; + i++; + } + return arrays2Polygon(array); + } + + /** + * 将array转为面 + * + * @param array array 例如[[1,2],[3,4],[6,6]]转为POLYGON ((1 2, 3 4, 6 6, 1 2)) + * @return Polygon + */ + public static Polygon arrays2Polygon(double[][] array) { + boolean endSame = array[0][0] == array[array.length - 1][0] && array[0][1] == array[array.length - 1][1]; + Coordinate[] coords = new Coordinate[endSame ? array.length : array.length + 1]; + for (int i = 0; i < array.length; i ++) { + double[] xy = array[i]; + coords[i] = new Coordinate(xy[0], xy[1]); + } + if (!endSame) { + coords[coords.length - 1] = coords[0].copy(); + } + return geometryFactory.createPolygon(coords); + } + + /** + * 将string转为线，例如str2Line("1,2;3,4;6,6",",",";")转为POLYGON ((1 2, 3 4, 6 6, 1 2)) + * + * @param str str + * @param xyRegex xy坐标间的分隔符 + * @param pointRegex 点之间的分隔符 + * @return Polygon + */ + public static Polygon str2Polygon(String str, String xyRegex, String pointRegex){ + String[] points = str.trim().split(pointRegex); + double[][] array = new double[points.length][]; + int i = 0; + for (String point: points) { + String[] xy = point.trim().split(xyRegex); + array[i] = new double[]{Double.parseDouble(xy[0]),Double.parseDouble(xy[1])}; + i++; + } + return arrays2Polygon(array); + } + + + +} diff --git a/giscat-vector/giscat-vector-pojo/src/test/java/org/wowtools/giscat/vector/pojo/converter/FoolStyleFeatureConverterTest.java b/giscat-vector/giscat-vector-pojo/src/test/java/org/wowtools/giscat/vector/pojo/converter/FoolStyleFeatureConverterTest.java new file mode 100644 index 0000000..e12a254 --- /dev/null +++ b/giscat-vector/giscat-vector-pojo/src/test/java/org/wowtools/giscat/vector/pojo/converter/FoolStyleFeatureConverterTest.java @@ -0,0 +1,82 @@ +package org.wowtools.giscat.vector.pojo.converter; + +import junit.framework.TestCase; +import org.junit.Assert; +import org.locationtech.jts.geom.LineString; +import org.locationtech.jts.geom.Point; +import org.locationtech.jts.geom.Polygon; + +import java.util.List; + +public class FoolStyleFeatureConverterTest extends TestCase { + + public void testXy2Point() { + Point point = FoolStyleFeatureConverter.xy2Point(20, 30); + Assert.assertEquals("POINT (20 30)", point.toText()); + } + + public void testList2Point() { + Point point = FoolStyleFeatureConverter.list2Point(List.of(20d, 30d)); + Assert.assertEquals("POINT (20 30)", point.toText()); + } + + public void testArray2Point() { + Point point = FoolStyleFeatureConverter.array2Point(new double[]{20, 30}); + Assert.assertEquals("POINT (20 30)", point.toText()); + } + + public void testStr2Point() { + Point point = FoolStyleFeatureConverter.str2Point("20 30", " "); + Assert.assertEquals("POINT (20 30)", point.toText()); + } + + public void testList2Line() { + LineString line = FoolStyleFeatureConverter.list2Line(List.of(1d, 2d, 3d, 4d)); + Assert.assertEquals("LINESTRING (1 2, 3 4)", line.toText()); + } + + public void testArray2Line() { + LineString line = FoolStyleFeatureConverter.array2Line(new double[]{1, 2, 3, 4}); + Assert.assertEquals("LINESTRING (1 2, 3 4)", line.toText()); + } + + public void testLists2Line() { + LineString line = FoolStyleFeatureConverter.lists2Line(List.of(new double[]{1, 2}, new double[]{3, 4})); + Assert.assertEquals("LINESTRING (1 2, 3 4)", line.toText()); + } + + public void testArrays2Line() { + LineString line = FoolStyleFeatureConverter.arrays2Line(new double[][]{new double[]{1, 2}, new double[]{3, 4}}); + Assert.assertEquals("LINESTRING (1 2, 3 4)", line.toText()); + } + + public void testStr2Line() { + LineString line = FoolStyleFeatureConverter.str2Line("1,2;3,4",",",";"); + Assert.assertEquals("LINESTRING (1 2, 3 4)", line.toText()); + } + + public void testList2Polygon() { + Polygon polygon = FoolStyleFeatureConverter.list2Polygon(List.of(1d, 2d, 3d, 4d, 6d, 6d)); + Assert.assertEquals("POLYGON ((1 2, 3 4, 6 6, 1 2))", polygon.toText()); + } + + public void testArray2Polygon() { + Polygon polygon = FoolStyleFeatureConverter.array2Polygon(new double[]{1d, 2d, 3d, 4d, 6d, 6d}); + Assert.assertEquals("POLYGON ((1 2, 3 4, 6 6, 1 2))", polygon.toText()); + } + + public void testLists2Polygon() { + Polygon polygon = FoolStyleFeatureConverter.lists2Polygon(List.of(new double[]{1,2},new double[]{3,4},new double[]{6,6})); + Assert.assertEquals("POLYGON ((1 2, 3 4, 6 6, 1 2))", polygon.toText()); + } + + public void testArrays2Polygon() { + Polygon polygon = FoolStyleFeatureConverter.arrays2Polygon(new double[][]{new double[]{1,2},new double[]{3,4},new double[]{6,6}}); + Assert.assertEquals("POLYGON ((1 2, 3 4, 6 6, 1 2))", polygon.toText()); + } + + public void testStr2Polygon() { + Polygon polygon = FoolStyleFeatureConverter.str2Polygon("1 2,3 4,6 6, 1 2"," ",","); + Assert.assertEquals("POLYGON ((1 2, 3 4, 6 6, 1 2))", polygon.toText()); + } +} diff --git a/giscat-vector/giscat-vector-pojo/src/test/java/org/wowtools/giscat/vector/pojo/converter/MakePbfTest1.java b/giscat-vector/giscat-vector-pojo/src/test/java/org/wowtools/giscat/vector/pojo/converter/MakePbfTest1.java new file mode 100644 index 0000000..5165794 --- /dev/null +++ b/giscat-vector/giscat-vector-pojo/src/test/java/org/wowtools/giscat/vector/pojo/converter/MakePbfTest1.java @@ -0,0 +1,69 @@ +package org.wowtools.giscat.vector.pojo.converter; + +import org.wowtools.giscat.vector.pojo.Feature; +import org.wowtools.giscat.vector.pojo.FeatureCollection; +import org.wowtools.giscat.vector.pojo.util.SampleData; + +import java.io.*; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +/** + * @author liuyu + * @date 2022/4/1 + */ +public class MakePbfTest1 { + + + public static void main(String[] args) throws Exception { + Map p = Map.of("a", List.of("1", 2.5d, true, "x", Map.of("xx", 111)), "b", "测试"); + FeatureCollection fc = new FeatureCollection(); + fc.setHeaders(Map.of( + "version",1.1, + "list",List.of(1,"ss",1.23) + + )); + List features = new LinkedList<>(); + fc.setFeatures(features); + features.add(new Feature(SampleData.point, p)); + features.add(new Feature(SampleData.lineString, p)); + features.add(new Feature(SampleData.polygon1, p)); + features.add(new Feature(SampleData.polygon2, p)); + features.add(new Feature(SampleData.polygon3, p)); + features.add(new Feature(SampleData.multiPoint, p)); + features.add(new Feature(SampleData.multiLineString, p)); + features.add(new Feature(SampleData.multiPolygon1, p)); + features.add(new Feature(SampleData.multiPolygon2, p)); + features.add(new Feature(SampleData.geometryCollection, p)); + byte[] bytes = ProtoFeatureConverter.featureCollection2Proto(fc); + save2File("D:/_test/1/testbytes.pbf", bytes); + System.out.println(GeoJsonFeatureConverter.toGeoJson(fc).toGeoJsonString()); + } + + + public static boolean save2File(String fname, byte[] msg) { + OutputStream fos = null; + try { + File file = new File(fname); + file.mkdirs(); + System.out.println(file.getAbsolutePath()); + fos = new FileOutputStream(file); + fos.write(msg); + fos.flush(); + return true; + } catch (FileNotFoundException e) { + return false; + } catch (IOException e) { + File parent; + return false; + } finally { + if (fos != null) { + try { + fos.close(); + } catch (IOException e) { + } + } + } + } +} diff --git a/giscat-vector/giscat-vector-pojo/src/test/java/org/wowtools/giscat/vector/pojo/converter/Test1.java b/giscat-vector/giscat-vector-pojo/src/test/java/org/wowtools/giscat/vector/pojo/converter/Test1.java index 0791554..8449d32 100644 --- a/giscat-vector/giscat-vector-pojo/src/test/java/org/wowtools/giscat/vector/pojo/converter/Test1.java +++ b/giscat-vector/giscat-vector-pojo/src/test/java/org/wowtools/giscat/vector/pojo/converter/Test1.java @@ -1,69 +1,32 @@ package org.wowtools.giscat.vector.pojo.converter; -import org.wowtools.giscat.vector.pojo.Feature; -import org.wowtools.giscat.vector.pojo.FeatureCollection; -import org.wowtools.giscat.vector.pojo.util.SampleData; +import org.wowtools.giscat.vector.pojo.converter.FoolStyleFeatureConverter; +import org.locationtech.jts.geom.*; -import java.io.*; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -/** - * @author liuyu - * @date 2022/4/1 - */ public class Test1 { + public static void main(String[] args) { + + { + Polygon p1 = FoolStyleFeatureConverter.array2Polygon(new double[]{150, 330, 260, 380, 380, 240}); + Polygon p2 = FoolStyleFeatureConverter.array2Polygon(new double[]{190, 260, 269, 325, 290, 160}); + System.out.println("p1与p2是否相交 "+p1.intersects(p2));//true + System.out.println("p1与p2的交集 "+p1.intersection(p2));//POLYGON ((274.56738768718805 281.25623960066554, 234.76427923844062 296.83136899365365, 269 325, 274.56738768718805 281.25623960066554)) + System.out.println("交集的面积 "+p1.intersection(p2).getArea());//827.2124277609248 + System.out.println("交集的周长 "+p1.intersection(p2).getLength());//131.17314524175666 + } + { + LineString l1 = FoolStyleFeatureConverter.array2Line(new double[]{190, 180, 170, 240, 250, 320, 323, 205}); + Polygon p1 = FoolStyleFeatureConverter.array2Polygon(new double[]{150, 330, 260, 380, 380, 240}); + System.out.println("线与多边形是否相交 " + l1.intersects(p1));//true + System.out.println("线与多边形的交集 "+l1.intersection(p1));//LINESTRING (229.0625 299.0625, 250 320, 274.60261569416497 281.2424547283702) + System.out.println("相交线段长度 "+l1.intersection(p1).getLength());//75.51691528550752 + } - public static void main(String[] args) throws Exception { - Map p = Map.of("a", List.of("1", 2.5d, true, "x", Map.of("xx", 111)), "b", "测试"); - FeatureCollection fc = new FeatureCollection(); - fc.setHeaders(Map.of( - "version",1.1, - "list",List.of(1,"ss",1.23) - - )); - List features = new LinkedList<>(); - fc.setFeatures(features); - features.add(new Feature(SampleData.point, p)); - features.add(new Feature(SampleData.lineString, p)); - features.add(new Feature(SampleData.polygon1, p)); - features.add(new Feature(SampleData.polygon2, p)); - features.add(new Feature(SampleData.polygon3, p)); - features.add(new Feature(SampleData.multiPoint, p)); - features.add(new Feature(SampleData.multiLineString, p)); - features.add(new Feature(SampleData.multiPolygon1, p)); - features.add(new Feature(SampleData.multiPolygon2, p)); - features.add(new Feature(SampleData.geometryCollection, p)); - byte[] bytes = ProtoFeatureConverter.featureCollection2Proto(fc); - save2File("D:/_test/1/testbytes.pbf", bytes); - System.out.println(GeoJsonFeatureConverter.toGeoJson(fc).toGeoJsonString()); - } - - - public static boolean save2File(String fname, byte[] msg) { - OutputStream fos = null; - try { - File file = new File(fname); - file.mkdirs(); - System.out.println(file.getAbsolutePath()); - fos = new FileOutputStream(file); - fos.write(msg); - fos.flush(); - return true; - } catch (FileNotFoundException e) { - return false; - } catch (IOException e) { - File parent; - return false; - } finally { - if (fos != null) { - try { - fos.close(); - } catch (IOException e) { - } - } + { + Point p = FoolStyleFeatureConverter.xy2Point(250, 310); + Polygon p1 = FoolStyleFeatureConverter.array2Polygon(new double[]{150, 330, 260, 380, 380, 240}); + System.out.println("点与多边形是否相交 " + p.intersects(p1));//true } } } diff --git a/giscat-vector/giscat-vector-rocksrtree/pom.xml b/giscat-vector/giscat-vector-rocksrtree/pom.xml index ac7c827..e3e052f 100644 --- a/giscat-vector/giscat-vector-rocksrtree/pom.xml +++ b/giscat-vector/giscat-vector-rocksrtree/pom.xml @@ -18,8 +18,12 @@ rocksdbjni + org.wowtools giscat-vector-util + + org.wowtools + catframe-common diff --git a/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/Branch.java b/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/Branch.java index a89b493..2f338bc 100644 --- a/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/Branch.java +++ b/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/Branch.java @@ -166,8 +166,9 @@ Node add(final RectNd t, TreeTransaction tx) { } else { final int bestLeaf = chooseLeaf(t, tRect, tx); - child[bestLeaf] = getChild(bestLeaf, tx).add(t, tx).id; - mbr = mbr.getMbr(getChild(bestLeaf, tx).getBound()); + Node c = getChild(bestLeaf, tx); + child[bestLeaf] = c.add(t, tx).id; + mbr = mbr.getMbr(c.getBound()); tx.put(id, this); return this; } diff --git a/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/RTree.java b/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/RTree.java index e4f407b..30ba615 100644 --- a/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/RTree.java +++ b/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/RTree.java @@ -30,9 +30,6 @@ * #L% */ -import com.google.protobuf.InvalidProtocolBufferException; -import org.rocksdb.RocksDB; -import org.rocksdb.RocksDBException; import org.wowtools.giscat.vector.pojo.Feature; import java.nio.charset.StandardCharsets; @@ -48,14 +45,12 @@ *

* Created by jcairns on 4/30/15.

*/ -public final class RTree{ +public final class RTree { public static final byte[] TreeDbKey = "T".getBytes(StandardCharsets.UTF_8); private static final double EPSILON = 1e-12; - protected String root = null; - private final TreeBuilder builder; protected RTree(TreeBuilder builder) { @@ -67,10 +62,11 @@ protected RTree(TreeBuilder builder) { * * @param rect 输入范围 * @param consumer 查询结果消费者，若accept返回false，则终止查询过程 + * @param tx 事务 */ public void contains(RectNd rect, FeatureConsumer consumer, TreeTransaction tx) { - if (root != null) { - builder.getNode(root, tx).contains(rect, consumer, tx); + if (builder.rootId != null) { + builder.getNode(builder.rootId, tx).contains(rect, consumer, tx); } } @@ -80,10 +76,11 @@ public void contains(RectNd rect, FeatureConsumer consumer, TreeTransaction tx) * * @param rect 输入范围 * @param consumer 查询结果消费者，若accept返回false，则终止查询过程 + * @param tx 事务 */ public void intersects(RectNd rect, FeatureConsumer consumer, TreeTransaction tx) { - if (root != null) { - builder.getNode(root, tx).intersects(rect, consumer, tx); + if (builder.rootId != null) { + builder.getNode(builder.rootId, tx).intersects(rect, consumer, tx); } } @@ -103,16 +100,17 @@ public void add(Feature feature, TreeTransaction tx) { /** * Add the data entry to the SpatialSearch structure * - * @param t Data entry to be added + * @param t Data entry to be added + * @param tx 事务 */ protected void add(final RectNd t, TreeTransaction tx) { - if (root != null) { - Node node = builder.getNode(root, tx); + if (builder.rootId != null) { + Node node = builder.getNode(builder.rootId, tx); Node newNode = node.add(t, tx); - root = newNode.id; + builder.rootId = newNode.id; } else { - root = builder.newLeaf(tx).id; - builder.getNode(root, tx).add(t, tx); + builder.rootId = builder.newLeaf(tx).id; + builder.getNode(builder.rootId, tx).add(t, tx); } } @@ -157,8 +155,8 @@ protected void add(final RectNd t, TreeTransaction tx) { * @return entry count */ public int getEntryCount(TreeTransaction tx) { - if (root != null) { - return builder.getNode(root, tx).totalSize(tx); + if (builder.rootId != null) { + return builder.getNode(builder.rootId, tx).totalSize(tx); } return 0; } @@ -177,8 +175,8 @@ static boolean isEqual(final double a, final double b, final double eps) { * @param consumer - callback for each element */ protected void forEach(Consumer consumer, TreeTransaction tx) { - if (root != null) { - builder.getNode(root, tx).forEach(consumer,tx); + if (builder.rootId != null) { + builder.getNode(builder.rootId, tx).forEach(consumer, tx); } } @@ -186,12 +184,12 @@ protected Stats collectStats(TreeTransaction tx) { Stats stats = new Stats(); stats.setMaxFill(builder.mMax); stats.setMinFill(builder.mMin); - builder.getNode(root, tx).collectStats(stats, 0,tx); + builder.getNode(builder.rootId, tx).collectStats(stats, 0, tx); return stats; } Node getRoot(TreeTransaction tx) { - return builder.getNode(root, tx); + return builder.getNode(builder.rootId, tx); } @@ -199,7 +197,7 @@ protected byte[] toBytes() { RocksRtreePb.RTreePb.Builder rtreeBuilder = RocksRtreePb.RTreePb.newBuilder(); rtreeBuilder.setMMax(builder.mMax); rtreeBuilder.setMMin(builder.mMin); - rtreeBuilder.setRootId(root); + rtreeBuilder.setRootId(builder.rootId); return rtreeBuilder.build().toByteArray(); } } diff --git a/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/RectNd.java b/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/RectNd.java index 489d8aa..74f1d71 100644 --- a/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/RectNd.java +++ b/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/RectNd.java @@ -90,6 +90,7 @@ public String toString() { return min + " " + max; } + private RectNd getMbrCache; /** * Calculate the resulting mbr when combining param HyperRect with this HyperRect * @@ -97,6 +98,9 @@ public String toString() { * @return new HyperRect representing mbr of both HyperRects combined */ public RectNd getMbr(RectNd r) { + if (null != getMbrCache) { + return getMbrCache; + } int dim = min.xs.length; double[] min = new double[dim]; double[] max = new double[dim]; @@ -108,7 +112,8 @@ public RectNd getMbr(RectNd r) { throw new RuntimeException(e); } } - return new RectNd(new PointNd(min), new PointNd(max)); + getMbrCache = new RectNd(new PointNd(min), new PointNd(max)); + return getMbrCache; } /** @@ -195,31 +200,42 @@ boolean intersects(RectNd r2) { return true; } + private double costCache = -1; /** * Calculate the "cost" of this HyperRect - usually the area across all dimensions * * @return - cost */ double cost() { + if (costCache >= 0) { + return costCache; + } double res = 1; for (int i = 0; i < min.getNDim(); i++) { res = res * (max.getCoord(i) - min.getCoord(i)); } - return Math.abs(res); + + costCache = Math.abs(res); + return costCache; } + private double perimeterCache = -1; /** * Calculate the perimeter of this HyperRect - across all dimesnions * * @return - perimeter */ double perimeter() { + if (perimeterCache >= 0) { + return perimeterCache; + } double n = Math.pow(2, getNDim()); double p = 0.0; final int nD = this.getNDim(); for (int d = 0; d < nD; d++) { p += n * this.getRange(d); } + perimeterCache = p; return p; } } diff --git a/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/RocksRtreePb.java b/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/RocksRtreePb.java index 2786a95..b6d42bb 100644 --- a/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/RocksRtreePb.java +++ b/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/RocksRtreePb.java @@ -2,7 +2,6 @@ // source: definition/RocksRtree.proto package org.wowtools.giscat.vector.rocksrtree; - public final class RocksRtreePb { private RocksRtreePb() {} public static void registerAllExtensions( diff --git a/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/TreeBuilder.java b/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/TreeBuilder.java index 6de6268..bdb8fe4 100644 --- a/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/TreeBuilder.java +++ b/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/TreeBuilder.java @@ -31,28 +31,32 @@ */ import com.google.protobuf.InvalidProtocolBufferException; +import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.rocksdb.*; import org.rocksdb.util.SizeUnit; import org.wowtools.giscat.vector.pojo.Feature; import org.wowtools.giscat.vector.pojo.FeatureCollection; +import org.wowtools.giscat.vector.util.analyse.Bbox; import java.io.Closeable; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Function; import static org.wowtools.giscat.vector.rocksrtree.RTree.TreeDbKey; /** - * Created by jcairns on 4/30/15. + * RTree构建器，利用TreeBuilder构建出RTree，继而进行后续的修改、查询操作 */ +@Slf4j public class TreeBuilder implements Closeable { -// private final Map branchMap = new HashMap<>(); - -// private final Map leafMap = new HashMap<>(); - + private final ReadWriteLock lock = new ReentrantReadWriteLock(); private long leafIdIndex = 0; private long branchIdIndex = 0; @@ -60,12 +64,17 @@ public class TreeBuilder implements Closeable { protected final int mMin; protected final int mMax; + private final int cache1Size; private final RocksDB db; private final RTree rTree; + protected String rootId; + private final Function featureRectNdFunction; + private final Map protoAbleCaches2; + private static Options buildDefaultRocksDbOptions() { Options options = new Options(); final Filter bloomFilter = new BloomFilter(10); @@ -94,27 +103,55 @@ private static Options buildDefaultRocksDbOptions() { } - public TreeBuilder(String fileDir, @Nullable Options options, int mMin, int mMax, Function featureRectNdFunction) { - if (null == options) { - options = buildDefaultRocksDbOptions(); - } - try { - db = RocksDB.open(options, fileDir); - } catch (RocksDBException e) { - throw new RuntimeException(e); - } - this.mMin = mMin; - this.mMax = mMax; - rTree = new RTree(this); - this.featureRectNdFunction = featureRectNdFunction; + public static final class TreeBuilderConfig { + /** + * rocksdb配置参数 + */ + public @Nullable Options options; + /** + * 每个非叶子节点最少有几个子节点 + */ + public int mMin = 16; + /** + * 每个非叶子节点最多有几个子节点 + */ + public int mMax = 64; + + /** + * 缓存的节点数，数值越大从磁盘读数据的概率越小，但越吃内存 + */ + public int cacheSize = 100000; + + /** + * 如何取得feature的外接矩形，可以从geometry或属性入手进行构建，例如，默认值是取二维矩形范围: + * 
+         *  (feature) -> {
+         *                 Bbox bbox = new Bbox(feature.getGeometry());
+         *                 return new RectNd(new double[]{bbox.xmin, bbox.ymin}, new double[]{bbox.xmax, bbox.ymax});
+         *      }
+         *
+ */ + public @Nullable Function featureRectNdFunction; } - public TreeBuilder(String fileDir, @Nullable Options options, Function featureRectNdFunction) { + + /** + * 获得一个TreeBuilder示例 + * + * @param dir RTree持久化存储到本地磁盘的路径 + * @param config rtree的配置信息 + * @see TreeBuilderConfig + */ + public TreeBuilder(@NotNull String dir, @Nullable TreeBuilderConfig config) { + if (null == config) { + config = new TreeBuilderConfig(); + } + Options options = config.options; if (null == options) { options = buildDefaultRocksDbOptions(); } try { - db = RocksDB.open(options, fileDir); + db = RocksDB.open(options, dir); } catch (RocksDBException e) { throw new RuntimeException(e); } @@ -124,20 +161,56 @@ public TreeBuilder(String fileDir, @Nullable Options options, Function featureRectNdFunction)构造"); - } - RocksRtreePb.RTreePb pbTree; - try { - pbTree = RocksRtreePb.RTreePb.parseFrom(bytes); - } catch (InvalidProtocolBufferException e) { - throw new RuntimeException(e); + if (config.mMin < 2) { + log.warn("mMin的值过小，自动调整为2"); + config.mMin = 2; + } + if (config.mMax < config.mMin) { + log.warn("mMax的值过小，自动调整为{}", config.mMin); + config.mMax = config.mMin; + } + mMin = config.mMin; + mMax = config.mMax; + } else { + RocksRtreePb.RTreePb pbTree; + try { + pbTree = RocksRtreePb.RTreePb.parseFrom(bytes); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + mMin = pbTree.getMMin(); + mMax = pbTree.getMMax(); + log.debug("rtree已存在，使用已有值 mMin {} mMax {}", mMin, mMax); + rootId = pbTree.getRootId(); } - mMin = pbTree.getMMin(); - mMax = pbTree.getMMax(); + rTree = new RTree(this); - rTree.root = pbTree.getRootId(); - this.featureRectNdFunction = featureRectNdFunction; + if (null == config.featureRectNdFunction) { + featureRectNdFunction = (feature) -> { + Bbox bbox = new Bbox(feature.getGeometry()); + return new RectNd(new double[]{bbox.xmin, bbox.ymin}, new double[]{bbox.xmax, bbox.ymax}); + }; + } else { + featureRectNdFunction = config.featureRectNdFunction; + } + + if (config.cacheSize < 128) { + log.warn("cacheSize过小，调整为128"); + config.cacheSize = 128; + } + final int cacheSize = config.cacheSize; + cache1Size = config.cacheSize / 10; + + protoAbleCaches2 = new LinkedHashMap(cacheSize, 0.75f, true) { + private static final long serialVersionUID = 1L; + + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > cacheSize; + } + }; } public RTree getRTree() { @@ -151,7 +224,7 @@ protected RectNd buildFeatureRect(Feature feature) { } public TreeTransaction newTx() { - return new TreeTransaction(db, this); + return new TreeTransaction(db, this, cache1Size, protoAbleCaches2, lock); } protected void clearCache() { diff --git a/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/TreeTransaction.java b/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/TreeTransaction.java index e04ded9..2587331 100644 --- a/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/TreeTransaction.java +++ b/giscat-vector/giscat-vector-rocksrtree/src/main/java/org/wowtools/giscat/vector/rocksrtree/TreeTransaction.java @@ -18,7 +18,9 @@ import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.Map; +import java.util.concurrent.locks.ReadWriteLock; import static org.wowtools.giscat.vector.rocksrtree.RTree.TreeDbKey; @@ -30,6 +32,20 @@ */ public class TreeTransaction implements AutoCloseable { + private static final byte[] nullBytes = new byte[0]; + + private static final ProtoAble nullProtoAble = new ProtoAble(null, null) { + @Override + public void fill(byte[] bytes) { + + } + + @Override + protected byte[] toBytes() { + return nullBytes; + } + }; + private final RocksDB db; private final WriteOptions writeOpt; private final WriteBatch batch; @@ -41,12 +57,32 @@ public class TreeTransaction implements AutoCloseable { private final String treeRootId; - protected TreeTransaction(RocksDB db, TreeBuilder builder) { + private final ReadWriteLock lock; + + private boolean commited = false; + + + //TODO 一二级缓存设计考虑 + private final Map protoAbleCaches2; + private final Map protoAbleCaches1; + + protected TreeTransaction(RocksDB db, TreeBuilder builder, int cache1Size, Map protoAbleCaches2, ReadWriteLock lock) { + protoAbleCaches1 = new LinkedHashMap(cache1Size, 0.75f, true) { + private static final long serialVersionUID = 1L; + + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > cache1Size; + } + }; this.db = db; this.builder = builder; + this.protoAbleCaches2 = protoAbleCaches2; writeOpt = new WriteOptions(); batch = new WriteBatch(); - treeRootId = builder.getRTree().root; + treeRootId = builder.rootId; + this.lock = lock; + lock.readLock().lock(); } protected void put(String key, ProtoAble value) { @@ -69,18 +105,49 @@ protected T get(Class t, String key) { if (null != txb) { return txb; } + ProtoAble cache; + cache = protoAbleCaches1.get(key); + if (null != cache) { + if (cache == nullProtoAble) { + return null; + } else { + return (T) cache; + } + } + + cache = protoAbleCaches2.get(key); + if (null != cache) { + if (cache == nullProtoAble) { + return null; + } else { + return (T) cache; + } + } + + byte[] bytes; try { bytes = db.get(key.getBytes(StandardCharsets.UTF_8)); } catch (RocksDBException e) { throw new RuntimeException(e); } - txb = ProtoAble.fromBytes(t, builder, key, bytes); - return txb; + if (null == bytes) { + protoAbleCaches1.put(key, nullProtoAble); + return null; + } else { + txb = ProtoAble.fromBytes(t, builder, key, bytes); + protoAbleCaches1.put(key, txb); + return txb; + } + } public void commit() { + if (commited) { + throw new RuntimeException("事务已经提交过一次了"); + } + commited = true; try { for (Map.Entry e : txAdded.entrySet()) { batch.put(e.getKey().getBytes(StandardCharsets.UTF_8), e.getValue().toBytes()); @@ -88,24 +155,45 @@ public void commit() { for (String k : txDeleted) { batch.delete(k.getBytes(StandardCharsets.UTF_8)); } - if (null == treeRootId || !treeRootId.equals(builder.getRTree().root)) { + if (null == treeRootId || !treeRootId.equals(builder.rootId)) { //rtree发生过变化，存储一次rtree batch.put(TreeDbKey, builder.getRTree().toBytes()); } - db.write(writeOpt, batch); + lock.readLock().unlock(); + lock.writeLock().lock(); + try { + db.write(writeOpt, batch); + protoAbleCaches2.putAll(txAdded); + for (String s : txDeleted) { + protoAbleCaches2.remove(s); + } + } finally { + lock.writeLock().unlock(); + } } catch (RocksDBException e) { throw new RuntimeException(e); } } public void rollback() { - close(); + txAdded.clear(); + txDeleted.clear(); + protoAbleCaches1.clear(); } @Override public void close() { - + if (!commited) { + lock.readLock().unlock(); + if (protoAbleCaches1.size() > 0) { + lock.writeLock().lock(); + protoAbleCaches2.putAll(protoAbleCaches1); + lock.writeLock().unlock(); + } + } + batch.close(); + writeOpt.close(); } diff --git a/giscat-vector/giscat-vector-rocksrtree/src/test/java/org/wowtools/giscat/vector/rocksrtreetest/Test.java b/giscat-vector/giscat-vector-rocksrtree/src/test/java/org/wowtools/giscat/vector/rocksrtreetest/Test.java index cd60a8c..10db089 100644 --- a/giscat-vector/giscat-vector-rocksrtree/src/test/java/org/wowtools/giscat/vector/rocksrtreetest/Test.java +++ b/giscat-vector/giscat-vector-rocksrtree/src/test/java/org/wowtools/giscat/vector/rocksrtreetest/Test.java @@ -19,6 +19,8 @@ * @date 2023/3/23 */ public class Test { + + public static void deleteFolder(File folder) { if (folder.isDirectory()) { File[] files = folder.listFiles(); // 获取文件夹中的所有文件和子文件夹 @@ -33,10 +35,10 @@ public static void deleteFolder(File folder) { private static void add(String dir, Function featureRectNdFunction, GeometryFactory geometryFactory) { deleteFolder(new File(dir)); - int num = 12345; - int txSize = 1000; + int num = 10000; + int txSize = 2000; - TreeBuilder builder = new TreeBuilder(dir, null, 2, 64, featureRectNdFunction); + TreeBuilder builder = new TreeBuilder(dir, null); RTree pTree = builder.getRTree(); long t = System.currentTimeMillis(); @@ -46,8 +48,9 @@ private static void add(String dir, Function featureRectNdFunct pTree.add(new Feature(point), tx); if (i % txSize == 0) { tx.commit(); + tx.close(); tx = builder.newTx(); - System.out.println("add "+i); + System.out.println("add " + i); } } tx.commit(); @@ -56,8 +59,8 @@ private static void add(String dir, Function featureRectNdFunct builder.close(); } - private static void query(String dir, Function featureRectNdFunction) { - TreeBuilder builder = new TreeBuilder(dir, null, featureRectNdFunction); + private static void query(String dir, Function featureRectNdFunction) throws Exception { + TreeBuilder builder = new TreeBuilder(dir, null); final RectNd rect = new RectNd(new double[]{1.9, 1.9}, new double[]{8.1, 8.1}); RTree pTree = builder.getRTree(); @@ -74,7 +77,7 @@ public boolean accept(Feature f) { try (TreeTransaction tx = builder.newTx()) { pTree.contains(rect, consumer, tx); } - System.out.println("query success,cost " + (System.currentTimeMillis() - t)); + System.out.println("1 query success,cost " + (System.currentTimeMillis() - t)); Assert.equals(7, res.size()); System.out.println(res.size()); @@ -85,10 +88,21 @@ public boolean accept(Feature f) { Assert.isTrue(feature.getGeometry().getCoordinate().y >= 2); Assert.isTrue(feature.getGeometry().getCoordinate().y <= 8); } -// builder.close(); + + for (int i = 0; i < 10; i++) { + res.clear(); + t = System.currentTimeMillis(); + try (TreeTransaction tx = builder.newTx()) { + pTree.contains(rect, consumer, tx); + } + System.out.println("2 query success,cost " + (System.currentTimeMillis() - t)); + Assert.equals(7, res.size()); + } + + builder.close(); } - public static void main(String[] args) { + public static void main(String[] args) throws Exception { String dir = "D:\\_tmp\\1\\rocksrtree"; Function featureRectNdFunction = (feature) -> { diff --git a/pom.xml b/pom.xml index c06a269..32cca7b 100644 --- a/pom.xml +++ b/pom.xml @@ -215,6 +215,9 @@ createdate + + **/RocksRtreePb.java +