From 43f2e608b2cac8cc70808f2cc00be565484bc8e3 Mon Sep 17 00:00:00 2001 From: godotg Date: Sat, 21 Oct 2023 21:24:01 +0800 Subject: [PATCH] ref[storage]: refactor lambda of storage --- .../zfoo/storage/manager/AbstractStorage.java | 221 +++++++++++++++++ .../com/zfoo/storage/manager/StorageInt.java | 31 ++- .../com/zfoo/storage/manager/StorageLong.java | 31 ++- .../zfoo/storage/manager/StorageManager.java | 2 +- .../zfoo/storage/manager/StorageObject.java | 230 ++---------------- 5 files changed, 281 insertions(+), 234 deletions(-) create mode 100644 storage/src/main/java/com/zfoo/storage/manager/AbstractStorage.java diff --git a/storage/src/main/java/com/zfoo/storage/manager/AbstractStorage.java b/storage/src/main/java/com/zfoo/storage/manager/AbstractStorage.java new file mode 100644 index 000000000..7497ef607 --- /dev/null +++ b/storage/src/main/java/com/zfoo/storage/manager/AbstractStorage.java @@ -0,0 +1,221 @@ +/* + * Copyright (C) 2020 The zfoo Authors + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except + * in compliance with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and limitations under the License. + */ + +package com.zfoo.storage.manager; + +import com.zfoo.protocol.collection.CollectionUtils; +import com.zfoo.protocol.util.*; +import com.zfoo.storage.interpreter.ResourceInterpreter; +import com.zfoo.storage.model.IStorage; +import com.zfoo.storage.model.IdDef; +import com.zfoo.storage.model.IndexDef; +import com.zfoo.storage.util.function.Func1; +import com.zfoo.storage.util.lambda.*; +import org.springframework.lang.Nullable; +import org.springframework.util.ConcurrentReferenceHashMap; + +import java.io.InputStream; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Proxy; +import java.util.*; + +/** + * @author godotg + */ +public abstract class AbstractStorage implements IStorage { + // 非唯一索引 + protected Map>> indexMap = new HashMap<>(); + // 唯一索引 + protected Map> uniqueIndexMap = new HashMap<>(); + + protected Class clazz; + protected IdDef idDef; + protected Map indexDefMap; + // EN: unused configuration tables will clear data to save memory. + // CN: 没有被使用的配置表会清除data数据，以达到节省内存的目的 + protected boolean recycle = true; + + private ConcurrentReferenceHashMap, String> funcCaches = new ConcurrentReferenceHashMap<>(); + + public static AbstractStorage parse(InputStream inputStream, Class resourceClazz, String suffix) { + var idDef = IdDef.valueOf(resourceClazz); + var indexDefMap = Collections.unmodifiableMap(IndexDef.createResourceIndexes(resourceClazz)); + + try { + var list = ResourceInterpreter.read(inputStream, resourceClazz, suffix); + // 校验id是否重复 + var set = new HashSet<>(); + for (var value : list) { + var id = ReflectionUtils.getField(idDef.getField(), value); + if (id == null) { + throw new RuntimeException("There is an item with an unconfigured id in the static resource"); + } + if (set.contains(id)) { + throw new RuntimeException(StringUtils.format("Duplicate [id:{}] of static resource [resource:{}]", id, resourceClazz.getSimpleName())); + } + set.add(id); + } + + var idType = idDef.getField().getType(); + if (idType == int.class || idType == Integer.class) { + return new StorageInt<>(resourceClazz, idDef, indexDefMap, list); + } else if (idType == long.class || idType == Long.class) { + return new StorageLong<>(resourceClazz, idDef, indexDefMap, list); + } else { + return new StorageObject<>(resourceClazz, idDef, indexDefMap, list); + } + } catch (Throwable e) { + throw new RuntimeException(e.getMessage(), e); + } finally { + IOUtils.closeIO(inputStream); + } + } + + public AbstractStorage(Class clazz, IdDef idDef, Map indexDefMap, List values) { + this.clazz = clazz; + this.idDef = idDef; + this.indexDefMap = indexDefMap; + for (var value : values) { + // 添加资源 + @SuppressWarnings("unchecked") + var v = (V) value; + // 添加索引 + for (var def : indexDefMap.values()) { + // 使用field的名称作为索引的名称 + var indexKey = def.getField().getName(); + var indexValue = ReflectionUtils.getField(def.getField(), v); + if (def.isUnique()) { + var uniqueIndex = uniqueIndexMap.computeIfAbsent(indexKey, it -> new HashMap<>(values.size())); + if (uniqueIndex.put(indexValue, v) != null) { + throw new RuntimeException(StringUtils.format("Duplicate unique index [index:{}][value:{}] of static resource [class:{}]", indexKey, indexValue, clazz.getName())); + } + } else { + var index = indexMap.computeIfAbsent(indexKey, it -> new HashMap<>(values.size())); + var list = index.computeIfAbsent(indexValue, it -> new ArrayList()); + list.add(v); + } + } + } + } + + @Override + public List getIndexes(Func1 func, INDEX index) { + String indexName = getMethodToField(func); + var indexValues = indexMap.get(indexName); + AssertionUtils.notNull(indexValues, "The index of [indexName:{}] does not exist in the static resource [resource:{}]", indexName, clazz.getSimpleName()); + var values = indexValues.get(index); + if (CollectionUtils.isEmpty(values)) { + return Collections.emptyList(); + } + return values; + } + + @Nullable + @Override + public V getUniqueIndex(Func1 func, INDEX index) { + String uniqueIndexName = getMethodToField(func); + var indexValueMap = uniqueIndexMap.get(uniqueIndexName); + AssertionUtils.notNull(indexValueMap, "There is no a unique index for [uniqueIndexName:{}] in the static resource [resource:{}]", uniqueIndexName, clazz.getSimpleName()); + var value = indexValueMap.get(index); + return value; + } + + @Override + public void recycleStorage() { + recycle = true; + indexMap = null; + uniqueIndexMap = null; + idDef = null; + indexDefMap = null; + } + + @Override + public boolean isRecycle() { + return recycle; + } + + @Override + public void setRecycle(boolean recycle) { + this.recycle = recycle; + } + + @Override + public IdDef getIdDef() { + return idDef; + } + + private String getMethodToField(Func1 func) { + var indexName = funcCaches.get(func); + if (indexName != null) { + return indexName; + } + + // 1. IDEA 调试模式下 lambda 表达式是一个代理 + if (func instanceof Proxy) { + try { + var lambda = new IdeaProxyLambdaMeta((Proxy) func); + indexName = FieldUtils.getMethodToField(clazz, lambda.getImplMethodName()); + } catch (Exception e) { + } + } + + // 2. 反射读取 + if (indexName == null) { + try { + var method = func.getClass().getDeclaredMethod("writeReplace"); + ReflectionUtils.makeAccessible(method); + var lambda = new ReflectLambdaMeta((java.lang.invoke.SerializedLambda) method.invoke(func)); + indexName = FieldUtils.getMethodToField(clazz, lambda.getImplMethodName()); + } catch (InvocationTargetException | NoSuchMethodException | IllegalAccessException e) { + } + } + + // 3. 反射失败使用序列化的方式读取 + if (indexName == null) { + try { + var lambda = new ShadowLambdaMeta(SerializedLambda.extract(func)); + indexName = FieldUtils.getMethodToField(clazz, lambda.getImplMethodName()); + } catch (Exception e) { + } + } + + // 4. 通过将func带入到dataMap中求解，适合GraalVM环境中 + if (indexName == null) { + try { + var fields = clazz.getDeclaredFields(); + Arrays.stream(fields).forEach(ReflectionUtils::makeAccessible); + for (var value : getAll()) { + var r = func.apply(value); + var valueFields = Arrays.stream(fields) + .map(it -> ReflectionUtils.getField(it, value)) + .filter(it -> it.equals(r) && it.getClass() == r.getClass()) + .toList(); + // 如果只有一个能匹配到func的返回值则就是这个方法 + if (valueFields.size() == 1) { + for (var field : fields) { + if (!ReflectionUtils.getField(field, value).equals(r)) { + continue; + } + indexName = field.getName(); + break; + } + break; + } + } + } catch (Exception e) { + } + } + + funcCaches.put(func, indexName); + return indexName; + } +} diff --git a/storage/src/main/java/com/zfoo/storage/manager/StorageInt.java b/storage/src/main/java/com/zfoo/storage/manager/StorageInt.java index 3822ae0e4..3d82995a6 100644 --- a/storage/src/main/java/com/zfoo/storage/manager/StorageInt.java +++ b/storage/src/main/java/com/zfoo/storage/manager/StorageInt.java @@ -13,33 +13,38 @@ package com.zfoo.storage.manager; import com.zfoo.protocol.util.AssertionUtils; +import com.zfoo.protocol.util.ReflectionUtils; +import com.zfoo.protocol.util.StringUtils; +import com.zfoo.storage.model.IdDef; +import com.zfoo.storage.model.IndexDef; import io.netty.util.collection.IntObjectHashMap; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Map; /** * @author godotg */ -public class StorageInt extends StorageObject { +public class StorageInt extends AbstractStorage { private IntObjectHashMap dataMap; - public StorageInt(StorageObject storage) { - this.dataMap = new IntObjectHashMap(storage.size()); - @SuppressWarnings("unchecked") - var map = (Map) storage.getData(); - this.dataMap.putAll(map); - super.indexMap = storage.indexMap; - super.uniqueIndexMap = storage.uniqueIndexMap; - super.clazz = storage.clazz; - super.idDef = storage.idDef; - super.indexDefMap = storage.indexDefMap; - super.recycle = storage.recycle; - storage.recycleStorage(); + public StorageInt(Class clazz, IdDef idDef, Map indexDefMap, List values) { + super(clazz, idDef, indexDefMap, values); + + this.dataMap = new IntObjectHashMap<>(values.size()); + for (var value : values) { + @SuppressWarnings("unchecked") + var id = (K) ReflectionUtils.getField(idDef.getField(), value); + @SuppressWarnings("unchecked") + var v = (V) value; + dataMap.put((Integer) id, v); + } } + @Override public boolean contain(K id) { return contain((int) id); diff --git a/storage/src/main/java/com/zfoo/storage/manager/StorageLong.java b/storage/src/main/java/com/zfoo/storage/manager/StorageLong.java index 2c1abd8b6..cc5b9c22c 100644 --- a/storage/src/main/java/com/zfoo/storage/manager/StorageLong.java +++ b/storage/src/main/java/com/zfoo/storage/manager/StorageLong.java @@ -13,33 +13,38 @@ package com.zfoo.storage.manager; import com.zfoo.protocol.util.AssertionUtils; +import com.zfoo.protocol.util.ReflectionUtils; +import com.zfoo.storage.model.IdDef; +import com.zfoo.storage.model.IndexDef; +import io.netty.util.collection.IntObjectHashMap; import io.netty.util.collection.LongObjectHashMap; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Map; /** * @author godotg */ -public class StorageLong extends StorageObject { +public class StorageLong extends AbstractStorage { private LongObjectHashMap dataMap; - public StorageLong(StorageObject storage) { - this.dataMap = new LongObjectHashMap(storage.size()); - @SuppressWarnings("unchecked") - var map = (Map) storage.getData(); - this.dataMap.putAll(map); - super.indexMap = storage.indexMap; - super.uniqueIndexMap = storage.uniqueIndexMap; - super.clazz = storage.clazz; - super.idDef = storage.idDef; - super.indexDefMap = storage.indexDefMap; - super.recycle = storage.recycle; - storage.recycleStorage(); + public StorageLong(Class clazz, IdDef idDef, Map indexDefMap, List values) { + super(clazz, idDef, indexDefMap, values); + + this.dataMap = new LongObjectHashMap<>(values.size()); + for (var value : values) { + @SuppressWarnings("unchecked") + var id = (K) ReflectionUtils.getField(idDef.getField(), value); + @SuppressWarnings("unchecked") + var v = (V) value; + dataMap.put((Long) id, v); + } } + @Override public boolean contain(K id) { return contain((long) id); diff --git a/storage/src/main/java/com/zfoo/storage/manager/StorageManager.java b/storage/src/main/java/com/zfoo/storage/manager/StorageManager.java index 37ec662d2..6e747c64a 100644 --- a/storage/src/main/java/com/zfoo/storage/manager/StorageManager.java +++ b/storage/src/main/java/com/zfoo/storage/manager/StorageManager.java @@ -135,7 +135,7 @@ public void initBefore() { var clazz = definition.getClazz(); var resource = definition.getResource(); var fileExtName = FileUtils.fileExtName(resource.getFilename()); - var storage = StorageObject.parse(resource.getInputStream(), clazz, fileExtName); + var storage = AbstractStorage.parse(resource.getInputStream(), clazz, fileExtName); storageMap.putIfAbsent(clazz, storage); } } catch (Exception e) { diff --git a/storage/src/main/java/com/zfoo/storage/manager/StorageObject.java b/storage/src/main/java/com/zfoo/storage/manager/StorageObject.java index a8ca028b6..13760a35a 100644 --- a/storage/src/main/java/com/zfoo/storage/manager/StorageObject.java +++ b/storage/src/main/java/com/zfoo/storage/manager/StorageObject.java @@ -13,271 +13,87 @@ package com.zfoo.storage.manager; import com.zfoo.protocol.collection.CollectionUtils; -import com.zfoo.protocol.util.*; -import com.zfoo.storage.interpreter.ResourceInterpreter; -import com.zfoo.storage.model.IStorage; +import com.zfoo.protocol.util.AssertionUtils; +import com.zfoo.protocol.util.ReflectionUtils; import com.zfoo.storage.model.IdDef; import com.zfoo.storage.model.IndexDef; -import com.zfoo.storage.util.function.Func1; -import com.zfoo.storage.util.lambda.*; -import org.springframework.lang.Nullable; -import org.springframework.util.ConcurrentReferenceHashMap; +import io.netty.util.collection.LongObjectHashMap; -import java.io.InputStream; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Proxy; import java.util.*; /** * @author godotg */ -public class StorageObject implements IStorage { - // all storage data - private Map dataMap; - // 非唯一索引 - protected Map>> indexMap = new HashMap<>(); - // 唯一索引 - protected Map> uniqueIndexMap = new HashMap<>(); - - protected Class clazz; - protected IdDef idDef; - protected Map indexDefMap; - // EN: unused configuration tables will clear data to save memory. - // CN: 没有被使用的配置表会清除data数据，以达到节省内存的目的 - protected boolean recycle = true; - - private ConcurrentReferenceHashMap, String> funcCaches = new ConcurrentReferenceHashMap<>(); - - public static StorageObject parse(InputStream inputStream, Class resourceClazz, String suffix) { - var idDef = IdDef.valueOf(resourceClazz); - var indexDefMap = Collections.unmodifiableMap(IndexDef.createResourceIndexes(resourceClazz)); +public class StorageObject extends AbstractStorage { - try { - var list = ResourceInterpreter.read(inputStream, resourceClazz, suffix); - var storage = new StorageObject<>(resourceClazz, idDef, indexDefMap, list); - - var idType = idDef.getField().getType(); - if (idType == int.class || idType == Integer.class) { - return new StorageInt<>(storage); - } else if (idType == long.class || idType == Long.class) { - return new StorageLong<>(storage); - } else { - return storage; - } - - } catch (Throwable e) { - throw new RuntimeException(e.getMessage(), e); - } finally { - IOUtils.closeIO(inputStream); - } - } - - protected StorageObject() { - } + private Map dataMap; public StorageObject(Class clazz, IdDef idDef, Map indexDefMap, List values) { + super(clazz, idDef, indexDefMap, values); this.dataMap = new HashMap<>(CollectionUtils.capacity(values.size())); - this.clazz = clazz; - this.idDef = idDef; - this.indexDefMap = indexDefMap; for (var value : values) { @SuppressWarnings("unchecked") var id = (K) ReflectionUtils.getField(idDef.getField(), value); - - if (id == null) { - throw new RuntimeException("There is an item with an unconfigured id in the static resource"); - } - if (dataMap.containsKey(id)) { - throw new RuntimeException(StringUtils.format("Duplicate [id:{}] of static resource [resource:{}]", id, clazz.getSimpleName())); - } - // 添加资源 @SuppressWarnings("unchecked") var v = (V) value; - dataMap.put(id, v); - // 添加索引 - for (var def : indexDefMap.values()) { - // 使用field的名称作为索引的名称 - var indexKey = def.getField().getName(); - var indexValue = ReflectionUtils.getField(def.getField(), v); - if (def.isUnique()) { - var uniqueIndex = uniqueIndexMap.computeIfAbsent(indexKey, it -> new HashMap<>(values.size())); - if (uniqueIndex.put(indexValue, v) != null) { - throw new RuntimeException(StringUtils.format("Duplicate unique index [index:{}][value:{}] of static resource [class:{}]", indexKey, indexValue, clazz.getName())); - } - } else { - var index = indexMap.computeIfAbsent(indexKey, it -> new HashMap<>(values.size())); - var list = index.computeIfAbsent(indexValue, it -> new ArrayList()); - list.add(v); - } - } + dataMap.put((K) id, v); } } + @Override public boolean contain(K id) { - return dataMap.containsKey(id); + return contain((long) id); } @Override public boolean contain(int id) { - @SuppressWarnings("unchecked") - var key = (K) Integer.valueOf(id); - return contain(key); + return contain((long) id); } @Override public boolean contain(long id) { - @SuppressWarnings("unchecked") - var key = (K) Long.valueOf(id); - return contain(key); + return dataMap.containsKey(id); } @Override public V get(K id) { - V result = dataMap.get(id); - AssertionUtils.notNull(result, "The static resource represented as [id:{}] in the static resource [resource:{}] does not exist", id, clazz.getSimpleName()); - return result; + return get((long) id); } @Override public V get(int id) { - @SuppressWarnings("unchecked") - var key = (K) Integer.valueOf(id); - return get(key); + return get((long) id); } @Override public V get(long id) { - @SuppressWarnings("unchecked") - var key = (K) Long.valueOf(id); - return get(key); + V result = dataMap.get(id); + AssertionUtils.notNull(result, "The static resource represented as [id:{}] in the static resource [resource:{}] does not exist", id, clazz.getSimpleName()); + return result; } @Override public Collection getAll() { - return dataMap.values(); + return dataMap.values().stream().toList(); } @Override public Map getData() { - return Collections.unmodifiableMap(dataMap); - } - - @Override - public List getIndexes(Func1 func, INDEX index) { - String indexName = getMethodToField(func); - var indexValues = indexMap.get(indexName); - AssertionUtils.notNull(indexValues, "The index of [indexName:{}] does not exist in the static resource [resource:{}]", indexName, clazz.getSimpleName()); - var values = indexValues.get(index); - if (CollectionUtils.isEmpty(values)) { - return Collections.emptyList(); - } - return values; + @SuppressWarnings("unchecked") + var map = (Map) Collections.unmodifiableMap(dataMap); + return map; } - @Nullable @Override - public V getUniqueIndex(Func1 func, INDEX index) { - String uniqueIndexName = getMethodToField(func); - var indexValueMap = uniqueIndexMap.get(uniqueIndexName); - AssertionUtils.notNull(indexValueMap, "There is no a unique index for [uniqueIndexName:{}] in the static resource [resource:{}]", uniqueIndexName, clazz.getSimpleName()); - var value = indexValueMap.get(index); - return value; + public int size() { + return dataMap.size(); } @Override public void recycleStorage() { - recycle = true; + super.recycleStorage(); dataMap = null; - indexMap = null; - uniqueIndexMap = null; - idDef = null; - indexDefMap = null; - } - - @Override - public boolean isRecycle() { - return recycle; - } - - @Override - public void setRecycle(boolean recycle) { - this.recycle = recycle; - } - - @Override - public IdDef getIdDef() { - return idDef; - } - - @Override - public int size() { - return dataMap.size(); } - private String getMethodToField(Func1 func) { - var indexName = funcCaches.get(func); - if (indexName != null) { - return indexName; - } - - // 1. IDEA 调试模式下 lambda 表达式是一个代理 - if (func instanceof Proxy) { - try { - var lambda = new IdeaProxyLambdaMeta((Proxy) func); - indexName = FieldUtils.getMethodToField(clazz, lambda.getImplMethodName()); - } catch (Exception e) { - } - } - - // 2. 反射读取 - if (indexName == null) { - try { - var method = func.getClass().getDeclaredMethod("writeReplace"); - ReflectionUtils.makeAccessible(method); - var lambda = new ReflectLambdaMeta((java.lang.invoke.SerializedLambda) method.invoke(func)); - indexName = FieldUtils.getMethodToField(clazz, lambda.getImplMethodName()); - } catch (InvocationTargetException | NoSuchMethodException | IllegalAccessException e) { - } - } - - // 3. 反射失败使用序列化的方式读取 - if (indexName == null) { - try { - var lambda = new ShadowLambdaMeta(SerializedLambda.extract(func)); - indexName = FieldUtils.getMethodToField(clazz, lambda.getImplMethodName()); - } catch (Exception e) { - } - } - - // 4. 通过将func带入到dataMap中求解，适合GraalVM环境中 - if (indexName == null) { - try { - var fields = clazz.getDeclaredFields(); - Arrays.stream(fields).forEach(ReflectionUtils::makeAccessible); - for (var value : dataMap.values()) { - var r = func.apply(value); - var valueFields = Arrays.stream(fields) - .map(it -> ReflectionUtils.getField(it, value)) - .filter(it -> it.equals(r) && it.getClass() == r.getClass()) - .toList(); - // 如果只有一个能匹配到func的返回值则就是这个方法 - if (valueFields.size() == 1) { - for (var field : fields) { - if (!ReflectionUtils.getField(field, value).equals(r)) { - continue; - } - indexName = field.getName(); - break; - } - break; - } - } - } catch (Exception e) { - } - } - - funcCaches.put(func, indexName); - return indexName; - } }