diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/IginxWorker.java b/core/src/main/java/cn/edu/tsinghua/iginx/IginxWorker.java index 9a4d9fd3df..97e3089014 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/IginxWorker.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/IginxWorker.java @@ -837,16 +837,19 @@ public Status registerTask(RegisterTaskReq req) { Predicate ruleNameFilter = FilePermissionRuleNameFilters.transformerRulesWithDefault(); - Predicate sourceChecker = + FilePermissionManager.Checker sourceChecker = FilePermissionManager.getInstance() .getChecker(null, ruleNameFilter, FileAccessType.EXECUTE); - File sourceFile = new File(filePath); - if (!sourceChecker.test(sourceFile.toPath())) { + Optional sourceCheckedPath = sourceChecker.normalize(filePath); + + if (!sourceCheckedPath.isPresent()) { errorMsg = String.format("Register file %s has no execute permission", filePath); LOGGER.error(errorMsg); return RpcUtils.FAILURE.setMessage(errorMsg); } + + File sourceFile = sourceCheckedPath.get().toFile(); if (!sourceFile.exists()) { errorMsg = String.format("Register file not exist in declared path, path=%s", filePath); LOGGER.error(errorMsg); @@ -890,19 +893,21 @@ public Status registerTask(RegisterTaskReq req) { String fileName = sourceFile.getName(); String destPath = String.join(File.separator, config.getDefaultUDFDir(), "python_scripts", fileName); - File destFile = new File(destPath); - if (destFile.exists()) { - errorMsg = String.format("Register file(s) already exist, name=%s", fileName); + FilePermissionManager.Checker destChecker = + FilePermissionManager.getInstance().getChecker(null, ruleNameFilter, FileAccessType.WRITE); + + Optional destCheckedPath = destChecker.normalize(destPath); + if (!destCheckedPath.isPresent()) { + errorMsg = String.format("Register file %s has no write permission", destPath); LOGGER.error(errorMsg); return RpcUtils.FAILURE.setMessage(errorMsg); } - Predicate destChecker = - FilePermissionManager.getInstance().getChecker(null, ruleNameFilter, FileAccessType.WRITE); + File destFile = destCheckedPath.get().toFile(); - if (!destChecker.test(destFile.toPath())) { - errorMsg = String.format("Register file %s has no write permission", destPath); + if (destFile.exists()) { + errorMsg = String.format("Register file(s) already exist, name=%s", fileName); LOGGER.error(errorMsg); return RpcUtils.FAILURE.setMessage(errorMsg); } @@ -984,25 +989,25 @@ public Status dropTask(DropTaskReq req) { + "python_scripts" + File.separator + transformTaskMeta.getFileName(); - File file = new File(filePath); - - if (!file.exists()) { - metaManager.dropTransformTask(name); - errorMsg = String.format("Register file not exist, path=%s", filePath); - LOGGER.error(errorMsg); - return RpcUtils.FAILURE.setMessage(errorMsg); - } - String pythonDir = config.getDefaultUDFDir() + File.separator + "python_scripts"; Predicate ruleNameFilter = FilePermissionRuleNameFilters.transformerRulesWithDefault(); - Predicate destChecker = + FilePermissionManager.Checker destChecker = FilePermissionManager.getInstance().getChecker(null, ruleNameFilter, FileAccessType.WRITE); + Optional normalizedFile = destChecker.normalize(filePath); - Path pythonDirPath = Paths.get(pythonDir); - if (!destChecker.test(pythonDirPath)) { + if (!normalizedFile.isPresent()) { errorMsg = String.format( - "User has no write permission in target directory, udf %s cannot be dropped.", name); + "User has no write permission in target directory, task %s cannot be dropped.", name); + LOGGER.error(errorMsg); + return RpcUtils.FAILURE.setMessage(errorMsg); + } + + File file = normalizedFile.get().toFile(); + + if (!file.exists()) { + metaManager.dropTransformTask(name); + errorMsg = String.format("Register file not exist, path=%s", filePath); LOGGER.error(errorMsg); return RpcUtils.FAILURE.setMessage(errorMsg); } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/auth/FilePermissionManager.java b/core/src/main/java/cn/edu/tsinghua/iginx/auth/FilePermissionManager.java index c512087765..30889636fc 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/auth/FilePermissionManager.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/auth/FilePermissionManager.java @@ -4,6 +4,7 @@ import cn.edu.tsinghua.iginx.conf.FilePermissionConfig; import cn.edu.tsinghua.iginx.conf.entity.FilePermissionDescriptor; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.*; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Predicate; @@ -32,7 +33,57 @@ public static FilePermissionManager getInstance() { filePermissionConfig.onReload(this::reload); } - public Predicate getChecker( + public interface Checker { + boolean test(Path path); + + default Optional normalize(String path) { + Optional p = cheatNormalize(Paths.get(path)); + if (!p.isPresent()) { + return Optional.empty(); + } + Path cheated = p.get(); + if (!test(cheated)) { + return Optional.empty(); + } + return Optional.of(cheated); + } + + default boolean testRelativePath(String path) { + return !path.contains(".."); + } + + /** + * cheat CodeQL to not complain about path traversal vulnerability. This method should not be + * used except you know what you are doing. + * + * @param path the path to normalize + * @return the normalized path if it is safe, otherwise empty + */ + @Deprecated + default Optional cheatNormalize(Path path) { + Path p = path.toAbsolutePath(); + // split path nodes + String root = p.getRoot().toString(); + String[] nodes = new String[p.getNameCount()]; + for (int i = 0; i < p.getNameCount(); i++) { + nodes[i] = p.getName(i).toString(); + } + // check if any node contains "." + if (!testRelativePath(root)) { + return Optional.empty(); + } + for (String node : nodes) { + if (!testRelativePath(node)) { + return Optional.empty(); + } + } + // rebuild path + Path rebuiltPath = Paths.get(root, nodes); + return Optional.of(rebuiltPath); + } + } + + public Checker getChecker( @Nullable String user, Predicate ruleNameFilter, FileAccessType... type) { return path -> { UserRules userRules = rules.get(); diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/auth/utils/FilePermissionRuleNameFilters.java b/core/src/main/java/cn/edu/tsinghua/iginx/auth/utils/FilePermissionRuleNameFilters.java index ac4d0cf81b..606d314658 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/auth/utils/FilePermissionRuleNameFilters.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/auth/utils/FilePermissionRuleNameFilters.java @@ -23,4 +23,12 @@ public static Predicate udfRules() { public static Predicate udfRulesWithDefault() { return udfRules().or(defaultRules()); } + + public static Predicate filesystemRules() { + return ruleName -> ruleName.startsWith("filesystem"); + } + + public static Predicate filesystemRulesWithDefault() { + return filesystemRules().or(defaultRules()); + } } diff --git a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/FileAppendWriter.java b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/FileAppendWriter.java index 4f9686cd49..3b45d467b5 100644 --- a/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/FileAppendWriter.java +++ b/core/src/main/java/cn/edu/tsinghua/iginx/transform/data/FileAppendWriter.java @@ -24,13 +24,26 @@ public class FileAppendWriter extends ExportWriter { private boolean hasWriteHeader; - public FileAppendWriter(String fileName) { - this.fileName = fileName; + public FileAppendWriter(String name) { + this.fileName = normalizeFileName(name).toString(); this.hasWriteHeader = false; File file = new File(fileName); createFileIfNotExist(file); } + private Path normalizeFileName(String fileName) { + Predicate ruleNameFilter = FilePermissionRuleNameFilters.transformerRulesWithDefault(); + + FilePermissionManager.Checker checker = + FilePermissionManager.getInstance().getChecker(null, ruleNameFilter, FileAccessType.WRITE); + + return checker + .normalize(fileName) + .orElseThrow( + () -> + new SecurityException("transformer has no permission to write file: " + fileName)); + } + @Override public void write(BatchData batchData) { if (!hasWriteHeader) { @@ -50,20 +63,13 @@ public void write(BatchData batchData) { } private void createFileIfNotExist(File file) { - Predicate ruleNameFilter = FilePermissionRuleNameFilters.transformerRulesWithDefault(); - - Predicate pathChecker = - FilePermissionManager.getInstance().getChecker(null, ruleNameFilter, FileAccessType.WRITE); - if (!pathChecker.test(file.toPath())) { - throw new SecurityException( - ("transformer has no permission to write file: " + file.getAbsolutePath())); - } if (!file.exists()) { LOGGER.info("File not exists, create it..."); // get and create parent dir - if (!file.getParentFile().exists()) { - System.out.println("Parent dir not exists, create it..."); - file.getParentFile().mkdirs(); + File normalizeParentFile = normalizeFileName(file.getParentFile().getPath()).toFile(); + if (!normalizeParentFile.exists()) { + LOGGER.info("Parent dir not exists, create it..."); + normalizeParentFile.mkdirs(); } try { // create file diff --git a/core/src/main/resources/file-permission.properties b/core/src/main/resources/file-permission.properties index da82880cc3..e95a5a5592 100644 --- a/core/src/main/resources/file-permission.properties +++ b/core/src/main/resources/file-permission.properties @@ -75,10 +75,11 @@ # The rule name is used to distinguish different rules. Different modules # determine the subset of rules used by filter. In details: # -# * the udf module uses rules starting with "udf" -# * the transform module uses rules starting with "transformer". # * the default module uses rules starting with "default". # + the rules of the default module is applied to all modules. +# * the udf module uses rules starting with "udf" +# * the transform module uses rules starting with "transformer". +# * the filesystem driver uses rules starting with "filesystem". # # Rule Application Order: # 1. For the same user, the rules are applied in the order of the configuration diff --git a/core/src/test/java/cn/edu/tsinghua/iginx/auth/FilePermissionManagerTest.java b/core/src/test/java/cn/edu/tsinghua/iginx/auth/FilePermissionManagerTest.java index b78b847577..849af83d5c 100644 --- a/core/src/test/java/cn/edu/tsinghua/iginx/auth/FilePermissionManagerTest.java +++ b/core/src/test/java/cn/edu/tsinghua/iginx/auth/FilePermissionManagerTest.java @@ -8,7 +8,7 @@ import cn.edu.tsinghua.iginx.conf.FilePermissionConfig; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.function.Predicate; +import java.util.Optional; import org.apache.commons.configuration2.ex.ConfigurationException; import org.junit.Test; @@ -31,22 +31,23 @@ public void testCheckNull() throws ConfigurationException { config.reload(); FilePermissionManager manager = new FilePermissionManager(config); try { - Predicate predicate = manager.getChecker(null, null, FileAccessType.READ); + FilePermissionManager.Checker predicate = manager.getChecker(null, null, FileAccessType.READ); predicate.test(Paths.get("test")); fail("Should throw NullPointerException"); } catch (NullPointerException ignored) { } try { - Predicate predicate = manager.getChecker(null, defaultRules(), null); + FilePermissionManager.Checker predicate = manager.getChecker(null, defaultRules(), null); predicate.test(Paths.get("test")); fail("Should throw NullPointerException"); } catch (NullPointerException ignored) { } - Predicate predicate = manager.getChecker(null, defaultRules(), FileAccessType.READ); + FilePermissionManager.Checker predicate = + manager.getChecker(null, defaultRules(), FileAccessType.READ); try { - predicate.test(null); + predicate.test((Path) null); fail("Should throw NullPointerException"); } catch (NullPointerException ignored) { } @@ -58,7 +59,7 @@ public void testCheckFallback() throws ConfigurationException { config.reload(); FilePermissionManager manager = new FilePermissionManager(config); - Predicate predicate = + FilePermissionManager.Checker predicate = manager.getChecker(null, udfRulesWithDefault(), FileAccessType.EXECUTE); assertTrue(predicate.test(Paths.get("udf.py"))); assertFalse(predicate.test(Paths.get("udf.sh"))); @@ -70,7 +71,7 @@ public void testCheckDefaultModule() throws ConfigurationException { config.reload(); FilePermissionManager manager = new FilePermissionManager(config); - Predicate predicate = + FilePermissionManager.Checker predicate = manager.getChecker(null, udfRulesWithDefault(), FileAccessType.EXECUTE); assertTrue(predicate.test(Paths.get("test.bat"))); } @@ -81,11 +82,11 @@ public void testCheckDefaultUser() throws ConfigurationException { config.reload(); FilePermissionManager manager = new FilePermissionManager(config); - Predicate predicate = + FilePermissionManager.Checker predicate = manager.getChecker("root", udfRulesWithDefault(), FileAccessType.EXECUTE); assertTrue(predicate.test(Paths.get("test.sh"))); - Predicate rootPredicate = + FilePermissionManager.Checker rootPredicate = manager.getChecker("test", udfRulesWithDefault(), FileAccessType.EXECUTE); assertFalse(rootPredicate.test(Paths.get("test.sh"))); } @@ -96,7 +97,7 @@ public void testCheckDefault() throws ConfigurationException { config.reload(); FilePermissionManager manager = new FilePermissionManager(config); - Predicate predicate = + FilePermissionManager.Checker predicate = manager.getChecker(null, udfRulesWithDefault(), FileAccessType.READ); assertTrue(predicate.test(Paths.get("test.py"))); } @@ -107,7 +108,7 @@ public void testCheckEmpty() throws ConfigurationException { config.reload(); FilePermissionManager manager = new FilePermissionManager(config); - Predicate predicate = manager.getChecker(null, udfRulesWithDefault()); + FilePermissionManager.Checker predicate = manager.getChecker(null, udfRulesWithDefault()); assertTrue(predicate.test(Paths.get("test.sh"))); } @@ -117,7 +118,7 @@ public void testCheckMultiple() throws ConfigurationException { config.reload(); FilePermissionManager manager = new FilePermissionManager(config); - Predicate predicate = + FilePermissionManager.Checker predicate = manager.getChecker( null, udfRulesWithDefault(), FileAccessType.EXECUTE, FileAccessType.WRITE); assertFalse(predicate.test(Paths.get("test.py"))); @@ -129,7 +130,7 @@ public void testCheckChinese() throws ConfigurationException { config.reload(); FilePermissionManager manager = new FilePermissionManager(config); - Predicate predicate = + FilePermissionManager.Checker predicate = manager.getChecker( "测试用户", ruleName -> ruleName.equals("第一条规则"), @@ -137,4 +138,13 @@ public void testCheckChinese() throws ConfigurationException { FileAccessType.WRITE); assertFalse(predicate.test(Paths.get("不允许.py"))); } + + @Test + public void testNormalize() { + FilePermissionManager.Checker checker = path -> true; + + Optional p = checker.normalize("test"); + assertTrue(p.isPresent()); + assertEquals(Paths.get("test").toAbsolutePath(), p.get()); + } } diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/FileSystemManager.java b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/FileSystemManager.java index 86ed1884a5..436fca62d2 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/FileSystemManager.java +++ b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/exec/FileSystemManager.java @@ -3,6 +3,7 @@ import static cn.edu.tsinghua.iginx.engine.logical.utils.PathUtils.MAX_CHAR; import static cn.edu.tsinghua.iginx.filesystem.shared.Constant.*; +import cn.edu.tsinghua.iginx.auth.entity.FileAccessType; import cn.edu.tsinghua.iginx.engine.physical.storage.utils.TagKVUtils; import cn.edu.tsinghua.iginx.engine.shared.KeyRange; import cn.edu.tsinghua.iginx.engine.shared.operator.tag.TagFilter; @@ -11,23 +12,20 @@ import cn.edu.tsinghua.iginx.filesystem.file.entity.FileMeta; import cn.edu.tsinghua.iginx.filesystem.query.entity.FileSystemResultTable; import cn.edu.tsinghua.iginx.filesystem.query.entity.Record; +import cn.edu.tsinghua.iginx.filesystem.tools.FilePathUtils; import cn.edu.tsinghua.iginx.filesystem.tools.MemoryPool; import cn.edu.tsinghua.iginx.thrift.DataType; import cn.edu.tsinghua.iginx.utils.Pair; import java.io.File; import java.io.IOException; -import java.nio.file.DirectoryStream; -import java.nio.file.FileVisitResult; -import java.nio.file.FileVisitor; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; import java.util.*; import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,13 +52,13 @@ public FileSystemManager(Map params) { fileOperator = new DefaultFileOperator(); } - /** ******************** 查询相关 ******************** */ + /** ***************** 查询相关 ******************** */ public List readFile( File file, TagFilter tagFilter, List keyRanges, boolean isDummy) throws IOException { List res = new ArrayList<>(); // 首先通过tagFilter和file,找到所有有关的文件列表 - List files = getFilesWithTagFilter(file, tagFilter, isDummy); + List files = getFilesWithTagFilter(file, tagFilter, isDummy, false); for (File f : files) { List records = new ArrayList<>(); // 根据keyRange过滤 @@ -80,9 +78,9 @@ public List readFile( return res; } - private List getFilesWithTagFilter(File file, TagFilter tagFilter, boolean isDummy) - throws IOException { - List associatedFiles = getAssociatedFiles(file, isDummy); + private List getFilesWithTagFilter( + File file, TagFilter tagFilter, boolean isDummy, boolean isWrite) throws IOException { + List associatedFiles = getAssociatedFiles(file, isDummy, isWrite); if (isDummy) { return associatedFiles; } @@ -189,10 +187,15 @@ private List readSingleFile(File file, long startKey, long endKey, boole return records; } - /** ******************** 写入相关 ******************** */ + /** ****************** 写入相关 ******************** */ public synchronized void writeFiles( List files, List> recordsList, List> tagsList) throws IOException { + files = + files.stream() + .map(f -> FilePathUtils.normalize(f, FileAccessType.WRITE)) + .collect(Collectors.toList()); + for (int i = 0; i < files.size(); i++) { writeFile(files.get(i), recordsList.get(i), tagsList.get(i)); } @@ -220,7 +223,7 @@ private synchronized void writeFile(File file, List records, Map tags) throws IOException { - for (File f : getAssociatedFiles(file, false)) { + for (File f : getAssociatedFiles(file, false, true)) { FileMeta fileMeta = getFileMeta(f); if ((tags == null || tags.isEmpty()) && fileMeta.getTags().isEmpty()) { return f; @@ -254,7 +257,7 @@ private File determineFileID(File file) throws IOException { // 获取文件id,例如 a.iginx5,则其id就是5 private int getFileID(File file) throws IOException { - List files = getAssociatedFiles(file, false); + List files = getAssociatedFiles(file, false, true); if (files.isEmpty()) { return -1; } @@ -271,7 +274,7 @@ private int getFileID(File file) throws IOException { return Collections.max(nums); } - /** ******************** 删除相关 ******************** */ + /** ****************** 删除相关 ******************** */ public void deleteFile(File file) throws IOException { deleteFiles(Collections.singletonList(file), null); } @@ -285,7 +288,7 @@ public void deleteFile(File file) throws IOException { public void deleteFiles(List files, TagFilter filter) throws IOException { for (File file : files) { try { - for (File f : getFilesWithTagFilter(file, filter, false)) { + for (File f : getFilesWithTagFilter(file, filter, false, true)) { fileOperator.delete(f); fileMetaMap.remove(f.getAbsolutePath()); } @@ -298,7 +301,7 @@ public void deleteFiles(List files, TagFilter filter) throws IOException { public void trimFilesContent(List files, TagFilter tagFilter, long startKey, long endKey) throws IOException { for (File file : files) { - List fileList = getFilesWithTagFilter(file, tagFilter, false); + List fileList = getFilesWithTagFilter(file, tagFilter, false, true); if (fileList.isEmpty()) { LOGGER.warn("cant trim the file that not exist!"); continue; @@ -310,13 +313,15 @@ public void trimFilesContent(List files, TagFilter tagFilter, long startKe } // 返回和file文件相关的所有文件 - private List getAssociatedFiles(File file, boolean isDummy) throws IOException { + private List getAssociatedFiles(File file, boolean isDummy, boolean isWrite) + throws IOException { List associatedFiles = new ArrayList<>(); try { String filePath = file.getAbsolutePath(); if (!filePath.contains(WILDCARD) && isDummy) { - if (file.isFile() && file.exists()) { - associatedFiles.add(file); + File checkedFile = FilePathUtils.normalize(file); + if (checkedFile.isFile() && checkedFile.exists()) { + associatedFiles.add(checkedFile); } } else { // filePath.contains(WILDCARD) || !isDummy File root; @@ -326,6 +331,7 @@ private List getAssociatedFiles(File file, boolean isDummy) throws IOExcep } else { // !isDummy root = file.getParentFile(); } + root = FilePathUtils.normalize(root); if (isDummy) { regex = filePath.replaceAll("[$^{}\\\\]", "\\\\$0").replaceAll("[*]", ".*"); } else { @@ -362,10 +368,14 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) { throw new IOException( String.format("get associated files of %s failure: %s", file.getAbsolutePath(), e)); } - return associatedFiles; + return associatedFiles.stream() + .map(f -> FilePathUtils.normalize(f, isWrite ? FileAccessType.WRITE : FileAccessType.READ)) + .collect(Collectors.toList()); } public List getAllFiles(File dir, boolean containsEmptyDir) { + dir = FilePathUtils.normalize(dir, FileAccessType.READ); + List res = new ArrayList<>(); try { Files.walkFileTree( @@ -404,6 +414,8 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) { // 返回字典序最大和最小的文件路径,可能是目录 public Pair getBoundaryOfFiles(File dir) { + dir = FilePathUtils.normalize(dir, FileAccessType.READ); + File[] files = dir.listFiles(); if (files == null || files.length == 0) { LOGGER.error("{} is empty", dir.getAbsolutePath()); @@ -419,6 +431,8 @@ public Pair getBoundaryOfFiles(File dir) { } public FileMeta getFileMeta(File file) { + file = FilePathUtils.normalize(file, FileAccessType.READ); + try { FileMeta fileMeta; String filePath = file.getAbsolutePath(); @@ -441,7 +455,7 @@ private boolean isDirEmpty(Path dir) throws IOException { } } - /** ******************** 资源控制 ******************** */ + /** ***************** 资源控制 ******************** */ public MemoryPool getMemoryPool() { return memoryPool; } diff --git a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/FilePathUtils.java b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/FilePathUtils.java index d8c71c3059..95a04f610d 100644 --- a/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/FilePathUtils.java +++ b/dataSources/filesystem/src/main/java/cn/edu/tsinghua/iginx/filesystem/tools/FilePathUtils.java @@ -2,8 +2,30 @@ import static cn.edu.tsinghua.iginx.filesystem.shared.Constant.*; +import cn.edu.tsinghua.iginx.auth.FilePermissionManager; +import cn.edu.tsinghua.iginx.auth.entity.FileAccessType; +import cn.edu.tsinghua.iginx.auth.utils.FilePermissionRuleNameFilters; +import java.io.File; +import java.nio.file.Path; +import java.util.Optional; +import java.util.function.Predicate; + public class FilePathUtils { + public static File normalize(File file, FileAccessType... type) throws SecurityException { + + Predicate ruleNameFilter = FilePermissionRuleNameFilters.filesystemRulesWithDefault(); + + FilePermissionManager.Checker sourceChecker = + FilePermissionManager.getInstance().getChecker(null, ruleNameFilter, type); + + Optional sourceCheckedPath = sourceChecker.normalize(file.getPath()); + if (!sourceCheckedPath.isPresent()) { + throw new SecurityException("filesystem has no permission to access file: " + file); + } + return sourceCheckedPath.get().toFile(); + } + public static String toIginxPath(String root, String storageUnit, String path) { if (path == null && storageUnit == null || storageUnit.equals(WILDCARD)) { return root; diff --git a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/StringUtils.java b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/StringUtils.java index c6a0278513..0453342c2a 100644 --- a/shared/src/main/java/cn/edu/tsinghua/iginx/utils/StringUtils.java +++ b/shared/src/main/java/cn/edu/tsinghua/iginx/utils/StringUtils.java @@ -22,10 +22,13 @@ import java.time.DateTimeException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Objects; +import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; public class StringUtils { @@ -61,9 +64,9 @@ public static int compare(String ts, String border, boolean isStart) { } /** - * @return 返回值为 0 表示包含,>0 表示在这个序列在 border 前,<0 表示 ts 在 border 后 * @param ts 时间序列(可能等于/含有*,不可能为null) * @param border 前缀式时间范围 + * @return 返回值为 0 表示包含,>0 表示在这个序列在 border 前,<0 表示 ts 在 border 后 */ public static int compare(String ts, String border) { // *.*.* @@ -113,23 +116,29 @@ public static String reformatPath(String path) { return path; } - public static boolean match(String string, String regex) { - return Pattern.matches(StringUtils.reformatColumnName(regex), string); + public static boolean match(String string, String iginxPattern) { + return toColumnMatcher(iginxPattern).test(string); + } + + public static Predicate toColumnMatcher(String iginxPattern) { + Objects.requireNonNull(iginxPattern); + + if (iginxPattern.equals("*")) { + return s -> true; + } + if (!iginxPattern.contains("*")) { + return iginxPattern::equals; + } + Pattern pattern = Pattern.compile(toRegexExpr(iginxPattern)); + return s -> pattern.matcher(s).matches(); } - private static String reformatColumnName(String name) { - if (!name.contains("*")) return Pattern.quote(name); - name = name.replaceAll("[.]", "[.]"); - name = name.replaceAll("[*]", ".*"); - name = name.replaceAll("[(]", "[(]"); - name = name.replaceAll("[)]", "[)]"); - name = name.replaceAll("[{]", "[{]"); - name = name.replaceAll("[}]", "[}]"); - name = name.replaceAll("[+]", "[\\\\+]"); - name = name.replaceAll("\\$", "[\\$]"); - name = name.replaceAll("\\^", "[\\\\^]"); - name = name.replaceAll("\\\\", "\\\\\\\\"); - return name; + public static String toRegexExpr(String iginxPattern) { + Objects.requireNonNull(iginxPattern); + + return Arrays.stream(iginxPattern.split("[*]+", -1)) + .map(s -> s.isEmpty() ? "" : Pattern.quote(s)) + .collect(Collectors.joining(".*")); } public static boolean isContainSpecialChar(String str) { diff --git a/shared/src/test/java/cn/edu/tsinghua/iginx/utils/StringUtilsTest.java b/shared/src/test/java/cn/edu/tsinghua/iginx/utils/StringUtilsTest.java index 046785e7e9..90cad48265 100644 --- a/shared/src/test/java/cn/edu/tsinghua/iginx/utils/StringUtilsTest.java +++ b/shared/src/test/java/cn/edu/tsinghua/iginx/utils/StringUtilsTest.java @@ -16,4 +16,25 @@ public void reformatPathTest() { String result = StringUtils.reformatPath(str); assertEquals(result, "\\.\\^\\$\\{\\}\\+\\?\\(\\)\\[\\]\\|\\\\\\\\.*"); } + + @Test + public void toRegexExpr() { + assertEquals("\\Qa.b.c\\E", StringUtils.toRegexExpr("a.b.c")); + assertEquals(".*", StringUtils.toRegexExpr("*")); + assertEquals(".*\\Q.b.c\\E", StringUtils.toRegexExpr("*.b.c")); + assertEquals("\\Qa.\\E.*\\Q.c\\E", StringUtils.toRegexExpr("a.*.c")); + assertEquals("\\Qa.b.\\E.*", StringUtils.toRegexExpr("a.b.*")); + assertEquals("\\Qa.\\E.*\\Q.\\E.*", StringUtils.toRegexExpr("a.*.*")); + assertEquals(".*\\Q.b.\\E.*", StringUtils.toRegexExpr("*.b.*")); + assertEquals(".*\\Q.\\E.*\\Q.c\\E", StringUtils.toRegexExpr("*.*.c")); + assertEquals(".*\\Q.\\E.*\\Q.\\E.*", StringUtils.toRegexExpr("*.*.*")); + assertEquals(".*", StringUtils.toRegexExpr("**")); + assertEquals(".*\\Q.b.c\\E", StringUtils.toRegexExpr("**.b.c")); + assertEquals("\\Qa.\\E.*\\Q.c\\E", StringUtils.toRegexExpr("a.**.c")); + assertEquals("\\Qa.b.\\E.*", StringUtils.toRegexExpr("a.b.**")); + assertEquals("\\Qa.\\E.*\\Q.\\E.*", StringUtils.toRegexExpr("a.**.**")); + assertEquals(".*\\Q.b.\\E.*", StringUtils.toRegexExpr("**.b.**")); + assertEquals(".*\\Q.\\E.*\\Q.c\\E", StringUtils.toRegexExpr("**.**.c")); + assertEquals(".*\\Q.\\E.*\\Q.\\E.*", StringUtils.toRegexExpr("**.**.**")); + } }