From 4e1626975d8ed5a45b8fb192cd77d40b554c8a62 Mon Sep 17 00:00:00 2001 From: Andrea Aime Date: Mon, 8 Apr 2024 19:42:31 +0200 Subject: [PATCH] Align FileBlobStore with parameter handling practice of S3/Azure --- .../storage/blobstore/file/FileBlobStore.java | 152 +++++++++++++----- .../storage/blobstore/file/FilePathUtils.java | 19 +++ 2 files changed, 130 insertions(+), 41 deletions(-) diff --git a/geowebcache/core/src/main/java/org/geowebcache/storage/blobstore/file/FileBlobStore.java b/geowebcache/core/src/main/java/org/geowebcache/storage/blobstore/file/FileBlobStore.java index 1f5c66b4d..39f3b18d5 100644 --- a/geowebcache/core/src/main/java/org/geowebcache/storage/blobstore/file/FileBlobStore.java +++ b/geowebcache/core/src/main/java/org/geowebcache/storage/blobstore/file/FileBlobStore.java @@ -14,15 +14,19 @@ */ package org.geowebcache.storage.blobstore.file; +import static java.util.Objects.isNull; import static org.geowebcache.storage.blobstore.file.FilePathUtils.filteredGridSetId; import static org.geowebcache.storage.blobstore.file.FilePathUtils.filteredLayerName; import static org.geowebcache.util.FileUtils.listFilesNullSafe; +import static org.geowebcache.util.TMSKeyBuilder.PARAMETERS_METADATA_OBJECT_PREFIX; +import static org.geowebcache.util.TMSKeyBuilder.PARAMETERS_METADATA_OBJECT_SUFFIX; import com.google.common.base.Preconditions; import java.io.File; import java.io.FileFilter; import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.io.UncheckedIOException; import java.io.UnsupportedEncodingException; import java.net.URLDecoder; @@ -34,6 +38,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Properties; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -66,6 +71,10 @@ /** See BlobStore interface description for details */ public class FileBlobStore implements BlobStore { + interface Writer { + void write(File file) throws IOException; + } + private static Logger log = Logging.getLogger(org.geowebcache.storage.blobstore.file.FileBlobStore.class.getName()); @@ -372,7 +381,7 @@ private File getLayerPath(String layerName) { /** Delete a particular tile */ @Override public boolean delete(TileObject stObj) throws StorageException { - File fh = getFileHandleTile(stObj, null); + File fh = getFileHandleTile(stObj, false); boolean ret = false; // we call fh.length() here to check whether the file exists and its length in a single // operation cause lots of calls to exists() may raise the file system cache usage to the @@ -466,7 +475,7 @@ public void postVisitDirectory(File dir) { */ @Override public boolean get(TileObject stObj) throws StorageException { - File fh = getFileHandleTile(stObj, null); + File fh = getFileHandleTile(stObj, false); if (!fh.exists()) { stObj.setStatus(Status.MISS); return false; @@ -482,17 +491,11 @@ public boolean get(TileObject stObj) throws StorageException { /** Store a tile. */ @Override public void put(TileObject stObj) throws StorageException { - - // an update to ParameterMap file is required !!! - Runnable upm = - () -> { - this.persistParameterMap(stObj); - }; - final File fh = getFileHandleTile(stObj, upm); + final File fh = getFileHandleTile(stObj, true); final long oldSize = fh.length(); final boolean existed = oldSize > 0; - writeFile(fh, stObj, existed); + writeTile(fh, stObj, existed); // mark the last modification as the tile creation time if set, otherwise // we'll leave it to the writing time @@ -507,6 +510,8 @@ public void put(TileObject stObj) throws StorageException { } } + putParametersMetadata(stObj.getLayerName(), stObj.getParametersId(), stObj.getParameters()); + /* * this is important because listeners may be tracking tile existence */ @@ -518,8 +523,40 @@ public void put(TileObject stObj) throws StorageException { } } - private File getFileHandleTile(TileObject stObj, Runnable onTileFolderCreation) + private void putParametersMetadata( + String layerName, String parametersId, Map parameters) throws StorageException { + assert (isNull(parametersId) == isNull(parameters)); + if (isNull(parametersId)) { + return; + } + File parametersFile = parametersFile(layerName, parametersId); + if (parametersFile.exists()) return; + + writeFile( + parametersFile, + false, + file -> { + Properties properties = new Properties(); + parameters.forEach(properties::setProperty); + try (OutputStream os = new FileOutputStream(file)) { + properties.store(os, "Parameters values for identifier: " + parametersId); + } + }); + } + + private File parametersFile(String layerName, String parametersId) { + String path = + FilePathUtils.buildPath( + this.path, + layerName, + PARAMETERS_METADATA_OBJECT_PREFIX + + parametersId + + PARAMETERS_METADATA_OBJECT_SUFFIX); + return new File(path); + } + + private File getFileHandleTile(TileObject stObj, boolean createParent) throws StorageException { final MimeType mimeType; try { mimeType = MimeType.createFromFormat(stObj.getBlobFormat()); @@ -535,12 +572,10 @@ private File getFileHandleTile(TileObject stObj, Runnable onTileFolderCreation) throw new StorageException("Failed to compute file path", e); } - // check if it's required to create tile folder - if (onTileFolderCreation != null) { + if (createParent) { log.fine("Creating parent tile folder and updating ParameterMap"); File parent = tilePath.getParentFile(); mkdirs(parent, stObj); - onTileFolderCreation.run(); } return tilePath; @@ -553,16 +588,32 @@ private Resource readFile(File fh) throws StorageException { return new FileResource(fh); } - private void writeFile(File target, TileObject stObj, boolean existed) throws StorageException { + private void writeTile(File target, TileObject stObj, boolean existed) throws StorageException { + writeFile( + target, + existed, + file -> { + try (FileOutputStream fos = new FileOutputStream(file); + FileChannel channel = fos.getChannel()) { + stObj.getBlob().transferTo(channel); + } + }); + } + + /** + * Writes into the target file by first creating a temporary file, filling it with the writer, + * and then renaming it to the target file. + * + * @throws StorageException + */ + private void writeFile(File target, boolean existed, Writer writer) throws StorageException { // first write to temp file tmp.mkdirs(); File temp = new File(tmp, tmpGenerator.newName()); try { - // open the output stream and read the blob into the tile - try (FileOutputStream fos = new FileOutputStream(temp); - FileChannel channel = fos.getChannel()) { - stObj.getBlob().transferTo(channel); + try { + writer.write(temp); } catch (IOException ioe) { throw new StorageException(ioe.getMessage() + " for " + target.getAbsolutePath()); } @@ -578,9 +629,7 @@ private void writeFile(File target, TileObject stObj, boolean existed) throws St temp = null; } } - } finally { - if (temp != null) { log.warning( "Tile " @@ -591,15 +640,6 @@ private void writeFile(File target, TileObject stObj, boolean existed) throws St } } - protected void persistParameterMap(TileObject stObj) { - if (Objects.nonNull(stObj.getParametersId())) { - putLayerMetadata( - stObj.getLayerName(), - "parameters." + stObj.getParametersId(), - ParametersUtils.getKvp(stObj.getParameters())); - } - } - @Override public void clear() throws StorageException { throw new StorageException("Not implemented yet!"); @@ -726,6 +766,10 @@ public boolean deleteByParametersId(String layerName, String parametersId) return false; } + // delete the parameter file + parametersFile(layerName, parametersId).delete(); + + // delete the caches File[] parameterCaches = listFilesNullSafe( layerPath, @@ -779,6 +823,9 @@ public boolean isParameterIdCached(String layerName, final String parametersId) @Override public Map>> getParametersMapping(String layerName) { + Set parameterIds = getParameterIds(layerName); + + // for backwards compatibility, check the parameters in the metadata file Map p; try { p = layerMetadata.getLayerMetadata(layerName); @@ -786,18 +833,41 @@ public Map>> getParametersMapping(String la log.fine("Optimistic read of metadata mappings failed"); return null; } - return getParameterIds(layerName).stream() + Map>> result = + parameterIds.stream() + .collect( + Collectors.toMap( + (id) -> id, + (id) -> { + String kvp = p.get("parameters." + id); + if (Objects.isNull(kvp)) { + return Optional.empty(); + } + kvp = urlDecUtf8(kvp); + return Optional.of(ParametersUtils.getMap(kvp)); + })); + + // go look for the current parameter files too though, and overwrite the legacy metadata + for (String parameterId : parameterIds) { + File file = parametersFile(layerName, parameterId); + if (file.exists()) { + try { + Properties properties = new Properties(); + properties.load(Files.newInputStream(file.toPath())); + result.put(parameterId, Optional.of(propertiesToMap(properties))); + } catch (IOException e) { + throw new RuntimeException("Failed to read parameters file", e); + } + } + } + + return result; + } + + private static Map propertiesToMap(Properties properties) { + return properties.entrySet().stream() .collect( - Collectors.toMap( - (id) -> id, - (id) -> { - String kvp = p.get("parameters." + id); - if (Objects.isNull(kvp)) { - return Optional.empty(); - } - kvp = urlDecUtf8(kvp); - return Optional.of(ParametersUtils.getMap(kvp)); - })); + Collectors.toMap(e -> e.getKey().toString(), e -> e.getValue().toString())); } static final int paramIdLength = diff --git a/geowebcache/core/src/main/java/org/geowebcache/storage/blobstore/file/FilePathUtils.java b/geowebcache/core/src/main/java/org/geowebcache/storage/blobstore/file/FilePathUtils.java index b6f8ab5e1..ea19da4d3 100644 --- a/geowebcache/core/src/main/java/org/geowebcache/storage/blobstore/file/FilePathUtils.java +++ b/geowebcache/core/src/main/java/org/geowebcache/storage/blobstore/file/FilePathUtils.java @@ -14,6 +14,8 @@ */ package org.geowebcache.storage.blobstore.file; +import java.io.File; + public class FilePathUtils { public static String gridsetZoomLevelDir(String gridSetId, long zoomLevel) { @@ -116,4 +118,21 @@ public static void appendGridsetZoomLevelDir(String gridSetId, long z, StringBui path.append('_'); zeroPadder(z, 2, path); } + + /** + * Returns a path built from a root and a list of components. The components are appended to the + * root with a {@link File#separatorChar} in between. The root is trusted not to need escaping, + * all other bits are filtered. + */ + public static String buildPath(String root, String... components) { + StringBuilder path = new StringBuilder(256); + path.append(root); + + for (String component : components) { + path.append(File.separatorChar); + appendFiltered(component, path); + } + + return path.toString(); + } }