add onDiskSize to core status API (#223)

* add onDiskSize to core status API * tidy * add back indexDiskSize * fix typo in Sizes RamUsageEstimator * add some needed conditionals to onDiskSize calcs * tidy * add backingDirectory to SizeAccountingIndexInput * tidy
cowpaths · Oct 1, 2024 · 00aa918 · 00aa918
1 parent c4ffc35
commit 00aa918
Show file tree

Hide file tree

Showing 7 changed files with 591 additions and 50 deletions.
diff --git a/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java b/solr/core/src/java/org/apache/solr/core/DirectoryFactory.java
@@ -166,6 +166,18 @@ public long size(Directory directory) throws IOException {
     return sizeOfDirectory(directory);
   }
 
+  /**
+   * @param directory to calculate size of
+   * @return size in bytes on disk, regardless of compression, etc
+   * @throws IOException on low level IO error
+   */
+  public long onDiskSize(Directory directory) throws IOException {
+    if (directory instanceof OnDiskSizeDirectory) {
+      return onDiskSizeOfDirectory(directory);
+    }
+    return sizeOfDirectory(directory);
+  }
+
   /**
    * @param path to calculate size of
    * @return size in bytes
@@ -267,6 +279,12 @@ public boolean isAbsolute(String path) {
 
   public interface SizeAware {
     long size() throws IOException;
+
+    long onDiskSize() throws IOException;
+  }
+
+  public interface OnDiskSizeDirectory {
+    long onDiskFileLength(String name) throws IOException;
   }
 
   public static long sizeOfDirectory(Directory directory) throws IOException {
@@ -295,6 +313,35 @@ public static long sizeOf(Directory directory, String file) throws IOException {
     }
   }
 
+  public static long onDiskSizeOfDirectory(Directory directory) throws IOException {
+    if (directory instanceof SizeAware) {
+      return ((SizeAware) directory).onDiskSize();
+    }
+    final String[] files = directory.listAll();
+    long size = 0;
+
+    for (final String file : files) {
+      size += onDiskSizeOf(directory, file);
+      if (size < 0) {
+        break;
+      }
+    }
+
+    return size;
+  }
+
+  public static long onDiskSizeOf(Directory directory, String file) throws IOException {
+    if (directory instanceof DirectoryFactory.OnDiskSizeDirectory) {
+      try {
+        return ((OnDiskSizeDirectory) directory).onDiskFileLength(file);
+      } catch (IOException e) {
+        // could be a race, file no longer exists, access denied, is a directory, etc.
+        return 0;
+      }
+    }
+    return directory.fileLength(file);
+  }
+
   /** Delete the files in the Directory */
   public static boolean empty(Directory dir) {
     boolean isSuccess = true;

diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -530,11 +530,49 @@ public long getIndexSize() {
     }
   }
 
+  public long getOnDiskSize() {
+    SolrRequestInfo requestInfo = SolrRequestInfo.getRequestInfo();
+    if (requestInfo != null) {
+      return (Long)
+          requestInfo
+              .getReq()
+              .getContext()
+              .computeIfAbsent(cachedOnDiskIndexSizeKeyName(), key -> calculateOnDiskSize());
+    } else {
+      return calculateOnDiskSize();
+    }
+  }
+
+  private long calculateOnDiskSize() {
+    Directory dir;
+    long size = 0;
+    try {
+      if (directoryFactory.exists(getIndexDir())) {
+        dir =
+            directoryFactory.get(
+                getIndexDir(), DirContext.DEFAULT, solrConfig.indexConfig.lockType);
+        try {
+          size = directoryFactory.onDiskSize(dir);
+        } finally {
+          directoryFactory.release(dir);
+        }
+      }
+    } catch (IOException e) {
+      log.error("IO error while trying to get the size of the Directory", e);
+    }
+    return size;
+  }
+
   private String cachedIndexSizeKeyName() {
     // avoid collision when we put index sizes for multiple cores in the same metrics request
     return "indexSize_" + getName();
   }
 
+  private String cachedOnDiskIndexSizeKeyName() {
+    // avoid collision when we put index sizes for multiple cores in the same metrics request
+    return "onDiskIndexSize_" + getName();
+  }
+
   public int getSegmentCount() {
     try {
       return withSearcher(

diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminOperation.java
@@ -380,8 +380,14 @@ public static NamedList<Object> getCoreStatus(
                 SimpleOrderedMap<Object> indexInfo =
                     LukeRequestHandler.getIndexInfo(searcher.get().getIndexReader());
                 long size = core.getIndexSize();
+                long onDiskSize = core.getOnDiskSize();
+                String readableSize = NumberUtils.readableSize(size);
+                String readableOnDiskSize =
+                    size == onDiskSize ? readableSize : NumberUtils.readableSize(onDiskSize);
                 indexInfo.add("sizeInBytes", size);
                 indexInfo.add("size", NumberUtils.readableSize(size));
+                indexInfo.add("onDiskSizeInBytes", onDiskSize);
+                indexInfo.add("onDiskSize", readableOnDiskSize);
                 info.add("index", indexInfo);
               } finally {
                 searcher.decref();

diff --git a/solr/core/src/java/org/apache/solr/storage/CompressingDirectory.java b/solr/core/src/java/org/apache/solr/storage/CompressingDirectory.java
@@ -39,8 +39,10 @@
 import org.apache.lucene.store.OutputStreamDataOutput;
 import org.apache.lucene.util.CollectionUtil;
 import org.apache.lucene.util.compress.LZ4;
+import org.apache.solr.core.DirectoryFactory;
 
-public class CompressingDirectory extends FSDirectory {
+public class CompressingDirectory extends FSDirectory
+    implements DirectoryFactory.OnDiskSizeDirectory {
 
   /**
    * Reference to {@code com.sun.nio.file.ExtendedOpenOption.DIRECT} by reflective class and enum
@@ -167,6 +169,11 @@ public long fileLength(String name) throws IOException {
     return readLengthFromHeader(path);
   }
 
+  @Override
+  public long onDiskFileLength(String name) throws IOException {
+    return Files.size(directoryPath.resolve(name));
+  }
+
   public static long readLengthFromHeader(Path path) throws IOException {
     if (Files.size(path) < Long.BYTES) {
       return 0;
@@ -243,7 +250,11 @@ public int transferTo(DataOutput out) throws IOException {
     }
   }
 
-  static final class DirectIOIndexOutput extends IndexOutput {
+  public interface SizeReportingIndexOutput {
+    long getBytesWritten();
+  }
+
+  static final class DirectIOIndexOutput extends IndexOutput implements SizeReportingIndexOutput {
     private final byte[] compressBuffer = new byte[COMPRESSION_BLOCK_SIZE];
     private final LZ4.FastCompressionHashTable ht = new LZ4.FastCompressionHashTable();
     private final ByteBuffer preBuffer;
@@ -264,6 +275,7 @@ static final class DirectIOIndexOutput extends IndexOutput {
     static final int HEADER_SIZE = 16; // 16 bytes
 
     private long filePos;
+    private long bytesWritten;
     private boolean isOpen;
 
     private final DirectBufferPool initialBlockBufferPool;
@@ -296,6 +308,7 @@ public DirectIOIndexOutput(
       preBuffer = ByteBuffer.wrap(compressBuffer);
       this.initialBlockBufferPool = initialBlockBufferPool;
       initialBlock = initialBlockBufferPool.get();
+      bytesWritten = HEADER_SIZE;
 
       // allocate space for the header
       buffer.position(buffer.position() + HEADER_SIZE);
@@ -340,6 +353,7 @@ private void dump() throws IOException {
 
       LZ4.compressWithDictionary(compressBuffer, 0, 0, COMPRESSION_BLOCK_SIZE, out, ht);
       int nextBlockSize = out.resetSize();
+      bytesWritten += nextBlockSize;
       blockDeltas.writeZInt(nextBlockSize - prevBlockSize);
       prevBlockSize = nextBlockSize;
       filePos += COMPRESSION_BLOCK_SIZE;
@@ -355,6 +369,7 @@ private void flush() throws IOException {
         LZ4.compressWithDictionary(compressBuffer, 0, 0, preBufferRemaining, out, ht);
       }
       int blockMapFooterSize = blockDeltas.transferTo(out);
+      bytesWritten += out.resetSize();
       if (wroteBlock) {
         writeHelper.flush(buffer, true);
         initialBlock.putLong(0, filePos);
@@ -377,6 +392,11 @@ private void flush() throws IOException {
       }
     }
 
+    @Override
+    public long getBytesWritten() {
+      return bytesWritten;
+    }
+
     private final SizeTrackingDataOutput out = new SizeTrackingDataOutput();
 
     private void writeBlock() throws IOException {