HBASE-28969 Move HFileLink file creations to SFT

apache · Nov 12, 2024 · 356d620 · 356d620
1 parent f3208d9
commit 356d620
Show file tree

Hide file tree

Showing 10 changed files with 129 additions and 187 deletions.
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/HFileLink.java
@@ -21,7 +21,6 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.TableName;
@@ -290,175 +289,6 @@ public static String createHFileLinkName(final TableName tableName, final String
     return s;
   }
 
-  /**
-   * Create a new HFileLink
-   * <p>
-   * It also adds a back-reference to the hfile back-reference directory to simplify the
-   * reference-count and the cleaning process.
-   * @param conf            {@link Configuration} to read for the archive directory name
-   * @param fs              {@link FileSystem} on which to write the HFileLink
-   * @param dstFamilyPath   - Destination path (table/region/cf/)
-   * @param hfileRegionInfo - Linked HFile Region Info
-   * @param hfileName       - Linked HFile name
-   * @return the file link name.
-   * @throws IOException on file or parent directory creation failure.
-   */
-  public static String create(final Configuration conf, final FileSystem fs,
-    final Path dstFamilyPath, final RegionInfo hfileRegionInfo, final String hfileName)
-    throws IOException {
-    return create(conf, fs, dstFamilyPath, hfileRegionInfo, hfileName, true);
-  }
-
-  /**
-   * Create a new HFileLink
-   * <p>
-   * It also adds a back-reference to the hfile back-reference directory to simplify the
-   * reference-count and the cleaning process.
-   * @param conf            {@link Configuration} to read for the archive directory name
-   * @param fs              {@link FileSystem} on which to write the HFileLink
-   * @param dstFamilyPath   - Destination path (table/region/cf/)
-   * @param hfileRegionInfo - Linked HFile Region Info
-   * @param hfileName       - Linked HFile name
-   * @param createBackRef   - Whether back reference should be created. Defaults to true.
-   * @return the file link name.
-   * @throws IOException on file or parent directory creation failure.
-   */
-  public static String create(final Configuration conf, final FileSystem fs,
-    final Path dstFamilyPath, final RegionInfo hfileRegionInfo, final String hfileName,
-    final boolean createBackRef) throws IOException {
-    TableName linkedTable = hfileRegionInfo.getTable();
-    String linkedRegion = hfileRegionInfo.getEncodedName();
-    return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, createBackRef);
-  }
-
-  /**
-   * Create a new HFileLink
-   * <p>
-   * It also adds a back-reference to the hfile back-reference directory to simplify the
-   * reference-count and the cleaning process.
-   * @param conf          {@link Configuration} to read for the archive directory name
-   * @param fs            {@link FileSystem} on which to write the HFileLink
-   * @param dstFamilyPath - Destination path (table/region/cf/)
-   * @param linkedTable   - Linked Table Name
-   * @param linkedRegion  - Linked Region Name
-   * @param hfileName     - Linked HFile name
-   * @return the file link name.
-   * @throws IOException on file or parent directory creation failure.
-   */
-  public static String create(final Configuration conf, final FileSystem fs,
-    final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion,
-    final String hfileName) throws IOException {
-    return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, true);
-  }
-
-  /**
-   * Create a new HFileLink. In the event of link creation failure, this method throws an
-   * IOException, so that the calling upper laying can decide on how to proceed with this.
-   * <p>
-   * It also adds a back-reference to the hfile back-reference directory to simplify the
-   * reference-count and the cleaning process.
-   * @param conf          {@link Configuration} to read for the archive directory name
-   * @param fs            {@link FileSystem} on which to write the HFileLink
-   * @param dstFamilyPath - Destination path (table/region/cf/)
-   * @param linkedTable   - Linked Table Name
-   * @param linkedRegion  - Linked Region Name
-   * @param hfileName     - Linked HFile name
-   * @param createBackRef - Whether back reference should be created. Defaults to true.
-   * @return the file link name.
-   * @throws IOException on file or parent directory creation failure.
-   */
-  public static String create(final Configuration conf, final FileSystem fs,
-    final Path dstFamilyPath, final TableName linkedTable, final String linkedRegion,
-    final String hfileName, final boolean createBackRef) throws IOException {
-    String familyName = dstFamilyPath.getName();
-    String regionName = dstFamilyPath.getParent().getName();
-    String tableName =
-      CommonFSUtils.getTableName(dstFamilyPath.getParent().getParent()).getNameAsString();
-
-    return create(conf, fs, dstFamilyPath, familyName, tableName, regionName, linkedTable,
-      linkedRegion, hfileName, createBackRef);
-  }
-
-  /**
-   * Create a new HFileLink
-   * <p>
-   * It also adds a back-reference to the hfile back-reference directory to simplify the
-   * reference-count and the cleaning process.
-   * @param conf          {@link Configuration} to read for the archive directory name
-   * @param fs            {@link FileSystem} on which to write the HFileLink
-   * @param dstFamilyPath - Destination path (table/region/cf/)
-   * @param dstTableName  - Destination table name
-   * @param dstRegionName - Destination region name
-   * @param linkedTable   - Linked Table Name
-   * @param linkedRegion  - Linked Region Name
-   * @param hfileName     - Linked HFile name
-   * @param createBackRef - Whether back reference should be created. Defaults to true.
-   * @return the file link name.
-   * @throws IOException on file or parent directory creation failure
-   */
-  public static String create(final Configuration conf, final FileSystem fs,
-    final Path dstFamilyPath, final String familyName, final String dstTableName,
-    final String dstRegionName, final TableName linkedTable, final String linkedRegion,
-    final String hfileName, final boolean createBackRef) throws IOException {
-    String name = createHFileLinkName(linkedTable, linkedRegion, hfileName);
-    String refName = createBackReferenceName(dstTableName, dstRegionName);
-
-    // Make sure the destination directory exists
-    fs.mkdirs(dstFamilyPath);
-
-    // Make sure the FileLink reference directory exists
-    Path archiveStoreDir =
-      HFileArchiveUtil.getStoreArchivePath(conf, linkedTable, linkedRegion, familyName);
-    Path backRefPath = null;
-    if (createBackRef) {
-      Path backRefssDir = getBackReferencesDir(archiveStoreDir, hfileName);
-      fs.mkdirs(backRefssDir);
-
-      // Create the reference for the link
-      backRefPath = new Path(backRefssDir, refName);
-      fs.createNewFile(backRefPath);
-    }
-    try {
-      // Create the link
-      if (fs.createNewFile(new Path(dstFamilyPath, name))) {
-        return name;
-      }
-    } catch (IOException e) {
-      LOG.error("couldn't create the link=" + name + " for " + dstFamilyPath, e);
-      // Revert the reference if the link creation failed
-      if (createBackRef) {
-        fs.delete(backRefPath, false);
-      }
-      throw e;
-    }
-    throw new IOException(
-      "File link=" + name + " already exists under " + dstFamilyPath + " folder.");
-  }
-
-  /**
-   * Create a new HFileLink starting from a hfileLink name
-   * <p>
-   * It also adds a back-reference to the hfile back-reference directory to simplify the
-   * reference-count and the cleaning process.
-   * @param conf          {@link Configuration} to read for the archive directory name
-   * @param fs            {@link FileSystem} on which to write the HFileLink
-   * @param dstFamilyPath - Destination path (table/region/cf/)
-   * @param hfileLinkName - HFileLink name (it contains hfile-region-table)
-   * @param createBackRef - Whether back reference should be created. Defaults to true.
-   * @return the file link name.
-   * @throws IOException on file or parent directory creation failure.
-   */
-  public static String createFromHFileLink(final Configuration conf, final FileSystem fs,
-    final Path dstFamilyPath, final String hfileLinkName, final boolean createBackRef)
-    throws IOException {
-    Matcher m = LINK_NAME_PATTERN.matcher(hfileLinkName);
-    if (!m.matches()) {
-      throw new IllegalArgumentException(hfileLinkName + " is not a valid HFileLink name!");
-    }
-    return create(conf, fs, dstFamilyPath, TableName.valueOf(m.group(1), m.group(2)), m.group(3),
-      m.group(4), createBackRef);
-  }
-
   /**
    * Create the back reference name
    */

diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
@@ -684,8 +684,7 @@ public Path splitStoreFile(RegionInfo hri, String familyName, HStoreFile f, byte
           hfileName = m.group(4);
         }
         // must create back reference here
-        HFileLink.create(conf, fs, splitDir, familyName, hri.getTable().getNameAsString(),
-          hri.getEncodedName(), linkedTable, linkedRegion, hfileName, true);
+        tracker.createHFileLink(linkedTable, linkedRegion, hfileName, true);
         Path path =
           new Path(splitDir, HFileLink.createHFileLinkName(linkedTable, linkedRegion, hfileName));
         LOG.info("Created linkFile:" + path.toString() + " for child: " + hri.getEncodedName()

diff --git a/...src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java b/...src/main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTracker.java
@@ -22,6 +22,7 @@
 import java.util.List;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
 import org.apache.hadoop.hbase.io.Reference;
 import org.apache.hadoop.hbase.regionserver.CreateStoreFileWriterParams;
@@ -119,4 +120,30 @@ StoreFileInfo getStoreFileInfo(final FileStatus fileStatus, final Path initialPa
   StoreFileInfo getStoreFileInfo(final Path initialPath, final boolean primaryReplica)
     throws IOException;
 
+  /**
+   * Create a new HFileLink
+   * <p>
+   * It also adds a back-reference to the hfile back-reference directory to simplify the
+   * reference-count and the cleaning process.
+   * @param hfileLinkName - HFileLink name (it contains hfile-region-table)
+   * @param createBackRef - Whether back reference should be created. Defaults to true.
+   * @return the file link name.
+   * @throws IOException on file or parent directory creation failure.
+   */
+  String createHFileLink(final TableName linkedTable, final String linkedRegion,
+    final String hfileName, final boolean createBackRef) throws IOException;
+
+  /**
+   * Create a new HFileLink starting from a hfileLink name
+   * <p>
+   * It also adds a back-reference to the hfile back-reference directory to simplify the
+   * reference-count and the cleaning process.
+   * @param hfileLinkName - HFileLink name (it contains hfile-region-table)
+   * @param createBackRef - Whether back reference should be created. Defaults to true.
+   * @return the file link name.
+   * @throws IOException on file or parent directory creation failure.
+   */
+  String createFromHFileLink(final String hfileName, final boolean createBackRef)
+    throws IOException;
+
 }
diff --git a/...main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java b/...main/java/org/apache/hadoop/hbase/regionserver/storefiletracker/StoreFileTrackerBase.java
@@ -25,12 +25,14 @@
 import java.io.InputStream;
 import java.util.Collection;
 import java.util.List;
+import java.util.regex.Matcher;
 import org.apache.commons.io.IOUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
 import org.apache.hadoop.hbase.io.HFileLink;
@@ -48,6 +50,7 @@
 import org.apache.hadoop.hbase.regionserver.StoreUtils;
 import org.apache.hadoop.hbase.util.CommonFSUtils;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.hbase.util.HFileArchiveUtil;
 import org.apache.yetus.audience.InterfaceAudience;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -319,6 +322,57 @@ public StoreFileInfo getStoreFileInfo(FileStatus fileStatus, Path initialPath,
       isPrimaryReplica);
   }
 
+  public String createHFileLink(final TableName linkedTable, final String linkedRegion,
+    final String hfileName, final boolean createBackRef) throws IOException {
+    String name = HFileLink.createHFileLinkName(linkedTable, linkedRegion, hfileName);
+    String refName = HFileLink.createBackReferenceName(ctx.getTableName().toString(),
+      ctx.getRegionInfo().getEncodedName());
+
+    FileSystem fs = ctx.getRegionFileSystem().getFileSystem();
+    // Make sure the destination directory exists
+    fs.mkdirs(ctx.getFamilyStoreDirectoryPath());
+
+    // Make sure the FileLink reference directory exists
+    Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(conf, linkedTable, linkedRegion,
+      ctx.getFamily().getNameAsString());
+    Path backRefPath = null;
+    if (createBackRef) {
+      Path backRefssDir = HFileLink.getBackReferencesDir(archiveStoreDir, hfileName);
+      fs.mkdirs(backRefssDir);
+
+      // Create the reference for the link
+      backRefPath = new Path(backRefssDir, refName);
+      fs.createNewFile(backRefPath);
+    }
+    try {
+      // Create the link
+      if (fs.createNewFile(new Path(ctx.getFamilyStoreDirectoryPath(), name))) {
+        return name;
+      }
+    } catch (IOException e) {
+      LOG.error("couldn't create the link=" + name + " for " + ctx.getFamilyStoreDirectoryPath(),
+        e);
+      // Revert the reference if the link creation failed
+      if (createBackRef) {
+        fs.delete(backRefPath, false);
+      }
+      throw e;
+    }
+    throw new IOException("File link=" + name + " already exists under "
+      + ctx.getFamilyStoreDirectoryPath() + " folder.");
+
+  }
+
+  public String createFromHFileLink(final String hfileLinkName, final boolean createBackRef)
+    throws IOException {
+    Matcher m = HFileLink.LINK_NAME_PATTERN.matcher(hfileLinkName);
+    if (!m.matches()) {
+      throw new IllegalArgumentException(hfileLinkName + " is not a valid HFileLink name!");
+    }
+    return createHFileLink(TableName.valueOf(m.group(1), m.group(2)), m.group(3), m.group(4),
+      createBackRef);
+  }
+
   /**
    * For primary replica, we will call load once when opening a region, and the implementation could
    * choose to do some cleanup work. So here we use {@code readOnly} to indicate that whether you

diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/snapshot/RestoreSnapshotHelper.java
@@ -726,11 +726,12 @@ private String restoreStoreFile(final Path familyDir, final RegionInfo regionInf
     final StoreFileTracker tracker) throws IOException {
     String hfileName = storeFile.getName();
     if (HFileLink.isHFileLink(hfileName)) {
-      return HFileLink.createFromHFileLink(conf, fs, familyDir, hfileName, createBackRef);
+      return tracker.createFromHFileLink(hfileName, createBackRef);
     } else if (StoreFileInfo.isReference(hfileName)) {
       return restoreReferenceFile(familyDir, regionInfo, storeFile, tracker);
     } else {
-      return HFileLink.create(conf, fs, familyDir, regionInfo, hfileName, createBackRef);
+      return tracker.createHFileLink(regionInfo.getTable(), regionInfo.getEncodedName(), hfileName,
+        createBackRef);
     }
   }
 

diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/RegionSplitter.java
@@ -50,6 +50,8 @@
 import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
 import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTracker;
+import org.apache.hadoop.hbase.regionserver.storefiletracker.StoreFileTrackerFactory;
 import org.apache.yetus.audience.InterfaceAudience;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -680,7 +682,9 @@ static LinkedList<Pair<byte[], byte[]>> splitScan(LinkedList<Pair<byte[], byte[]
             // Check every Column Family for that region -- check does not have references.
             boolean refFound = false;
             for (ColumnFamilyDescriptor c : htd.getColumnFamilies()) {
-              if ((refFound = regionFs.hasReferences(c.getNameAsString()))) {
+              StoreFileTracker sft = StoreFileTrackerFactory
+                .create(regionFs.getFileSystem().getConf(), htd, c, regionFs);
+              if ((refFound = sft.hasReferences())) {
                 break;
               }
             }

diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestPrefetch.java
@@ -473,14 +473,15 @@ private void testPrefetchWhenHFileLink(Consumer<Cacheable> test) throws Exceptio
     HRegionFileSystem dstRegionFs = HRegionFileSystem.createRegionOnFileSystem(testConf, fs,
       CommonFSUtils.getTableDir(testDir, dstHri.getTable()), dstHri);
     Path dstPath = new Path(regionFs.getTableDir(), new Path(dstHri.getRegionNameAsString(), "cf"));
-    HFileLink.create(testConf, this.fs, dstPath, hri, storeFilePath.getName());
-    Path linkFilePath =
-      new Path(dstPath, HFileLink.createHFileLinkName(hri, storeFilePath.getName()));
-
     StoreFileTracker sft = StoreFileTrackerFactory.create(testConf, false,
       StoreContext.getBuilder()
         .withFamilyStoreDirectoryPath(new Path(dstRegionFs.getRegionDir(), "cf"))
+        .withColumnFamilyDescriptor(ColumnFamilyDescriptorBuilder.of("cf"))
         .withRegionFileSystem(dstRegionFs).build());
+    sft.createHFileLink(hri.getTable(), hri.getEncodedName(), storeFilePath.getName(), true);
+    Path linkFilePath =
+      new Path(dstPath, HFileLink.createHFileLinkName(hri, storeFilePath.getName()));
+
     // Try to open store file from link
     StoreFileInfo storeFileInfo = sft.getStoreFileInfo(linkFilePath, true);
     HStoreFile hsf = new HStoreFile(storeFileInfo, BloomType.NONE, cacheConf);