Skip to content

Commit

Permalink
Add period at the start of the hashed prefix
Browse files Browse the repository at this point in the history
  • Loading branch information
@ashking94
ashking94 committed May 24, 2024
1 parent 2700327 commit c0cc3de
Show file tree
Hide file tree
Showing 3 changed files with 96 additions and 24 deletions.
4 changes: 3 additions & 1 deletion server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@
@ExperimentalApi
public class RemoteStoreEnums {

static final char REMOTE_STORE_DEDICATED_PREFIX_START_CHAR = '.';

/**
* Categories of the data in Remote store.
*/
Expand Down Expand Up @@ -112,7 +114,7 @@ boolean requiresHashAlgorithm() {
public BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm) {
assert Objects.nonNull(hashAlgorithm) : "hashAlgorithm is expected to be non-null";
return BlobPath.cleanPath()
.add(hashAlgorithm.hash(pathInput))
.add(REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + hashAlgorithm.hash(pathInput))
.add(pathInput.basePath())
.add(pathInput.indexUUID())
.add(pathInput.shardId())
Expand Down
6 changes: 3 additions & 3 deletions server/src/test/java/org/opensearch/index/remote/RemoteIndexPathTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ public void testToXContentWithSegmentRepo() throws IOException {
xContentBuilder = indexPath.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
xContentBuilder.endObject();
String expected =
"{\"version\":\"1\",\"index_uuid\":\"djjsid73he8yd7usduh\",\"shard_count\":2,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"segments\":[\"data\",\"metadata\",\"lock_files\"]},\"paths\":[\"9BmBinD5HYs/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/data/\",\"ExCNOD8_5ew/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/data/\",\"z8wtf0yr2l4/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/metadata/\",\"VheHVwFlExE/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/metadata/\",\"IgFKbsDeUpQ/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/lock_files/\",\"pA3gy_GZtns/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/lock_files/\"]}";
"{\"version\":\"1\",\"index_uuid\":\"djjsid73he8yd7usduh\",\"shard_count\":2,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"segments\":[\"data\",\"metadata\",\"lock_files\"]},\"paths\":[\".9BmBinD5HYs/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/data/\",\".ExCNOD8_5ew/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/data/\",\".z8wtf0yr2l4/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/metadata/\",\".VheHVwFlExE/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/metadata/\",\".IgFKbsDeUpQ/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/lock_files/\",\".pA3gy_GZtns/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/lock_files/\"]}";
assertEquals(expected, xContentBuilder.toString());
}

Expand All @@ -68,7 +68,7 @@ public void testToXContentForTranslogRepoOnly() throws IOException {
xContentBuilder = indexPath.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
xContentBuilder.endObject();
String expected =
"{\"version\":\"1\",\"index_uuid\":\"djjsid73he8yd7usduh\",\"shard_count\":2,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"translog\":[\"data\",\"metadata\"]},\"paths\":[\"2EaVODaKBck/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/translog/data/\",\"dTS2VqEOUNo/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/translog/data/\",\"PVNKNGonmZw/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/translog/metadata/\",\"NXmt0Y6NjA8/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/translog/metadata/\"]}";
"{\"version\":\"1\",\"index_uuid\":\"djjsid73he8yd7usduh\",\"shard_count\":2,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"translog\":[\"data\",\"metadata\"]},\"paths\":[\".2EaVODaKBck/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/translog/data/\",\".dTS2VqEOUNo/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/translog/data/\",\".PVNKNGonmZw/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/translog/metadata/\",\".NXmt0Y6NjA8/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/translog/metadata/\"]}";
assertEquals(expected, xContentBuilder.toString());
}

Expand All @@ -92,7 +92,7 @@ public void testToXContentForBothRepos() throws IOException {
xContentBuilder = indexPath.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
xContentBuilder.endObject();
String expected =
"{\"version\":\"1\",\"index_uuid\":\"csbdqiu8a7sdnjdks\",\"shard_count\":3,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"translog\":[\"data\",\"metadata\"],\"segments\":[\"data\",\"metadata\",\"lock_files\"]},\"paths\":[\"Cjo0F6kNjYk/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/data/\",\"kpayyhxct1I/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/data/\",\"p2RlgnHeIgc/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/data/\",\"gkPIurBtB1w/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/metadata/\",\"Y4YhlbxAB1c/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/metadata/\",\"HYc8fyVPouI/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/metadata/\",\"igzyZCz1ysI/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/lock_files/\",\"uEluEiYmptk/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/lock_files/\",\"TfAD8f06_7A/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/lock_files/\",\"QqKEpasbEGs/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/translog/data/\",\"sNyoimoe1Bw/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/translog/data/\",\"d4YQtONfq50/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/translog/data/\",\"zLr4UXjK8T4/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/translog/metadata/\",\"_s8i7ZmlXGE/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/translog/metadata/\",\"tvtD3-k5ISg/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/translog/metadata/\"]}";
"{\"version\":\"1\",\"index_uuid\":\"csbdqiu8a7sdnjdks\",\"shard_count\":3,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"translog\":[\"data\",\"metadata\"],\"segments\":[\"data\",\"metadata\",\"lock_files\"]},\"paths\":[\".Cjo0F6kNjYk/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/data/\",\".kpayyhxct1I/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/data/\",\".p2RlgnHeIgc/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/data/\",\".gkPIurBtB1w/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/metadata/\",\".Y4YhlbxAB1c/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/metadata/\",\".HYc8fyVPouI/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/metadata/\",\".igzyZCz1ysI/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/lock_files/\",\".uEluEiYmptk/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/lock_files/\",\".TfAD8f06_7A/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/lock_files/\",\".QqKEpasbEGs/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/translog/data/\",\".sNyoimoe1Bw/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/translog/data/\",\".d4YQtONfq50/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/translog/data/\",\".zLr4UXjK8T4/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/translog/metadata/\",\"._s8i7ZmlXGE/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/translog/metadata/\",\".tvtD3-k5ISg/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/translog/metadata/\"]}";
assertEquals(expected, xContentBuilder.toString());
}

Expand Down
110 changes: 90 additions & 20 deletions server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,15 @@ public void testGeneratePathForHashedPrefixType() {
BlobPath result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
assertTrue(
result.buildAsString()
.startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()))
.startsWith(
String.join(
SEPARATOR,
RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_BASE64.hash(pathInput),
basePath,
dataCategory.getName(),
dataType.getName()
)
)
);

// assert with exact value for known base path
Expand All @@ -180,7 +188,7 @@ public void testGeneratePathForHashedPrefixType() {
.dataType(dataType)
.build();
result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
assertEquals("DgSI70IciXs/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/data/", result.buildAsString());
assertEquals(".DgSI70IciXs/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/data/", result.buildAsString());

// Translog Metadata
dataType = METADATA;
Expand All @@ -194,7 +202,15 @@ public void testGeneratePathForHashedPrefixType() {
result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
assertTrue(
result.buildAsString()
.startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()))
.startsWith(
String.join(
SEPARATOR,
RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_BASE64.hash(pathInput),
basePath,
dataCategory.getName(),
dataType.getName()
)
)
);

// assert with exact value for known base path
Expand All @@ -206,7 +222,7 @@ public void testGeneratePathForHashedPrefixType() {
.dataType(dataType)
.build();
result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
assertEquals("oKU5SjILiy4/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/metadata/", result.buildAsString());
assertEquals(".oKU5SjILiy4/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/metadata/", result.buildAsString());

// Translog Lock files - This is a negative case where the assertion will trip.
dataType = LOCK_FILES;
Expand Down Expand Up @@ -242,7 +258,15 @@ public void testGeneratePathForHashedPrefixType() {
result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
assertTrue(
result.buildAsString()
.startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()))
.startsWith(
String.join(
SEPARATOR,
RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_BASE64.hash(pathInput),
basePath,
dataCategory.getName(),
dataType.getName()
)
)
);

// assert with exact value for known base path
Expand All @@ -254,7 +278,7 @@ public void testGeneratePathForHashedPrefixType() {
.dataType(dataType)
.build();
result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
assertEquals("AUBRfCIuWdk/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/data/", result.buildAsString());
assertEquals(".AUBRfCIuWdk/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/data/", result.buildAsString());

// Segment Metadata
dataType = METADATA;
Expand All @@ -268,7 +292,15 @@ public void testGeneratePathForHashedPrefixType() {
result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
assertTrue(
result.buildAsString()
.startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()))
.startsWith(
String.join(
SEPARATOR,
RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_BASE64.hash(pathInput),
basePath,
dataCategory.getName(),
dataType.getName()
)
)
);

// assert with exact value for known base path
Expand All @@ -280,7 +312,7 @@ public void testGeneratePathForHashedPrefixType() {
.dataType(dataType)
.build();
result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
assertEquals("erwR-G735Uw/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/metadata/", result.buildAsString());
assertEquals(".erwR-G735Uw/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/metadata/", result.buildAsString());

// Segment Lockfiles
dataType = LOCK_FILES;
Expand All @@ -294,7 +326,15 @@ public void testGeneratePathForHashedPrefixType() {
result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
assertTrue(
result.buildAsString()
.startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()))
.startsWith(
String.join(
SEPARATOR,
RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_BASE64.hash(pathInput),
basePath,
dataCategory.getName(),
dataType.getName()
)
)
);

// assert with exact value for known base path
Expand All @@ -306,7 +346,7 @@ public void testGeneratePathForHashedPrefixType() {
.dataType(dataType)
.build();
result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
assertEquals("KeYDIk0mJXI/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/lock_files/", result.buildAsString());
assertEquals(".KeYDIk0mJXI/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/lock_files/", result.buildAsString());
}

public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() {
Expand Down Expand Up @@ -334,7 +374,13 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() {
assertTrue(
result.buildAsString()
.startsWith(
String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())
String.join(
SEPARATOR,
RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_COMPOSITE_1.hash(pathInput),
basePath,
dataCategory.getName(),
dataType.getName()
)
)
);

Expand All @@ -350,7 +396,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() {
.dataType(dataType)
.build();
result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1);
assertEquals("D10000001001000/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/data/", result.buildAsString());
assertEquals(".D10000001001000/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/data/", result.buildAsString());

// Translog Metadata
dataType = METADATA;
Expand All @@ -365,7 +411,13 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() {
assertTrue(
result.buildAsString()
.startsWith(
String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())
String.join(
SEPARATOR,
RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_COMPOSITE_1.hash(pathInput),
basePath,
dataCategory.getName(),
dataType.getName()
)
)
);

Expand All @@ -379,7 +431,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() {
.build();
result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1);
assertEquals(
"o00101001010011/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/metadata/",
".o00101001010011/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/metadata/",
result.buildAsString()
);

Expand Down Expand Up @@ -418,7 +470,13 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() {
assertTrue(
result.buildAsString()
.startsWith(
String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())
String.join(
SEPARATOR,
RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_COMPOSITE_1.hash(pathInput),
basePath,
dataCategory.getName(),
dataType.getName()
)
)
);

Expand All @@ -431,7 +489,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() {
.dataType(dataType)
.build();
result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1);
assertEquals("A01010000000101/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/data/", result.buildAsString());
assertEquals(".A01010000000101/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/data/", result.buildAsString());

// Segment Metadata
dataType = METADATA;
Expand All @@ -446,7 +504,13 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() {
assertTrue(
result.buildAsString()
.startsWith(
String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())
String.join(
SEPARATOR,
RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_COMPOSITE_1.hash(pathInput),
basePath,
dataCategory.getName(),
dataType.getName()
)
)
);

Expand All @@ -460,7 +524,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() {
.build();
result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1);
assertEquals(
"e10101111000001/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/metadata/",
".e10101111000001/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/metadata/",
result.buildAsString()
);

Expand All @@ -477,7 +541,13 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() {
assertTrue(
result.buildAsString()
.startsWith(
String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())
String.join(
SEPARATOR,
RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_COMPOSITE_1.hash(pathInput),
basePath,
dataCategory.getName(),
dataType.getName()
)
)
);

Expand All @@ -491,7 +561,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() {
.build();
result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1);
assertEquals(
"K01111001100000/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/lock_files/",
".K01111001100000/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/lock_files/",
result.buildAsString()
);
}
Expand Down

0 comments on commit c0cc3de

Please sign in to comment.