Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Experimental changes no review required #13525

Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@
@ExperimentalApi
public class RemoteStoreEnums {

static final char REMOTE_STORE_DEDICATED_PREFIX_START_CHAR = '.';

/**
* Categories of the data in Remote store.
*/
Expand Down Expand Up @@ -112,7 +114,7 @@ boolean requiresHashAlgorithm() {
public BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm) {
assert Objects.nonNull(hashAlgorithm) : "hashAlgorithm is expected to be non-null";
return BlobPath.cleanPath()
.add(hashAlgorithm.hash(pathInput))
.add(REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + hashAlgorithm.hash(pathInput))
.add(pathInput.basePath())
.add(pathInput.indexUUID())
.add(pathInput.shardId())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ public void testToXContentWithSegmentRepo() throws IOException {
xContentBuilder = indexPath.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
xContentBuilder.endObject();
String expected =
"{\"version\":\"1\",\"index_uuid\":\"djjsid73he8yd7usduh\",\"shard_count\":2,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"segments\":[\"data\",\"metadata\",\"lock_files\"]},\"paths\":[\"9BmBinD5HYs/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/data/\",\"ExCNOD8_5ew/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/data/\",\"z8wtf0yr2l4/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/metadata/\",\"VheHVwFlExE/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/metadata/\",\"IgFKbsDeUpQ/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/lock_files/\",\"pA3gy_GZtns/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/lock_files/\"]}";
"{\"version\":\"1\",\"index_uuid\":\"djjsid73he8yd7usduh\",\"shard_count\":2,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"segments\":[\"data\",\"metadata\",\"lock_files\"]},\"paths\":[\".9BmBinD5HYs/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/data/\",\".ExCNOD8_5ew/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/data/\",\".z8wtf0yr2l4/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/metadata/\",\".VheHVwFlExE/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/metadata/\",\".IgFKbsDeUpQ/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/lock_files/\",\".pA3gy_GZtns/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/lock_files/\"]}";
assertEquals(expected, xContentBuilder.toString());
}

Expand All @@ -68,7 +68,7 @@ public void testToXContentForTranslogRepoOnly() throws IOException {
xContentBuilder = indexPath.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
xContentBuilder.endObject();
String expected =
"{\"version\":\"1\",\"index_uuid\":\"djjsid73he8yd7usduh\",\"shard_count\":2,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"translog\":[\"data\",\"metadata\"]},\"paths\":[\"2EaVODaKBck/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/translog/data/\",\"dTS2VqEOUNo/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/translog/data/\",\"PVNKNGonmZw/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/translog/metadata/\",\"NXmt0Y6NjA8/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/translog/metadata/\"]}";
"{\"version\":\"1\",\"index_uuid\":\"djjsid73he8yd7usduh\",\"shard_count\":2,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"translog\":[\"data\",\"metadata\"]},\"paths\":[\".2EaVODaKBck/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/translog/data/\",\".dTS2VqEOUNo/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/translog/data/\",\".PVNKNGonmZw/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/translog/metadata/\",\".NXmt0Y6NjA8/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/translog/metadata/\"]}";
assertEquals(expected, xContentBuilder.toString());
}

Expand All @@ -92,7 +92,7 @@ public void testToXContentForBothRepos() throws IOException {
xContentBuilder = indexPath.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
xContentBuilder.endObject();
String expected =
"{\"version\":\"1\",\"index_uuid\":\"csbdqiu8a7sdnjdks\",\"shard_count\":3,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"translog\":[\"data\",\"metadata\"],\"segments\":[\"data\",\"metadata\",\"lock_files\"]},\"paths\":[\"Cjo0F6kNjYk/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/data/\",\"kpayyhxct1I/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/data/\",\"p2RlgnHeIgc/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/data/\",\"gkPIurBtB1w/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/metadata/\",\"Y4YhlbxAB1c/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/metadata/\",\"HYc8fyVPouI/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/metadata/\",\"igzyZCz1ysI/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/lock_files/\",\"uEluEiYmptk/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/lock_files/\",\"TfAD8f06_7A/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/lock_files/\",\"QqKEpasbEGs/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/translog/data/\",\"sNyoimoe1Bw/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/translog/data/\",\"d4YQtONfq50/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/translog/data/\",\"zLr4UXjK8T4/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/translog/metadata/\",\"_s8i7ZmlXGE/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/translog/metadata/\",\"tvtD3-k5ISg/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/translog/metadata/\"]}";
"{\"version\":\"1\",\"index_uuid\":\"csbdqiu8a7sdnjdks\",\"shard_count\":3,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"translog\":[\"data\",\"metadata\"],\"segments\":[\"data\",\"metadata\",\"lock_files\"]},\"paths\":[\".Cjo0F6kNjYk/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/data/\",\".kpayyhxct1I/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/data/\",\".p2RlgnHeIgc/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/data/\",\".gkPIurBtB1w/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/metadata/\",\".Y4YhlbxAB1c/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/metadata/\",\".HYc8fyVPouI/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/metadata/\",\".igzyZCz1ysI/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/lock_files/\",\".uEluEiYmptk/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/lock_files/\",\".TfAD8f06_7A/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/lock_files/\",\".QqKEpasbEGs/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/translog/data/\",\".sNyoimoe1Bw/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/translog/data/\",\".d4YQtONfq50/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/translog/data/\",\".zLr4UXjK8T4/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/translog/metadata/\",\"._s8i7ZmlXGE/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/translog/metadata/\",\".tvtD3-k5ISg/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/translog/metadata/\"]}";
assertEquals(expected, xContentBuilder.toString());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,15 @@ public void testGeneratePathForHashedPrefixType() {
BlobPath result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
assertTrue(
result.buildAsString()
.startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()))
.startsWith(
String.join(
SEPARATOR,
RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_BASE64.hash(pathInput),
basePath,
dataCategory.getName(),
dataType.getName()
)
)
);

// assert with exact value for known base path
Expand All @@ -180,7 +188,7 @@ public void testGeneratePathForHashedPrefixType() {
.dataType(dataType)
.build();
result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
assertEquals("DgSI70IciXs/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/data/", result.buildAsString());
assertEquals(".DgSI70IciXs/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/data/", result.buildAsString());

// Translog Metadata
dataType = METADATA;
Expand All @@ -194,7 +202,15 @@ public void testGeneratePathForHashedPrefixType() {
result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
assertTrue(
result.buildAsString()
.startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()))
.startsWith(
String.join(
SEPARATOR,
RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_BASE64.hash(pathInput),
basePath,
dataCategory.getName(),
dataType.getName()
)
)
);

// assert with exact value for known base path
Expand All @@ -206,7 +222,7 @@ public void testGeneratePathForHashedPrefixType() {
.dataType(dataType)
.build();
result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
assertEquals("oKU5SjILiy4/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/metadata/", result.buildAsString());
assertEquals(".oKU5SjILiy4/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/metadata/", result.buildAsString());

// Translog Lock files - This is a negative case where the assertion will trip.
dataType = LOCK_FILES;
Expand Down Expand Up @@ -242,7 +258,15 @@ public void testGeneratePathForHashedPrefixType() {
result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
assertTrue(
result.buildAsString()
.startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()))
.startsWith(
String.join(
SEPARATOR,
RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_BASE64.hash(pathInput),
basePath,
dataCategory.getName(),
dataType.getName()
)
)
);

// assert with exact value for known base path
Expand All @@ -254,7 +278,7 @@ public void testGeneratePathForHashedPrefixType() {
.dataType(dataType)
.build();
result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
assertEquals("AUBRfCIuWdk/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/data/", result.buildAsString());
assertEquals(".AUBRfCIuWdk/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/data/", result.buildAsString());

// Segment Metadata
dataType = METADATA;
Expand All @@ -268,7 +292,15 @@ public void testGeneratePathForHashedPrefixType() {
result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
assertTrue(
result.buildAsString()
.startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()))
.startsWith(
String.join(
SEPARATOR,
RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_BASE64.hash(pathInput),
basePath,
dataCategory.getName(),
dataType.getName()
)
)
);

// assert with exact value for known base path
Expand All @@ -280,7 +312,7 @@ public void testGeneratePathForHashedPrefixType() {
.dataType(dataType)
.build();
result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
assertEquals("erwR-G735Uw/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/metadata/", result.buildAsString());
assertEquals(".erwR-G735Uw/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/metadata/", result.buildAsString());

// Segment Lockfiles
dataType = LOCK_FILES;
Expand All @@ -294,7 +326,15 @@ public void testGeneratePathForHashedPrefixType() {
result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
assertTrue(
result.buildAsString()
.startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()))
.startsWith(
String.join(
SEPARATOR,
RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_BASE64.hash(pathInput),
basePath,
dataCategory.getName(),
dataType.getName()
)
)
);

// assert with exact value for known base path
Expand All @@ -306,7 +346,7 @@ public void testGeneratePathForHashedPrefixType() {
.dataType(dataType)
.build();
result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
assertEquals("KeYDIk0mJXI/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/lock_files/", result.buildAsString());
assertEquals(".KeYDIk0mJXI/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/lock_files/", result.buildAsString());
}

public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() {
Expand Down Expand Up @@ -334,7 +374,13 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() {
assertTrue(
result.buildAsString()
.startsWith(
String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())
String.join(
SEPARATOR,
RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_COMPOSITE_1.hash(pathInput),
basePath,
dataCategory.getName(),
dataType.getName()
)
)
);

Expand All @@ -350,7 +396,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() {
.dataType(dataType)
.build();
result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1);
assertEquals("D10000001001000/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/data/", result.buildAsString());
assertEquals(".D10000001001000/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/data/", result.buildAsString());

// Translog Metadata
dataType = METADATA;
Expand All @@ -365,7 +411,13 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() {
assertTrue(
result.buildAsString()
.startsWith(
String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())
String.join(
SEPARATOR,
RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_COMPOSITE_1.hash(pathInput),
basePath,
dataCategory.getName(),
dataType.getName()
)
)
);

Expand All @@ -379,7 +431,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() {
.build();
result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1);
assertEquals(
"o00101001010011/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/metadata/",
".o00101001010011/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/metadata/",
result.buildAsString()
);

Expand Down Expand Up @@ -418,7 +470,13 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() {
assertTrue(
result.buildAsString()
.startsWith(
String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())
String.join(
SEPARATOR,
RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_COMPOSITE_1.hash(pathInput),
basePath,
dataCategory.getName(),
dataType.getName()
)
)
);

Expand All @@ -431,7 +489,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() {
.dataType(dataType)
.build();
result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1);
assertEquals("A01010000000101/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/data/", result.buildAsString());
assertEquals(".A01010000000101/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/data/", result.buildAsString());

// Segment Metadata
dataType = METADATA;
Expand All @@ -446,7 +504,13 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() {
assertTrue(
result.buildAsString()
.startsWith(
String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())
String.join(
SEPARATOR,
RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_COMPOSITE_1.hash(pathInput),
basePath,
dataCategory.getName(),
dataType.getName()
)
)
);

Expand All @@ -460,7 +524,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() {
.build();
result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1);
assertEquals(
"e10101111000001/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/metadata/",
".e10101111000001/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/metadata/",
result.buildAsString()
);

Expand All @@ -477,7 +541,13 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() {
assertTrue(
result.buildAsString()
.startsWith(
String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())
String.join(
SEPARATOR,
RemoteStoreEnums.REMOTE_STORE_DEDICATED_PREFIX_START_CHAR + FNV_1A_COMPOSITE_1.hash(pathInput),
basePath,
dataCategory.getName(),
dataType.getName()
)
)
);

Expand All @@ -491,7 +561,7 @@ public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() {
.build();
result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1);
assertEquals(
"K01111001100000/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/lock_files/",
".K01111001100000/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/lock_files/",
result.buildAsString()
);
}
Expand Down
Loading