Skip to content

Commit

Permalink
Merge pull request opensearch-project#806 from wrijeff/scaling
Browse files Browse the repository at this point in the history
Updated hash ring vnode count to more evenly distribute load.
  • Loading branch information
wrijeff authored Aug 11, 2021
2 parents ce6031e + 9457c8d commit 88e0236
Show file tree
Hide file tree
Showing 4 changed files with 11 additions and 10 deletions.
12 changes: 6 additions & 6 deletions data-prepper-core/integrationTest.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -170,9 +170,9 @@ task rawSpanEndToEndTest(type: Test) {
dependsOn build
dependsOn startOdfeDockerContainer
def createDataPrepper1Task = createDataPrepperDockerContainer(
"rawSpanDataPrepper1", "data-prepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_YAML}")
"rawSpanDataPrepper1", "dataprepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_YAML}")
def createDataPrepper2Task = createDataPrepperDockerContainer(
"rawSpanDataPrepper2", "data-prepper2", 21891, 4901, "/app/${RAW_SPAN_PIPELINE_YAML}")
"rawSpanDataPrepper2", "dataprepper2", 21891, 4901, "/app/${RAW_SPAN_PIPELINE_YAML}")
def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer)
def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer)
dependsOn startDataPrepper1Task
Expand Down Expand Up @@ -207,9 +207,9 @@ task rawSpanCompatibilityEndToEndTest(type: Test) {
dependsOn build
dependsOn startOdfeDockerContainer
def createDataPrepper1Task = createDataPrepperDockerContainer(
"rawSpanDataPrepperFromBuild", "data-prepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}")
"rawSpanDataPrepperFromBuild", "dataprepper1", 21890, 4900, "/app/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}")
def createDataPrepper2Task = createDataPrepperDockerContainerFromPullImage(
"rawSpanDataPrepperFromPull", "data-prepper2", 21891, 4901, "src/integrationTest/resources/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}")
"rawSpanDataPrepperFromPull", "dataprepper2", 21891, 4901, "src/integrationTest/resources/${RAW_SPAN_PIPELINE_LATEST_RELEASE_YAML}")
def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer)
def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer)
dependsOn startDataPrepper1Task
Expand Down Expand Up @@ -244,9 +244,9 @@ task serviceMapEndToEndTest(type: Test) {
dependsOn build
dependsOn startOdfeDockerContainer
def createDataPrepper1Task = createDataPrepperDockerContainer(
"serviceMapDataPrepper1", "data-prepper1", 21890, 4900, "/app/${SERVICE_MAP_PIPELINE_YAML}")
"serviceMapDataPrepper1", "dataprepper1", 21890, 4900, "/app/${SERVICE_MAP_PIPELINE_YAML}")
def createDataPrepper2Task = createDataPrepperDockerContainer(
"serviceMapDataPrepper2", "data-prepper2", 21891, 4901, "/app/${SERVICE_MAP_PIPELINE_YAML}")
"serviceMapDataPrepper2", "dataprepper2", 21891, 4901, "/app/${SERVICE_MAP_PIPELINE_YAML}")
def startDataPrepper1Task = startDataPrepperDockerContainer(createDataPrepper1Task as DockerCreateContainer)
def startDataPrepper2Task = startDataPrepperDockerContainer(createDataPrepper2Task as DockerCreateContainer)
dependsOn startDataPrepper1Task
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ entry-pipeline:
prepper:
- peer_forwarder:
discovery_mode: "static"
static_endpoints: ["data-prepper1", "data-prepper2"]
static_endpoints: ["dataprepper1", "dataprepper2"]
ssl: false
sink:
- pipeline:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ entry-pipeline:
prepper:
- peer_forwarder:
discovery_mode: "static"
static_endpoints: ["data-prepper1", "data-prepper2"]
static_endpoints: ["dataprepper1", "dataprepper2"]
ssl: false
sink:
- pipeline:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
public class PeerForwarderConfig {
public static final String TIME_OUT = "time_out";
public static final String MAX_NUM_SPANS_PER_REQUEST = "span_agg_count";
public static final int NUM_VIRTUAL_NODES = 10;
public static final int NUM_VIRTUAL_NODES = 128;
public static final String TARGET_PORT = "target_port";
public static final String DISCOVERY_MODE = "discovery_mode";
public static final String DOMAIN_NAME = "domain_name";
Expand All @@ -23,6 +23,7 @@ public class PeerForwarderConfig {
private static final String USE_ACM_CERT_FOR_SSL = "useAcmCertForSSL";
private static final boolean DEFAULT_USE_ACM_CERT_FOR_SSL = false;
private static final int DEFAULT_TARGET_PORT = 21890;
private static final int DEFAULT_TIMEOUT_SECONDS = 2;
private static final String ACM_CERT_ISSUE_TIME_OUT_MILLIS = "acmCertIssueTimeOutMillis";
private static final int DEFAULT_ACM_CERT_ISSUE_TIME_OUT_MILLIS = 120000;
private static final String ACM_CERT_ARN = "acmCertificateArn";
Expand Down Expand Up @@ -84,7 +85,7 @@ public static PeerForwarderConfig buildConfig(final PluginSetting pluginSetting)
return new PeerForwarderConfig(
peerClientPool,
hashRing,
pluginSetting.getIntegerOrDefault(TIME_OUT, 3),
pluginSetting.getIntegerOrDefault(TIME_OUT, DEFAULT_TIMEOUT_SECONDS),
pluginSetting.getIntegerOrDefault(MAX_NUM_SPANS_PER_REQUEST, 48));
}

Expand Down

0 comments on commit 88e0236

Please sign in to comment.