Skip to content

Commit

Permalink
Test - Lower resource limits on cpu & memory
Browse files Browse the repository at this point in the history
  • Loading branch information
@michalxo
michalxo committed Oct 21, 2024
1 parent 2c333a4 commit 27880c7
Show file tree
Hide file tree
Showing 4 changed files with 75 additions and 14 deletions.
4 changes: 4 additions & 0 deletions common/src/main/java/io/brokerqe/claire/Constants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ public interface Constants {
String TMP_DIR_SYSTEM = System.getProperty("java.io.tmpdir");
String DATE_FORMAT = "yyyy-MM-dd_HH-mm-ss";

String CPU = "cpu";
String MEMORY = "memory";

// Test tags
String TAG_JAAS = "jaas";
String TAG_JDBC = "jdbc";
Expand Down Expand Up @@ -189,6 +192,7 @@ public interface Constants {
String MARIADB_DRIVER_URL = "https://dlm.mariadb.com/2912798/Connectors/java/connector-java-3.1.4/mariadb-java-client-3.1.4.jar";
String MSSQL_DRIVER_URL = "https://download.microsoft.com/download/a/9/1/a91534b0-ed8c-4501-b491-e1dd0a20335a/sqljdbc_12.2.0.0_enu.zip";
String ORACLE_DRIVER_URL = "https://download.oracle.com/otn-pub/otn_software/jdbc/2110/ojdbc11.jar";

enum SECRETSOURCE {
CERT_MANAGER,
TRUST_MANAGER,
Expand Down
53 changes: 52 additions & 1 deletion operator-suite/src/main/java/io/brokerqe/claire/KubeClient.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@
import io.fabric8.kubernetes.api.model.batch.v1.Job;
import io.fabric8.kubernetes.api.model.batch.v1.JobList;
import io.fabric8.kubernetes.api.model.batch.v1.JobStatus;
import io.fabric8.kubernetes.api.model.metrics.v1beta1.ContainerMetrics;
import io.fabric8.kubernetes.api.model.metrics.v1beta1.NodeMetrics;
import io.fabric8.kubernetes.api.model.metrics.v1beta1.NodeMetricsList;
import io.fabric8.kubernetes.api.model.metrics.v1beta1.PodMetrics;
import io.fabric8.kubernetes.api.model.metrics.v1beta1.PodMetricsList;
import io.fabric8.kubernetes.api.model.networking.v1.Ingress;
import io.fabric8.kubernetes.client.Config;
import io.fabric8.kubernetes.client.KubernetesClient;
Expand Down Expand Up @@ -55,9 +60,9 @@
import java.time.Instant;
import java.time.ZoneOffset;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
import java.util.Map;
import java.util.Base64;
import java.util.NoSuchElementException;
import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
Expand All @@ -66,6 +71,7 @@
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.is;

@SuppressWarnings({"checkstyle:ClassFanOutComplexity"})
public class KubeClient {
private final KubernetesClient client;
private final KubernetesPlatform platform;
Expand Down Expand Up @@ -957,6 +963,51 @@ public void deleteConfigMap(String namespaceName, String configMapName, boolean
LOGGER.info("[{}] Deleted config map {}", namespaceName, configMapName);
}

// ========================================================
// ---------> Kubernetes cluster metrics methods <---------
// ========================================================
public NodeMetricsList getKubernetesNodesMetrics() {
return getKubernetesClient().top().nodes().metrics();
}

public NodeMetrics getKubernetesNodeMetrics(String nodeName) {
return getKubernetesClient().top().nodes().withName(nodeName).metric();
}

public void printKubernetesNodesMetrics() {
NodeMetricsList nodeMetricsList = getKubernetesNodesMetrics();
StringBuilder metricsLine = new StringBuilder();
nodeMetricsList.getItems().forEach(nodeMetrics -> {
String name = nodeMetrics.getMetadata().getName();
String usage = nodeMetrics.getUsage().toString();
metricsLine.append(name).append(":").append(usage).append("\n");
});
LOGGER.info("Currently used node resources:\n{}", metricsLine);
}

public PodMetricsList getKubernetesPodsMetrics(String namespace) {
return getKubernetesClient().top().pods().inNamespace(namespace).metrics();
}

public PodMetrics getKubernetesPodMetrics(String namespace, String podName) {
return getKubernetesClient().top().pods().metrics(namespace, podName);
}

public void printKubernetesPodsMetrics(String namespace) {
PodMetricsList podMetricsList = getKubernetesPodsMetrics(namespace);
StringBuilder metricsLine = new StringBuilder();
podMetricsList.getItems().forEach(podMetrics -> {
String name = podMetrics.getMetadata().getName();
metricsLine.append("pod: ").append(name);
for (ContainerMetrics c : podMetrics.getContainers()) {
String cname = c.getName();
String usage = c.getUsage().toString();
metricsLine.append("container: ").append(cname).append(":").append(usage).append("\n");
}
});
LOGGER.info("[{}] Currently used pod resources:\n{}", namespace, metricsLine);
}

// =============================
// ---------> HELPERS <---------
// =============================
Expand Down
26 changes: 16 additions & 10 deletions operator-suite/src/test/java/io/brokerqe/claire/configuration/BrokerConfigurationTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,9 +232,11 @@ void verifyResourceRequest() {

@Test
void verifyResourceLimits() {
getClient().printKubernetesNodesMetrics();
getClient().printKubernetesPodsMetrics(testNamespace);
Map<String, IntOrString> requestedResourceLimits = new HashMap<>();
IntOrString cpuValue = new IntOrString("400m");
IntOrString memValue = new IntOrString("512M");
IntOrString cpuValue = new IntOrString("250m");
IntOrString memValue = new IntOrString("300Mi");
requestedResourceLimits.put("cpu", cpuValue);
requestedResourceLimits.put("memory", memValue);
ActiveMQArtemis broker = new ActiveMQArtemisBuilder()
Expand All @@ -250,7 +252,7 @@ void verifyResourceLimits() {
.endDeploymentplanResources()
.endDeploymentPlan()
.endSpec().build();
ResourceManager.createArtemis(testNamespace, broker, true);
ResourceManager.createArtemis(testNamespace, broker, true, Constants.DURATION_2_MINUTES);
Pod brokerPod = getClient().getPod(testNamespace, testBrokerName + "-ss-0");
Map<String, Quantity> limits = brokerPod.getSpec().getContainers().get(0).getResources().getLimits();
verifyResourceRequestValues("limit", limits, cpuValue, memValue);
Expand All @@ -260,9 +262,10 @@ void verifyResourceLimits() {

@Test
void verifyResourceUpdates() {
getClient().printKubernetesNodesMetrics();
Map<String, IntOrString> requestedResources = new HashMap<>();
IntOrString cpuValue = new IntOrString("400m");
IntOrString memValue = new IntOrString("512M");
IntOrString cpuValue = new IntOrString("300m");
IntOrString memValue = new IntOrString("350Mi");
requestedResources.put("cpu", cpuValue);
requestedResources.put("memory", memValue);
ActiveMQArtemis broker = new ActiveMQArtemisBuilder()
Expand All @@ -279,15 +282,16 @@ void verifyResourceUpdates() {
.endDeploymentplanResources()
.endDeploymentPlan()
.endSpec().build();
broker = ResourceManager.createArtemis(testNamespace, broker, true);
broker = ResourceManager.createArtemis(testNamespace, broker, true, Constants.DURATION_2_MINUTES);
Pod brokerPod = getClient().getPod(testNamespace, testBrokerName + "-ss-0");
Map<String, Quantity> limits = brokerPod.getSpec().getContainers().get(0).getResources().getLimits();
Map<String, Quantity> requests = brokerPod.getSpec().getContainers().get(0).getResources().getRequests();
verifyResourceRequestValues("limit", limits, cpuValue, memValue);
verifyResourceRequestValues("request", requests, cpuValue, memValue);
getClient().printKubernetesPodsMetrics(testNamespace);

cpuValue = new IntOrString("500m");
memValue = new IntOrString("768M");
cpuValue = new IntOrString("400m");
memValue = new IntOrString("400Mi");
requestedResources.put("cpu", cpuValue);
requestedResources.put("memory", memValue);
broker.getSpec().getDeploymentPlan().getResources().setLimits(requestedResources);
Expand All @@ -306,6 +310,7 @@ void verifyResourceUpdates() {

@Test
void verifyDefaultResourceRequests() {
getClient().printKubernetesNodesMetrics();
ActiveMQArtemis broker = ResourceManager.createArtemis(testNamespace, testBrokerName);
Pod brokerPod = getClient().getPod(testNamespace, testBrokerName + "-ss-0");
Map<String, Quantity> limits = brokerPod.getSpec().getContainers().get(0).getResources().getLimits();
Expand All @@ -314,9 +319,10 @@ void verifyDefaultResourceRequests() {
assertThat(String.format("Resource limits were applied by default: %s", limits), limits, aMapWithSize(0));
assertThat(String.format("Resource requests were applied by default: %s", requests), requests, aMapWithSize(0));

getClient().printKubernetesPodsMetrics(testNamespace);
Map<String, IntOrString> requestedResources = new HashMap<>();
IntOrString cpuValue = new IntOrString("400m");
IntOrString memValue = new IntOrString("512M");
IntOrString cpuValue = new IntOrString("250m");
IntOrString memValue = new IntOrString("256Mi");
requestedResources.put("cpu", cpuValue);
requestedResources.put("memory", memValue);
Resources resources = new Resources();
Expand Down
6 changes: 3 additions & 3 deletions operator-suite/src/test/java/io/brokerqe/claire/persistence/JdbcSingleDbTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ protected void deployPostgres() {
postgres.deployPostgres("db-jar-plugin");
}

void deployBrokerWithDB(int brokerSize, boolean waitForDeployment) {
void deployBrokerWithDatabase(int brokerSize, boolean waitForDeployment) {
getClient().createSecretEncodedData(testNamespace, LOGGER_SECRET_NAME, Map.of(ArtemisConstants.LOGGING_PROPERTIES_CONFIG_KEY, TestUtils.getFileContentAsBase64(LOGGER_FILE)), true);

String downloadDriverCommand = "mkdir -p /amq/init/config/extra-libs && wget -O /amq/init/config/extra-libs/postgresql.jar %s".formatted(Constants.POSTGRESQL_DRIVER_URL);
Expand Down Expand Up @@ -147,7 +147,7 @@ void deployBrokerWithDB(int brokerSize, boolean waitForDeployment) {

@Test
void testSingleBrokerMessaging() {
deployBrokerWithDB(1, true);
deployBrokerWithDatabase(1, true);
ActiveMQArtemisAddress myAddress = ResourceManager.createArtemisAddress(testNamespace, "testx", "testx");
Pod brokerPod = getClient().getFirstPodByPrefixName(testNamespace, brokerName);
int msgsExpected = 10;
Expand Down Expand Up @@ -176,7 +176,7 @@ void testJdbcLock() {
String acquiredLockLog = "AMQ221035: Primary Server Obtained primary lock";
String waitingForLockLog = "AMQ221034: Waiting indefinitely to obtain primary lock";

deployBrokerWithDB(2, false);
deployBrokerWithDatabase(2, false);
TestUtils.waitFor("First broker pod to run", Constants.DURATION_5_SECONDS, Constants.DURATION_2_MINUTES, () -> {
List<Pod> pods = getClient().listPodsByPrefixName(testNamespace, brokerName);
Boolean[] bools = {false, false};
Expand Down

0 comments on commit 27880c7

Please sign in to comment.