Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Default env and site #313

Merged
merged 3 commits into from
Aug 22, 2023
Merged

Default env and site #313

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c6b81ed
Add default constructor to fix deserialization error
jradhakrishnan Aug 22, 2023
fce9503
Set `AWSAccount` in context instead of just tenant name
jradhakrishnan Aug 22, 2023
13bf83f
Change field and method name
jradhakrishnan Aug 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/main/java/ai/asserts/aws/MetricNameUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ public class MetricNameUtil {
public static final String SCRAPE_LATENCY_METRIC = "aws_exporter_milliseconds";
public static final String ASSERTS_ERROR_TYPE = "asserts_error_type";
public static final String TENANT = "tenant";
public static final String ENV = "asserts_env";
public static final String SITE = "asserts_site";
public static final String SCRAPE_ERROR_COUNT_METRIC = "aws_exporter_error_total";
public static final String SCRAPE_OPERATION_LABEL = "operation";
public static final String SCRAPE_REGION_LABEL = "region";
Expand Down
15 changes: 8 additions & 7 deletions src/main/java/ai/asserts/aws/TaskExecutorUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
*/
package ai.asserts.aws;

import ai.asserts.aws.account.AWSAccount;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.stereotype.Component;
Expand All @@ -21,24 +22,24 @@ public class TaskExecutorUtil {
private final TaskThreadPool taskThreadPool;

private final AWSApiCallRateLimiter rateLimiter;
private static final ThreadLocal<String> tenantName = new ThreadLocal<>();
private static final ThreadLocal<AWSAccount> accountDetails = new ThreadLocal<>();


public TaskExecutorUtil(@Qualifier("aws-api-calls-thread-pool") TaskThreadPool taskThreadPool, AWSApiCallRateLimiter rateLimiter) {
this.taskThreadPool = taskThreadPool;
this.rateLimiter = rateLimiter;
}

public <T> Future<T> executeTenantTask(String tenant, TenantTask<T> task) {
public <T> Future<T> executeAccountTask(AWSAccount accountDetails, TenantTask<T> task) {
return taskThreadPool.getExecutorService().submit(() -> {
tenantName.set(tenant);
TaskExecutorUtil.accountDetails.set(accountDetails);
try {
return rateLimiter.call(task);
} catch (Exception e) {
log.error("Failed to execute tenant task for tenant:" + tenant, e);
log.error("Failed to execute tenant task for tenant:" + accountDetails, e);
return task.getReturnValueWhenError();
} finally {
tenantName.remove();
TaskExecutorUtil.accountDetails.remove();
}
});
}
Expand All @@ -53,7 +54,7 @@ public <K> void awaitAll(List<Future<K>> futures, Consumer<K> consumer) {
});
}

public String getTenant() {
return tenantName.get();
public AWSAccount getAccountDetails() {
return accountDetails.get();
}
}
16 changes: 9 additions & 7 deletions src/main/java/ai/asserts/aws/account/AWSAccount.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import lombok.Builder;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.Setter;
import lombok.ToString;
import lombok.experimental.SuperBuilder;
Expand All @@ -21,22 +22,23 @@
@EqualsAndHashCode
@ToString
@SuperBuilder
@NoArgsConstructor
public class AWSAccount {
@Setter
private String tenant;
// Use different json field name to match property from API Response
@JsonProperty("accountID")
private final String accountId;
private final String name;
private String accountId;
private String name;
@ToString.Exclude
private final String accessId;
private String accessId;
@ToString.Exclude
private final String secretKey;
private String secretKey;
// Use different json field name to match property from API Response
@JsonProperty("assumeRoleARN")
private final String assumeRole;
private final String externalId;
private final boolean paused;
private String assumeRole;
private String externalId;
private boolean paused;
@Builder.Default
private final Set<String> regions = new TreeSet<>();

Expand Down
4 changes: 3 additions & 1 deletion src/main/java/ai/asserts/aws/account/SingleInstanceAccountProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import lombok.AllArgsConstructor;
import lombok.EqualsAndHashCode;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.ToString;
import lombok.experimental.SuperBuilder;
import lombok.extern.slf4j.Slf4j;
Expand Down Expand Up @@ -97,8 +98,8 @@ private Set<AWSAccount> getAccountsInternal() {
AWSAccount ac = new AWSAccount(tenantName, accountId, null, null, null,
regions);
accountRegions.putIfAbsent(ac.getAccountId(), ac);
log.info("Scraping AWS Accounts {}", accountRegions);
}
log.info("Scraping AWS Accounts {}", accountRegions);
return Sets.newHashSet(accountRegions.values());
}

Expand All @@ -111,6 +112,7 @@ String getTenantName() {
@Getter
@SuperBuilder
@ToString
@NoArgsConstructor
public static class CloudwatchConfigs {
List<AWSAccount> cloudWatchConfigs;
}
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/ai/asserts/aws/cloudwatch/alarms/AlarmFetcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ public void update() {
continue;
}
accountRegion.getRegions().forEach(region ->
futures.add(taskExecutorUtil.executeTenantTask(accountRegion.getTenant(),
futures.add(taskExecutorUtil.executeAccountTask(accountRegion,
new CollectionBuilderTask<Sample>() {
@Override
public List<Sample> call() {
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/ai/asserts/aws/cloudwatch/query/MetricQueryProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ Map<String, Map<String, Map<Integer, List<MetricQuery>>>> getQueriesInternal() {
log.info("Will discover metrics and build metric queries for tenant {}, account {}",
accountRegion.getTenant(), accountRegion.getAccountId());
accountRegion.getRegions().forEach(region -> futures.add(
taskExecutorUtil.executeTenantTask(accountRegion.getTenant(), new SimpleTenantTask<Void>() {
taskExecutorUtil.executeAccountTask(accountRegion, new SimpleTenantTask<Void>() {
@Override
public Void call() {
buildQueries(scrapeConfig, region, accountRegion, account, queriesByAccount);
Expand Down
9 changes: 5 additions & 4 deletions src/main/java/ai/asserts/aws/exporter/ApiGatewayToLambdaBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ public void update() {
List<Future<List<Sample>>> futures = new ArrayList<>();
for (AWSAccount accountRegion : accountProvider.getAccounts()) {
accountRegion.getRegions().forEach(region ->
futures.add(taskExecutorUtil.executeTenantTask(accountRegion.getTenant(),
futures.add(taskExecutorUtil.executeAccountTask(accountRegion,
new CollectionBuilderTask<Sample>() {
@Override
public List<Sample> call() {
Expand Down Expand Up @@ -182,18 +182,19 @@ private void captureIntegrations(ApiGatewayClient client, Set<ResourceRelation>
}
Matcher matcher = LAMBDA_URI_PATTERN.matcher(uri);
if (matcher.matches()) {
String tenant = taskExecutorUtil.getAccountDetails().getTenant();
ResourceRelation resourceRelation = ResourceRelation.builder()
.tenant(taskExecutorUtil.getTenant())
.tenant(tenant)
.from(Resource.builder()
.tenant(taskExecutorUtil.getTenant())
.tenant(tenant)
.type(ApiGateway)
.name(restApi.name())
.id(restApi.id())
.region(region)
.account(accountId)
.build())
.to(Resource.builder()
.tenant(taskExecutorUtil.getTenant())
.tenant(tenant)
.type(LambdaFunction)
.name(matcher.group(4))
.region(matcher.group(2))
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/ai/asserts/aws/exporter/DynamoDBExporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ public void update() {
List<Future<List<Sample>>> futures = new ArrayList<>();
accountProvider.getAccounts().forEach(account -> account.getRegions().forEach(region ->
futures.add(
taskExecutorUtil.executeTenantTask(account.getTenant(), new CollectionBuilderTask<Sample>() {
taskExecutorUtil.executeAccountTask(account, new CollectionBuilderTask<Sample>() {
@Override
public List<Sample> call() {
return buildSamples(region, account, allSamples);
Expand Down
11 changes: 6 additions & 5 deletions src/main/java/ai/asserts/aws/exporter/EC2ToEBSVolumeExporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,14 +103,14 @@ public void update() {
List<Future<List<Sample>>> futures = new ArrayList<>();
List<Future<List<ResourceRelation>>> volumeFutures = new ArrayList<>();
accountProvider.getAccounts().forEach(awsAccount -> awsAccount.getRegions().forEach(region -> {
futures.add(taskExecutorUtil.executeTenantTask(awsAccount.getTenant(), new CollectionBuilderTask<Sample>() {
futures.add(taskExecutorUtil.executeAccountTask(awsAccount, new CollectionBuilderTask<Sample>() {
@Override
public List<Sample> call() {
return buildEC2InstanceMetrics(region, awsAccount);
}
}));
volumeFutures.add(
taskExecutorUtil.executeTenantTask(awsAccount.getTenant(),
taskExecutorUtil.executeAccountTask(awsAccount,
new CollectionBuilderTask<ResourceRelation>() {
@Override
public List<ResourceRelation> call() {
Expand Down Expand Up @@ -156,17 +156,18 @@ private List<ResourceRelation> buildResourceRelations(AWSAccount awsAccount, Str
newAttachedVolumes.addAll(resp.volumes().stream()
.flatMap(volume -> volume.attachments().stream())
.map(volumeAttachment -> {
String tenant = taskExecutorUtil.getAccountDetails().getTenant();
Resource ec2Instance = Resource.builder()
.tenant(taskExecutorUtil.getTenant())
.tenant(tenant)
.account(accountId)
.region(region)
.type(EC2Instance)
.name(volumeAttachment.instanceId())
.build();
return ResourceRelation.builder()
.tenant(taskExecutorUtil.getTenant())
.tenant(tenant)
.from(Resource.builder()
.tenant(taskExecutorUtil.getTenant())
.tenant(tenant)
.account(accountId)
.region(region)
.type(EBSVolume)
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/ai/asserts/aws/exporter/ECSTaskProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ public void run() {
for (AWSAccount account : accountProvider.getAccounts()) {
ScrapeConfig scrapeConfig = scrapeConfigProvider.getScrapeConfig(account.getTenant());
for (String region : account.getRegions()) {
taskExecutorUtil.executeTenantTask(account.getTenant(), new SimpleTenantTask<Void>() {
taskExecutorUtil.executeAccountTask(account, new SimpleTenantTask<Void>() {
@Override
public Void call() {
Map<Resource, List<Resource>> clusterWiseNewTasks = new HashMap<>();
Expand Down
5 changes: 3 additions & 2 deletions src/main/java/ai/asserts/aws/exporter/ECSTaskUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,9 +236,10 @@ private LabelsBuilder getLabelsBuilder(AWSAccount account, Resource cluster, Opt

LabelsBuilder labelsBuilder;
taskSubnetMap.computeIfAbsent(taskResource.getName(), k -> getSubnetDetails(task, taskResource));
String tenant = taskExecutorUtil.getAccountDetails().getTenant();
if (service.isPresent()) {
labelsBuilder = Labels.builder()
.tenant(taskExecutorUtil.getTenant())
.tenant(tenant)
.workload(service.get())
.taskId(taskResource.getName())
.pod(service.get() + "-" + taskResource.getName())
Expand All @@ -254,7 +255,7 @@ private LabelsBuilder getLabelsBuilder(AWSAccount account, Resource cluster, Opt
.metricsPath("/metrics");
} else {
labelsBuilder = Labels.builder()
.tenant(taskExecutorUtil.getTenant())
.tenant(tenant)
.workload(taskDefResource.getName())
.taskId(taskResource.getName())
.pod(taskDefResource.getName() + "-" + taskResource.getName())
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/ai/asserts/aws/exporter/EMRExporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ public void update() {
List<Future<List<Sample>>> futures = new ArrayList<>();
accountProvider.getAccounts().forEach(account -> account.getRegions().forEach(region ->
futures.add(
taskExecutorUtil.executeTenantTask(account.getTenant(), new CollectionBuilderTask<Sample>() {
taskExecutorUtil.executeAccountTask(account, new CollectionBuilderTask<Sample>() {
@Override
public List<Sample> call() {
return buildSamples(region, account);
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/ai/asserts/aws/exporter/KinesisAnalyticsExporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ public void update() {
List<Future<List<Sample>>> futures = new ArrayList<>();
accountProvider.getAccounts().forEach(account -> account.getRegions().forEach(region ->
futures.add(
taskExecutorUtil.executeTenantTask(account.getTenant(), new CollectionBuilderTask<Sample>() {
taskExecutorUtil.executeAccountTask(account, new CollectionBuilderTask<Sample>() {
@Override
public List<Sample> call() {
return buildSamples(region, account);
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/ai/asserts/aws/exporter/KinesisFirehoseExporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ public void update() {
List<Future<List<Sample>>> futures = new ArrayList<>();
accountProvider.getAccounts().forEach(account -> account.getRegions().forEach(region ->
futures.add(
taskExecutorUtil.executeTenantTask(account.getTenant(), new CollectionBuilderTask<Sample>() {
taskExecutorUtil.executeAccountTask(account, new CollectionBuilderTask<Sample>() {
@Override
public List<Sample> call() {
return buildMetricSamples(region, account);
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/ai/asserts/aws/exporter/KinesisStreamExporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ public void update() {
List<Sample> allSamples = new ArrayList<>();
List<Future<List<Sample>>> futures = new ArrayList<>();
accountProvider.getAccounts().forEach(account -> account.getRegions().forEach(region ->
futures.add(taskExecutorUtil.executeTenantTask(account.getTenant(), new CollectionBuilderTask<Sample>() {
futures.add(taskExecutorUtil.executeAccountTask(account, new CollectionBuilderTask<Sample>() {
@Override
public List<Sample> call() {
return buildSamples(region, account);
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/ai/asserts/aws/exporter/LBToASGRelationBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ public void updateRouting() {
List<Future<List<Sample>>> futures = new ArrayList<>();
for (AWSAccount accountRegion : accountProvider.getAccounts()) {
accountRegion.getRegions().forEach(region ->
futures.add(taskExecutorUtil.executeTenantTask(accountRegion.getTenant(),
futures.add(taskExecutorUtil.executeAccountTask(accountRegion,
new CollectionBuilderTask<Sample>() {
@Override
public List<Sample> call() {
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/ai/asserts/aws/exporter/LBToECSRoutingBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ public void run() {
Set<ResourceRelation> newRouting = new HashSet<>();
List<Future<Set<ResourceRelation>>> futures = new ArrayList<>();
accountProvider.getAccounts().forEach(awsAccount -> awsAccount.getRegions().forEach(region ->
futures.add(taskExecutorUtil.executeTenantTask(awsAccount.getTenant(),
futures.add(taskExecutorUtil.executeAccountTask(awsAccount,
new SimpleTenantTask<Set<ResourceRelation>>() {
@Override
public Set<ResourceRelation> call() {
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/ai/asserts/aws/exporter/LBToLambdaRoutingBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public Set<ResourceRelation> getRoutings() {
List<Future<Pair<Set<ResourceRelation>, Set<Resource>>>> futures = new ArrayList<>();
for (AWSAccount accountRegion : accountProvider.getAccounts()) {
accountRegion.getRegions().forEach(region ->
futures.add(taskExecutorUtil.executeTenantTask(accountRegion.getTenant(),
futures.add(taskExecutorUtil.executeAccountTask(accountRegion,
new SimpleTenantTask<Pair<Set<ResourceRelation>, Set<Resource>>>() {
@Override
public Pair<Set<ResourceRelation>, Set<Resource>> call() {
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/ai/asserts/aws/exporter/LambdaCapacityExporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,8 @@ private List<MetricFamilySamples> getMetrics() {
String account = accountRegion.getAccountId();
Map<String, Map<String, LambdaFunction>> byRegion = byAccountByRegion.getOrDefault(account,
Collections.emptyMap());
byRegion.forEach((region, functions) -> futures.add(taskExecutorUtil.executeTenantTask(
accountRegion.getTenant(), new SimpleTenantTask<Map<String, List<Sample>>>() {
byRegion.forEach((region, functions) -> futures.add(taskExecutorUtil.executeAccountTask(
accountRegion, new SimpleTenantTask<Map<String, List<Sample>>>() {
@Override
public Map<String, List<Sample>> call() {
return buildSamples(region, accountRegion, account, accountLimitMetric,
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/ai/asserts/aws/exporter/LambdaEventSourceExporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ private List<MetricFamilySamples> getMappings() {
ScrapeConfig scrapeConfig = scrapeConfigProvider.getScrapeConfig(accountRegion.getTenant());
Optional<NamespaceConfig> lambdaConfig = scrapeConfig.getLambdaConfig();
lambdaConfig.ifPresent(namespaceConfig -> accountRegion.getRegions().forEach(region ->
futures.add(taskExecutorUtil.executeTenantTask(accountRegion.getTenant(),
futures.add(taskExecutorUtil.executeAccountTask(accountRegion,
new SimpleTenantTask<Map<String, List<Sample>>>() {
@Override
public Map<String, List<Sample>> call() {
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/ai/asserts/aws/exporter/LambdaInvokeConfigExporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ List<MetricFamilySamples> getInvokeConfigs() {
Map<String, Map<String, LambdaFunction>> byRegion =
new ConcurrentHashMap<>(byAccountByRegion.get(account));
byRegion.forEach((region, byARN) ->
futures.add(taskExecutorUtil.executeTenantTask(accountRegion.getTenant(),
futures.add(taskExecutorUtil.executeAccountTask(accountRegion,
new CollectionBuilderTask<Sample>() {
@Override
public List<Sample> call() {
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/ai/asserts/aws/exporter/LoadBalancerExporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ public void update() {
List<Future<Pair<List<Sample>, List<Sample>>>> futures = new ArrayList<>();
accountProvider.getAccounts().forEach(awsAccount -> awsAccount.getRegions().forEach(region -> {
ScrapeConfig scrapeConfig = scrapeConfigProvider.getScrapeConfig(awsAccount.getTenant());
futures.add(taskExecutorUtil.executeTenantTask(awsAccount.getTenant(),
futures.add(taskExecutorUtil.executeAccountTask(awsAccount,
new SimpleTenantTask<Pair<List<Sample>, List<Sample>>>() {
@Override
public Pair<List<Sample>, List<Sample>> call() {
Expand Down
Loading
Loading