Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Emit stats for the teamcity file share #63

Merged
merged 2 commits into from
Feb 13, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions source/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ public static async Task Main(string[] args)
services.AddHostedService<TeamCityBuildArtifactScraper>();
services.AddHostedService<TeamCityQueueWaitScraper>();
services.AddHostedService<TeamCityMutedTestsScraper>();
services.AddHostedService<TeamCityDiskSpaceScraper>();
})
.UseConsoleLifetime()
.Build();
Expand Down
87 changes: 87 additions & 0 deletions source/Scrapers/TeamCityDiskSpaceScraper.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
#nullable enable
using System;
using System.Threading;
using System.Threading.Tasks;
using Azure.Identity;
using Azure.ResourceManager;
using Azure.ResourceManager.Resources;
using Azure.ResourceManager.Storage;
using Microsoft.Extensions.Configuration;
using Prometheus.Client;
using Serilog;

namespace TeamCityBuildStatsScraper.Scrapers
{
record StorageStatistics(long TotalCapacity, long UsedCapacity)
{
public long AvailableCapacity => TotalCapacity - UsedCapacity;
}

class TeamCityDiskSpaceScraper : BackgroundService
{
readonly IMetricFactory metricFactory;
readonly IConfiguration configuration;
readonly ILogger logger;

public TeamCityDiskSpaceScraper(IMetricFactory metricFactory, IConfiguration configuration, ILogger logger)
: base(logger.ForContext("Scraper", nameof(TeamCityDiskSpaceScraper)))
{
this.metricFactory = metricFactory;
this.configuration = configuration;
this.logger = logger;
}

protected override TimeSpan DelayBetweenScrapes => TimeSpan.FromMinutes(5);

protected override async Task Scrape(CancellationToken cancellationToken)
{
var stats = await RetrieveFileStorageMetrics(cancellationToken);

var totalCapacityGauge = metricFactory.CreateGauge("build_storage_total_capacity", "Total capacity of the file share");
var usedCapacityGauge = metricFactory.CreateGauge("build_storage_used_capacity", "Used capacity of the file share");
var availableCapacityGauge = metricFactory.CreateGauge("build_storage_available_capacity", "Available capacity on the file share");

totalCapacityGauge.Set(stats?.TotalCapacity ?? 0);
usedCapacityGauge.Set(stats?.UsedCapacity ?? 0);
availableCapacityGauge.Set(stats?.AvailableCapacity ?? 0);

Logger.Debug("TeamCity Disk Space - Total Capacity {TotalCapacity}, Used Capacity {UsedCapacity}, Available Capacity {AvailableCapacity}",
stats?.TotalCapacity,
stats?.UsedCapacity,
stats?.AvailableCapacity);
}

async Task<StorageStatistics?> RetrieveFileStorageMetrics(CancellationToken cancellationToken)
{
logger.Verbose("Retrieving Azure file storage metrics");

try
{
var resourceGroupName = configuration.GetValue<string>("AZURE_FILE_SHARE_RESOURCE_GROUP_NAME");
var storageAccountName = configuration.GetValue<string>("AZURE_FILE_SHARE_STORAGE_ACCOUNT_NAME");
var subscriptionId = configuration.GetValue<string>("AZURE_FILE_SHARE_SUBSCRIPTION_ID");
var shareName = configuration.GetValue<string>("AZURE_FILE_SHARE_STORAGE_SHARE_NAME");

var client = new ArmClient(new DefaultAzureCredential());
var subscription = await client.GetSubscriptions().GetAsync(subscriptionId, cancellationToken);
var resourceGroups = subscription.Value.GetResourceGroups();
ResourceGroupResource resourceGroup = await resourceGroups.GetAsync(resourceGroupName, cancellationToken);
StorageAccountResource storageAccount = await resourceGroup.GetStorageAccountAsync(storageAccountName, cancellationToken: cancellationToken);

var fileService = storageAccount.GetFileService();
FileShareResource fileShare = await fileService.GetFileShareAsync(shareName, expand:"stats", cancellationToken: cancellationToken);
var shareProperties = fileShare.Data;
var totalCapacityInBytes = shareProperties.ShareQuota.GetValueOrDefault() * 1024L * 1024L * 1024L;
var shareUsageInBytes = shareProperties.ShareUsageBytes.GetValueOrDefault();

return new StorageStatistics(totalCapacityInBytes, shareUsageInBytes);
}
catch (Exception ex)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How should this behave if we forget to configure the environment values above?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it will return null, and then we'll get 0's coming out of it, which I think we could alert on.
I considered going 💥, but thought this was wiser

{
logger.Warning(ex, "Failed to get Azure file storage metrics");
}
return null;
}
}

}
2 changes: 2 additions & 0 deletions source/TeamCityBuildStatsScraper.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Azure.Identity" Version="1.10.4" />
<PackageReference Include="Azure.ResourceManager.Storage" Version="1.2.0" />
<PackageReference Include="Polly" Version="8.3.0" />
<PackageReference Include="Prometheus.Client" Version="5.2.0" />
<PackageReference Include="Prometheus.Client.DependencyInjection" Version="1.3.0" />
Expand Down