Skip to content

Commit

Permalink
Initial pass at adding domain support for Pipeline Artifacts
Browse files Browse the repository at this point in the history
  • Loading branch information
Brian Barthel committed Aug 4, 2023
1 parent 5a8ed1f commit d488bbc
Show file tree
Hide file tree
Showing 5 changed files with 222 additions and 157 deletions.
1 change: 1 addition & 0 deletions src/Agent.Plugins/Artifact/PipelineArtifactConstants.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,5 +18,6 @@ public class PipelineArtifactConstants
public const string FileShareArtifact = "filepath";
public const string CustomPropertiesPrefix = "user-";
public const string HashType = "HashType";
public const string DomainId = "DomainId";
}
}
118 changes: 77 additions & 41 deletions src/Agent.Plugins/Artifact/PipelineArtifactProvider.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
using Microsoft.VisualStudio.Services.WebApi;
using Microsoft.VisualStudio.Services.Content.Common;
using Microsoft.VisualStudio.Services.BlobStore.Common;
using Microsoft.VisualStudio.Services.BlobStore.Common.Telemetry;

namespace Agent.Plugins
{
Expand All @@ -37,12 +38,19 @@ public async Task DownloadSingleArtifactAsync(
CancellationToken cancellationToken,
AgentTaskPluginExecutionContext context)
{
// if properties doesn't have it, use the default domain for backward compatibility
IDomainId domainId = WellKnownDomainIds.DefaultDomainId;
if(buildArtifact.Resource.Properties.TryGetValue(PipelineArtifactConstants.DomainId, out string domainIdString))
{
domainId = DomainIdFactory.Create(domainIdString);
}

var (dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClientAsync(
this.context.IsSystemDebugTrue(),
(str) => this.context.Output(str),
this.connection,
DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context),
WellKnownDomainIds.DefaultDomainId,
domainId,
Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.PipelineArtifact,
context,
cancellationToken);
Expand Down Expand Up @@ -85,49 +93,77 @@ public async Task DownloadMultipleArtifactsAsync(
CancellationToken cancellationToken,
AgentTaskPluginExecutionContext context)
{
var (dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClientAsync(
this.context.IsSystemDebugTrue(),
(str) => this.context.Output(str),
this.connection,
DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context),
WellKnownDomainIds.DefaultDomainId,
Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.PipelineArtifact,
context,
cancellationToken);
// create clients and group artifacts for each domain:
Dictionary<IDomainId, (DedupManifestArtifactClient Client, BlobStoreClientTelemetry Telemetry, Dictionary<string, DedupIdentifier> ArtifactDictionary)> dedupManifestClients =
new Dictionary<IDomainId, (DedupManifestArtifactClient Client, BlobStoreClientTelemetry Telemetry, Dictionary<string, DedupIdentifier> ArtifactDictionary)>();

using (clientTelemetry)
foreach(var buildArtifact in buildArtifacts)
{
// if properties doesn't have it, use the default domain for backward compatibility
IDomainId domainId = WellKnownDomainIds.DefaultDomainId;
if(buildArtifact.Resource.Properties.TryGetValue(PipelineArtifactConstants.DomainId, out string domainIdString))
{
domainId = DomainIdFactory.Create(domainIdString);
}
// Have we already created the clients for this domain?
if(dedupManifestClients.ContainsKey(domainId)) {
// Clients already created for this domain, Just add the artifact to the list:
dedupManifestClients[domainId].ArtifactDictionary.Add(buildArtifact.Name, DedupIdentifier.Create(buildArtifact.Resource.Data));
}
else
{
// create the clients:
var (dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance.CreateDedupManifestClientAsync(
this.context.IsSystemDebugTrue(),
(str) => this.context.Output(str),
this.connection,
DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context),
WellKnownDomainIds.DefaultDomainId,
Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.PipelineArtifact,
context,
cancellationToken);

// and create the artifact dictionary with the current artifact
var artifactDictionary = new Dictionary<string, DedupIdentifier>();
artifactDictionary.Add(buildArtifact.Name, DedupIdentifier.Create(buildArtifact.Resource.Data));

dedupManifestClients.Add(domainId, (dedupManifestClient, clientTelemetry, artifactDictionary));
}
}
foreach(var clientInfo in dedupManifestClients.Values)
{
var artifactNameAndManifestIds = buildArtifacts.ToDictionary(
keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong
elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data));
// 2) download to the target path
var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds(
artifactNameAndManifestIds,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters,
minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName);
using (clientInfo.Telemetry)
{
// 2) download to the target path
var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds(
clientInfo.ArtifactDictionary,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters,
minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName);

PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord<PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context));
await clientTelemetry.MeasureActionAsync(
record: downloadRecord,
actionAsync: async () =>
{
await AsyncHttpRetryHelper.InvokeVoidAsync(
async () =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true,
context: nameof(DownloadMultipleArtifactsAsync),
cancellationToken: cancellationToken,
continueOnCapturedContext: false);
});
// Send results to CustomerIntelligence
this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
PipelineArtifactActionRecord downloadRecord = clientInfo.Telemetry.CreateRecord<PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadMultipleArtifactsAsync), this.context));

await clientInfo.Telemetry.MeasureActionAsync(
record: downloadRecord,
actionAsync: async () =>
{
await AsyncHttpRetryHelper.InvokeVoidAsync(
async () =>
{
await clientInfo.Client.DownloadAsync(options, cancellationToken);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true,
context: nameof(DownloadMultipleArtifactsAsync),
cancellationToken: cancellationToken,
continueOnCapturedContext: false);
});
// Send results to CustomerIntelligence
this.context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
}
}
}
}
Expand Down
103 changes: 20 additions & 83 deletions src/Agent.Plugins/Artifact/PipelineArtifactServer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,26 @@ internal async Task UploadAsync(
IDictionary<string, string> properties,
CancellationToken cancellationToken)
{
// Get the client settings, if any:
var tracer = DedupManifestArtifactClientFactory.CreateArtifactsTracer(false, (str) => context.Output(str));
VssConnection connection = context.VssConnection;
var (dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance
.CreateDedupManifestClientAsync(
var clientSettings = await DedupManifestArtifactClientFactory.GetClientSettingsAsync(
connection,
Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.PipelineArtifact,
tracer,
cancellationToken);

// Get the default domain to use:
IDomainId domainId = DedupManifestArtifactClientFactory.GetDefaultDomainId(clientSettings, tracer);

var (dedupManifestClient, clientTelemetry) = DedupManifestArtifactClientFactory.Instance
.CreateDedupManifestClient(
context.IsSystemDebugTrue(),
(str) => context.Output(str),
connection,
DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context),
WellKnownDomainIds.DefaultDomainId,
Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.PipelineArtifact,
domainId,
clientSettings,
context,
cancellationToken);

Expand Down Expand Up @@ -84,7 +95,8 @@ internal async Task UploadAsync(
{ PipelineArtifactConstants.RootId, result.RootId.ValueString },
{ PipelineArtifactConstants.ProofNodes, StringUtil.ConvertToJson(result.ProofNodes.ToArray()) },
{ PipelineArtifactConstants.ArtifactSize, result.ContentSize.ToString() },
{ PipelineArtifactConstants.HashType, dedupManifestClient.HashType.Serialize() }
{ PipelineArtifactConstants.HashType, dedupManifestClient.HashType.Serialize() },
{ PipelineArtifactConstants.DomainId, domainId.Serialize() }
};

BuildArtifact buildArtifact = await AsyncHttpRetryHelper.InvokeAsync(
Expand Down Expand Up @@ -140,22 +152,11 @@ internal async Task DownloadAsync(
CancellationToken cancellationToken)
{
VssConnection connection = context.VssConnection;
var (dedupManifestClient, clientTelemetry) = await DedupManifestArtifactClientFactory.Instance
.CreateDedupManifestClientAsync(
context.IsSystemDebugTrue(),
(str) => context.Output(str),
connection,
DedupManifestArtifactClientFactory.Instance.GetDedupStoreClientMaxParallelism(context),
WellKnownDomainIds.DefaultDomainId,
Microsoft.VisualStudio.Services.BlobStore.WebApi.Contracts.Client.PipelineArtifact,
context,
cancellationToken);
PipelineArtifactProvider provider = new PipelineArtifactProvider(context, connection, tracer);

BuildServer buildServer = new(connection);

using (clientTelemetry)
// download all pipeline artifacts if artifact name is missing
{
if (downloadOptions == DownloadOptions.MultiDownload)
{
List<BuildArtifact> artifacts;
Expand Down Expand Up @@ -186,41 +187,7 @@ internal async Task DownloadAsync(
}
else
{
context.Output(StringUtil.Loc("DownloadingMultiplePipelineArtifacts", pipelineArtifacts.Count()));

var artifactNameAndManifestIds = pipelineArtifacts.ToDictionary(
keySelector: (a) => a.Name, // keys should be unique, if not something is really wrong
elementSelector: (a) => DedupIdentifier.Create(a.Resource.Data));
// 2) download to the target path
var options = DownloadDedupManifestArtifactOptions.CreateWithMultiManifestIds(
artifactNameAndManifestIds,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters,
minimatchFilterWithArtifactName: downloadParameters.MinimatchFilterWithArtifactName,
customMinimatchOptions: downloadParameters.CustomMinimatchOptions);

PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord<PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context));
await clientTelemetry.MeasureActionAsync(
record: downloadRecord,
actionAsync: async () =>
{
await AsyncHttpRetryHelper.InvokeVoidAsync(
async () =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true,
context: nameof(DownloadAsync),
cancellationToken: cancellationToken,
continueOnCapturedContext: false);
});

// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
await provider.DownloadMultipleArtifactsAsync(downloadParameters,artifacts, cancellationToken, context);
}
}
else if (downloadOptions == DownloadOptions.SingleDownload)
Expand All @@ -246,42 +213,12 @@ await AsyncHttpRetryHelper.InvokeVoidAsync(
{
throw new InvalidOperationException($"Invalid {nameof(downloadParameters.ProjectRetrievalOptions)}!");
}

var manifestId = DedupIdentifier.Create(buildArtifact.Resource.Data);
var options = DownloadDedupManifestArtifactOptions.CreateWithManifestId(
manifestId,
downloadParameters.TargetDirectory,
proxyUri: null,
minimatchPatterns: downloadParameters.MinimatchFilters,
customMinimatchOptions: downloadParameters.CustomMinimatchOptions);

PipelineArtifactActionRecord downloadRecord = clientTelemetry.CreateRecord<PipelineArtifactActionRecord>((level, uri, type) =>
new PipelineArtifactActionRecord(level, uri, type, nameof(DownloadAsync), context));
await clientTelemetry.MeasureActionAsync(
record: downloadRecord,
actionAsync: async () =>
{
await AsyncHttpRetryHelper.InvokeVoidAsync(
async () =>
{
await dedupManifestClient.DownloadAsync(options, cancellationToken);
},
maxRetries: 3,
tracer: tracer,
canRetryDelegate: e => true,
context: nameof(DownloadAsync),
cancellationToken: cancellationToken,
continueOnCapturedContext: false);
});

// Send results to CustomerIntelligence
context.PublishTelemetry(area: PipelineArtifactConstants.AzurePipelinesAgent, feature: PipelineArtifactConstants.PipelineArtifact, record: downloadRecord);
await provider.DownloadSingleArtifactAsync(downloadParameters, buildArtifact, cancellationToken, context);
}
else
{
throw new InvalidOperationException($"Invalid {nameof(downloadOptions)}!");
}
}
}

// Download for version 2. This decision was made because version 1 is sealed and we didn't want to break any existing customers.
Expand Down
Loading

0 comments on commit d488bbc

Please sign in to comment.