Skip to content
This repository has been archived by the owner on Feb 23, 2024. It is now read-only.

Commit

Permalink
When extracting single API only extract named values used the API (#558)
Browse files Browse the repository at this point in the history
  • Loading branch information
@GRichards05
GRichards05 authored Aug 9, 2021
1 parent 834617a commit 7911271
Show file tree
Hide file tree
Showing 4 changed files with 146 additions and 23 deletions.
64 changes: 64 additions & 0 deletions src/APIM_ARMTemplate/apimtemplate/Extractor/EntityExtractors/BackendExtractor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,5 +179,69 @@ public bool DoesBackendReferenceNamedValue(TemplateResource namedValueResource,
|| namedValue == backendTemplateResource.properties.description
|| namedValue == backendTemplateResource.properties.title);
}

public async Task<bool> IsNamedValueUsedInBackends(string apimname, string resourceGroup, string singleApiName, List<TemplateResource> apiTemplateResources, Extractor exc, string propertyName, string propertyDisplayName)
{
// isolate api and operation policy resources in the case of a single api extraction, as they may reference backends
var policyResources = apiTemplateResources.Where(resource => (resource.type == ResourceTypeConstants.APIPolicy || resource.type == ResourceTypeConstants.APIOperationPolicy || resource.type == ResourceTypeConstants.ProductPolicy));
var emptyNamedValueResources = new List<TemplateResource>();

// pull all backends for service
JObject oBackends = new JObject();
int skipNumberOfBackends = 0;

do
{
string backends = await GetBackendsAsync(apimname, resourceGroup, skipNumberOfBackends);
oBackends = JObject.Parse(backends);

foreach (var item in oBackends["value"])
{
var content = item.ToString();

// check if backend references the named value, credentials for example
if (content.Contains(string.Concat("{{", propertyName, "}}")) || content.Contains(string.Concat("{{", propertyDisplayName, "}}")))
{
//only true if this is a full extraction, or in the case of a single api, if it is referenced by one of the API policies
if (singleApiName == null)
{
return true;
}
else
{
// is this backend related to the single api?
// is backend used in the extracted policies for this API
// if backend id is referenced in policy
// or a named value is referenced in policy to a backend, we have already checked the policy for named value.

// check if this backend is used by any of the policies extracted
string backendName = ((JValue)item["name"]).Value.ToString();
string backend = await GetBackendDetailsAsync(apimname, resourceGroup, backendName);

// convert returned backend to template resource class
BackendTemplateResource backendTemplateResource = JsonConvert.DeserializeObject<BackendTemplateResource>(backend);

// we have already checked if the named value is used in a policy, we just need to confirm if the backend is referenced by this single api within the policy file
// this is why an empty named values must be passed to this method for validation
foreach (PolicyTemplateResource policyTemplateResource in policyResources)
{
string policyContent = ExtractorUtils.GetPolicyContent(exc, policyTemplateResource);

if (DoesPolicyReferenceBackend(policyContent, emptyNamedValueResources, backendName, backendTemplateResource))
{
// dont need to go through all policies and backends if the named values has already been found
return true;
}
}
}
}
}

skipNumberOfBackends += GlobalConstants.NumOfRecords;
}
while (oBackends["nextLink"] != null);

return false;
}
}
}
40 changes: 26 additions & 14 deletions src/APIM_ARMTemplate/apimtemplate/Extractor/EntityExtractors/MasterTemplateExtractor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -440,7 +440,8 @@ public string[] GenerateAPIRevisionDependencies(string curRevTemplate)
public async Task<Template> CreateMasterTemplateParameterValues(List<string> apisToExtract, Extractor exc,
Dictionary<string, object> apiLoggerId,
Dictionary<string, string> loggerResourceIds,
Dictionary<string, BackendApiParameters> backendParams)
Dictionary<string, BackendApiParameters> backendParams,
List<TemplateResource> propertyResources)
{
// used to create the parameter values for use in parameters file
// create empty template
Expand Down Expand Up @@ -523,14 +524,20 @@ public async Task<Template> CreateMasterTemplateParameterValues(List<string> api
{
JToken oProperty = JObject.Parse(extractedProperty);
string propertyName = ((JValue)oProperty["name"]).Value.ToString();
string fullPropertyResource = await pExc.GetPropertyDetailsAsync(exc.sourceApimName, exc.resourceGroup, propertyName);
PropertyTemplateResource propertyTemplateResource = JsonConvert.DeserializeObject<PropertyTemplateResource>(fullPropertyResource);
//Only add the property if it is not controlled by keyvault
if (propertyTemplateResource?.properties.keyVault == null)

// check if the property has been extracted as it is being used in a policy or backend
if (propertyResources.Count(item => item.name.Contains(propertyName)) > 0)
{
string propertyValue = propertyTemplateResource.properties.value;
string validPName = ExtractorUtils.GenValidParamName(propertyName, ParameterPrefix.Property);
namedValues.Add(validPName, propertyValue);
string fullPropertyResource = await pExc.GetPropertyDetailsAsync(exc.sourceApimName, exc.resourceGroup, propertyName);
PropertyTemplateResource propertyTemplateResource = JsonConvert.DeserializeObject<PropertyTemplateResource>(fullPropertyResource);

//Only add the property if it is not controlled by keyvault
if (propertyTemplateResource?.properties.keyVault == null)
{
string propertyValue = propertyTemplateResource.properties.value;
string validPName = ExtractorUtils.GenValidParamName(propertyName, ParameterPrefix.Property);
namedValues.Add(validPName, propertyValue);
}
}
}
TemplateObjectParameterProperties namedValueProperties = new TemplateObjectParameterProperties()
Expand All @@ -549,13 +556,18 @@ public async Task<Template> CreateMasterTemplateParameterValues(List<string> api
{
JToken oProperty = JObject.Parse(extractedProperty);
string propertyName = ((JValue)oProperty["name"]).Value.ToString();
string fullPropertyResource = await pExc.GetPropertyDetailsAsync(exc.sourceApimName, exc.resourceGroup, propertyName);
PropertyTemplateResource propertyTemplateResource = JsonConvert.DeserializeObject<PropertyTemplateResource>(fullPropertyResource);
if (propertyTemplateResource?.properties.keyVault != null)

// check if the property has been extracted as it is being used in a policy or backend
if (propertyResources.Count(item => item.name.Contains(propertyName)) > 0)
{
string propertyValue = propertyTemplateResource.properties.keyVault.secretIdentifier;
string validPName = ExtractorUtils.GenValidParamName(propertyName, ParameterPrefix.Property);
keyVaultNamedValues.Add(validPName, propertyValue);
string fullPropertyResource = await pExc.GetPropertyDetailsAsync(exc.sourceApimName, exc.resourceGroup, propertyName);
PropertyTemplateResource propertyTemplateResource = JsonConvert.DeserializeObject<PropertyTemplateResource>(fullPropertyResource);
if (propertyTemplateResource?.properties.keyVault != null)
{
string propertyValue = propertyTemplateResource.properties.keyVault.secretIdentifier;
string validPName = ExtractorUtils.GenValidParamName(propertyName, ParameterPrefix.Property);
keyVaultNamedValues.Add(validPName, propertyValue);
}
}
}
TemplateObjectParameterProperties keyVaultNamedValueProperties = new TemplateObjectParameterProperties()
Expand Down
57 changes: 52 additions & 5 deletions src/APIM_ARMTemplate/apimtemplate/Extractor/EntityExtractors/PropertyExtractor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ public async Task<string> GetPropertyDetailsAsync(string ApiManagementName, stri
return await CallApiManagementAsync(azToken, requestUrl);
}

public async Task<Template> GenerateNamedValuesTemplateAsync(string singleApiName, List<TemplateResource> apiTemplateResources, Extractor exc)
public async Task<Template> GenerateNamedValuesTemplateAsync(string singleApiName, List<TemplateResource> apiTemplateResources, Extractor exc, BackendExtractor backendExtractor, List<TemplateResource> loggerTemplateResources)
{
Template armTemplate = GenerateEmptyPropertyTemplateWithParameters();

Expand Down Expand Up @@ -82,6 +82,9 @@ public async Task<Template> GenerateNamedValuesTemplateAsync(string singleApiNam
// pull all named values (properties) for service
string[] properties = await GetPropertiesAsync(exc.sourceApimName, exc.resourceGroup);

// isolate api and operation policy resources in the case of a single api extraction, as they may reference named value
var policyResources = apiTemplateResources.Where(resource => (resource.type == ResourceTypeConstants.APIPolicy || resource.type == ResourceTypeConstants.APIOperationPolicy || resource.type == ResourceTypeConstants.ProductPolicy));

foreach (var extractedProperty in properties)
{
JToken oProperty = JObject.Parse(extractedProperty);
Expand Down Expand Up @@ -119,14 +122,58 @@ public async Task<Template> GenerateNamedValuesTemplateAsync(string singleApiNam
}
else
{
// TODO - if the user is executing a single api, extract all the named values used in the template resources
Console.WriteLine("'{0}' Named value found", propertyName);
templateResources.Add(propertyTemplateResource);
};
// if the user is executing a single api, extract all the named values used in the template resources
bool foundInPolicy = DoesPolicyReferenceNamedValue(exc, policyResources, propertyName, propertyTemplateResource);
bool foundInBackEnd = await backendExtractor.IsNamedValueUsedInBackends(exc.sourceApimName, exc.resourceGroup, singleApiName, apiTemplateResources, exc, propertyName, propertyTemplateResource.properties.displayName);
bool foundInLogger = DoesLoggerReferenceNamedValue(loggerTemplateResources, propertyName, propertyTemplateResource);

// check if named value is referenced in a backend
if (foundInPolicy || foundInBackEnd || foundInLogger)
{
// named value was used in policy, extract it
Console.WriteLine("'{0}' Named value found", propertyName);
templateResources.Add(propertyTemplateResource);
}
}
}

armTemplate.resources = templateResources.ToArray();
return armTemplate;
}

private bool DoesPolicyReferenceNamedValue(Extractor exc, IEnumerable<TemplateResource> policyResources, string propertyName, PropertyTemplateResource propertyTemplateResource)
{
// check if named value is referenced in a policy file
foreach (PolicyTemplateResource policyTemplateResource in policyResources)
{
string policyContent = ExtractorUtils.GetPolicyContent(exc, policyTemplateResource);

if (policyContent.Contains(string.Concat("{{", propertyTemplateResource.properties.displayName, "}}")) || policyContent.Contains(string.Concat("{{", propertyName, "}}")))
{
// dont need to go through all policies if the named value has already been found
return true;
}
}
return false;
}

private bool DoesLoggerReferenceNamedValue(IEnumerable<TemplateResource> loggerTemplateResources, string propertyName, PropertyTemplateResource propertyTemplateResource)
{
foreach(LoggerTemplateResource logger in loggerTemplateResources)
{
if (logger.properties.credentials != null)
{
if ((!string.IsNullOrEmpty(logger.properties.credentials.connectionString) && logger.properties.credentials.connectionString.Contains(propertyName)) ||
(!string.IsNullOrEmpty(logger.properties.credentials.instrumentationKey) && logger.properties.credentials.instrumentationKey.Contains(propertyName)) ||
(!string.IsNullOrEmpty(logger.properties.credentials.connectionString) && logger.properties.credentials.connectionString.Contains(propertyTemplateResource.properties.displayName)) ||
(!string.IsNullOrEmpty(logger.properties.credentials.instrumentationKey) && logger.properties.credentials.instrumentationKey.Contains(propertyTemplateResource.properties.displayName)))
{
// dont need to go through all loggers if the named value has already been found
return true;
}
}
}
return false;
}
}
}
8 changes: 4 additions & 4 deletions src/APIM_ARMTemplate/apimtemplate/Extractor/Utilities/ExtractorUtils.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,22 +107,22 @@ public static async Task GenerateTemplates(
Template productAPITemplate = await productAPIExtractor.GenerateAPIProductsARMTemplateAsync(singleApiName, multipleApiNames, exc);
Template apiTagTemplate = await apiTagExtractor.GenerateAPITagsARMTemplateAsync(singleApiName, multipleApiNames, exc);
List<TemplateResource> productTemplateResources = productTemplate.resources.ToList();
Template namedValueTemplate = await propertyExtractor.GenerateNamedValuesTemplateAsync(singleApiName, apiTemplateResources, exc);
List<TemplateResource> loggerResources = loggerTemplate.resources.ToList();
Template namedValueTemplate = await propertyExtractor.GenerateNamedValuesTemplateAsync(singleApiName, apiTemplateResources, exc, backendExtractor, loggerResources);
Template tagTemplate = await tagExtractor.GenerateTagsTemplateAsync(sourceApim, resourceGroup, singleApiName, apiTemplateResources, productTemplateResources, policyXMLBaseUrl, policyXMLSasToken);
List<TemplateResource> namedValueResources = namedValueTemplate.resources.ToList();

Tuple<Template, Dictionary<string, BackendApiParameters>> backendResult = await backendExtractor.GenerateBackendsARMTemplateAsync(sourceApim, resourceGroup, singleApiName, apiTemplateResources, namedValueResources, exc);

Dictionary<string, string> loggerResourceIds = null;
if (exc.paramLogResourceId)
{
List<TemplateResource> loggerResources = loggerTemplate.resources.ToList();
{
loggerResourceIds = loggerExtractor.GetAllLoggerResourceIds(loggerResources);
loggerTemplate = loggerExtractor.SetLoggerResourceId(loggerTemplate);
}

// create parameters file
Template templateParameters = await masterTemplateExtractor.CreateMasterTemplateParameterValues(apisToExtract, exc, apiLoggerId, loggerResourceIds, backendResult.Item2);
Template templateParameters = await masterTemplateExtractor.CreateMasterTemplateParameterValues(apisToExtract, exc, apiLoggerId, loggerResourceIds, backendResult.Item2, namedValueResources);

// write templates to output file location
string apiFileName = fileNameGenerator.GenerateExtractorAPIFileName(singleApiName, fileNames.baseFileName);
Expand Down

0 comments on commit 7911271

Please sign in to comment.