Extractor features (#101)

* add schemas to readme

* cleanup readme, include space for extractor

* fix table of contents routing

* extract diagnostic resource and add to template

* update to version 2018-06-01-preview, update creator config values

* remove name property from creator diagnostic

* extract loggers, enable logger and diagnostic iterative deployment

* update creator readme

* pull named values and substitute them for previously hidden credentials

* add validation on additional creator config properties, write tests

* refactor - update file structure, namespaces, remove dup classes,

* comment out representations on req/res operation schema until schema templates are extracted

* Odaibert/operations policies and products (#100)

* Added Products extractor to [APIM-NAME]-products.json file

* Finished:
- [Extractor] Relationship/Association between APIs and products are not extracted #85
- [Extractor] Allow users to specify if they want to extract all APIs or a single API (using apiId) #78
- [Extractor] Extract policies at the operation level #80
- [Extractor] Split APIs into multiple API templates #77

src/APIM_ARMTemplate/.gitignore

@@ -37,3 +37,6 @@ msbuild.wrn
 
 # Visual Studio 2015
 .vs/
+
+# Local Testing
+*validTesting.yml

src/APIM_ARMTemplate/apimtemplate.test/CmdLine/CreateTests.cs

@@ -2,8 +2,9 @@
 using Xunit;
 using McMaster.Extensions.CommandLineUtils;
 using System.IO;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Create;
 
-namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates
+namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Test
 {
     public class CreateTests
     {
@@ -110,5 +111,41 @@ public void ShouldFailWithInvalidLinking()
             var ex = Assert.ThrowsAny<CommandParsingException>(() => createCommand.Execute(args));
             Assert.Contains("LinkTemplatesBaseUrl is required for linked templates", ex.Message);
         }
+
+        [Fact]
+        public void ShouldFailWithInvalidVersionSetDisplayName()
+        {
+            var createCommand = new CreateCommand();
+            string[] args = new string[] { "--configFile", String.Concat(this.configExamplesFolder, "invalidVersionSetDisplayName.yml") };
+            var ex = Assert.ThrowsAny<CommandParsingException>(() => createCommand.Execute(args));
+            Assert.Contains("Display name is required if an API Version Set is provided", ex.Message);
+        }
+
+        [Fact]
+        public void ShouldFailWithInvalidVersionSetVersioningScheme()
+        {
+            var createCommand = new CreateCommand();
+            string[] args = new string[] { "--configFile", String.Concat(this.configExamplesFolder, "invalidVersionSetVersioningScheme.yml") };
+            var ex = Assert.ThrowsAny<CommandParsingException>(() => createCommand.Execute(args));
+            Assert.Contains("Versioning scheme is required if an API Version Set is provided", ex.Message);
+        }
+
+        [Fact]
+        public void ShouldFailWithInvalidOperationPolicy()
+        {
+            var createCommand = new CreateCommand();
+            string[] args = new string[] { "--configFile", String.Concat(this.configExamplesFolder, "invalidOperationPolicy.yml") };
+            var ex = Assert.ThrowsAny<CommandParsingException>(() => createCommand.Execute(args));
+            Assert.Contains("Policy XML is required if an API operation is provided", ex.Message);
+        }
+
+        [Fact]
+        public void ShouldFailWithInvalidDiagnosticLoggerId()
+        {
+            var createCommand = new CreateCommand();
+            string[] args = new string[] { "--configFile", String.Concat(this.configExamplesFolder, "invalidDiagnosticLoggerId.yml") };
+            var ex = Assert.ThrowsAny<CommandParsingException>(() => createCommand.Execute(args));
+            Assert.Contains("LoggerId is required if an API diagnostic is provided", ex.Message);
+        }
     }
 }

src/APIM_ARMTemplate/apimtemplate.test/CmdLine/ExtractTests.cs

@@ -3,7 +3,7 @@
 using McMaster.Extensions.CommandLineUtils;
 
 
-namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates
+namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Test
 {
     public class ExtractTests
     {

src/APIM_ARMTemplate/apimtemplate.test/Creator/FileHandlerTests/FileWriterTests.cs

@@ -2,9 +2,9 @@
 using Xunit;
 using Newtonsoft.Json.Linq;
 using System;
-using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Create;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common;
 
-namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates
+namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Test
 {
     public class FileWriterTests
     {

src/APIM_ARMTemplate/apimtemplate.test/Creator/FileHandlerTests/OpenAPISpecReaderTests.cs

@@ -1,8 +1,8 @@
 using Xunit;
-using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Create;
 using Microsoft.OpenApi.Models;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common;
 
-namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates
+namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Test
 {
     public class OpenAPISpecReaderTests
     {

src/APIM_ARMTemplate/apimtemplate.test/Creator/TemplateCreatorFactories/APITemplateCreatorFactory.cs

@@ -1,5 +1,7 @@
-
-namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Create
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Create;
+
+namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Test
 {
     public class APITemplateCreatorFactory
     {

src/APIM_ARMTemplate/apimtemplate.test/Creator/TemplateCreatorFactories/APIVersionSetTemplateCreatorFactory.cs

@@ -1,5 +1,6 @@
-
-namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Create
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Create;
+
+namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Test
 {
     public class APIVersionSetTemplateCreatorFactory
     {

src/APIM_ARMTemplate/apimtemplate.test/Creator/TemplateCreatorFactories/MasterTemplateCreatorFactory.cs

@@ -1,5 +1,6 @@
-
-namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Create
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Create;
+
+namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Test
 {
     public class MasterTemplateCreatorFactory
     {

src/APIM_ARMTemplate/apimtemplate.test/Creator/TemplateCreatorFactories/PolicyTemplateCreatorFactory.cs

@@ -1,5 +1,7 @@
-
-namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Create
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Create;
+
+namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Test
 {
     public class PolicyTemplateCreatorFactory
     {

src/APIM_ARMTemplate/apimtemplate.test/Creator/TemplateCreatorTests/APITemplateCreatorTests.cs

@@ -1,9 +1,9 @@
 using Microsoft.OpenApi.Models;
-using System;
-using System.IO;
 using Xunit;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Create;
 
-namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Create
+namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Test
 {
     public class APITemplateCreatorTests
     {
@@ -23,6 +23,7 @@ public async void ShouldCreateInitialAPITemplateResourceFromCreatorConfig()
                     revision = "revision",
                     revisionDescription = "revisionDescription",
                     suffix = "suffix",
+                    subscriptionRequired = true,
                     authenticationSettings = new APITemplateAuthenticationSettings()
                     {
                         oAuth2 = new APITemplateOAuth2()
@@ -53,6 +54,7 @@ public async void ShouldCreateInitialAPITemplateResourceFromCreatorConfig()
             Assert.Equal(creatorConfig.api.revision, apiTemplateResource.properties.apiRevision);
             Assert.Equal(creatorConfig.api.revisionDescription, apiTemplateResource.properties.apiRevisionDescription);
             Assert.Equal(creatorConfig.api.suffix, apiTemplateResource.properties.path);
+            Assert.Equal(creatorConfig.api.subscriptionRequired, apiTemplateResource.properties.subscriptionRequired);
             Assert.Equal(creatorConfig.api.authenticationSettings.oAuth2.authorizationServerId, apiTemplateResource.properties.authenticationSettings.oAuth2.authorizationServerId);
             Assert.Equal(creatorConfig.api.authenticationSettings.oAuth2.scope, apiTemplateResource.properties.authenticationSettings.oAuth2.scope);
             Assert.Equal(creatorConfig.api.authenticationSettings.openid.openidProviderId, apiTemplateResource.properties.authenticationSettings.openid.openidProviderId);

src/APIM_ARMTemplate/apimtemplate.test/Creator/TemplateCreatorTests/APIVersionSetTemplateCreatorTests.cs

@@ -1,6 +1,8 @@
 using Xunit;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Create;
 
-namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Create
+namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Test
 {
     public class APIVersionSetTemplateCreatorTests
     {

src/APIM_ARMTemplate/apimtemplate.test/Creator/TemplateCreatorTests/DiagnosticTemplateCreatorTests.cs

@@ -1,6 +1,8 @@
 using Xunit;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Create;
 
-namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Create
+namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Test
 {
     public class DiagnosticTemplateCreatorTests
     {
@@ -13,56 +15,56 @@ public void ShouldCreateDiagnosticTemplateResourceFromCreatorConfig()
             {
                 api = new APIConfig()
                 {
-                    name = "name"
-                },
-                diagnostic = new DiagnosticTemplateProperties()
-                {
-                    alwaysLog = "alwaysLog",
-                    loggerId = "loggerId",
-                    sampling = new DiagnosticTemplateSampling()
-                    {
-                        samplingType = "samplingType",
-                        percentage = 100
-                    },
-                    frontend = new DiagnosticTemplateFrontendBackend()
+                    diagnostic = new DiagnosticConfig()
                     {
-                        request = new DiagnosticTemplateRequestResponse()
+                        name = "applicationinsights",
+                        alwaysLog = "alwaysLog",
+                        loggerId = "loggerId",
+                        sampling = new DiagnosticTemplateSampling()
                         {
-                            headers = new string[] { "frontendrequestheader" },
-                            body = new DiagnosticTemplateRequestResponseBody()
-                            {
-                                bytes = 512
-                            }
+                            samplingType = "samplingType",
+                            percentage = 100
                         },
-                        response = new DiagnosticTemplateRequestResponse()
+                        frontend = new DiagnosticTemplateFrontendBackend()
                         {
-                            headers = new string[] { "frontendresponseheader" },
-                            body = new DiagnosticTemplateRequestResponseBody()
+                            request = new DiagnosticTemplateRequestResponse()
                             {
-                                bytes = 512
-                            }
-                        }
-                    },
-                    backend = new DiagnosticTemplateFrontendBackend()
-                    {
-                        request = new DiagnosticTemplateRequestResponse()
-                        {
-                            headers = new string[] { "backendrequestheader" },
-                            body = new DiagnosticTemplateRequestResponseBody()
+                                headers = new string[] { "frontendrequestheader" },
+                                body = new DiagnosticTemplateRequestResponseBody()
+                                {
+                                    bytes = 512
+                                }
+                            },
+                            response = new DiagnosticTemplateRequestResponse()
                             {
-                                bytes = 512
+                                headers = new string[] { "frontendresponseheader" },
+                                body = new DiagnosticTemplateRequestResponseBody()
+                                {
+                                    bytes = 512
+                                }
                             }
                         },
-                        response = new DiagnosticTemplateRequestResponse()
+                        backend = new DiagnosticTemplateFrontendBackend()
                         {
-                            headers = new string[] { "backendresponseheader" },
-                            body = new DiagnosticTemplateRequestResponseBody()
+                            request = new DiagnosticTemplateRequestResponse()
                             {
-                                bytes = 512
+                                headers = new string[] { "backendrequestheader" },
+                                body = new DiagnosticTemplateRequestResponseBody()
+                                {
+                                    bytes = 512
+                                }
+                            },
+                            response = new DiagnosticTemplateRequestResponse()
+                            {
+                                headers = new string[] { "backendresponseheader" },
+                                body = new DiagnosticTemplateRequestResponseBody()
+                                {
+                                    bytes = 512
+                                }
                             }
-                        }
-                    },
-                    enableHttpCorrelationHeaders = true
+                        },
+                        enableHttpCorrelationHeaders = true
+                    }
                 }
             };
 
@@ -71,21 +73,21 @@ public void ShouldCreateDiagnosticTemplateResourceFromCreatorConfig()
             DiagnosticTemplateResource diagnosticTemplateResource = diagnosticTemplateCreator.CreateAPIDiagnosticTemplateResource(creatorConfig, dependsOn);
 
             // assert
-            Assert.Equal($"[concat(parameters('ApimServiceName'), '/{creatorConfig.api.name}/diagnostic')]", diagnosticTemplateResource.name);
+            Assert.Equal($"[concat(parameters('ApimServiceName'), '/{creatorConfig.api.name}/{creatorConfig.api.diagnostic.name}')]", diagnosticTemplateResource.name);
             Assert.Equal(dependsOn, diagnosticTemplateResource.dependsOn);
-            Assert.Equal(creatorConfig.diagnostic.alwaysLog, diagnosticTemplateResource.properties.alwaysLog);
-            Assert.Equal(creatorConfig.diagnostic.loggerId, diagnosticTemplateResource.properties.loggerId);
-            Assert.Equal(creatorConfig.diagnostic.enableHttpCorrelationHeaders, diagnosticTemplateResource.properties.enableHttpCorrelationHeaders);
-            Assert.Equal(creatorConfig.diagnostic.sampling.samplingType, diagnosticTemplateResource.properties.sampling.samplingType);
-            Assert.Equal(creatorConfig.diagnostic.sampling.percentage, diagnosticTemplateResource.properties.sampling.percentage);
-            Assert.Equal(creatorConfig.diagnostic.frontend.request.headers, diagnosticTemplateResource.properties.frontend.request.headers);
-            Assert.Equal(creatorConfig.diagnostic.frontend.request.body.bytes, diagnosticTemplateResource.properties.frontend.request.body.bytes);
-            Assert.Equal(creatorConfig.diagnostic.frontend.response.headers, diagnosticTemplateResource.properties.frontend.response.headers);
-            Assert.Equal(creatorConfig.diagnostic.frontend.response.body.bytes, diagnosticTemplateResource.properties.frontend.response.body.bytes);
-            Assert.Equal(creatorConfig.diagnostic.backend.request.headers, diagnosticTemplateResource.properties.backend.request.headers);
-            Assert.Equal(creatorConfig.diagnostic.backend.request.body.bytes, diagnosticTemplateResource.properties.backend.request.body.bytes);
-            Assert.Equal(creatorConfig.diagnostic.backend.response.headers, diagnosticTemplateResource.properties.backend.response.headers);
-            Assert.Equal(creatorConfig.diagnostic.backend.response.body.bytes, diagnosticTemplateResource.properties.backend.response.body.bytes);
+            Assert.Equal(creatorConfig.api.diagnostic.alwaysLog, diagnosticTemplateResource.properties.alwaysLog);
+            Assert.Equal(creatorConfig.api.diagnostic.loggerId, diagnosticTemplateResource.properties.loggerId);
+            Assert.Equal(creatorConfig.api.diagnostic.enableHttpCorrelationHeaders, diagnosticTemplateResource.properties.enableHttpCorrelationHeaders);
+            Assert.Equal(creatorConfig.api.diagnostic.sampling.samplingType, diagnosticTemplateResource.properties.sampling.samplingType);
+            Assert.Equal(creatorConfig.api.diagnostic.sampling.percentage, diagnosticTemplateResource.properties.sampling.percentage);
+            Assert.Equal(creatorConfig.api.diagnostic.frontend.request.headers, diagnosticTemplateResource.properties.frontend.request.headers);
+            Assert.Equal(creatorConfig.api.diagnostic.frontend.request.body.bytes, diagnosticTemplateResource.properties.frontend.request.body.bytes);
+            Assert.Equal(creatorConfig.api.diagnostic.frontend.response.headers, diagnosticTemplateResource.properties.frontend.response.headers);
+            Assert.Equal(creatorConfig.api.diagnostic.frontend.response.body.bytes, diagnosticTemplateResource.properties.frontend.response.body.bytes);
+            Assert.Equal(creatorConfig.api.diagnostic.backend.request.headers, diagnosticTemplateResource.properties.backend.request.headers);
+            Assert.Equal(creatorConfig.api.diagnostic.backend.request.body.bytes, diagnosticTemplateResource.properties.backend.request.body.bytes);
+            Assert.Equal(creatorConfig.api.diagnostic.backend.response.headers, diagnosticTemplateResource.properties.backend.response.headers);
+            Assert.Equal(creatorConfig.api.diagnostic.backend.response.body.bytes, diagnosticTemplateResource.properties.backend.response.body.bytes);
         }
     }
 }

src/APIM_ARMTemplate/apimtemplate.test/Creator/TemplateCreatorTests/MasterTemplateCreatorTests.cs

@@ -1,7 +1,9 @@
 using System.Collections.Generic;
 using Xunit;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Create;
 
-namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Create
+namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Test
 {
     public class MasterTemplateCreatorTests
     {

src/APIM_ARMTemplate/apimtemplate.test/Creator/TemplateCreatorTests/PolicyTemplateCreatorTests.cs

@@ -1,7 +1,9 @@
 using System.Collections.Generic;
 using Xunit;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Create;
 
-namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Create
+namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Test
 {
     public class PolicyTemplateCreatorTests
     {

src/APIM_ARMTemplate/apimtemplate.test/Creator/TemplateCreatorTests/ProductAPITemplateCreatorTests.cs

@@ -1,7 +1,9 @@
 using System.Collections.Generic;
 using Xunit;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Create;
 
-namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Create
+namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Test
 {
     public class ProductAPITemplateCreatorTests
     {