diff --git a/readme.md b/readme.md index 7531b538d..c561e16aa 100644 --- a/readme.md +++ b/readme.md @@ -2,7 +2,7 @@ Microsoft Azure Event Hubs

-

Microsoft Azure Event Hubs +

Azure Event Hubs samples

follow on Twitter

-We're happy to host our client SDKs and integrations on GitHub as free, open-source projects. We currently support: +This repository contains samples for Azure Event Hubs. -* [Event Hubs for Apache Kafka](https://github.com/Azure/azure-event-hubs-for-kafka) -* [.NET](https://github.com/azure/azure-event-hubs-dotnet) -* [Java](https://github.com/azure/azure-event-hubs-java) -* [C](https://github.com/azure/azure-event-hubs-c) -* [Node.js](https://github.com/azure/azure-event-hubs-node) -* [Python](https://github.com/azure/azure-event-hubs-python) -* [Go](https://github.com/azure/azure-event-hubs-go) -* [Apache Spark connector](https://github.com/Azure/azure-event-hubs-spark) +## Client SDK samples -If you have any feedback, questions, or comments, go to the respective repo and file an issue. **Your involvement is welcome and encouraged!** +> **Note:** This repository holds samples for the legacy libraries for Azure Event Hubs for .NET and Java developers. We highly recommend you to upgrade to the newer packages. + +.NET legacy packages + +- Microsoft.Azure.EventHubs (**legacy**) +- Microsoft.Azure.EventHubs.Processor (**legacy**) +- WindowsAzure.ServiceBus (**legacy**) + +Java legacy packages + +- com.microsoft.azure:azure-eventhubs (**legacy**) +- com.microsoft.azure:azure-eventhubs-eph (**legacy**) + +Samples for **latest** .NET packages for Azure Event Hubs -## What is Event Hubs? +- [Azure.Messaging.EventHubs](https://docs.microsoft.com/samples/azure/azure-sdk-for-net/azuremessagingeventhubs-samples/) +- [Azure.Messaging.EventHubs.Processor](https://docs.microsoft.com/samples/azure/azure-sdk-for-net/azuremessagingeventhubsprocessor-samples/) + +Samples for **latest** Java packages for Azure Event Hubs + +- [com.azure:azure-messaging-eventhubs](https://github.com/Azure/azure-sdk-for-java/tree/master/sdk/eventhubs/azure-messaging-eventhubs/src/samples) +- [com.azure:azure-messaging-eventhubs-checkpointstore-blob](https://github.com/Azure/azure-sdk-for-java/tree/master/sdk/eventhubs/azure-messaging-eventhubs-checkpointstore-blob/src/samples) + +You can find samples for other languages and frameworks in the following repositories: +- [JavaScript](https://github.com/Azure/azure-sdk-for-js/tree/master/sdk/eventhub/event-hubs/samples) +- [Python](https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/eventhub/azure-eventhub/samples) +- [C](https://github.com/azure/azure-event-hubs-c) +- [Go](https://github.com/azure/azure-event-hubs-go) +- [Apache Spark connector](https://github.com/Azure/azure-event-hubs-spark) +- [Event Hubs for Apache Kafka](https://github.com/Azure/azure-event-hubs-for-kafka) + +If you have any feedback, questions, or comments, go to the respective repo and file an issue. **Your involvement is welcome and encouraged!** -Azure Event Hubs is a hyper-scale data ingestion service, fully-managed by Microsoft, that enables you to collect, store and process trillions of events from websites, apps, IoT devices, and any stream of data. To learn more, please visit our [marketing page](https://azure.microsoft.com/services/event-hubs/). +## Management SDK samples +This repository also contains [management samples](https://github.com/Azure/azure-event-hubs/tree/master/samples/Management) that use the .NET management SDK, PowerShell, and CLI. -## Azure Event Hubs for Apache Kafka -Event Hubs now provides a Kafka endpoint that can be used by your existing Kafka-based applications as an alternative to running your own Kafka cluster. To learn more, check out the new [Azure Event Hubs for Apache Kafka](https://github.com/Azure/azure-event-hubs-for-kafka) repo! +## See also +To learn about Event Hubs, see our [Event Hubs documentation](https://docs.microsoft.com/en-us/azure/event-hubs/). diff --git a/samples/DotNet/AnomalyDetector/AnomalyDetector.sln b/samples/DotNet/Azure.Messaging.EventHubs/AnomalyDetector/AnomalyDetector.sln similarity index 100% rename from samples/DotNet/AnomalyDetector/AnomalyDetector.sln rename to samples/DotNet/Azure.Messaging.EventHubs/AnomalyDetector/AnomalyDetector.sln diff --git a/samples/DotNet/AnomalyDetector/Producer/EventGenerator.cs b/samples/DotNet/Azure.Messaging.EventHubs/AnomalyDetector/Producer/EventGenerator.cs similarity index 100% rename from samples/DotNet/AnomalyDetector/Producer/EventGenerator.cs rename to samples/DotNet/Azure.Messaging.EventHubs/AnomalyDetector/Producer/EventGenerator.cs diff --git a/samples/DotNet/AnomalyDetector/Producer/JsonHelper.cs b/samples/DotNet/Azure.Messaging.EventHubs/AnomalyDetector/Producer/JsonHelper.cs similarity index 100% rename from samples/DotNet/AnomalyDetector/Producer/JsonHelper.cs rename to samples/DotNet/Azure.Messaging.EventHubs/AnomalyDetector/Producer/JsonHelper.cs diff --git a/samples/DotNet/AnomalyDetector/Producer/Producer.csproj b/samples/DotNet/Azure.Messaging.EventHubs/AnomalyDetector/Producer/Producer.csproj similarity index 59% rename from samples/DotNet/AnomalyDetector/Producer/Producer.csproj rename to samples/DotNet/Azure.Messaging.EventHubs/AnomalyDetector/Producer/Producer.csproj index cf1926bfe..a0f3c2c23 100644 --- a/samples/DotNet/AnomalyDetector/Producer/Producer.csproj +++ b/samples/DotNet/Azure.Messaging.EventHubs/AnomalyDetector/Producer/Producer.csproj @@ -6,8 +6,8 @@ - - + + diff --git a/samples/DotNet/Azure.Messaging.EventHubs/AnomalyDetector/Producer/Program.cs b/samples/DotNet/Azure.Messaging.EventHubs/AnomalyDetector/Producer/Program.cs new file mode 100644 index 000000000..913bd895d --- /dev/null +++ b/samples/DotNet/Azure.Messaging.EventHubs/AnomalyDetector/Producer/Program.cs @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Producer +{ + using System; + using System.Text; + using System.Threading.Tasks; + using Azure.Messaging.EventHubs; + using Azure.Messaging.EventHubs.Producer; + using System.Collections.Generic; + using System.IO; + + public class Program + { + private const string EventHubNamespaceConnectionString = ""; + private const string EventHubName = ""; + private const string TransactionsDumpFile = "mocktransactions.csv"; + + private static EventHubProducerClient producerClient; + + static async Task Main() + { + // create an Event Hubs Producer client using the namespace connection string and the event hub name + producerClient = new EventHubProducerClient(EventHubNamespaceConnectionString, EventHubName); + + // send messages to the event hub + await SendMessagesToEventHubAsync(1000); + + await producerClient.CloseAsync(); + + Console.WriteLine("Press [enter] to exit."); + Console.ReadLine(); + } + + // Creates an Event Hub client and sends messages to the event hub. + private static async Task SendMessagesToEventHubAsync(int numMessagesToSend) + { + var eg = new EventGenerator(); + + IEnumerable transactions = eg.GenerateEvents(numMessagesToSend); + + if (File.Exists(TransactionsDumpFile)) + { + // exceptions not handled for brevity + File.Delete(TransactionsDumpFile); + } + + await File.AppendAllTextAsync( + TransactionsDumpFile, + $"CreditCardId,Timestamp,Location,Amount,Type{Environment.NewLine}"); + + int numSuccessfulMessages = 0; + try + { + // create a batch using the producer client + using (EventDataBatch eventBatch = await producerClient.CreateBatchAsync()) + { + foreach (var t in transactions) + { + // we don't send the transaction type as part of the message. + // that is up to the downstream analytics to figure out! + // we just pretty print them here so they can easily be compared with the downstream + // analytics results. + var message = t.Data.ToJson(); + + if (t.Type == TransactionType.Suspect) + { + var fc = Console.ForegroundColor; + Console.ForegroundColor = ConsoleColor.Yellow; + + Console.WriteLine($"Suspect transaction: {message}"); + + Console.ForegroundColor = fc; // reset to original + } + else + { + Console.WriteLine($"Regular transaction: {message}"); + } + + var line = $"{t.Data.CreditCardId},{t.Data.Timestamp.ToString("o")},{t.Data.Location},{t.Data.Amount},{t.Type}{Environment.NewLine}"; + + File.AppendAllText(TransactionsDumpFile, line); + + // add the message to the batch + eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes(message))); + numSuccessfulMessages++; + } + // send the batch of messages to the event hub using the producer object + await producerClient.SendAsync(eventBatch); + await Task.Delay(10); + } + } + catch (Exception ex) + { + Console.WriteLine($"{Environment.NewLine}Exception: {ex.Message}"); + } + Console.WriteLine(); + Console.WriteLine($"{numSuccessfulMessages} messages sent successfully."); + } + } +} + diff --git a/samples/DotNet/AnomalyDetector/Producer/Transaction.cs b/samples/DotNet/Azure.Messaging.EventHubs/AnomalyDetector/Producer/Transaction.cs similarity index 100% rename from samples/DotNet/AnomalyDetector/Producer/Transaction.cs rename to samples/DotNet/Azure.Messaging.EventHubs/AnomalyDetector/Producer/Transaction.cs diff --git a/samples/DotNet/AnomalyDetector/Producer/TransactionData.cs b/samples/DotNet/Azure.Messaging.EventHubs/AnomalyDetector/Producer/TransactionData.cs similarity index 100% rename from samples/DotNet/AnomalyDetector/Producer/TransactionData.cs rename to samples/DotNet/Azure.Messaging.EventHubs/AnomalyDetector/Producer/TransactionData.cs diff --git a/samples/DotNet/AnomalyDetector/Producer/TransactionType.cs b/samples/DotNet/Azure.Messaging.EventHubs/AnomalyDetector/Producer/TransactionType.cs similarity index 100% rename from samples/DotNet/AnomalyDetector/Producer/TransactionType.cs rename to samples/DotNet/Azure.Messaging.EventHubs/AnomalyDetector/Producer/TransactionType.cs diff --git a/samples/DotNet/Azure.Messaging.EventHubs/AnomalyDetector/Readme.md b/samples/DotNet/Azure.Messaging.EventHubs/AnomalyDetector/Readme.md new file mode 100644 index 000000000..6c6aeb0ad --- /dev/null +++ b/samples/DotNet/Azure.Messaging.EventHubs/AnomalyDetector/Readme.md @@ -0,0 +1,5 @@ + +# Visualize data anomalies in real-time events sent to Azure Event Hubs +In this sample, you run an application that creates and sends credit card transactions to an event hub. Then you read the stream of data in real-time with Azure Stream Analytics, which separates the valid transactions from the invalid transactions, and then use Power BI to visually identify the transactions that are tagged as invalid. + +For detailed information and steps for using this sample, see [this article](https://docs.microsoft.com/azure/event-hubs/event-hubs-tutorial-visualize-anomalies). \ No newline at end of file diff --git a/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/Global.asax b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/Global.asax new file mode 100644 index 000000000..21d85acb9 --- /dev/null +++ b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/Global.asax @@ -0,0 +1 @@ +<%@ Application Codebehind="Global.asax.cs" Inherits="ManagedIdentityWebApp.Global" Language="C#" %> diff --git a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/Global.asax.cs b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/Global.asax.cs similarity index 90% rename from samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/Global.asax.cs rename to samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/Global.asax.cs index 8e8cf42b3..e483b9141 100644 --- a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/Global.asax.cs +++ b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/Global.asax.cs @@ -5,7 +5,7 @@ using System.Web.Security; using System.Web.SessionState; -namespace EventHubsMSIDemoWebApp +namespace ManagedIdentityWebApp { public class Global : System.Web.HttpApplication { diff --git a/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/ManagedIdentityWebApp.csproj b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/ManagedIdentityWebApp.csproj new file mode 100644 index 000000000..ae30c2757 --- /dev/null +++ b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/ManagedIdentityWebApp.csproj @@ -0,0 +1,257 @@ + + + + + Debug + AnyCPU + + + 2.0 + {476F98B4-05BF-42B6-B0B4-E50CD5B71635} + {349c5851-65df-11da-9384-00065b846f21};{fae04ec0-301f-11d3-bf4b-00c04f79efbc} + Library + Properties + ManagedIdentityWebApp + ManagedIdentityWebApp + v4.7.2 + true + + + + + + + + + + + + true + full + false + bin\ + DEBUG;TRACE + prompt + 4 + 8.0 + + + true + pdbonly + true + bin\ + TRACE + prompt + 4 + 8.0 + + + + packages\Azure.Core.1.0.1\lib\netstandard2.0\Azure.Core.dll + + + packages\Azure.Identity.1.1.0\lib\netstandard2.0\Azure.Identity.dll + + + packages\Azure.Messaging.EventHubs.5.0.1\lib\netstandard2.0\Azure.Messaging.EventHubs.dll + + + packages\Microsoft.Azure.Amqp.2.4.3\lib\net45\Microsoft.Azure.Amqp.dll + + + packages\Microsoft.Bcl.AsyncInterfaces.1.1.0\lib\net461\Microsoft.Bcl.AsyncInterfaces.dll + + + packages\Microsoft.CodeDom.Providers.DotNetCompilerPlatform.2.0.1\lib\net45\Microsoft.CodeDom.Providers.DotNetCompilerPlatform.dll + + + + packages\Microsoft.Identity.Client.4.7.1\lib\net45\Microsoft.Identity.Client.dll + + + packages\Newtonsoft.Json.12.0.3\lib\net45\Newtonsoft.Json.dll + + + packages\System.Buffers.4.5.0\lib\netstandard2.0\System.Buffers.dll + + + + + packages\System.Diagnostics.DiagnosticSource.4.7.0\lib\net46\System.Diagnostics.DiagnosticSource.dll + + + + packages\System.IO.4.3.0\lib\net462\System.IO.dll + True + True + + + packages\System.Memory.4.5.3\lib\netstandard2.0\System.Memory.dll + + + packages\System.Net.Http.4.3.4\lib\net46\System.Net.Http.dll + True + True + + + packages\System.Net.WebSockets.4.3.0\lib\net46\System.Net.WebSockets.dll + True + True + + + packages\System.Net.WebSockets.Client.4.3.2\lib\net46\System.Net.WebSockets.Client.dll + True + True + + + + packages\System.Numerics.Vectors.4.6.0-preview5.19224.8\lib\net46\System.Numerics.Vectors.dll + + + packages\System.Reflection.TypeExtensions.4.7.0\lib\net461\System.Reflection.TypeExtensions.dll + + + packages\System.Runtime.4.3.1\lib\net462\System.Runtime.dll + True + True + + + packages\System.Runtime.CompilerServices.Unsafe.4.7.0\lib\netstandard2.0\System.Runtime.CompilerServices.Unsafe.dll + + + + packages\System.Runtime.Serialization.Primitives.4.3.0\lib\net46\System.Runtime.Serialization.Primitives.dll + True + True + + + + packages\System.Security.Cryptography.Algorithms.4.3.1\lib\net463\System.Security.Cryptography.Algorithms.dll + True + True + + + packages\System.Security.Cryptography.Encoding.4.3.0\lib\net46\System.Security.Cryptography.Encoding.dll + True + True + + + packages\System.Security.Cryptography.Primitives.4.3.0\lib\net46\System.Security.Cryptography.Primitives.dll + True + True + + + packages\System.Security.Cryptography.ProtectedData.4.7.0\lib\net461\System.Security.Cryptography.ProtectedData.dll + + + packages\System.Security.Cryptography.X509Certificates.4.3.2\lib\net461\System.Security.Cryptography.X509Certificates.dll + True + True + + + + packages\System.Text.Encodings.Web.4.7.0\lib\netstandard2.0\System.Text.Encodings.Web.dll + + + packages\System.Text.Json.4.7.0\lib\net461\System.Text.Json.dll + + + packages\System.Threading.Channels.4.7.0\lib\netstandard2.0\System.Threading.Channels.dll + + + packages\System.Threading.Tasks.Extensions.4.6.0-preview.18571.3\lib\netstandard2.0\System.Threading.Tasks.Extensions.dll + + + packages\System.ValueTuple.4.5.0\lib\net47\System.ValueTuple.dll + + + + + + + + + + + + + + + + + + + + + + + Web.config + + + Web.config + + + + + + + + Designer + + + + + SendReceive.aspx + ASPXCodeBehind + + + SendReceive.aspx + + + Global.asax + + + + + + + + + 10.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + + + + + + + + + True + True + 56779 + / + http://localhost:56779/ + False + False + + + False + + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + \ No newline at end of file diff --git a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp.sln b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/ManagedIdentityWebApp.sln similarity index 84% rename from samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp.sln rename to samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/ManagedIdentityWebApp.sln index aa57c11b9..99d261930 100644 --- a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp.sln +++ b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/ManagedIdentityWebApp.sln @@ -3,7 +3,7 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio 15 VisualStudioVersion = 15.0.27004.2010 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "EventHubsMSIDemoWebApp", "EventHubsMSIDemoWebApp\EventHubsMSIDemoWebApp.csproj", "{476F98B4-05BF-42B6-B0B4-E50CD5B71635}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ManagedIdentityWebApp", "ManagedIdentityWebApp.csproj", "{476F98B4-05BF-42B6-B0B4-E50CD5B71635}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/Properties/AssemblyInfo.cs b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/Properties/AssemblyInfo.cs similarity index 92% rename from samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/Properties/AssemblyInfo.cs rename to samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/Properties/AssemblyInfo.cs index 27de5cefb..eb184e7ac 100644 --- a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/Properties/AssemblyInfo.cs +++ b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/Properties/AssemblyInfo.cs @@ -5,11 +5,11 @@ // General Information about an assembly is controlled through the following // set of attributes. Change these attribute values to modify the information // associated with an assembly. -[assembly: AssemblyTitle("EventHubsMSIDemoWebApp")] +[assembly: AssemblyTitle("ManagedIdentityWebApp")] [assembly: AssemblyDescription("")] [assembly: AssemblyConfiguration("")] [assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("EventHubsMSIDemoWebApp")] +[assembly: AssemblyProduct("ManagedIdentityWebApp")] [assembly: AssemblyCopyright("Copyright © 2017")] [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] diff --git a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/README.md b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/README.md similarity index 100% rename from samples/DotNet/MSI/EventHubsMSIDemoWebApp/README.md rename to samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/README.md diff --git a/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/SendReceive.aspx b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/SendReceive.aspx new file mode 100644 index 000000000..cad3dbad9 --- /dev/null +++ b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/SendReceive.aspx @@ -0,0 +1,32 @@ +<%@ Page Async="true" Language="C#" AutoEventWireup="true" CodeBehind="SendReceive.aspx.cs" Inherits="ManagedIdentityWebApp.SendReceive" %> + + + + + + EventHubs Managed Identity Demo + + + +
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+ + +
+
+
+ + diff --git a/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/SendReceive.aspx.cs b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/SendReceive.aspx.cs new file mode 100644 index 000000000..6531373b7 --- /dev/null +++ b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/SendReceive.aspx.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + + +// Always add app to IAM roles +// Don't use on deployment slots but only on root +namespace ManagedIdentityWebApp +{ + using System; + using System.Text; + using System.Threading; + using Azure.Messaging.EventHubs; + using Azure.Messaging.EventHubs.Producer; + using Azure.Messaging.EventHubs.Consumer; + using Azure.Identity; + using System.Threading.Tasks; + + public partial class SendReceive : System.Web.UI.Page + { + protected void Page_Load(object sender, EventArgs e) + { + + } + + protected async void btnSend_Click(object sender, EventArgs e) + { + await using (EventHubProducerClient producerClient = new EventHubProducerClient(txtNamespace.Text, txtEventHub.Text, new DefaultAzureCredential())) + { + // create a batch + using (EventDataBatch eventBatch = await producerClient.CreateBatchAsync()) + { + + // add events to the batch. only one in this case. + eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes(txtData.Text))); + + // send the batch to the event hub + await producerClient.SendAsync(eventBatch); + } + + txtOutput.Text = $"{DateTime.Now} - SENT{Environment.NewLine}{txtOutput.Text}"; + } + } + protected async void btnReceive_Click(object sender, EventArgs e) + { + await using (var consumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, txtNamespace.Text, txtEventHub.Text, new DefaultAzureCredential())) + { + int eventsRead = 0; + try + { + using CancellationTokenSource cancellationSource = new CancellationTokenSource(); + cancellationSource.CancelAfter(TimeSpan.FromSeconds(5)); + + await foreach (PartitionEvent partitionEvent in consumerClient.ReadEventsAsync(cancellationSource.Token)) + { + txtOutput.Text = $"Event Read: { Encoding.UTF8.GetString(partitionEvent.Data.Body.ToArray()) }{ Environment.NewLine}" + txtOutput.Text; + eventsRead++; + } + } + catch (TaskCanceledException ex) + { + txtOutput.Text = $"Number of events read: {eventsRead}{ Environment.NewLine}" + txtOutput.Text; + } + } + } + } +} \ No newline at end of file diff --git a/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/SendReceive.aspx.designer.cs b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/SendReceive.aspx.designer.cs new file mode 100644 index 000000000..f376bf7cf --- /dev/null +++ b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/SendReceive.aspx.designer.cs @@ -0,0 +1,89 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace ManagedIdentityWebApp +{ + + + public partial class SendReceive + { + + /// + /// form1 control. + /// + /// + /// Auto-generated field. + /// To modify move field declaration from designer file to code-behind file. + /// + protected global::System.Web.UI.HtmlControls.HtmlForm form1; + + /// + /// txtNamespace control. + /// + /// + /// Auto-generated field. + /// To modify move field declaration from designer file to code-behind file. + /// + protected global::System.Web.UI.WebControls.TextBox txtNamespace; + + /// + /// txtEventHub control. + /// + /// + /// Auto-generated field. + /// To modify move field declaration from designer file to code-behind file. + /// + protected global::System.Web.UI.WebControls.TextBox txtEventHub; + + /// + /// txtData control. + /// + /// + /// Auto-generated field. + /// To modify move field declaration from designer file to code-behind file. + /// + protected global::System.Web.UI.WebControls.TextBox txtData; + + /// + /// btnSend control. + /// + /// + /// Auto-generated field. + /// To modify move field declaration from designer file to code-behind file. + /// + protected global::System.Web.UI.WebControls.Button btnSend; + + /// + /// btnReceive control. + /// + /// + /// Auto-generated field. + /// To modify move field declaration from designer file to code-behind file. + /// + protected global::System.Web.UI.WebControls.Button btnReceive; + + /// + /// txtOutput control. + /// + /// + /// Auto-generated field. + /// To modify move field declaration from designer file to code-behind file. + /// + protected global::System.Web.UI.WebControls.TextBox txtOutput; + + /// + /// hiddenStartingOffset control. + /// + /// + /// Auto-generated field. + /// To modify move field declaration from designer file to code-behind file. + /// + protected global::System.Web.UI.WebControls.HiddenField hiddenStartingOffset; + } +} diff --git a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/StyleSheet.css b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/StyleSheet.css similarity index 88% rename from samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/StyleSheet.css rename to samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/StyleSheet.css index 9b0b73b9e..cdaafb614 100644 --- a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/StyleSheet.css +++ b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/StyleSheet.css @@ -1,5 +1,5 @@ label { - width: 100px; + width: 540px; display: inline-block; } diff --git a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/Web.Debug.config b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/Web.Debug.config similarity index 100% rename from samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/Web.Debug.config rename to samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/Web.Debug.config diff --git a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/Web.Release.config b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/Web.Release.config similarity index 100% rename from samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/Web.Release.config rename to samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/Web.Release.config diff --git a/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/Web.config b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/Web.config new file mode 100644 index 000000000..59ccd8c22 --- /dev/null +++ b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/Web.config @@ -0,0 +1,98 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/packages.config b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/packages.config new file mode 100644 index 000000000..4af44b2bd --- /dev/null +++ b/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp/packages.config @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/EventHubsMSIDemo.aspx b/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/EventHubsMSIDemo.aspx deleted file mode 100644 index e61820b6f..000000000 --- a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/EventHubsMSIDemo.aspx +++ /dev/null @@ -1,35 +0,0 @@ -<%@ Page Language="C#" AutoEventWireup="true" CodeBehind="EventHubsMSIDemo.aspx.cs" Inherits="EventHubsMSIDemoWebApp.EventHubsMSIDemo" %> - - - - - - EventHubs MSI Demo - - - -
-
-
- -
-
- -
-
- -
-
- -
-
- -
-
- - -
-
-
- - diff --git a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/EventHubsMSIDemo.aspx.cs b/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/EventHubsMSIDemo.aspx.cs deleted file mode 100644 index cd076841b..000000000 --- a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/EventHubsMSIDemo.aspx.cs +++ /dev/null @@ -1,78 +0,0 @@ -using Microsoft.ServiceBus; -using Microsoft.ServiceBus.Messaging; -using Newtonsoft.Json.Linq; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Web; -using System.Web.UI; -using System.Web.UI.WebControls; - -// Always add app to IAM roles -// Don't use on deployment slots but only on root -namespace EventHubsMSIDemoWebApp -{ - public partial class EventHubsMSIDemo : System.Web.UI.Page - { - protected void Page_Load(object sender, EventArgs e) - { - - } - - protected void btnSend_Click(object sender, EventArgs e) - { - MessagingFactorySettings messagingFactorySettings = new MessagingFactorySettings - { - TokenProvider = TokenProvider.CreateManagedServiceIdentityTokenProvider(ServiceAudience.EventHubsAudience), - TransportType = TransportType.Amqp - }; - - MessagingFactory messagingFactory = MessagingFactory.Create($"sb://{txtNamespace.Text}.servicebus.windows.net/", - messagingFactorySettings); - - EventHubClient ehClient = messagingFactory.CreateEventHubClient(txtEventHub.Text); - ehClient.Send(new EventData(Encoding.UTF8.GetBytes(txtData.Text))); - ehClient.Close(); - messagingFactory.Close(); - } - - protected void btnReceive_Click(object sender, EventArgs e) - { - MessagingFactorySettings messagingFactorySettings = new MessagingFactorySettings - { - TokenProvider = TokenProvider.CreateManagedServiceIdentityTokenProvider(ServiceAudience.EventHubsAudience), - TransportType = TransportType.Amqp - }; - - messagingFactorySettings.AmqpTransportSettings.EnableLinkRedirect = false; - - MessagingFactory messagingFactory = MessagingFactory.Create($"sb://{txtNamespace.Text}.servicebus.windows.net/", - messagingFactorySettings); - - EventHubClient ehClient = messagingFactory.CreateEventHubClient(txtEventHub.Text); - EventHubConsumerGroup consumerGroup = ehClient.GetDefaultConsumerGroup(); - int partitions = int.Parse(txtPartitions.Text); - string[] Offsets = new string[partitions]; - if (!string.IsNullOrEmpty(hiddenStartingOffset.Value)) - { - Offsets = hiddenStartingOffset.Value.Split(','); - } - System.Threading.Tasks.Parallel.ForEach(Enumerable.Range(0, int.Parse(txtPartitions.Text)), partitionId => - { - EventHubReceiver receiver = consumerGroup.CreateReceiver($"{partitionId}", Offsets[partitionId] == null ? "-1" : Offsets[partitionId]); - EventData data = receiver.Receive(TimeSpan.FromSeconds(1)); - if (data != null) - { - Offsets[partitionId] = data.Offset; - txtReceivedData.Text += $"PartitionId: {partitionId} Seq#:{data.SequenceNumber} data:{Encoding.UTF8.GetString(data.GetBytes())}{Environment.NewLine}"; - } - receiver.Close(); - }); - - hiddenStartingOffset.Value = string.Join(",", Offsets); - ehClient.Close(); - messagingFactory.Close(); - } - } -} \ No newline at end of file diff --git a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp.csproj b/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp.csproj deleted file mode 100644 index 11622983d..000000000 --- a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp.csproj +++ /dev/null @@ -1,170 +0,0 @@ - - - - - - Debug - AnyCPU - - - 2.0 - {476F98B4-05BF-42B6-B0B4-E50CD5B71635} - {349c5851-65df-11da-9384-00065b846f21};{fae04ec0-301f-11d3-bf4b-00c04f79efbc} - Library - Properties - EventHubsMSIDemoWebApp - EventHubsMSIDemoWebApp - v4.6.1 - true - - - - - - - - - - - true - full - false - bin\ - DEBUG;TRACE - prompt - 4 - - - true - pdbonly - true - bin\ - TRACE - prompt - 4 - - - - ..\packages\Microsoft.Azure.Services.AppAuthentication.1.1.0-preview\lib\net452\Microsoft.Azure.Services.AppAuthentication.dll - - - ..\packages\Microsoft.CodeDom.Providers.DotNetCompilerPlatform.1.0.7\lib\net45\Microsoft.CodeDom.Providers.DotNetCompilerPlatform.dll - - - - ..\packages\Microsoft.IdentityModel.Clients.ActiveDirectory.3.14.2\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.dll - - - ..\packages\Microsoft.IdentityModel.Clients.ActiveDirectory.3.14.2\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.Platform.dll - - - ..\packages\WindowsAzure.ServiceBus.4.1.6\lib\net45\Microsoft.ServiceBus.dll - - - ..\packages\Newtonsoft.Json.11.0.1-beta1\lib\net45\Newtonsoft.Json.dll - - - ..\packages\System.IdentityModel.Tokens.Jwt.4.0.4.403061554\lib\net45\System.IdentityModel.Tokens.Jwt.dll - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Web.config - - - Web.config - - - - - - - - Designer - - - - - EventHubsMSIDemo.aspx - ASPXCodeBehind - - - EventHubsMSIDemo.aspx - - - Global.asax - - - - - - - - - 10.0 - $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) - - - - - - - - - True - True - 56779 - / - http://localhost:56779/ - False - False - - - False - - - - - - - This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - - - - - - - - \ No newline at end of file diff --git a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/Global.asax b/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/Global.asax deleted file mode 100644 index 6245438a1..000000000 --- a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/Global.asax +++ /dev/null @@ -1 +0,0 @@ -<%@ Application Codebehind="Global.asax.cs" Inherits="EventHubsMSIDemoWebApp.Global" Language="C#" %> diff --git a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/packages.config b/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/packages.config deleted file mode 100644 index aab789b1a..000000000 --- a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/packages.config +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - - - - - \ No newline at end of file diff --git a/samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/AnomalyDetector.sln b/samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/AnomalyDetector.sln new file mode 100644 index 000000000..13711caba --- /dev/null +++ b/samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/AnomalyDetector.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.27004.2005 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Producer", "Producer\Producer.csproj", "{8A768654-3ABD-4D6B-BDB3-C59CFC05C50A}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {8A768654-3ABD-4D6B-BDB3-C59CFC05C50A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8A768654-3ABD-4D6B-BDB3-C59CFC05C50A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8A768654-3ABD-4D6B-BDB3-C59CFC05C50A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8A768654-3ABD-4D6B-BDB3-C59CFC05C50A}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {73BDBF55-C72D-4F8D-999B-E8E068E3B5A4} + EndGlobalSection +EndGlobal diff --git a/samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/Producer/EventGenerator.cs b/samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/Producer/EventGenerator.cs new file mode 100644 index 000000000..6ec4d0c10 --- /dev/null +++ b/samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/Producer/EventGenerator.cs @@ -0,0 +1,127 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Producer +{ + using System; + using System.Collections.Generic; + + internal sealed class EventGenerator + { + // i.e. 1 in every 20 transactions is an anomaly + private const int AnomalyFactor = 20; + + private const int MaxAmount = 1000; + + private const int MaxDays = 30; + + private readonly Random random = new Random((int)DateTimeOffset.UtcNow.Ticks); + + private readonly DateTimeOffset startTime = new DateTimeOffset(DateTime.Now); + + private readonly List knownCreditCards = new List + { + "FC6018B3-B934-4FFF-B6A9-904C4B3082C7", + "71B3B20F-0BD1-4C1D-BA33-3264D904581C", + "B3B19755-C6E5-4E4A-B423-1B31DCCEB137", + "1AF29CA0-FBF2-48B6-9D0B-88FF331FADC1", + "039AA17C-2C5C-4A4F-B019-D416D0E4F5E6", + }; + + private readonly List knownLocations = new List + { + "Seattle", + "San Francisco", + "New York", + "Kahului", + "Miami", + }; + + public IEnumerable GenerateEvents(int count) + { + int counter = 0; + + var timestamp = startTime; + + while (counter < count) + { + foreach (var t in NewMockPurchases(timestamp)) + { + yield return t; + counter++; + } + + timestamp += TimeSpan.FromMinutes(10); + } + } + + /// + /// Returns a new mock transaction. In some cases where there is an anomaly, then it returns + /// 2 transactions. one regular and one anomaly. They differ in amounts, locations and timestamps, but have + /// the same credit card Id. + /// At the consumer side, since the timestamps are close together but the locations are different, these + /// 2 transactions are flagged as anomalous. + /// + /// + private IEnumerable NewMockPurchases(DateTimeOffset timestamp) + { + var maxIndex = Math.Min(knownCreditCards.Count, knownLocations.Count); + + var index = random.Next(0, maxIndex); + var cc = knownCreditCards[index]; + var location = knownLocations[index]; + + bool isAnomaly = (random.Next(0, AnomalyFactor) % AnomalyFactor) == 0; + + var purchases = new List(); + + var regularTransaction = new Transaction + { + Data = new TransactionData + { + Amount = random.Next(1, MaxAmount), + Location = location, + CreditCardId = cc, + Timestamp = timestamp, + }, + Type = TransactionType.Regular, + }; + + purchases.Add(regularTransaction); + + if (isAnomaly) + { + // change the location to something else + // now the transaction on a credit card is happening from a different location which is an anomaly! + + string newLocation = null; + + do + { + var newIndex = random.Next(0, knownLocations.Count); + newLocation = knownLocations[newIndex]; + + // while loop is - if by chance the "random" new location is the same as the original location + + } while (string.Equals(newLocation, location, StringComparison.OrdinalIgnoreCase)); + + var suspectTransaction = new Transaction + { + Data = new TransactionData + { + Amount = random.Next(1, MaxAmount), + Location = newLocation, + CreditCardId = cc, + Timestamp = timestamp + TimeSpan.FromSeconds(2), // suspect transaction time range is close to a regular transaction + }, + Type = TransactionType.Suspect, + }; + + purchases.Add(suspectTransaction); + } + + return purchases; + } + } +} + diff --git a/samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/Producer/JsonHelper.cs b/samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/Producer/JsonHelper.cs new file mode 100644 index 000000000..5e48472f4 --- /dev/null +++ b/samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/Producer/JsonHelper.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Producer +{ + using Newtonsoft.Json; + using Newtonsoft.Json.Serialization; + public static class JsonHelper + { + private static readonly JsonSerializerSettings publicSerializationSettings = CreatePublicSerializationSettings(); + + /// + /// Converts an object to its Json representation. + /// + public static string ToJson(this T value) + { + string json = JsonConvert.SerializeObject(value, typeof(T), publicSerializationSettings); + return json; + } + + /// + /// Converts a Json string to an object. + /// + public static T FromJson(this string value) + { + T @object = JsonConvert.DeserializeObject(value, publicSerializationSettings); + + return @object; + } + + private static JsonSerializerSettings CreatePublicSerializationSettings() + { + var settings = new JsonSerializerSettings(); + + settings.Converters.Add(new Newtonsoft.Json.Converters.StringEnumConverter()); + + return settings; + } + } +} + diff --git a/samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/Producer/Producer.csproj b/samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/Producer/Producer.csproj new file mode 100644 index 000000000..053c27992 --- /dev/null +++ b/samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/Producer/Producer.csproj @@ -0,0 +1,13 @@ + + + + Exe + netcoreapp2.0 + + + + + + + + diff --git a/samples/DotNet/AnomalyDetector/Producer/Program.cs b/samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/Producer/Program.cs similarity index 79% rename from samples/DotNet/AnomalyDetector/Producer/Program.cs rename to samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/Producer/Program.cs index ace75d409..e2b567055 100644 --- a/samples/DotNet/AnomalyDetector/Producer/Program.cs +++ b/samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/Producer/Program.cs @@ -6,15 +6,15 @@ namespace Producer using System; using System.Text; using System.Threading.Tasks; - using Microsoft.Azure.EventHubs; + using Azure.Messaging.EventHubs; using System.Collections.Generic; using System.IO; public class Program { - private const string EventHubConnectionString = "Event Hubs connection string"; + private const string EventHubConnectionString = ""; - private const string EventHubName = "Event Hub name"; + private const string EventHubName = ""; private const string TransactionsDumpFile = "mocktransactions.csv"; @@ -27,16 +27,10 @@ public static int Main(string[] args) private static async Task MainAsync(string[] args) { - // Creates an EventHubsConnectionStringBuilder object from a the connection string, and sets the EntityPath. - // Typically the connection string should have the Entity Path in it, but for the sake of this simple scenario - // we are using the connection string from the namespace. - var connectionStringBuilder = new EventHubsConnectionStringBuilder(EventHubConnectionString) - { - EntityPath = EventHubName, - }; - - eventHubClient = EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString()); + // create an Event Hubs client using the namespace connection string and the event hub name + eventHubClient = new EventHubClient(EventHubConnectionString, EventHubName); + // send messages to the event hub await SendMessagesToEventHubAsync(1000); await eventHubClient.CloseAsync(); @@ -64,6 +58,9 @@ private static async Task SendMessagesToEventHubAsync(int numMessagesToSend) TransactionsDumpFile, $"CreditCardId,Timestamp,Location,Amount,Type{Environment.NewLine}"); + // create a producer object that you can use to produce or send messages to the event hub + EventHubProducer producer = eventHubClient.CreateProducer(); + foreach (var t in transactions) { try @@ -93,7 +90,9 @@ private static async Task SendMessagesToEventHubAsync(int numMessagesToSend) File.AppendAllText(TransactionsDumpFile, line); var ed = new EventData(Encoding.UTF8.GetBytes(message)); - await eventHubClient.SendAsync(ed); + + // send the message to the event hub using the producer object + await producer.SendAsync(ed); } catch (Exception ex) { diff --git a/samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/Producer/Transaction.cs b/samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/Producer/Transaction.cs new file mode 100644 index 000000000..eb743f1c3 --- /dev/null +++ b/samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/Producer/Transaction.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Producer +{ + internal sealed class Transaction + { + public TransactionData Data { get; set; } + + public TransactionType Type { get; set; } + } +} + diff --git a/samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/Producer/TransactionData.cs b/samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/Producer/TransactionData.cs new file mode 100644 index 000000000..dd4af634f --- /dev/null +++ b/samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/Producer/TransactionData.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Producer +{ + using System; + + internal sealed class TransactionData + { + public string CreditCardId { get; set; } // like a credit card no. just simpler to use a Guid for our sample than a 16 digit no! + + public int Amount { get; set; } // simplified to int instead of double for now + + public string Location { get; set; } + + public DateTimeOffset Timestamp { get; set; } + } +} + diff --git a/samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/Producer/TransactionType.cs b/samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/Producer/TransactionType.cs new file mode 100644 index 000000000..6bccc741c --- /dev/null +++ b/samples/DotNet/Microsoft.Azure.EventHubs/AnomalyDetector/Producer/TransactionType.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Producer +{ + /// + /// A mechanism to identify the mock transactions that are generated on the client side. + /// This is not used downstream. But is used more for an easy (or visual) comparison of + /// anomalous data that the downstream analytics produces. + /// + internal enum TransactionType + { + Regular, + Suspect, + } +} + diff --git a/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/AzureEventHubsSDK/App.config b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/AzureEventHubsSDK/App.config new file mode 100644 index 000000000..e7aa69521 --- /dev/null +++ b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/AzureEventHubsSDK/App.config @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/samples/DotNet/Rbac/AzureEventHubsSDK/Program.cs b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/AzureEventHubsSDK/Program.cs similarity index 97% rename from samples/DotNet/Rbac/AzureEventHubsSDK/Program.cs rename to samples/DotNet/Microsoft.Azure.EventHubs/Rbac/AzureEventHubsSDK/Program.cs index 25f46903c..308113412 100644 --- a/samples/DotNet/Rbac/AzureEventHubsSDK/Program.cs +++ b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/AzureEventHubsSDK/Program.cs @@ -93,7 +93,8 @@ static async Task ClientAssertionCertScenarioAsync() var authResult = await app.AcquireTokenForClient(new string[] { $"{audience}/.default" }).ExecuteAsync(); return authResult.AccessToken; - }); + }, + $"https://login.microsoftonline.com/{TenantId}"); var ehClient = EventHubClient.CreateWithTokenProvider(new Uri($"sb://{EventHubNamespace}/"), EventHubName, tp); await SendReceiveAsync(ehClient); @@ -111,17 +112,13 @@ static async Task ClientCredentialsScenarioAsync() var authResult = await app.AcquireTokenForClient(new string[] { $"{audience}/.default" }).ExecuteAsync(); return authResult.AccessToken; - }); + }, + $"https://login.microsoftonline.com/{TenantId}"); var ehClient = EventHubClient.CreateWithTokenProvider(new Uri($"sb://{EventHubNamespace}/"), EventHubName, tp); await SendReceiveAsync(ehClient); } - static Task ControlPlaneAndDataPlaneMixedScenarioAsync() - { - return Task.FromResult(0); - } - static X509Certificate2 GetCertificate() { List locations = new List diff --git a/samples/DotNet/Rbac/AzureEventHubsSDK/Properties/AssemblyInfo.cs b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/AzureEventHubsSDK/Properties/AssemblyInfo.cs similarity index 100% rename from samples/DotNet/Rbac/AzureEventHubsSDK/Properties/AssemblyInfo.cs rename to samples/DotNet/Microsoft.Azure.EventHubs/Rbac/AzureEventHubsSDK/Properties/AssemblyInfo.cs diff --git a/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/AzureEventHubsSDK/README.md b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/AzureEventHubsSDK/README.md new file mode 100644 index 000000000..9c69c3576 --- /dev/null +++ b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/AzureEventHubsSDK/README.md @@ -0,0 +1,6 @@ +# Role based access sample with Microsoft.Azure.EventHubs SDK + +See [Authorize access to Event Hubs resources using Azure Active Directory](https://docs.microsoft.com/azure/event-hubs/authorize-access-azure-active-directory) for information on Role based access (RBAC). + +**Note**: This sample shows different ways to authenticate using Azure Active Directory with the legacy Event Hubs library `Microsoft.Azure.EventHubs`. +For the current version of the library `Azure.Messaging.EventHubs`, please see the [sample that makes use of credentials from Azure.Identity](https://github.com/Azure/azure-sdk-for-net/blob/master/sdk/eventhub/Azure.Messaging.EventHubs/samples/Sample06_IdentityAndSharedAccessCredentials.md). diff --git a/samples/DotNet/Rbac/AzureEventHubsSDK/RbacWithAzureEventHubsSdk.csproj b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/AzureEventHubsSDK/RbacWithAzureEventHubsSdk.csproj similarity index 78% rename from samples/DotNet/Rbac/AzureEventHubsSDK/RbacWithAzureEventHubsSdk.csproj rename to samples/DotNet/Microsoft.Azure.EventHubs/Rbac/AzureEventHubsSDK/RbacWithAzureEventHubsSdk.csproj index e3e283cd6..847992f99 100644 --- a/samples/DotNet/Rbac/AzureEventHubsSDK/RbacWithAzureEventHubsSdk.csproj +++ b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/AzureEventHubsSDK/RbacWithAzureEventHubsSdk.csproj @@ -40,34 +40,34 @@ packages\Microsoft.Azure.Amqp.2.4.2\lib\net45\Microsoft.Azure.Amqp.dll - - packages\Microsoft.Azure.EventHubs.3.0.0-dev.20190618.1\lib\net461\Microsoft.Azure.EventHubs.dll + + packages\Microsoft.Azure.EventHubs.4.0.0\lib\net461\Microsoft.Azure.EventHubs.dll - ..\..\..\..\..\..\CxCache\Microsoft.Azure.Services.AppAuthentication.1.2.0\lib\net461\Microsoft.Azure.Services.AppAuthentication.dll + packages\Microsoft.Azure.Services.AppAuthentication.1.2.0\lib\net461\Microsoft.Azure.Services.AppAuthentication.dll True packages\Microsoft.Identity.Client.4.0.0\lib\net45\Microsoft.Identity.Client.dll - ..\..\..\..\..\..\CxCache\Microsoft.IdentityModel.Clients.ActiveDirectory.3.19.4\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.dll + packages\Microsoft.IdentityModel.Clients.ActiveDirectory.3.19.4\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.dll True - ..\..\..\..\..\..\CxCache\Microsoft.IdentityModel.Clients.ActiveDirectory.3.19.4\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.Platform.dll + packages\Microsoft.IdentityModel.Clients.ActiveDirectory.3.19.4\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.Platform.dll True - ..\..\..\..\..\..\CxCache\Microsoft.IdentityModel.JsonWebTokens.5.5.0\lib\net461\Microsoft.IdentityModel.JsonWebTokens.dll + packages\Microsoft.IdentityModel.JsonWebTokens.5.5.0\lib\net461\Microsoft.IdentityModel.JsonWebTokens.dll True - ..\..\..\..\..\..\CxCache\Microsoft.IdentityModel.Logging.5.5.0\lib\net461\Microsoft.IdentityModel.Logging.dll + packages\Microsoft.IdentityModel.Logging.5.5.0\lib\net461\Microsoft.IdentityModel.Logging.dll True - ..\..\..\..\..\..\CxCache\Microsoft.IdentityModel.Tokens.5.5.0\lib\net461\Microsoft.IdentityModel.Tokens.dll + packages\Microsoft.IdentityModel.Tokens.5.5.0\lib\net461\Microsoft.IdentityModel.Tokens.dll True @@ -85,18 +85,18 @@ - ..\..\..\..\..\..\CxCache\System.IdentityModel.Tokens.Jwt.5.5.0\lib\net461\System.IdentityModel.Tokens.Jwt.dll + packages\System.IdentityModel.Tokens.Jwt.5.5.0\lib\net461\System.IdentityModel.Tokens.Jwt.dll True packages\System.Net.Http.4.3.4\lib\net46\System.Net.Http.dll - ..\..\..\..\..\..\CxCache\System.Net.WebSockets.4.3.0\lib\net46\System.Net.WebSockets.dll + packages\System.Net.WebSockets.4.3.0\lib\net46\System.Net.WebSockets.dll True - ..\..\..\..\..\..\CxCache\System.Net.WebSockets.Client.4.3.2\lib\net46\System.Net.WebSockets.Client.dll + packages\System.Net.WebSockets.Client.4.3.2\lib\net46\System.Net.WebSockets.Client.dll True @@ -104,11 +104,11 @@ - ..\..\..\..\..\..\CxCache\System.Runtime.Serialization.Primitives.4.3.0\lib\net46\System.Runtime.Serialization.Primitives.dll + packages\System.Runtime.Serialization.Primitives.4.3.0\lib\net46\System.Runtime.Serialization.Primitives.dll True - ..\..\..\..\..\..\CxCache\System.Security.Cryptography.Algorithms.4.3.1\lib\net461\System.Security.Cryptography.Algorithms.dll + packages\System.Security.Cryptography.Algorithms.4.3.1\lib\net461\System.Security.Cryptography.Algorithms.dll True @@ -118,7 +118,7 @@ packages\System.Security.Cryptography.Primitives.4.3.0\lib\net46\System.Security.Cryptography.Primitives.dll - ..\..\..\..\..\..\CxCache\System.Security.Cryptography.X509Certificates.4.3.2\lib\net461\System.Security.Cryptography.X509Certificates.dll + packages\System.Security.Cryptography.X509Certificates.4.3.2\lib\net461\System.Security.Cryptography.X509Certificates.dll True @@ -136,14 +136,15 @@ Designer - + + Designer + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - + \ No newline at end of file diff --git a/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/ManagedIdentityWebApp.sln b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/ManagedIdentityWebApp.sln new file mode 100644 index 000000000..99d261930 --- /dev/null +++ b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/ManagedIdentityWebApp.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.27004.2010 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ManagedIdentityWebApp", "ManagedIdentityWebApp.csproj", "{476F98B4-05BF-42B6-B0B4-E50CD5B71635}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {476F98B4-05BF-42B6-B0B4-E50CD5B71635}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {476F98B4-05BF-42B6-B0B4-E50CD5B71635}.Debug|Any CPU.Build.0 = Debug|Any CPU + {476F98B4-05BF-42B6-B0B4-E50CD5B71635}.Release|Any CPU.ActiveCfg = Release|Any CPU + {476F98B4-05BF-42B6-B0B4-E50CD5B71635}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {AE17B842-5FBE-4C43-B583-5441703DBCE9} + EndGlobalSection +EndGlobal diff --git a/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/Properties/AssemblyInfo.cs b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..eb184e7ac --- /dev/null +++ b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/Properties/AssemblyInfo.cs @@ -0,0 +1,35 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("ManagedIdentityWebApp")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("ManagedIdentityWebApp")] +[assembly: AssemblyCopyright("Copyright © 2017")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("476f98b4-05bf-42b6-b0b4-e50cd5b71635")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Revision and Build Numbers +// by using the '*' as shown below: +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/README.md b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/README.md new file mode 100644 index 000000000..c1c2af8d3 --- /dev/null +++ b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/README.md @@ -0,0 +1,6 @@ +# Managed Service Identity sample # + +**Note:** This sample uses the legacy Event Hubs library `Microsoft.Azure.EventHubs`. We strongly encourage you to use the current library `Azure.Messaging.EventHubs`. See the [corresponding sample](https://github.com/Azure/azure-event-hubs/tree/master/samples/DotNet/Azure.Messaging.EventHubs/ManagedIdentityWebApp) that uses the current library. + +For more information on Managed Service Identity (MSI) and how to run this sample follow this [link](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-managed-service-identity). + diff --git a/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/SendReceive.aspx b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/SendReceive.aspx new file mode 100644 index 000000000..afa25938d --- /dev/null +++ b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/SendReceive.aspx @@ -0,0 +1,35 @@ +<%@ Page Language="C#" AutoEventWireup="true" CodeBehind="SendReceive.aspx.cs" Inherits="ManagedIdentityWebApp.SendReceive" %> + + + + + + EventHubs Managed Identity Demo + + + +
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+ + +
+
+
+ + diff --git a/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/SendReceive.aspx.cs b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/SendReceive.aspx.cs new file mode 100644 index 000000000..609913342 --- /dev/null +++ b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/SendReceive.aspx.cs @@ -0,0 +1,69 @@ +using System; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.EventHubs; + +// Always add app to IAM roles +// Don't use on deployment slots but only on root +namespace ManagedIdentityWebApp +{ + public partial class SendReceive : System.Web.UI.Page + { + protected void Page_Load(object sender, EventArgs e) + { + + } + + protected void btnSend_Click(object sender, EventArgs e) + { + EventHubClient ehClient = EventHubClient.CreateWithManagedIdentity(new Uri($"sb://{txtNamespace.Text}.servicebus.windows.net/"), txtEventHub.Text); + ehClient.SendAsync(new EventData(Encoding.UTF8.GetBytes(txtData.Text))).GetAwaiter().GetResult(); + txtOutput.Text = $"{DateTime.Now} - SENT{Environment.NewLine}" + txtOutput.Text; + ehClient.Close(); + } + + protected void btnReceive_Click(object sender, EventArgs e) + { + EventHubClient ehClient = EventHubClient.CreateWithManagedIdentity(new Uri($"sb://{txtNamespace.Text}.servicebus.windows.net/"), txtEventHub.Text); + int partitions = int.Parse(txtPartitions.Text); + string[] lastOffsets = new string[partitions]; + + if (!string.IsNullOrEmpty(hiddenStartingOffset.Value)) + { + lastOffsets = hiddenStartingOffset.Value.Split(','); + } + + var totalReceived = 0; + + Parallel.ForEach(Enumerable.Range(0, int.Parse(txtPartitions.Text)), partitionId => + { + var receiver = ehClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, $"{partitionId}", lastOffsets[partitionId] == null ? EventPosition.FromStart() : EventPosition.FromOffset(lastOffsets[partitionId])); + var messages = receiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(15)).GetAwaiter().GetResult(); + + if (messages != null) + { + foreach (var message in messages) + { + txtOutput.Text = $"{DateTime.Now} - RECEIVED PartitionId: {partitionId} Seq#:{message.SystemProperties.SequenceNumber} data:{Encoding.UTF8.GetString(message.Body.Array)}{Environment.NewLine}" + txtOutput.Text; + lastOffsets[partitionId] = message.SystemProperties.Offset; + } + + Interlocked.Add(ref totalReceived, messages.Count()); + } + + receiver.Close(); + }); + + txtOutput.Text = $"{DateTime.Now} - RECEIVED TOTAL = {totalReceived}{Environment.NewLine}" + txtOutput.Text; + + if (totalReceived > 0) + { + hiddenStartingOffset.Value = string.Join(",", lastOffsets); + } + + ehClient.Close(); + } + } +} \ No newline at end of file diff --git a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/EventHubsMSIDemo.aspx.designer.cs b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/SendReceive.aspx.designer.cs similarity index 94% rename from samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/EventHubsMSIDemo.aspx.designer.cs rename to samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/SendReceive.aspx.designer.cs index 98a8eb487..00124e3c9 100644 --- a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/EventHubsMSIDemo.aspx.designer.cs +++ b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/SendReceive.aspx.designer.cs @@ -7,10 +7,10 @@ // //------------------------------------------------------------------------------ -namespace EventHubsMSIDemoWebApp { +namespace ManagedIdentityWebApp { - public partial class EventHubsMSIDemo { + public partial class SendReceive { /// /// form1 control. @@ -76,13 +76,13 @@ public partial class EventHubsMSIDemo { protected global::System.Web.UI.WebControls.Button btnReceive; /// - /// txtReceivedData control. + /// txtOutput control. /// /// /// Auto-generated field. /// To modify move field declaration from designer file to code-behind file. /// - protected global::System.Web.UI.WebControls.TextBox txtReceivedData; + protected global::System.Web.UI.WebControls.TextBox txtOutput; /// /// hiddenStartingOffset control. diff --git a/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/StyleSheet.css b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/StyleSheet.css new file mode 100644 index 000000000..18b4dd479 --- /dev/null +++ b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/StyleSheet.css @@ -0,0 +1,13 @@ +label { + width: 200px; + display: inline-block; +} + +text { + width: 300px; + display: block; +} + +input[type="submit"] { + width: 75px; +} diff --git a/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/Web.Debug.config b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/Web.Debug.config new file mode 100644 index 000000000..fae9cfefa --- /dev/null +++ b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/Web.Debug.config @@ -0,0 +1,30 @@ + + + + + + + + + + \ No newline at end of file diff --git a/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/Web.Release.config b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/Web.Release.config new file mode 100644 index 000000000..da6e960b8 --- /dev/null +++ b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/Web.Release.config @@ -0,0 +1,31 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/Web.config b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/Web.config similarity index 53% rename from samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/Web.config rename to samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/Web.config index 245b26e30..635d14d02 100644 --- a/samples/DotNet/MSI/EventHubsMSIDemoWebApp/EventHubsMSIDemoWebApp/Web.config +++ b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/Web.config @@ -4,23 +4,43 @@ https://go.microsoft.com/fwlink/?LinkId=169433 --> + - + - - - - - - + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/packages.config b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/packages.config new file mode 100644 index 000000000..c5c4cbbc5 --- /dev/null +++ b/samples/DotNet/Microsoft.Azure.EventHubs/Rbac/ManagedIdentityWebApp/packages.config @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/samples/DotNet/Microsoft.Azure.EventHubs/SampleEphReceiver/Program.cs b/samples/DotNet/Microsoft.Azure.EventHubs/SampleEphReceiver/Program.cs index 75d50e116..853e17449 100644 --- a/samples/DotNet/Microsoft.Azure.EventHubs/SampleEphReceiver/Program.cs +++ b/samples/DotNet/Microsoft.Azure.EventHubs/SampleEphReceiver/Program.cs @@ -15,7 +15,7 @@ public class Program private const string StorageContainerName = "Storage account container name"; private const string StorageAccountName = "Storage account name"; private const string StorageAccountKey = "Storage account key"; - + private static readonly string StorageConnectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", StorageAccountName, StorageAccountKey); public static void Main(string[] args) diff --git a/samples/DotNet/Microsoft.Azure.EventHubs/SampleEphReceiver/readme.md b/samples/DotNet/Microsoft.Azure.EventHubs/SampleEphReceiver/readme.md index efecf5b65..e7a9eac27 100644 --- a/samples/DotNet/Microsoft.Azure.EventHubs/SampleEphReceiver/readme.md +++ b/samples/DotNet/Microsoft.Azure.EventHubs/SampleEphReceiver/readme.md @@ -1,14 +1,16 @@ # Receive events with the Event Processor Host in .NET Standard -This sample shows how to write a .NET Core console application that receives a set of events from an event hub by using the **Event Processor Host** library. You can run the solution as-is, replacing the strings with your event hub and storage account values. The sample is also [available as a tutorial](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-dotnet-standard-getstarted-receive-eph). +**Note:** This sample uses the legacy Event Hubs library `Microsoft.Azure.EventHubs.Processor` to write a .NET Core console application that receives a set of events from an Event Hub instance. We strongly encourage you to refer to the [quick start that uses the new library `Azure.Messaging.EventHubs.Processor`](https://docs.microsoft.com/azure/event-hubs/event-hubs-dotnet-standard-getstarted-send#receive-events) instead. + +You can run the solution as-is, replacing the strings with your event hub and storage account values. ## Prerequisites -* [Microsoft Visual Studio 2015 or 2017](http://www.visualstudio.com). -* [.NET Core Visual Studio 2015 or 2017 tools](http://www.microsoft.com/net/core). -* An Azure subscription. -* [An event hub namespace and an event hub](event-hubs-quickstart-namespace-portal.md). -* An Azure Storage account. +- [Microsoft Visual Studio 2015 or 2017](http://www.visualstudio.com). +- [.NET Core Visual Studio 2015 or 2017 tools](http://www.microsoft.com/net/core). +- An [Azure subscription](https://azure.microsoft.com/free/). +- [An event hub namespace and an event hub](event-hubs-quickstart-namespace-portal.md). +- An Azure Storage account. ## Run the sample @@ -20,17 +22,19 @@ To run the sample, follow these steps: 4. Load the SampleEphReceiver.sln solution file into Visual Studio. 5. Add the [Microsoft.Azure.EventHubs](https://www.nuget.org/packages/Microsoft.Azure.EventHubs/) and [Microsoft.Azure.EventHubs.Processor](https://www.nuget.org/packages/Microsoft.Azure.EventHubs.Processor/) NuGet packages to the project. 6. In Program.cs, replace the following constants with the corresponding values for the event hub connection string, event hub name: - ```csharp - private const string EventHubConnectionString = "Event Hubs connection string"; - private const string EventHubName = "Event Hub name"; - ``` -7. Create a Storage account to host a blob container, needed for lease management by the Event Processor Host. + ```csharp + private const string EventHubConnectionString = "Event Hubs connection string"; + private const string EventHubName = "Event Hub name"; + ``` +7. Create a Storage account to host a blob container, needed for lease management by the Event Processor Host. 8. In Program.cs, replace the storage account container name, storage account name, and storage account key (the container will be created if not present): + ``` private const string StorageContainerName = "Storage account container name"; private const string StorageAccountName = "Storage account name"; private const string StorageAccountKey = "Storage account key"; ``` + 9. Run the program, and ensure that there are no errors. Congratulations! You have now received events from an event hub by using the Event Processor Host. To send events, see the [SampleSender](https://github.com/Azure/azure-event-hubs/tree/master/samples/DotNet/Microsoft.Azure.EventHubs/SampleSender) sample. diff --git a/samples/DotNet/Microsoft.Azure.EventHubs/SampleSender/readme.md b/samples/DotNet/Microsoft.Azure.EventHubs/SampleSender/readme.md index 0054911dc..1c2322946 100644 --- a/samples/DotNet/Microsoft.Azure.EventHubs/SampleSender/readme.md +++ b/samples/DotNet/Microsoft.Azure.EventHubs/SampleSender/readme.md @@ -1,29 +1,30 @@ # Send events to Azure Event Hubs in .NET Standard -This sample shows how to write a .NET Core console application that sends a set of events to an event hub. You can run the solution as-is, replacing the `EhConnectionString` and `EhEntityPath` strings with your event hub values. The sample is also [available as a tutorial](https://docs.microsoft.com/azure/event-hubs/event-hubs-dotnet-standard-getstarted-send). +**Note:** This sample uses the legacy Event Hubs library `Microsoft.Azure.EventHubs.Processor` to write a .NET Core console application that sends a set of events to an Event Hub instance. We strongly encourage you to refer to the [quick start that uses the new library `Azure.Messaging.EventHubs`](https://docs.microsoft.com/azure/event-hubs/event-hubs-dotnet-standard-getstarted-send#send-events) instead. + +You can run the solution as-is, replacing the strings with your Event Hub instance details like connection string and name. ## Prerequisites -* [Microsoft Visual Studio 2015 or 2017](http://www.visualstudio.com). -* [.NET Core Visual Studio 2015 or 2017 tools](http://www.microsoft.com/net/core). -* An Azure subscription. -* [An event hub namespace and an event hub](event-hubs-quickstart-namespace-portal.md). +- [Microsoft Visual Studio 2015 or 2017](http://www.visualstudio.com). +- [.NET Core SDK](http://www.microsoft.com/net/core). +- An [Azure subscription](https://azure.microsoft.com/free/). +- [An Event Hub namespace and an Event Hub](event-hubs-quickstart-namespace-portal.md). ## Run the sample To run the sample, follow these steps: 1. Clone or download this GitHub repo. -2. [Create an Event Hubs namespace and an event hub](https://docs.microsoft.com/azure/event-hubs/event-hubs-create). +2. [Create an Event Hubs namespace and an Event Hub](https://docs.microsoft.com/azure/event-hubs/event-hubs-create). 3. In Visual Studio, select **File**, then **Open Project/Soultion**. Navigate to the \azure-event-hubs\samples\DotNet\Microsoft.Azure.EventHubs\SampleSender folder. 4. Load the SampleSender.sln solution file into Visual Studio. 5. Add the [Microsoft.Azure.EventHubs](https://www.nuget.org/packages/Microsoft.Azure.EventHubs/) NuGet package to the project. -6. In Program.cs, replace the placeholders in brackets with the proper values that were obtained when creating the event hub. Make sure that the `Event Hubs connection string` is the namespace-level connection string, and not the event hub string: - ```csharp - private const string EhConnectionString = "Event Hubs connection string"; - private const string EhEntityPath = "Event Hub name"; - ``` +6. In Program.cs, replace the placeholders in brackets with the proper values that were obtained when creating the Event Hub. Make sure that the `Event Hubs connection string` is the namespace-level connection string, and not the Event Hub string: + ```csharp + private const string EhConnectionString = "Event Hubs connection string"; + private const string EhEntityPath = "Event Hub name"; + ``` 7. Run the program, and ensure that there are no errors. -Congratulations! You have now sent events to an event hub. To receive these events, see the [SampleEphReceiver](https://github.com/Azure/azure-event-hubs/tree/master/samples/DotNet/Microsoft.Azure.EventHubs/SampleEphReceiver) sample. - +Congratulations! You have now sent events to an Event Hub. To receive these events, see the [SampleEphReceiver](https://github.com/Azure/azure-event-hubs/tree/master/samples/DotNet/Microsoft.Azure.EventHubs/SampleEphReceiver) sample. diff --git a/samples/DotNet/ServiceFabricProcessor/ProgrammersGuide.md b/samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/ProgrammersGuide.md similarity index 100% rename from samples/DotNet/ServiceFabricProcessor/ProgrammersGuide.md rename to samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/ProgrammersGuide.md diff --git a/samples/DotNet/ServiceFabricProcessor/Readme.md b/samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/Readme.md similarity index 100% rename from samples/DotNet/ServiceFabricProcessor/Readme.md rename to samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/Readme.md diff --git a/samples/DotNet/ServiceFabricProcessor/SFProcessorSample.sln b/samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/SFProcessorSample.sln similarity index 100% rename from samples/DotNet/ServiceFabricProcessor/SFProcessorSample.sln rename to samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/SFProcessorSample.sln diff --git a/samples/DotNet/ServiceFabricProcessor/SFProcessorSample/ApplicationPackageRoot/ApplicationManifest.xml b/samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/SFProcessorSample/ApplicationPackageRoot/ApplicationManifest.xml similarity index 100% rename from samples/DotNet/ServiceFabricProcessor/SFProcessorSample/ApplicationPackageRoot/ApplicationManifest.xml rename to samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/SFProcessorSample/ApplicationPackageRoot/ApplicationManifest.xml diff --git a/samples/DotNet/ServiceFabricProcessor/SFProcessorSample/ApplicationParameters/Cloud.xml b/samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/SFProcessorSample/ApplicationParameters/Cloud.xml similarity index 100% rename from samples/DotNet/ServiceFabricProcessor/SFProcessorSample/ApplicationParameters/Cloud.xml rename to samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/SFProcessorSample/ApplicationParameters/Cloud.xml diff --git a/samples/DotNet/ServiceFabricProcessor/SFProcessorSample/ApplicationParameters/Local.1Node.xml b/samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/SFProcessorSample/ApplicationParameters/Local.1Node.xml similarity index 100% rename from samples/DotNet/ServiceFabricProcessor/SFProcessorSample/ApplicationParameters/Local.1Node.xml rename to samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/SFProcessorSample/ApplicationParameters/Local.1Node.xml diff --git a/samples/DotNet/ServiceFabricProcessor/SFProcessorSample/ApplicationParameters/Local.5Node.xml b/samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/SFProcessorSample/ApplicationParameters/Local.5Node.xml similarity index 100% rename from samples/DotNet/ServiceFabricProcessor/SFProcessorSample/ApplicationParameters/Local.5Node.xml rename to samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/SFProcessorSample/ApplicationParameters/Local.5Node.xml diff --git a/samples/DotNet/ServiceFabricProcessor/SFProcessorSample/PublishProfiles/Cloud.xml b/samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/SFProcessorSample/PublishProfiles/Cloud.xml similarity index 100% rename from samples/DotNet/ServiceFabricProcessor/SFProcessorSample/PublishProfiles/Cloud.xml rename to samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/SFProcessorSample/PublishProfiles/Cloud.xml diff --git a/samples/DotNet/ServiceFabricProcessor/SFProcessorSample/PublishProfiles/Local.1Node.xml b/samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/SFProcessorSample/PublishProfiles/Local.1Node.xml similarity index 100% rename from samples/DotNet/ServiceFabricProcessor/SFProcessorSample/PublishProfiles/Local.1Node.xml rename to samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/SFProcessorSample/PublishProfiles/Local.1Node.xml diff --git a/samples/DotNet/ServiceFabricProcessor/SFProcessorSample/PublishProfiles/Local.5Node.xml b/samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/SFProcessorSample/PublishProfiles/Local.5Node.xml similarity index 100% rename from samples/DotNet/ServiceFabricProcessor/SFProcessorSample/PublishProfiles/Local.5Node.xml rename to samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/SFProcessorSample/PublishProfiles/Local.5Node.xml diff --git a/samples/DotNet/ServiceFabricProcessor/SFProcessorSample/SFProcessorSample.sfproj b/samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/SFProcessorSample/SFProcessorSample.sfproj similarity index 100% rename from samples/DotNet/ServiceFabricProcessor/SFProcessorSample/SFProcessorSample.sfproj rename to samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/SFProcessorSample/SFProcessorSample.sfproj diff --git a/samples/DotNet/ServiceFabricProcessor/SFProcessorSample/Scripts/Deploy-FabricApplication.ps1 b/samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/SFProcessorSample/Scripts/Deploy-FabricApplication.ps1 similarity index 100% rename from samples/DotNet/ServiceFabricProcessor/SFProcessorSample/Scripts/Deploy-FabricApplication.ps1 rename to samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/SFProcessorSample/Scripts/Deploy-FabricApplication.ps1 diff --git a/samples/DotNet/ServiceFabricProcessor/SFProcessorSample/packages.config b/samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/SFProcessorSample/packages.config similarity index 100% rename from samples/DotNet/ServiceFabricProcessor/SFProcessorSample/packages.config rename to samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/SFProcessorSample/packages.config diff --git a/samples/DotNet/ServiceFabricProcessor/Stateful1/PackageRoot/Config/Settings.xml b/samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/Stateful1/PackageRoot/Config/Settings.xml similarity index 100% rename from samples/DotNet/ServiceFabricProcessor/Stateful1/PackageRoot/Config/Settings.xml rename to samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/Stateful1/PackageRoot/Config/Settings.xml diff --git a/samples/DotNet/ServiceFabricProcessor/Stateful1/PackageRoot/ServiceManifest.xml b/samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/Stateful1/PackageRoot/ServiceManifest.xml similarity index 100% rename from samples/DotNet/ServiceFabricProcessor/Stateful1/PackageRoot/ServiceManifest.xml rename to samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/Stateful1/PackageRoot/ServiceManifest.xml diff --git a/samples/DotNet/ServiceFabricProcessor/Stateful1/Program.cs b/samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/Stateful1/Program.cs similarity index 100% rename from samples/DotNet/ServiceFabricProcessor/Stateful1/Program.cs rename to samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/Stateful1/Program.cs diff --git a/samples/DotNet/ServiceFabricProcessor/Stateful1/SampleEventProcessor.cs b/samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/Stateful1/SampleEventProcessor.cs similarity index 100% rename from samples/DotNet/ServiceFabricProcessor/Stateful1/SampleEventProcessor.cs rename to samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/Stateful1/SampleEventProcessor.cs diff --git a/samples/DotNet/ServiceFabricProcessor/Stateful1/ServiceEventSource.cs b/samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/Stateful1/ServiceEventSource.cs similarity index 100% rename from samples/DotNet/ServiceFabricProcessor/Stateful1/ServiceEventSource.cs rename to samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/Stateful1/ServiceEventSource.cs diff --git a/samples/DotNet/ServiceFabricProcessor/Stateful1/Stateful1.cs b/samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/Stateful1/Stateful1.cs similarity index 100% rename from samples/DotNet/ServiceFabricProcessor/Stateful1/Stateful1.cs rename to samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/Stateful1/Stateful1.cs diff --git a/samples/DotNet/ServiceFabricProcessor/Stateful1/Stateful1.csproj b/samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/Stateful1/Stateful1.csproj similarity index 100% rename from samples/DotNet/ServiceFabricProcessor/Stateful1/Stateful1.csproj rename to samples/DotNet/Microsoft.Azure.EventHubs/ServiceFabricProcessor/Stateful1/Stateful1.csproj diff --git a/samples/DotNet/Microsoft.ServiceBus/ControlAndDataPlane/App.config b/samples/DotNet/Microsoft.ServiceBus/ControlAndDataPlane/App.config new file mode 100644 index 000000000..a515cf233 --- /dev/null +++ b/samples/DotNet/Microsoft.ServiceBus/ControlAndDataPlane/App.config @@ -0,0 +1,12 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/samples/DotNet/Microsoft.ServiceBus/ControlAndDataPlane/ControlAndDataPlane.csproj b/samples/DotNet/Microsoft.ServiceBus/ControlAndDataPlane/ControlAndDataPlane.csproj new file mode 100644 index 000000000..945dcb3f0 --- /dev/null +++ b/samples/DotNet/Microsoft.ServiceBus/ControlAndDataPlane/ControlAndDataPlane.csproj @@ -0,0 +1,88 @@ + + + + + Debug + AnyCPU + {1485E9E2-11C3-4601-929D-04618277EB9D} + Exe + ControlAndDataPlane + ControlAndDataPlane + v4.7.2 + 512 + true + true + + + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + latest + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + packages\Microsoft.Azure.Services.AppAuthentication.1.0.3\lib\net452\Microsoft.Azure.Services.AppAuthentication.dll + + + packages\Microsoft.Identity.Client.4.0.0\lib\net45\Microsoft.Identity.Client.dll + + + packages\Microsoft.IdentityModel.Clients.ActiveDirectory.3.19.8\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.dll + + + packages\Microsoft.IdentityModel.Clients.ActiveDirectory.3.19.8\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.Platform.dll + + + packages\WindowsAzure.ServiceBus.6.0.0\lib\net45\Microsoft.ServiceBus.dll + + + + + + + + packages\System.IdentityModel.Tokens.Jwt.4.0.4.403061554\lib\net45\System.IdentityModel.Tokens.Jwt.dll + + + + + + + + + + + + + + + + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + \ No newline at end of file diff --git a/samples/DotNet/Microsoft.ServiceBus/ControlAndDataPlane/ControlAndDataPlane.sln b/samples/DotNet/Microsoft.ServiceBus/ControlAndDataPlane/ControlAndDataPlane.sln new file mode 100644 index 000000000..5d25c2075 --- /dev/null +++ b/samples/DotNet/Microsoft.ServiceBus/ControlAndDataPlane/ControlAndDataPlane.sln @@ -0,0 +1,22 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 14 +VisualStudioVersion = 14.0.25420.1 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ControlAndDataPlane", "ControlAndDataPlane.csproj", "{1485E9E2-11C3-4601-929D-04618277EB9D}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {1485E9E2-11C3-4601-929D-04618277EB9D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {1485E9E2-11C3-4601-929D-04618277EB9D}.Debug|Any CPU.Build.0 = Debug|Any CPU + {1485E9E2-11C3-4601-929D-04618277EB9D}.Release|Any CPU.ActiveCfg = Release|Any CPU + {1485E9E2-11C3-4601-929D-04618277EB9D}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/samples/DotNet/Microsoft.ServiceBus/ControlAndDataPlane/Program.cs b/samples/DotNet/Microsoft.ServiceBus/ControlAndDataPlane/Program.cs new file mode 100644 index 000000000..400dce82d --- /dev/null +++ b/samples/DotNet/Microsoft.ServiceBus/ControlAndDataPlane/Program.cs @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace ControlAndDataPlane +{ + using System; + using System.Configuration; + using System.Linq; + using System.Text; + using System.Threading.Tasks; + using Microsoft.Identity.Client; + using Microsoft.ServiceBus; + using Microsoft.ServiceBus.Messaging; + + class Program + { + static readonly string TenantId = ConfigurationManager.AppSettings["tenantId"]; + static readonly string ClientId = ConfigurationManager.AppSettings["clientId"]; + static readonly string ClientSecret = ConfigurationManager.AppSettings["clientSecret"]; + static readonly string EventHubNamespace = ConfigurationManager.AppSettings["eventHubNamespaceFQDN"]; + + public static async Task Main(string[] args) + { + // Create token provider so that we can use it at both management and runtime clients. + TokenProvider tokenProvider = TokenProvider.CreateAzureActiveDirectoryTokenProvider( + async (audience, authority, state) => + { + IConfidentialClientApplication app = ConfidentialClientApplicationBuilder.Create(ClientId) + .WithAuthority(authority) + .WithClientSecret(ClientSecret) + .Build(); + + var authResult = await app.AcquireTokenForClient(new string[] { $"{audience}/.default" }).ExecuteAsync(); + return authResult.AccessToken; + }, + ServiceAudience.EventHubsAudience, + $"https://login.microsoftonline.com/{TenantId}"); + + var eventHubName = "testeh-" + Guid.NewGuid().ToString(); + + // Create NamespaceManger and EventHubClient with Azure Active Directory token provider. + var ehUri = new Uri($"sb://{EventHubNamespace}/"); + var namespaceManager = new NamespaceManager(ehUri, tokenProvider); + var messagingFactory = MessagingFactory.Create(ehUri, + new MessagingFactorySettings() + { + TokenProvider = tokenProvider, + TransportType = TransportType.Amqp + }); + var ehClient = messagingFactory.CreateEventHubClient(eventHubName); + + // Create a new event hub. + Console.WriteLine($"Creating event hub {eventHubName}"); + await namespaceManager.CreateEventHubAsync(eventHubName); + + // Send and receive a message. + await SendReceiveAsync(ehClient); + + // Delete event hub. + Console.WriteLine($"Deleting event hub {eventHubName}"); + await namespaceManager.DeleteEventHubAsync(eventHubName); + + Console.WriteLine("Press enter to exit"); + Console.ReadLine(); + } + + static async Task SendReceiveAsync(EventHubClient ehClient) + { + Console.WriteLine("Fetching eventhub description to discover partitions"); + var ehDesc = await ehClient.GetRuntimeInformationAsync(); + Console.WriteLine($"Discovered partitions as {string.Join(",", ehDesc.PartitionIds)}"); + + var defaultConsumerGroup = ehClient.GetDefaultConsumerGroup(); + var receiveTasks = ehDesc.PartitionIds.Select(async partitionId => + { + Console.WriteLine($"Initiating receiver on partition {partitionId}"); + var receiver = await defaultConsumerGroup.CreateReceiverAsync(partitionId, EventPosition.FromStart()); + + while(true) + { + var events = await receiver.ReceiveAsync(1, TimeSpan.FromSeconds(15)); + if (events == null || events.Count() == 0) + { + break; + } + + Console.WriteLine($"Received from partition {partitionId} with message content '" + + Encoding.UTF8.GetString(events.First().GetBytes()) + "'"); + } + + await receiver.CloseAsync(); + }).ToList(); + + await Task.Delay(5000); + + Console.WriteLine("Sending single event"); + await ehClient.SendAsync(new EventData(Encoding.UTF8.GetBytes("Hello!"))); + Console.WriteLine("Send done"); + + Console.WriteLine("Waiting for receivers to complete"); + await Task.WhenAll(receiveTasks); + Console.WriteLine("All receivers completed"); + } + } +} diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/DWDumper/Properties/AssemblyInfo.cs b/samples/DotNet/Microsoft.ServiceBus/ControlAndDataPlane/Properties/AssemblyInfo.cs similarity index 83% rename from samples/e2e/EventHubsCaptureEventGridDemo/DWDumper/Properties/AssemblyInfo.cs rename to samples/DotNet/Microsoft.ServiceBus/ControlAndDataPlane/Properties/AssemblyInfo.cs index d12e26ed9..e82f585be 100644 --- a/samples/e2e/EventHubsCaptureEventGridDemo/DWDumper/Properties/AssemblyInfo.cs +++ b/samples/DotNet/Microsoft.ServiceBus/ControlAndDataPlane/Properties/AssemblyInfo.cs @@ -1,16 +1,15 @@ using System.Reflection; -using System.Runtime.CompilerServices; using System.Runtime.InteropServices; // General Information about an assembly is controlled through the following // set of attributes. Change these attribute values to modify the information // associated with an assembly. -[assembly: AssemblyTitle("DWDumper")] +[assembly: AssemblyTitle("ControlAndDataPlane")] [assembly: AssemblyDescription("")] [assembly: AssemblyConfiguration("")] [assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("DWDumper")] -[assembly: AssemblyCopyright("Copyright © 2017")] +[assembly: AssemblyProduct("ControlAndDataPlane")] +[assembly: AssemblyCopyright("Copyright © 2019")] [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] @@ -20,7 +19,7 @@ [assembly: ComVisible(false)] // The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("c9e1d3c8-0c0d-45eb-a3b5-15f95efc01e3")] +[assembly: Guid("5c908c55-400f-4041-99ff-eb2ead0ae8d3")] // Version information for an assembly consists of the following four values: // diff --git a/samples/DotNet/Rbac/EventHubsSenderReceiverRbac/README.md b/samples/DotNet/Microsoft.ServiceBus/ControlAndDataPlane/README.md similarity index 74% rename from samples/DotNet/Rbac/EventHubsSenderReceiverRbac/README.md rename to samples/DotNet/Microsoft.ServiceBus/ControlAndDataPlane/README.md index 9e4cd2c30..6a3556155 100644 --- a/samples/DotNet/Rbac/EventHubsSenderReceiverRbac/README.md +++ b/samples/DotNet/Microsoft.ServiceBus/ControlAndDataPlane/README.md @@ -1,3 +1,3 @@ -# Role based access sample # +# Role based access sample with WindowsAzure.ServiceBus SDK # For more information on Role based access (RBAC) and how to run this sample follow [this](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-role-based-access-control) link. diff --git a/samples/DotNet/Microsoft.ServiceBus/ControlAndDataPlane/packages.config b/samples/DotNet/Microsoft.ServiceBus/ControlAndDataPlane/packages.config new file mode 100644 index 000000000..768225688 --- /dev/null +++ b/samples/DotNet/Microsoft.ServiceBus/ControlAndDataPlane/packages.config @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/samples/DotNet/EventHubReliableSend/EventHubReliableSend/EventHubReliableSend.csproj b/samples/DotNet/Microsoft.ServiceBus/EventHubReliableSend/EventHubReliableSend.csproj similarity index 79% rename from samples/DotNet/EventHubReliableSend/EventHubReliableSend/EventHubReliableSend.csproj rename to samples/DotNet/Microsoft.ServiceBus/EventHubReliableSend/EventHubReliableSend.csproj index 0f0e6449e..08ccb1e80 100644 --- a/samples/DotNet/EventHubReliableSend/EventHubReliableSend/EventHubReliableSend.csproj +++ b/samples/DotNet/Microsoft.ServiceBus/EventHubReliableSend/EventHubReliableSend.csproj @@ -73,28 +73,25 @@ - ..\..\..\..\..\..\CxCache\Microsoft.Azure.Services.AppAuthentication.1.0.3\lib\net452\Microsoft.Azure.Services.AppAuthentication.dll + packages\Microsoft.Azure.Services.AppAuthentication.1.0.3\lib\net452\Microsoft.Azure.Services.AppAuthentication.dll - ..\..\..\..\..\..\CxCache\Microsoft.IdentityModel.Clients.ActiveDirectory.3.19.8\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.dll + packages\Microsoft.IdentityModel.Clients.ActiveDirectory.3.19.8\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.dll - ..\..\..\..\..\..\CxCache\Microsoft.IdentityModel.Clients.ActiveDirectory.3.19.8\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.Platform.dll + packages\Microsoft.IdentityModel.Clients.ActiveDirectory.3.19.8\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.Platform.dll - ..\..\..\..\..\..\CxCache\WindowsAzure.ServiceBus.5.0.0\lib\net46\Microsoft.ServiceBus.dll - - - ..\..\..\..\..\enl\AppServer\bins\retail\amd64\Appserver\Test\Microsoft.ServiceBus.TestHooks.dll + packages\WindowsAzure.ServiceBus.5.2.0\lib\net45\Microsoft.ServiceBus.dll - ..\packages\Microsoft.WindowsAzure.ConfigurationManager.3.2.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll + packages\Microsoft.WindowsAzure.ConfigurationManager.3.2.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll - ..\..\..\..\..\..\CxCache\System.IdentityModel.Tokens.Jwt.4.0.4.403061554\lib\net45\System.IdentityModel.Tokens.Jwt.dll + packages\System.IdentityModel.Tokens.Jwt.4.0.4.403061554\lib\net45\System.IdentityModel.Tokens.Jwt.dll @@ -140,12 +137,12 @@ - + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - + - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/samples/DotNet/Rbac/AzureEventHubsSDK/README.md b/samples/DotNet/Rbac/AzureEventHubsSDK/README.md deleted file mode 100644 index 41ced41b3..000000000 --- a/samples/DotNet/Rbac/AzureEventHubsSDK/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Role based access sample with Microsoft.Azure.EventHubs SDK # - -For more information on Role based access (RBAC) and how to run this sample follow [this](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-role-based-access-control) link. diff --git a/samples/Java/Basic/AdvancedSendOptions/pom.xml b/samples/Java/Basic/AdvancedSendOptions/pom.xml index b0bde814f..4cb5b21de 100644 --- a/samples/Java/Basic/AdvancedSendOptions/pom.xml +++ b/samples/Java/Basic/AdvancedSendOptions/pom.xml @@ -43,16 +43,16 @@ - + - com.microsoft.azure - azure-eventhubs - 2.3.0 - - + com.microsoft.azure + azure-eventhubs + 3.0.2 + + com.google.code.gson gson 2.8.2 - + diff --git a/samples/Java/Basic/AdvancedSendOptions/src/main/java/com/microsoft/azure/eventhubs/samples/AdvancedSendOptions/AdvancedSendOptions.java b/samples/Java/Basic/AdvancedSendOptions/src/main/java/com/microsoft/azure/eventhubs/samples/AdvancedSendOptions/AdvancedSendOptions.java index 4d5fedc7b..b7e6a77a7 100644 --- a/samples/Java/Basic/AdvancedSendOptions/src/main/java/com/microsoft/azure/eventhubs/samples/AdvancedSendOptions/AdvancedSendOptions.java +++ b/samples/Java/Basic/AdvancedSendOptions/src/main/java/com/microsoft/azure/eventhubs/samples/AdvancedSendOptions/AdvancedSendOptions.java @@ -46,7 +46,7 @@ public static void main(String[] args) // Each EventHubClient instance spins up a new TCP/SSL connection, which is expensive. // It is always a best practice to reuse these instances. The following sample shows the same. - final EventHubClient ehClient = EventHubClient.createSync(connStr.toString(), executorService); + final EventHubClient ehClient = EventHubClient.createFromConnectionStringSync(connStr.toString(), executorService); PartitionSender sender = null; try { diff --git a/samples/Java/Basic/AsyncSend/README.md b/samples/Java/Basic/AsyncSend/README.md new file mode 100644 index 000000000..2cd77923a --- /dev/null +++ b/samples/Java/Basic/AsyncSend/README.md @@ -0,0 +1,68 @@ +# Send events asynchronously to Azure Event Hubs using Java + +To run the sample, you need to edit the [sample code](src/main/java/com/microsoft/azure/eventhubs/samples/asyncsend/AsyncSend.java) and provide the following information: + +```java + final String namespaceName = "----EventHubsNamespaceName-----"; + final String eventHubName = "----EventHubName-----"; + final String sasKeyName = "-----SharedAccessSignatureKeyName-----"; + final String sasKey = "---SharedAccessSignatureKey----"; +``` + +## Prerequisites + +Please refer to the [overview README](../../readme.md) for prerequisites and setting up the sample environment, including creating an Event Hubs cloud namespace and an Event Hub. + +## Build and run + +The sample can be built independently with + +```bash +mvn clean package +``` + +and then run with (or just from VS Code or another Java IDE) + +```bash +java -jar ./target/asyncsend-1.0.0-jar-with-dependencies.jar +``` + +## Understanding Async Send + +This sample demonstrates using the asynchronous send() method on the EventHubClient class. + +The corresponding sendSync() method is pretty simple: it waits for the the send operation to +be complete, then returns, indicating success, or throws, indicating an error. Unfortunately, it's +not fast, because it means waiting for the message to go across the network to the service, for +the service to store the message, and then for the result to come back across the network. + +Switching to the asynchronous send() means your code must be more +sophisticated. By its nature, an asynchronous method just starts an operation and does not wait +for it to finish, so your code can be doing something useful while the operation proceeds in the +background. However, that means that the method itself cannot tell +you anything about the result of the operation, because there is no result yet when the method +returns. Java provides a standard class, CompletableFuture, for asynchronous methods to return, +which allows the calling code to obtain the result of the operation at a later time, and the Event +Hubs Java client makes use of that class. CompletableFuture offers some sophisticated abilities, +but this sample simply retrieves the result by calling get(), which will return when the operation +finishes (or has already finished) successfully, or throw an exception if there was an error. + +One of the things your code can be doing while waiting for a send operation to finish is to prepare +and send additional messages, which this sample demonstrates. This is known as concurrent sending +and it is a good way to increase your throughput. The sample sends three groups of 100 messages each, +with one concurrent send, ten concurrent sends, and thirty concurrent sends respectively. It uses a +queue to save the CompletableFuture instances returned by send(), and when the queue grows larger +than the desired number of concurrent operations, it waits for the oldest CompletableFuture to +complete. One concurrent send is equivalent to making a synchronous call: each send must wait for the +previous one to finish. With ten concurrent sends, the average wait time and total execution time is +much smaller, because by the time the 11th send is attempted, the first send is mostly done, and the +same for the 12th send and the second, etc. With thirty concurrent sends, the average wait time is +nearly 0, because the first send is already finished by the time the 31st is attempted, etc. The +useful number of concurrent calls is limited by memory usage, CPU usage, network congestion, and other +factors, and we do not recommend trying more than about 100. + +Error handling is also different when using asynchronous calls. Supposing the get() on the CompletableFuture +throws an error, indicating a failure, but you still want that message and would like to retry sending +it -- how do you know what message failed? This sample demonstrates one possible approach, by storing +the original message with the CompletableFuture. There are many possibilities and which is best +depends on the structure of your application. \ No newline at end of file diff --git a/samples/Java/Basic/AsyncSend/pom.xml b/samples/Java/Basic/AsyncSend/pom.xml new file mode 100644 index 000000000..2b9760505 --- /dev/null +++ b/samples/Java/Basic/AsyncSend/pom.xml @@ -0,0 +1,53 @@ + + 4.0.0 + asyncsend + 1.0.0 + asyncsend + jar + + UTF-8 + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.6.1 + + 1.8 + 1.8 + true + lines,vars,source + + + + maven-assembly-plugin + + + package + + single + + + + + + jar-with-dependencies + + + + com.microsoft.azure.eventhubs.samples.AsyncSend.AsyncSend + + + + + + + + + com.microsoft.azure + azure-eventhubs + 3.1.1 + + + diff --git a/samples/Java/Basic/AsyncSend/src/main/java/com/microsoft/azure/eventhubs/samples/AsyncSend/AsyncSend.java b/samples/Java/Basic/AsyncSend/src/main/java/com/microsoft/azure/eventhubs/samples/AsyncSend/AsyncSend.java new file mode 100644 index 000000000..38bfe47e5 --- /dev/null +++ b/samples/Java/Basic/AsyncSend/src/main/java/com/microsoft/azure/eventhubs/samples/AsyncSend/AsyncSend.java @@ -0,0 +1,112 @@ +/* + * Copyright (c) Microsoft. All rights reserved. + * Licensed under the MIT license. See LICENSE file in the project root for full license information. + */ +package com.microsoft.azure.eventhubs.samples.AsyncSend; + +import com.microsoft.azure.eventhubs.ConnectionStringBuilder; +import com.microsoft.azure.eventhubs.EventData; +import com.microsoft.azure.eventhubs.EventHubClient; +import com.microsoft.azure.eventhubs.EventHubException; + +import java.io.IOException; +import java.nio.charset.Charset; +import java.time.Instant; +import java.util.LinkedList; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + +public class AsyncSend { + + public static void main(String[] args) + throws EventHubException, ExecutionException, InterruptedException, IOException { + + final ConnectionStringBuilder connStr = new ConnectionStringBuilder() + .setNamespaceName("Your Event Hubs namespace name") // to target National clouds - use .setEndpoint(URI) + .setEventHubName("Your event hub") + .setSasKeyName("Your policy name") + .setSasKey("Your primary SAS key"); + + // The Executor handles all asynchronous tasks and this is passed to the + // EventHubClient instance. This enables the user to segregate their thread + // pool based on the work load. This pool can then be shared across multiple + // EventHubClient instances. + final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(4); + + // Each EventHubClient instance spins up a new TCP/SSL connection, which is + // expensive. It is always a best practice to reuse these instances. + final EventHubClient ehClient = EventHubClient.createFromConnectionStringSync(connStr.toString(), executorService); + + send100Messages(ehClient, 1); + System.out.println(); + + send100Messages(ehClient, 10); + System.out.println(); + + send100Messages(ehClient, 30); + System.out.println(); + + ehClient.closeSync(); + executorService.shutdown(); + } + + public static void send100Messages(final EventHubClient ehClient, final int inFlightMax) { + System.out.println("Sending 100 messages with " + inFlightMax + " concurrent sends"); + + LinkedList inFlight = new LinkedList(); + long totalWait = 0L; + long totalStart = Instant.now().toEpochMilli(); + + for (int i = 0; i < 100; i++) { + String payload = "Message " + Integer.toString(i); + byte[] payloadBytes = payload.getBytes(Charset.defaultCharset()); + MessageAndResult mar = new MessageAndResult(i, EventData.create(payloadBytes)); + long startWait = Instant.now().toEpochMilli(); + + if (inFlight.size() >= inFlightMax) { + MessageAndResult oldest = inFlight.remove(); + try { + oldest.result.get(); + //System.out.println("Completed send of message " + oldest.messageNumber + ": succeeded"); + } catch (InterruptedException | ExecutionException e) { + System.out.println("Completed send of message " + oldest.messageNumber + ": failed: " + e.toString()); + } + } + + long waitTime = Instant.now().toEpochMilli() - startWait; + totalWait += waitTime; + //System.out.println("Blocked time waiting to send (ms): " + waitTime); + mar.result = ehClient.send(mar.message); + //System.out.println("Started send of message " + mar.messageNumber); + inFlight.add(mar); + } + + System.out.println("All sends started"); + + while (inFlight.size() > 0) { + MessageAndResult oldest = inFlight.remove(); + try { + oldest.result.get(); + //System.out.println("Completed send of message " + oldest.messageNumber + ": succeeded"); + } catch (InterruptedException | ExecutionException e) { + System.out.println("Completed send of message " + oldest.messageNumber + ": failed: " + e.toString()); + } + } + + System.out.println("All sends completed, average blocked time (ms): " + (totalWait / 100L)); + System.out.println("Total time to send 100 messages (ms): " + (Instant.now().toEpochMilli() - totalStart)); + } + + private static class MessageAndResult { + public final int messageNumber; + public final EventData message; + public CompletableFuture result; + + public MessageAndResult(final int messageNumber, final EventData message) { + this.messageNumber = messageNumber; + this.message = message; + } + } +} diff --git a/samples/Java/Basic/EventProcessorSample/.classpath b/samples/Java/Basic/EventProcessorSample/.classpath deleted file mode 100644 index db2a016f9..000000000 --- a/samples/Java/Basic/EventProcessorSample/.classpath +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/samples/Java/Basic/EventProcessorSample/.project b/samples/Java/Basic/EventProcessorSample/.project deleted file mode 100644 index 03043695d..000000000 --- a/samples/Java/Basic/EventProcessorSample/.project +++ /dev/null @@ -1,23 +0,0 @@ - - - EventProcessorSample - - - - - - org.eclipse.jdt.core.javabuilder - - - - - org.eclipse.m2e.core.maven2Builder - - - - - - org.eclipse.jdt.core.javanature - org.eclipse.m2e.core.maven2Nature - - diff --git a/samples/Java/Basic/EventProcessorSample/pom.xml b/samples/Java/Basic/EventProcessorSample/pom.xml index 7f8149c0d..f66303de8 100644 --- a/samples/Java/Basic/EventProcessorSample/pom.xml +++ b/samples/Java/Basic/EventProcessorSample/pom.xml @@ -43,26 +43,26 @@ - + - com.microsoft.azure - azure-eventhubs - 2.3.0 - - - com.microsoft.azure - azure-eventhubs-eph - 2.5.0 - - - log4j - log4j - 1.2.17 - + com.microsoft.azure + azure-eventhubs + 3.0.2 + + + com.microsoft.azure + azure-eventhubs-eph + 3.0.2 + + + log4j + log4j + 1.2.17 + org.slf4j slf4j-log4j12 1.7.25 - + diff --git a/samples/Java/Basic/EventProcessorSample/src/main/java/com/microsoft/azure/eventhubs/samples/eventprocessorsample/EventProcessorSample.java b/samples/Java/Basic/EventProcessorSample/src/main/java/com/microsoft/azure/eventhubs/samples/eventprocessorsample/EventProcessorSample.java index d46b88eb8..38c651918 100644 --- a/samples/Java/Basic/EventProcessorSample/src/main/java/com/microsoft/azure/eventhubs/samples/eventprocessorsample/EventProcessorSample.java +++ b/samples/Java/Basic/EventProcessorSample/src/main/java/com/microsoft/azure/eventhubs/samples/eventprocessorsample/EventProcessorSample.java @@ -28,7 +28,7 @@ public static void main(String args[]) throws InterruptedException, ExecutionExc // account. All instances of EventProcessorHost which will be consuming from the same Event Hub and consumer // group must use the same Azure Storage account and container name. String consumerGroupName = "$Default"; - String namespaceName = "----ServiceBusNamespaceName----"; + String namespaceName = "----EventHubNamespaceName----"; String eventHubName = "----EventHubName----"; String sasKeyName = "----SharedAccessSignatureKeyName----"; String sasKey = "----SharedAccessSignatureKey----"; @@ -46,13 +46,11 @@ public static void main(String args[]) throws InterruptedException, ExecutionExc // Create the instance of EventProcessorHost using the most basic constructor. This constructor uses Azure Storage for // persisting partition leases and checkpoints. The host name, which identifies the instance of EventProcessorHost, must be unique. // You can use a plain UUID, or use the createHostName utility method which appends a UUID to a supplied string. - EventProcessorHost host = new EventProcessorHost( - EventProcessorHost.createHostName(hostNamePrefix), - eventHubName, - consumerGroupName, - eventHubConnectionString.toString(), - storageConnectionString, - storageContainerName); + EventProcessorHost host = EventProcessorHost.EventProcessorHostBuilder + .newBuilder(EventProcessorHost.createHostName(hostNamePrefix), consumerGroupName) + .useAzureStorageCheckpointLeaseManager(storageConnectionString, storageContainerName, null) + .useEventHubConnectionString(eventHubConnectionString.toString(), eventHubName) + .build(); // Registering an event processor class with an instance of EventProcessorHost starts event processing. The host instance // obtains leases on some partitions of the Event Hub, possibly stealing some from other host instances, in a way that diff --git a/samples/Java/Basic/Log4jAppenderSample/README.md b/samples/Java/Basic/Log4jAppenderSample/README.md new file mode 100644 index 000000000..c2b73a695 --- /dev/null +++ b/samples/Java/Basic/Log4jAppenderSample/README.md @@ -0,0 +1,14 @@ +# EventHub Appender Java Sample + +This sample demonstrates the use of EventHubAppender for streaming/sending log4j2 logs to Azure Event Hub. +The EventHubAppender extension (Log4j plugin) code can be found at https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/eventhubs/microsoft-azure-eventhubs-extensions + +Note : The above extension depends on [this version of Azure Event Hubs library](https://github.com/Azure/azure-sdk-for-java/tree/main/sdk/eventhubs/microsoft-azure-eventhubs). + +Steps to run the project : + 1. Clone the repository. + 2. Update the value of 'eventHubConnectionString' in src/main/resources/log4j2.xml file. (This is the config file) + 3. Update the value of 'classpathPrefix' element in the pom.xml file of the project + 4. Run 'mvn clean' + 5. Run 'mvn package' + 6. Run 'java -cp target/my-app-1.0-SNAPSHOT.jar:/*: com.eventhubappendersample.app.App' diff --git a/samples/Java/Basic/Log4jAppenderSample/pom.xml b/samples/Java/Basic/Log4jAppenderSample/pom.xml new file mode 100644 index 000000000..8e1bbe499 --- /dev/null +++ b/samples/Java/Basic/Log4jAppenderSample/pom.xml @@ -0,0 +1,74 @@ + + 4.0.0 + log4jeventhubappendersample + log4jeventhubappendersample + jar + 1.0.0 + + UTF-8 + + + + org.apache.logging.log4j + log4j-api + 2.13.0 + + + org.apache.logging.log4j + log4j-core + [2.13.2,) + + + com.microsoft.azure + azure-eventhubs-extensions + 3.3.0 + + + org.slf4j + slf4j-api + 1.7.32 + + + + org.slf4j + slf4j-simple + 1.7.32 + + + + + + + maven-compiler-plugin + 3.8.0 + + 1.8 + 1.8 + true + lines,vars,source + + + + maven-assembly-plugin + + + package + + single + + + + + + jar-with-dependencies + + + + com.microsoft.azure.eventhubs.samples.log4jeventhubappendersample.Log4JEventHubAppenderSample + + + + + + + diff --git a/samples/Java/Basic/Log4jAppenderSample/src/main/java/com/microsoft/azure/eventhubs/samples/log4jeventhubappendersample/Log4JEventHubAppenderSample.java b/samples/Java/Basic/Log4jAppenderSample/src/main/java/com/microsoft/azure/eventhubs/samples/log4jeventhubappendersample/Log4JEventHubAppenderSample.java new file mode 100644 index 000000000..b4ac6b84a --- /dev/null +++ b/samples/Java/Basic/Log4jAppenderSample/src/main/java/com/microsoft/azure/eventhubs/samples/log4jeventhubappendersample/Log4JEventHubAppenderSample.java @@ -0,0 +1,29 @@ +package com.microsoft.azure.eventhubs.samples.log4jeventhubappendersample; + +import java.util.concurrent.TimeUnit; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * Hello world! + * + */ +public class Log4JEventHubAppenderSample +{ + private static final Logger logger = LogManager.getLogger(Log4JEventHubAppenderSample.class); + public static void main( String[] args ) + { + int i; + for(i=0; i<=10;i++) + { + try { + TimeUnit.SECONDS.sleep(1); + logger.error("Hello World " + i); + } + + catch(InterruptedException e) + { + } + } + } +} diff --git a/samples/Java/Basic/Log4jAppenderSample/src/main/resources/log4j2.xml b/samples/Java/Basic/Log4jAppenderSample/src/main/resources/log4j2.xml new file mode 100644 index 000000000..8f163e613 --- /dev/null +++ b/samples/Java/Basic/Log4jAppenderSample/src/main/resources/log4j2.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/samples/Java/Basic/ReceiveByDateTime/pom.xml b/samples/Java/Basic/ReceiveByDateTime/pom.xml index 3108cd866..1124ea6be 100644 --- a/samples/Java/Basic/ReceiveByDateTime/pom.xml +++ b/samples/Java/Basic/ReceiveByDateTime/pom.xml @@ -43,11 +43,11 @@ - + - com.microsoft.azure - azure-eventhubs - 2.3.0 - - + com.microsoft.azure + azure-eventhubs + 3.0.2 + + diff --git a/samples/Java/Basic/ReceiveByDateTime/src/main/java/com/microsoft/azure/eventhubs/samples/receivebydatetime/ReceiveByDateTime.java b/samples/Java/Basic/ReceiveByDateTime/src/main/java/com/microsoft/azure/eventhubs/samples/receivebydatetime/ReceiveByDateTime.java index bb73e1af2..57f72ce6b 100644 --- a/samples/Java/Basic/ReceiveByDateTime/src/main/java/com/microsoft/azure/eventhubs/samples/receivebydatetime/ReceiveByDateTime.java +++ b/samples/Java/Basic/ReceiveByDateTime/src/main/java/com/microsoft/azure/eventhubs/samples/receivebydatetime/ReceiveByDateTime.java @@ -31,7 +31,7 @@ public static void main(String[] args) .setSasKey("---SharedAccessSignatureKey----"); final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(4); - final EventHubClient ehClient = EventHubClient.createSync(connStr.toString(), executorService); + final EventHubClient ehClient = EventHubClient.createFromConnectionStringSync(connStr.toString(), executorService); final EventHubRuntimeInformation eventHubInfo = ehClient.getRuntimeInformation().get(); final String partitionId = eventHubInfo.getPartitionIds()[0]; // get first partition's id diff --git a/samples/Java/Basic/ReceiveUsingOffset/pom.xml b/samples/Java/Basic/ReceiveUsingOffset/pom.xml index 806d116b4..669acc71b 100644 --- a/samples/Java/Basic/ReceiveUsingOffset/pom.xml +++ b/samples/Java/Basic/ReceiveUsingOffset/pom.xml @@ -43,11 +43,11 @@ - + - com.microsoft.azure - azure-eventhubs - 2.3.0 - - + com.microsoft.azure + azure-eventhubs + 3.0.2 + + diff --git a/samples/Java/Basic/ReceiveUsingOffset/src/main/java/com/microsoft/azure/eventhubs/samples/receiveusingoffset/ReceiveUsingOffset.java b/samples/Java/Basic/ReceiveUsingOffset/src/main/java/com/microsoft/azure/eventhubs/samples/receiveusingoffset/ReceiveUsingOffset.java index 9890dc64f..83335768d 100644 --- a/samples/Java/Basic/ReceiveUsingOffset/src/main/java/com/microsoft/azure/eventhubs/samples/receiveusingoffset/ReceiveUsingOffset.java +++ b/samples/Java/Basic/ReceiveUsingOffset/src/main/java/com/microsoft/azure/eventhubs/samples/receiveusingoffset/ReceiveUsingOffset.java @@ -29,7 +29,7 @@ public static void main(String[] args) .setSasKey("---SharedAccessSignatureKey----"); final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(4); - final EventHubClient ehClient = EventHubClient.createSync(connStr.toString(), executorService); + final EventHubClient ehClient = EventHubClient.createFromConnectionStringSync(connStr.toString(), executorService); final EventHubRuntimeInformation eventHubInfo = ehClient.getRuntimeInformation().get(); final String partitionId = eventHubInfo.getPartitionIds()[0]; // get first partition's id diff --git a/samples/Java/Basic/ReceiveUsingSequenceNumber/pom.xml b/samples/Java/Basic/ReceiveUsingSequenceNumber/pom.xml index 82e583349..b91950b98 100644 --- a/samples/Java/Basic/ReceiveUsingSequenceNumber/pom.xml +++ b/samples/Java/Basic/ReceiveUsingSequenceNumber/pom.xml @@ -43,11 +43,11 @@ - + - com.microsoft.azure - azure-eventhubs - 2.3.0 - - + com.microsoft.azure + azure-eventhubs + 3.0.2 + + diff --git a/samples/Java/Basic/ReceiveUsingSequenceNumber/src/main/java/com/microsoft/azure/eventhubs/samples/receiveusingsequencenumber/ReceiveUsingSequenceNumber.java b/samples/Java/Basic/ReceiveUsingSequenceNumber/src/main/java/com/microsoft/azure/eventhubs/samples/receiveusingsequencenumber/ReceiveUsingSequenceNumber.java index 8027db35a..77de83a5c 100644 --- a/samples/Java/Basic/ReceiveUsingSequenceNumber/src/main/java/com/microsoft/azure/eventhubs/samples/receiveusingsequencenumber/ReceiveUsingSequenceNumber.java +++ b/samples/Java/Basic/ReceiveUsingSequenceNumber/src/main/java/com/microsoft/azure/eventhubs/samples/receiveusingsequencenumber/ReceiveUsingSequenceNumber.java @@ -30,7 +30,7 @@ public static void main(String[] args) .setSasKey("---SharedAccessSignatureKey----"); final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(4); - final EventHubClient ehClient = EventHubClient.createSync(connStr.toString(), executorService); + final EventHubClient ehClient = EventHubClient.createFromConnectionStringSync(connStr.toString(), executorService); final EventHubRuntimeInformation eventHubInfo = ehClient.getRuntimeInformation().get(); final String partitionId = eventHubInfo.getPartitionIds()[0]; // get first partition's id diff --git a/samples/Java/Basic/SendBatch/pom.xml b/samples/Java/Basic/SendBatch/pom.xml index e4df26e64..97b8a0e94 100644 --- a/samples/Java/Basic/SendBatch/pom.xml +++ b/samples/Java/Basic/SendBatch/pom.xml @@ -43,16 +43,16 @@ - - - com.microsoft.azure - azure-eventhubs - 2.3.0 - + + + com.microsoft.azure + azure-eventhubs + 3.0.2 + com.google.code.gson gson [2.8.2,] - + diff --git a/samples/Java/Basic/SendBatch/src/main/java/com/microsoft/azure/eventhubs/samples/sendbatch/SendBatch.java b/samples/Java/Basic/SendBatch/src/main/java/com/microsoft/azure/eventhubs/samples/sendbatch/SendBatch.java index 686eb4a21..899034ca5 100644 --- a/samples/Java/Basic/SendBatch/src/main/java/com/microsoft/azure/eventhubs/samples/sendbatch/SendBatch.java +++ b/samples/Java/Basic/SendBatch/src/main/java/com/microsoft/azure/eventhubs/samples/sendbatch/SendBatch.java @@ -31,7 +31,7 @@ public static void main(String[] args) final Gson gson = new GsonBuilder().create(); final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(4); - final EventHubClient sender = EventHubClient.createSync(connStr.toString(), executorService); + final EventHubClient sender = EventHubClient.createFromConnectionStringSync(connStr.toString(), executorService); try { for (int batchNumber = 0; batchNumber < 10; batchNumber++) { diff --git a/samples/Java/Basic/SimpleProxy/pom.xml b/samples/Java/Basic/SimpleProxy/pom.xml index baf58c6e6..b403f18ed 100644 --- a/samples/Java/Basic/SimpleProxy/pom.xml +++ b/samples/Java/Basic/SimpleProxy/pom.xml @@ -44,11 +44,11 @@ - - com.microsoft.azure - azure-eventhubs - 2.3.0 - + + com.microsoft.azure + azure-eventhubs + 3.0.2 + com.google.code.gson gson diff --git a/samples/Java/Basic/SimpleProxy/src/main/java/com.microsoft.azure.eventhubs.samples.SimpleProxy/SimpleProxy.java b/samples/Java/Basic/SimpleProxy/src/main/java/com.microsoft.azure.eventhubs.samples.SimpleProxy/SimpleProxy.java index 83b9bf864..7326604dd 100644 --- a/samples/Java/Basic/SimpleProxy/src/main/java/com.microsoft.azure.eventhubs.samples.SimpleProxy/SimpleProxy.java +++ b/samples/Java/Basic/SimpleProxy/src/main/java/com.microsoft.azure.eventhubs.samples.SimpleProxy/SimpleProxy.java @@ -79,7 +79,7 @@ protected PasswordAuthentication getPasswordAuthentication() { connStr.setTransportType(TransportType.AMQP_WEB_SOCKETS); final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(4); - final EventHubClient ehClient = EventHubClient.createSync(connStr.toString(), executorService); + final EventHubClient ehClient = EventHubClient.createFromConnectionStringSync(connStr.toString(), executorService); final Gson gson = new GsonBuilder().create(); PartitionSender sender = null; diff --git a/samples/Java/Basic/SimpleSend/README.md b/samples/Java/Basic/SimpleSend/README.md index 285fa2798..76c1d4200 100644 --- a/samples/Java/Basic/SimpleSend/README.md +++ b/samples/Java/Basic/SimpleSend/README.md @@ -2,7 +2,7 @@ The [Send events to Azure Event Hubs using Java](https://docs.microsoft.com/azure/event-hubs/event-hubs-java-get-started-send) tutorial walks you through ingesting into your event hub using Java with this code. -To run the sample, you need to edit the [sample code](src/main/java/com/microsoft/azure/eventhubs/samples/simplesend/SimpleSend.java) and provide the following information: +To run the sample, you need to edit the [sample code](src/main/java/com/microsoft/azure/eventhubs/samples/SimpleSend/SimpleSend.java) and provide the following information: ```java final String namespaceName = "----EventHubsNamespaceName-----"; diff --git a/samples/Java/Basic/SimpleSend/pom.xml b/samples/Java/Basic/SimpleSend/pom.xml index 3bdffea7a..16bb783cb 100644 --- a/samples/Java/Basic/SimpleSend/pom.xml +++ b/samples/Java/Basic/SimpleSend/pom.xml @@ -43,16 +43,16 @@ - + - com.microsoft.azure - azure-eventhubs - 2.3.0 - - + com.microsoft.azure + azure-eventhubs + 3.0.2 + + com.google.code.gson gson 2.8.5 - + diff --git a/samples/Java/Basic/SimpleSend/src/main/java/com/microsoft/azure/eventhubs/samples/SimpleSend/SimpleSend.java b/samples/Java/Basic/SimpleSend/src/main/java/com/microsoft/azure/eventhubs/samples/SimpleSend/SimpleSend.java index c4cccbb78..57908cd94 100644 --- a/samples/Java/Basic/SimpleSend/src/main/java/com/microsoft/azure/eventhubs/samples/SimpleSend/SimpleSend.java +++ b/samples/Java/Basic/SimpleSend/src/main/java/com/microsoft/azure/eventhubs/samples/SimpleSend/SimpleSend.java @@ -40,7 +40,7 @@ public static void main(String[] args) // Each EventHubClient instance spins up a new TCP/SSL connection, which is expensive. // It is always a best practice to reuse these instances. The following sample shows this. - final EventHubClient ehClient = EventHubClient.createSync(connStr.toString(), executorService); + final EventHubClient ehClient = EventHubClient.createFromConnectionStringSync(connStr.toString(), executorService); try { for (int i = 0; i < 100; i++) { diff --git a/samples/Java/Benchmarks/AutoScaleOnIngress/pom.xml b/samples/Java/Benchmarks/AutoScaleOnIngress/pom.xml index 7c70ccad6..ec1a5cc43 100644 --- a/samples/Java/Benchmarks/AutoScaleOnIngress/pom.xml +++ b/samples/Java/Benchmarks/AutoScaleOnIngress/pom.xml @@ -43,11 +43,11 @@ - + - com.microsoft.azure - azure-eventhubs - 2.3.0 - - + com.microsoft.azure + azure-eventhubs + 3.0.2 + + diff --git a/samples/Java/Benchmarks/AutoScaleOnIngress/src/main/java/com/microsoft/azure/eventhubs/samples/autoscaleoningress/EventHubClientPool.java b/samples/Java/Benchmarks/AutoScaleOnIngress/src/main/java/com/microsoft/azure/eventhubs/samples/autoscaleoningress/EventHubClientPool.java index e78718105..702123e13 100644 --- a/samples/Java/Benchmarks/AutoScaleOnIngress/src/main/java/com/microsoft/azure/eventhubs/samples/autoscaleoningress/EventHubClientPool.java +++ b/samples/Java/Benchmarks/AutoScaleOnIngress/src/main/java/com/microsoft/azure/eventhubs/samples/autoscaleoningress/EventHubClientPool.java @@ -34,7 +34,7 @@ public CompletableFuture initialize() throws IOException, EventHubExceptio final CompletableFuture[] createSenders = new CompletableFuture[this.poolSize]; for (int count = 0; count < poolSize; count++) { final int clientsIndex = count; - createSenders[count] = EventHubClient.create(this.connectionString, executorService).thenAccept(new Consumer() { + createSenders[count] = EventHubClient.createFromConnectionString(this.connectionString, executorService).thenAccept(new Consumer() { @Override public void accept(EventHubClient eventHubClient) { clients[clientsIndex] = eventHubClient; diff --git a/samples/Java/Benchmarks/IngressBenchmark/pom.xml b/samples/Java/Benchmarks/IngressBenchmark/pom.xml index 3d4d04917..1298bec57 100644 --- a/samples/Java/Benchmarks/IngressBenchmark/pom.xml +++ b/samples/Java/Benchmarks/IngressBenchmark/pom.xml @@ -43,11 +43,11 @@ - + - com.microsoft.azure - azure-eventhubs - 2.3.0 - - + com.microsoft.azure + azure-eventhubs + 3.0.2 + + diff --git a/samples/Java/Benchmarks/IngressBenchmark/src/main/java/com/microsoft/azure/eventhubs/samples/ingressbenchmark/EventHubClientPool.java b/samples/Java/Benchmarks/IngressBenchmark/src/main/java/com/microsoft/azure/eventhubs/samples/ingressbenchmark/EventHubClientPool.java index b8a2dc128..8ec98807a 100644 --- a/samples/Java/Benchmarks/IngressBenchmark/src/main/java/com/microsoft/azure/eventhubs/samples/ingressbenchmark/EventHubClientPool.java +++ b/samples/Java/Benchmarks/IngressBenchmark/src/main/java/com/microsoft/azure/eventhubs/samples/ingressbenchmark/EventHubClientPool.java @@ -34,7 +34,7 @@ public CompletableFuture initialize() throws IOException, EventHubExceptio final CompletableFuture[] createSenders = new CompletableFuture[this.poolSize]; for (int count = 0; count < poolSize; count++) { final int clientsIndex = count; - createSenders[count] = EventHubClient.create(this.connectionString, executorService).thenAccept(new Consumer() { + createSenders[count] = EventHubClient.createFromConnectionString(this.connectionString, executorService).thenAccept(new Consumer() { @Override public void accept(EventHubClient eventHubClient) { clients[clientsIndex] = eventHubClient; diff --git a/samples/Java/README.md b/samples/Java/README.md index 158c4012c..a773808ef 100644 --- a/samples/Java/README.md +++ b/samples/Java/README.md @@ -1,32 +1,34 @@ # Azure Event Hubs Java samples -Azure Event Hubs is a highly scalable data streaming platform and event ingestion service, capable of receiving and processing millions of events per second. The samples present here enables Java developers to easily ingest and process events from your event hub. +**Note:**: The samples present here enables Java developers to easily ingest and process events from an instance of [Azure Event Hubs](https://docs.microsoft.com/azure/event-hubs/event-hubs-about) using the legacy libraries `com.microsoft.azure:azure-eventhubs` and `com.microsoft.azure:azure-eventhubs-eph`. We strongly recommend you to use the current library `com.azure:azure-messaging-eventhubs` instead. + +- [Java samples for current Azure Event Hubs client library](https://docs.microsoft.com/en-us/samples/azure/azure-sdk-for-java/eventhubs-samples/) +- [Java samples for current Azure Event Hubs Checkpoint Store client library](https://docs.microsoft.com/samples/azure/azure-sdk-for-java/eventhubs-samples/) ## Prerequisites -1. The samples depend on the Java JDK 1.8 and are built using Maven. You can [download Maven](https://maven.apache.org/download.cgi). [Install and configure Maven](https://maven.apache.org/install.html). The sample require Maven version > 3.3.9. -2. You need an Azure Subscription, if you do not have one - create a [free account](https://azure.microsoft.com/free/?ref=microsoft.com&utm_source=microsoft.com&utm_medium=docs&utm_campaign=visualstudio) before you begin -3. [An Event Hubs namespace and an event hub where you ingest the data](https://docs.microsoft.com/azure/event-hubs/event-hubs-create) -4. [A SAS key to access the event hub](https://docs.microsoft.com/azure/event-hubs/event-hubs-create#SAS) +1. The samples depend on the Java JDK 1.8 and are built using Maven. You can [download Maven](https://maven.apache.org/download.cgi). [Install and configure Maven](https://maven.apache.org/install.html). The sample require Maven version > 3.3.9. +2. You need an Azure Subscription, if you do not have one - create a [free account](https://azure.microsoft.com/free/?ref=microsoft.com&utm_source=microsoft.com&utm_medium=docs&utm_campaign=visualstudio) before you begin +3. [An Event Hubs namespace and an event hub where you ingest the data](https://docs.microsoft.com/azure/event-hubs/event-hubs-create) +4. [A SAS key to access the event hub](https://docs.microsoft.com/azure/event-hubs/event-hubs-create#SAS) ### Sending events -* **SendBatch** - The [SendBatch](./Basic/SendBatch) sample illustrates how to ingest batches of events into your event hub. -* **SimpleSend** - The [SimpleSend](./Basic/SimpleSend) sample illustrates how to ingest events into your event hub. -* **AdvanceSendOptions** - The [AdvancedSendOptions](./Basic/AdvancedSendOptions) sample illustrates the various options available with Event Hubs to ingest events. - -### Processing events +- **SendBatch** - The [SendBatch](./Basic/SendBatch) sample illustrates how to ingest batches of events into your event hub. +- **SimpleSend** - The [SimpleSend](./Basic/SimpleSend) sample illustrates how to ingest events into your event hub. +- **AdvanceSendOptions** - The [AdvancedSendOptions](./Basic/AdvancedSendOptions) sample illustrates the various options available with Event Hubs to ingest events. -* **ReceiveByDateTime** - The [ReceiveByDateTime](./Basic/ReceiveByDateTime) sample illustrates how to receive events from an event hub partition using a specific date-time offset. -* **ReceiveUsingOffset** - The [ReceiveUsingOffset](./Basic/ReceiveUsingOffset) sample illustrates how to receive events from an event hub partition using a specific data offset. -* **ReceiveUsingSequenceNumber** - The [ReceiveUsingSequenceNumber](./Basic/ReceiveUsingSequenceNumber) sample illustrates how can receive from an event hub partitions using a sequence number. -* **EventProcessorSample** - The [EventProcessorSample](./Basic/EventProcessorSample) sample illustrates how to receive events from an event hub using the event processor host, which provides automatic partition selection and fail-over across multiple concurrent receivers. +### Processing events +- **ReceiveByDateTime** - The [ReceiveByDateTime](./Basic/ReceiveByDateTime) sample illustrates how to receive events from an event hub partition using a specific date-time offset. +- **ReceiveUsingOffset** - The [ReceiveUsingOffset](./Basic/ReceiveUsingOffset) sample illustrates how to receive events from an event hub partition using a specific data offset. +- **ReceiveUsingSequenceNumber** - The [ReceiveUsingSequenceNumber](./Basic/ReceiveUsingSequenceNumber) sample illustrates how can receive from an event hub partitions using a sequence number. +- **EventProcessorSample** - The [EventProcessorSample](./Basic/EventProcessorSample) sample illustrates how to receive events from an event hub using the event processor host, which provides automatic partition selection and fail-over across multiple concurrent receivers. ### Benchmarks -* **AutoScaleOnIngress** - The [AutoScaleOnIngress](./Benchmarks/AutoScaleOnIngress) sample illustrates how an event hub can automatically scale up on high loads. The sample will send events at a rate that just exceed the configured rate of an event hub, causing the event hub to scale up. -* **IngressBenchmark** - The [IngressBenchmark](./Benchmarks/IngressBenchmark) sample allows measuring the ingress rate. +- **AutoScaleOnIngress** - The [AutoScaleOnIngress](./Benchmarks/AutoScaleOnIngress) sample illustrates how an event hub can automatically scale up on high loads. The sample will send events at a rate that just exceed the configured rate of an event hub, causing the event hub to scale up. +- **IngressBenchmark** - The [IngressBenchmark](./Benchmarks/IngressBenchmark) sample allows measuring the ingress rate. ## Build and run @@ -40,4 +42,4 @@ The samples are dropped into the respective sample's ./target subfolder. The bui ```bash java -jar ./target/{project}-1.0.0-jar-with-dependencies.jar -``` \ No newline at end of file +``` diff --git a/samples/Java/Rbac/README.md b/samples/Java/Rbac/README.md new file mode 100644 index 000000000..7aa4c31df --- /dev/null +++ b/samples/Java/Rbac/README.md @@ -0,0 +1,42 @@ +# Role based access sample with Azure Event Hubs Java SDK + +For general information on using Role based access (RBAC) with Azure Event Hubs, see the [documentation](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-role-based-access-control). + +This sample uses the [Microsoft Authentication Library (MSAL) for Java](https://github.com/AzureAD/microsoft-authentication-library-for-java) to obtain tokens from Azure Active Directory (AAD). + +## Prerequisites + +Please refer to the Java sample overview README for setting up the sample environment, including creating an Event Hubs cloud namespace and an event hub. + +The specific AAD pattern used in this sample is ["Authenticate an appliction"](https://docs.microsoft.com/en-us/azure/event-hubs/authenticate-application). Please follow the steps described to +create an application (client) id and application (client) secret, obtain your directory (tenant) id, and assign the application the Data Owner role on your event hub. + +Once you have performed the previous steps, edit SendReceive.java to provide the necessary information. + +```java + final java.net.URI namespace = new java.net.URI("----EventHubsNamespace---.servicebus.windows.net"); // to target National clouds, change domain name too + final String eventhub = "----EventHubName----"; + final String authority = "https://login.windows.net/----replaceWithTenantIdGuid----"; + final String clientId = "----replaceWithClientIdGuid----"; // not needed to run with Managed Identity + final String clientSecret = "----replaceWithClientSecret----"; // not needed to run with Managed Identity +``` + +The Azure Event Hubs Java SDK also has limited built-in support for Managed Identity: specifically, when running in a virtual machine with a system-assigned managed identity, the SDK can +obtain and use that identity to perform role based access. This sample can demonstrate that ability when run in an +[appropriately-configured virtual machine](https://docs.microsoft.com/en-us/azure/active-directory/managed-identities-azure-resources/qs-configure-portal-windows-vm) and the managed identity +[has been assigned the Data Owner role on your event hub.](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-managed-service-identity) + + +## Build and run + +The sample can be built independently with + +```bash +mvn clean package +``` + +and then run with (or just from VS Code or another Java IDE) + +```bash +java -jar ./target/send-1.0.0-jar-with-dependencies.jar +``` \ No newline at end of file diff --git a/samples/Java/Rbac/pom.xml b/samples/Java/Rbac/pom.xml new file mode 100644 index 000000000..e2f5e8aff --- /dev/null +++ b/samples/Java/Rbac/pom.xml @@ -0,0 +1,63 @@ + + 4.0.0 + rbac + 1.0.0 + rbac + jar + + UTF-8 + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.6.1 + + 1.8 + 1.8 + true + lines,vars,source + + + + maven-assembly-plugin + + + package + + single + + + + + + jar-with-dependencies + + + + com.microsoft.azure.eventhubs.samples.rbac.SendReceive + + + + + + + + + com.microsoft.azure + azure-eventhubs + 3.0.2 + + + com.google.code.gson + gson + 2.8.5 + + + com.microsoft.azure + msal4j + 0.5.0-preview + + + diff --git a/samples/Java/Rbac/src/main/java/com/microsoft/azure/eventhubs/samples/rbac/AuthCallback.java b/samples/Java/Rbac/src/main/java/com/microsoft/azure/eventhubs/samples/rbac/AuthCallback.java new file mode 100644 index 000000000..943eb6421 --- /dev/null +++ b/samples/Java/Rbac/src/main/java/com/microsoft/azure/eventhubs/samples/rbac/AuthCallback.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) Microsoft. All rights reserved. + * Licensed under the MIT license. See LICENSE file in the project root for full license information. + */ + +package com.microsoft.azure.eventhubs.samples.rbac; + +import java.util.Collections; +import java.util.concurrent.CompletableFuture; + +import com.microsoft.aad.msal4j.*; +import com.microsoft.azure.eventhubs.AzureActiveDirectoryTokenProvider; + +class AuthCallback implements AzureActiveDirectoryTokenProvider.AuthenticationCallback { + final private String clientId; + final private String clientSecret; + + public AuthCallback(final String clientId, final String clientSecret) { + this.clientId = clientId; + this.clientSecret = clientSecret; + } + + @Override + public CompletableFuture acquireToken(String audience, String authority, Object state) { + try { + ConfidentialClientApplication app = ConfidentialClientApplication.builder(this.clientId, new ClientSecret(this.clientSecret)) + .authority(authority) + .build(); + ClientCredentialParameters parameters = ClientCredentialParameters.builder(Collections.singleton(audience + ".default")).build(); + return app.acquireToken(parameters).thenApply((authResult) -> { + return authResult.accessToken(); + }); + } catch (Exception e) { + CompletableFuture failed = new CompletableFuture(); + failed.completeExceptionally(e); + return failed; + } + } +} diff --git a/samples/Java/Rbac/src/main/java/com/microsoft/azure/eventhubs/samples/rbac/CustomTokenProvider.java b/samples/Java/Rbac/src/main/java/com/microsoft/azure/eventhubs/samples/rbac/CustomTokenProvider.java new file mode 100644 index 000000000..2ae948abc --- /dev/null +++ b/samples/Java/Rbac/src/main/java/com/microsoft/azure/eventhubs/samples/rbac/CustomTokenProvider.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) Microsoft. All rights reserved. + * Licensed under the MIT license. See LICENSE file in the project root for full license information. + */ + +package com.microsoft.azure.eventhubs.samples.rbac; + +import java.text.ParseException; +import java.time.Duration; +import java.util.Collections; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; + +import com.microsoft.aad.msal4j.*; +import com.microsoft.azure.eventhubs.*; +import com.microsoft.azure.eventhubs.impl.ClientConstants; + +class CustomTokenProvider implements ITokenProvider { + final private String authority; + final private String audience = ClientConstants.EVENTHUBS_AUDIENCE; + final private String clientId; + final private String clientSecret; + + public CustomTokenProvider(final String authority, final String clientId, final String clientSecret) { + this.authority = authority; + this.clientId = clientId; + this.clientSecret = clientSecret; + } + + @Override + public CompletableFuture getToken(String resource, Duration timeout) { + try { + ConfidentialClientApplication app = ConfidentialClientApplication.builder(this.clientId, new ClientSecret(this.clientSecret)) + .authority(authority) + .build(); + ClientCredentialParameters parameters = ClientCredentialParameters.builder(Collections.singleton(audience + ".default")).build(); + return app.acquireToken(parameters) + .thenApply((authResult) -> { + try { + return new JsonSecurityToken(authResult.accessToken(), resource); + } catch (ParseException e) { + throw new CompletionException(e); + } + }); + } + catch (Exception e) { + CompletableFuture failed = new CompletableFuture(); + failed.completeExceptionally(e); + return failed; + } + } +} + diff --git a/samples/Java/Rbac/src/main/java/com/microsoft/azure/eventhubs/samples/rbac/SendReceive.java b/samples/Java/Rbac/src/main/java/com/microsoft/azure/eventhubs/samples/rbac/SendReceive.java new file mode 100644 index 000000000..1dfbc777d --- /dev/null +++ b/samples/Java/Rbac/src/main/java/com/microsoft/azure/eventhubs/samples/rbac/SendReceive.java @@ -0,0 +1,189 @@ +/* + * Copyright (c) Microsoft. All rights reserved. + * Licensed under the MIT license. See LICENSE file in the project root for full license information. + */ + +package com.microsoft.azure.eventhubs.samples.rbac; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.microsoft.azure.eventhubs.AzureActiveDirectoryTokenProvider; +import com.microsoft.azure.eventhubs.ConnectionStringBuilder; +import com.microsoft.azure.eventhubs.EventData; +import com.microsoft.azure.eventhubs.EventHubClient; +import com.microsoft.azure.eventhubs.EventHubException; +import com.microsoft.azure.eventhubs.EventPosition; +import com.microsoft.azure.eventhubs.PartitionReceiver; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.charset.Charset; +import java.time.Instant; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; + +public class SendReceive { + + final java.net.URI namespace = new java.net.URI("----EventHubsNamespace---.servicebus.windows.net"); // to target National clouds, change domain name too + final String eventhub = "----EventHubName----"; + final String authority = "https://login.windows.net/----replaceWithTenantIdGuid----"; + final String clientId = "----replaceWithClientIdGuid----"; // not needed to run with Managed Identity + final String clientSecret = "----replaceWithClientSecret----"; // not needed to run with Managed Identity + + public SendReceive() throws URISyntaxException { + } + + public static int main(String[] args) + throws EventHubException, ExecutionException, InterruptedException, IOException, URISyntaxException { + + SendReceive ss = new SendReceive(); + return ss.run(args); + } + + private int run(String[] args) throws IOException { + + System.out.println("Choose an action:"); + System.out.println("[A] Authenticate via Managed Identity"); + System.out.println("[B] Get AAD token using AuthCallback"); + System.out.println("[C] Get AAD token using AzureActiveDirectoryTokenProvider to wrap AuthCallback"); + System.out.println("[D] Get AAD token using ITokenProvider implementation"); + + char key = (char)System.in.read(); + char keyPressed = Character.toUpperCase(key); + + try { + switch (keyPressed) { + case 'A': + managedIdentityScenario(); + break; + case 'B': + useAuthCallback(); + break; + case 'C': + useAADTokenProvider(); + break; + case 'D': + useCustomTokenProvider(); + break; + default: + System.out.println("Unknown command, press enter to exit"); + System.in.read(); + return -1; + } + } + catch (Exception ex) { + System.out.println("Error during execution. Exception: " + ex.toString()); + return -1; + } + + return 0; + } + + private ScheduledExecutorService getScheduledExecutorService() { + // The Executor handles all asynchronous tasks and this is passed to the EventHubClient instance. + // This enables the user to segregate their thread pool based on the work load. + // This pool can then be shared across multiple EventHubClient instances. + // The following sample uses a single thread executor, as there is only one EventHubClient instance, + // handling different flavors of ingestion to Event Hubs here. + final ScheduledExecutorService executorService = Executors.newScheduledThreadPool(4); + return executorService; + } + + private void managedIdentityScenario() throws IOException, EventHubException { + + final ConnectionStringBuilder connStr = new ConnectionStringBuilder() + .setEndpoint(this.namespace) + .setEventHubName(this.eventhub) + .setAuthentication(ConnectionStringBuilder.MANAGED_IDENTITY_AUTHENTICATION); + ScheduledExecutorService executorService = getScheduledExecutorService(); + + final EventHubClient ehClient = EventHubClient.createFromConnectionStringSync(connStr.toString(), executorService); + + sendReceive(ehClient, executorService); + } + + private void useAuthCallback() throws IOException, InterruptedException, ExecutionException, EventHubException { + + final AuthCallback callback = new AuthCallback(clientId, clientSecret); + ScheduledExecutorService executorService = getScheduledExecutorService(); + + final EventHubClient ehClient = EventHubClient.createWithAzureActiveDirectory(namespace, eventhub, callback, authority, executorService, null).get(); + + sendReceive(ehClient, executorService); + } + + private void useAADTokenProvider() throws IOException, InterruptedException, ExecutionException, EventHubException { + + final AuthCallback callback = new AuthCallback(clientId, clientSecret); + ScheduledExecutorService executorService = getScheduledExecutorService(); + + final AzureActiveDirectoryTokenProvider aadTokenProvider = new AzureActiveDirectoryTokenProvider(callback, authority, null); + + final EventHubClient ehClient = EventHubClient.createWithTokenProvider(namespace, eventhub, aadTokenProvider, executorService, null).get(); + + sendReceive(ehClient, executorService); + } + + private void useCustomTokenProvider() throws IOException, InterruptedException, ExecutionException, EventHubException { + + final CustomTokenProvider tokenProvider = new CustomTokenProvider(authority, clientId, clientSecret); + ScheduledExecutorService executorService = getScheduledExecutorService(); + + final EventHubClient ehClient = EventHubClient.createWithTokenProvider(namespace, eventhub, tokenProvider, executorService, null).get(); + + sendReceive(ehClient, executorService); + } + + private void sendReceive(EventHubClient ehClient, ScheduledExecutorService executorService) throws IOException, EventHubException { + try { + final Gson gson = new GsonBuilder().create(); + + for (int i = 0; i < 100; i++) { + + String payload = "Message " + Integer.toString(i); + byte[] payloadBytes = gson.toJson(payload).getBytes(Charset.defaultCharset()); + EventData sendEvent = EventData.create(payloadBytes); + ehClient.sendSync(sendEvent); + } + + System.out.println(Instant.now() + ": Send Complete..."); + + final PartitionReceiver receiver = ehClient.createReceiverSync( + EventHubClient.DEFAULT_CONSUMER_GROUP_NAME, + "0", + EventPosition.fromStartOfStream()); + + Iterable receivedEvents = receiver.receiveSync(100); + while (true) { + int batchSize = 0; + if (receivedEvents != null) { + for (final EventData receivedEvent : receivedEvents) { + if (receivedEvent.getBytes() != null) + System.out.println(String.format("Message Payload: %s", new String(receivedEvent.getBytes(), Charset.defaultCharset()))); + + System.out.println(String.format("Offset: %s, SeqNo: %s, EnqueueTime: %s", + receivedEvent.getSystemProperties().getOffset(), + receivedEvent.getSystemProperties().getSequenceNumber(), + receivedEvent.getSystemProperties().getEnqueuedTime())); + batchSize++; + } + } + else { + break; + } + + System.out.println(String.format("ReceivedBatch Size: %s", batchSize)); + receivedEvents = receiver.receiveSync(100); + } + + System.out.println(Instant.now() + ": Receive Complete..."); + + System.out.println("Press Enter to stop."); + System.in.read(); + } finally { + ehClient.closeSync(); + executorService.shutdown(); + } + } +} diff --git a/samples/Java/pom.xml b/samples/Java/pom.xml index 8dceb9e1a..9df62a3b4 100644 --- a/samples/Java/pom.xml +++ b/samples/Java/pom.xml @@ -29,7 +29,9 @@ Basic/AdvancedSendOptions Basic/SendBatch Basic/SimpleSend - Basic/SimpleProxy + Basic/AsyncSend + Basic/SimpleProxy + Basic/Log4jAppenderSample Benchmarks/AutoScaleOnIngress Benchmarks/IngressBenchmark diff --git a/samples/Management/CLI/AddschematoSchemaGroups/AD.sh b/samples/Management/CLI/AddschematoSchemaGroups/AD.sh new file mode 100644 index 000000000..7c2b05c18 --- /dev/null +++ b/samples/Management/CLI/AddschematoSchemaGroups/AD.sh @@ -0,0 +1,9 @@ +#Step 1- first step is to get the bearer token using client credential flow. We should set the resource to be URL encoded string for eventhubs.azure.net +response=$(curl -X POST -d 'grant_type=client_credentials&client_id=&client_secret=&resource=https%3A%2F%2Feventhubs.azure.net' https://login.microsoftonline.com//oauth2/token) + +#formatting token to drop "" quotes and suffixing bearer for final use +token="Bearer `echo $response | jq ."access_token" | tr -d '"'`" + +#Step 2-Making a REST call to dataplane endpoint to add schema to schema groups +curl -X PUT -d '{"namespace": "com.azure.schemaregistry.samples","type": "record","name": "Order","fields": [{"name": "id","type": "string"},{"name": "amount","type": "double"}]}' -H "Content-Type:application/json" -H "Authorization:$token" -H "Serialization-Type:Avro" \ +'https://.servicebus.windows.net/$schemagroups//schemas/?api-version=2020-09-01-preview' \ No newline at end of file diff --git a/samples/Management/CLI/AddschematoSchemaGroups/ReadMe.md b/samples/Management/CLI/AddschematoSchemaGroups/ReadMe.md new file mode 100644 index 000000000..a11ba8f02 --- /dev/null +++ b/samples/Management/CLI/AddschematoSchemaGroups/ReadMe.md @@ -0,0 +1,43 @@ +# Addition of Schema under Schema groups + +With release of [Schema registry](https://docs.microsoft.com/en-us/azure/event-hubs/schema-registry-overview#what-is-azure-schema-registry) feature in Event Hubs, it becomes easier to store different kinds of schemas under schema groups and have seamless interaction between producer and consumer application. Since addition of schema under schema groups is a data plane operation ( not intended to hit management endpoint) hence, there is no publicly available PS Cmdlet to abstract this functionality. + +This sample is meant to serve as reference code for adding schemas under schema groups using Bash. + +# Prerequisites + +Please ensure that following steps are followed before following the sample code: + +1. Local machine with linux/mac OS or you could use Azure CloudShell via https://shell.azure.com + +2. Create AD application for authentication,here are additional details: https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal +3. Please keep a note of Application ID, secret that you had created in last step ( you would be using this in PS sample) +4. Assign the service principal Schema registry contributor role (To know more about how to assign a role: https://docs.microsoft.com/en-us/azure/role-based-access-control/role-assignments-portal?tabs=current ), so that it could make changes to schema groups. Here is quick read about Schema registry contributor role and permissions it posseses : https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles#schema-registry-contributor-preview +5. Finally, update the following placeholders in sample code with information associated to your environment: +- Application_ID: Application ID of AD Application +- Application_Secret: Secret value associated to Application +- Tenant_Id: Tenant ID of Associated AD tenant +- Namespace_Name: Name of Event Hubs namespace +- SchemaGroup_Name: Name of schema groups under event hub namespace +- Schema_Name: Name of schema to be added under Schema group + +# Script Overview + +This PS sample can be spliced into two REST calls that are being made. + +1. Once we have the AD application details as talked above, we would make first API call to fetch the authorization token needed to successfully make schema addition call. +2. After fetching and formatting the bearer token, we trigger another API call to add schema under schema groups. Once this code runs successfully, you should be able to find the newly added schema under schema groups on Azure Portal. + +# Additional Reference: + +Here are some additional reference links that could be helpful: + +- Schema Registry in Event Hubs - https://docs.microsoft.com/en-us/azure/event-hubs/schema-registry-overview +- Event hubs Overview- https://docs.microsoft.com/en-us/azure/event-hubs/ + +# Conclusion + +We would love to hear about your feedback about this sample. We would be happy to see any pull requests if in case you are interested to contribute to our community. + +Stay tuned for future updates from Azure Messaging team. Happy scripting! + diff --git a/samples/Management/DotNet/EventHubsManagementSample.sln b/samples/Management/DotNet/EventHubsManagementSample/EventHubsManagementSample.sln similarity index 100% rename from samples/Management/DotNet/EventHubsManagementSample.sln rename to samples/Management/DotNet/EventHubsManagementSample/EventHubsManagementSample.sln diff --git a/samples/Management/DotNet/README.md b/samples/Management/DotNet/EventHubsManagementSample/README.md similarity index 100% rename from samples/Management/DotNet/README.md rename to samples/Management/DotNet/EventHubsManagementSample/README.md diff --git a/samples/DotNet/GeoDRClient/GeoDRClient.sln b/samples/Management/DotNet/GeoDRClient/GeoDRClient.sln similarity index 100% rename from samples/DotNet/GeoDRClient/GeoDRClient.sln rename to samples/Management/DotNet/GeoDRClient/GeoDRClient.sln diff --git a/samples/DotNet/GeoDRClient/GeoDRClient/App.config b/samples/Management/DotNet/GeoDRClient/GeoDRClient/App.config similarity index 100% rename from samples/DotNet/GeoDRClient/GeoDRClient/App.config rename to samples/Management/DotNet/GeoDRClient/GeoDRClient/App.config diff --git a/samples/DotNet/GeoDRClient/GeoDRClient/GeoDRClient.csproj b/samples/Management/DotNet/GeoDRClient/GeoDRClient/GeoDRClient.csproj similarity index 100% rename from samples/DotNet/GeoDRClient/GeoDRClient/GeoDRClient.csproj rename to samples/Management/DotNet/GeoDRClient/GeoDRClient/GeoDRClient.csproj diff --git a/samples/DotNet/GeoDRClient/GeoDRClient/GeoDRConfig.cs b/samples/Management/DotNet/GeoDRClient/GeoDRClient/GeoDRConfig.cs similarity index 100% rename from samples/DotNet/GeoDRClient/GeoDRClient/GeoDRConfig.cs rename to samples/Management/DotNet/GeoDRClient/GeoDRClient/GeoDRConfig.cs diff --git a/samples/DotNet/GeoDRClient/GeoDRClient/GeoDRSampleConfig1.json b/samples/Management/DotNet/GeoDRClient/GeoDRClient/GeoDRSampleConfig1.json similarity index 100% rename from samples/DotNet/GeoDRClient/GeoDRClient/GeoDRSampleConfig1.json rename to samples/Management/DotNet/GeoDRClient/GeoDRClient/GeoDRSampleConfig1.json diff --git a/samples/DotNet/GeoDRClient/GeoDRClient/GeoDisasterRecoveryClient.cs b/samples/Management/DotNet/GeoDRClient/GeoDRClient/GeoDisasterRecoveryClient.cs similarity index 100% rename from samples/DotNet/GeoDRClient/GeoDRClient/GeoDisasterRecoveryClient.cs rename to samples/Management/DotNet/GeoDRClient/GeoDRClient/GeoDisasterRecoveryClient.cs diff --git a/samples/DotNet/GeoDRClient/GeoDRClient/JsonHelper.cs b/samples/Management/DotNet/GeoDRClient/GeoDRClient/JsonHelper.cs similarity index 100% rename from samples/DotNet/GeoDRClient/GeoDRClient/JsonHelper.cs rename to samples/Management/DotNet/GeoDRClient/GeoDRClient/JsonHelper.cs diff --git a/samples/DotNet/GeoDRClient/GeoDRClient/Program.cs b/samples/Management/DotNet/GeoDRClient/GeoDRClient/Program.cs similarity index 100% rename from samples/DotNet/GeoDRClient/GeoDRClient/Program.cs rename to samples/Management/DotNet/GeoDRClient/GeoDRClient/Program.cs diff --git a/samples/DotNet/GeoDRClient/GeoDRClient/packages.config b/samples/Management/DotNet/GeoDRClient/GeoDRClient/packages.config similarity index 100% rename from samples/DotNet/GeoDRClient/GeoDRClient/packages.config rename to samples/Management/DotNet/GeoDRClient/GeoDRClient/packages.config diff --git a/samples/DotNet/GeoDRClient/readme.md b/samples/Management/DotNet/GeoDRClient/readme.md similarity index 96% rename from samples/DotNet/GeoDRClient/readme.md rename to samples/Management/DotNet/GeoDRClient/readme.md index 30a6c02a5..f3d264d40 100644 --- a/samples/DotNet/GeoDRClient/readme.md +++ b/samples/Management/DotNet/GeoDRClient/readme.md @@ -52,7 +52,7 @@ For more information on creating a Service Principal, refer to the following art The Geo DR actions could be * CreatePairing -For creating a paired region. After this, you should see metadata (i.e. event hubs, consumer groups, throughput units etc. replicated to the secondary namespace). +For creating a paired region. After this, you should see metadata (i.e. Event Hubs, consumer groups, throughput units etc. replicated to the secondary namespace). * FailOver Simulating a failover. After this action, the secondary namespace becomes the primary @@ -64,7 +64,7 @@ For breaking the pairing between a primary and secondary namespace For deleting an alias, that contains information about the primary-secondary pairing * GetConnectionStrings -In a Geo DR enabled namespace, the Event Hubs should be accessed only via the alias. This is because, the alias can point to either the primary event hub or the failed over event hub. This way, the user does not have to adjust the connection strings in his/her apps to point to a different event hub in the case of a failover. +In a Geo DR enabled namespace, the Event Hubs should be accessed only via the alias. This is because, the alias can point to either the primary Event Hub or the failed over Event Hub. This way, the user does not have to adjust the connection strings in his/her apps to point to a different Event Hub in the case of a failover. Examples * EventHubsGeoDRManagementSample.exe CreatePairing GeoDRSampleConfig.json @@ -86,7 +86,7 @@ The following section describes the steps for performing Geo-diaster recovery, d. Changing the names of an alias is not allowed. e. Changing the secondary namespace is not allowed. 3. Create an alias and provide the primary and secondary namespaces to complete the pairing. -4. Get the required connection strings on the alias to connect to your event hubs. +4. Get the required connection strings on the alias to connect to your Event Hubs. 5. Once the namespaces are paired with an alias, the metadata is replicated periodically in both namespaces. **Note:** Creating a pairing, failing over, breaking the pairing, deleting the alias have all retries build in. All before mentioned operations will retry 10 times with 10 minutes in between each attempt. @@ -95,7 +95,7 @@ The following section describes the steps for performing Geo-diaster recovery, After this step, the seconday namespace becomes the primary namespace. 1. Initiate a fail-over. This step is only performed on the secondary namespace. The geo-pairing is broken and the alias now points to the secondary namespace. **Note:** The Failover can take a few minutes to complete. -2. Senders and receivers still connect to the event hubs using the alias. The failover does not disrupt the connection. +2. Senders and receivers still connect to the Event Hubs using the alias. The failover does not disrupt the connection. 3. Because the pairing is broken, the old primary namespace no longer has a replication status associated with it. 4. The metadata synchronization between the primary and secondary namespaces also stops diff --git a/samples/Management/PowerShell/AddingSchematoSchemagroups/Addingschematoschemagroup.ps1 b/samples/Management/PowerShell/AddingSchematoSchemagroups/Addingschematoschemagroup.ps1 new file mode 100644 index 000000000..889bbe217 --- /dev/null +++ b/samples/Management/PowerShell/AddingSchematoSchemagroups/Addingschematoschemagroup.ps1 @@ -0,0 +1,52 @@ + +#Step 1 - Pass in AD Application details below to be able to fetch authorization token against legitimate Service Principal. +$Fields = @{ + grant_type = "client_credentials" + client_id = "" + resource = "https://eventhubs.azure.net" + client_secret = "" + }; + + #update tenant ID to the correct tenant where you would like to get authorization token from + $response = Invoke-RestMethod –Uri "https://login.microsoftonline.com//oauth2/token" –ContentType "application/x-www-form-urlencoded" –Method POST –Body $Fields + + #response would have bearer token in the properties that we would use to trigger next call. + $token = $response.access_token + + #Step 2- You can declare schema JSON associated to schema in $Body variable below + $Body = @" + { + "namespace": "com.azure.schemaregistry.samples", + "type": "record", + "name": "Order", + "fields": [ + { + "name": "id", + "type": "string" + }, + { + "name": "amount", + "type": "double" + } + ] +} +"@ + +#Step 3- This Uri needs to have details added- namespace name, schema group name and schema name respectively. + +$NamespaceName = '' +$SchemaGroupName = '' +$SchemaName = '' + + +$uri = "https://$NamespaceName.servicebus.windows.net/$schemagroups/$SchemaGroupName/schemas/$SchemaName`?api-version=2020-09-01-preview" + +#Step 4- These headers would be sent along with the API Call +$headers = @{ + "Content-Type" = "application/atom+xml;type=entry;charset=utf-8" + "Serialization-Type" = "Avro" + "Authorization" = "Bearer " + $token +} + +#Step 5- This step shows the final call that we make to add schema under the schema groups, +Invoke-RestMethod -Method "PUT" -Uri $uri -Headers $headers -Body $Body -ContentType "application/json" diff --git a/samples/Management/PowerShell/AddingSchematoSchemagroups/ReadMe.md b/samples/Management/PowerShell/AddingSchematoSchemagroups/ReadMe.md new file mode 100644 index 000000000..4dd04e09e --- /dev/null +++ b/samples/Management/PowerShell/AddingSchematoSchemagroups/ReadMe.md @@ -0,0 +1,44 @@ +# Addition of Schema under Schema groups + +With release of [Schema registry](https://docs.microsoft.com/en-us/azure/event-hubs/schema-registry-overview#what-is-azure-schema-registry) feature in Event Hubs, it becomes easier to store different kinds of schemas under schema groups and have seamless interaction between producer and consumer application. Since addition of schema under schema groups is a data plane operation ( not intended to hit management endpoint) hence, there is no publicly available PS Cmdlet to abstract this functionality. + +This sample is meant to serve as reference code for adding schemas under schema groups using PowerShell. + +# Prerequisites + +Please ensure that following steps are followed before following the sample code: + +1. Updated PS module in Windows machine or you could use Azure CloudShell via https://shell.azure.com + +2. Create AD application for authentication,here are additional details: https://docs.microsoft.com/en-us/azure/active-directory/develop/howto-create-service-principal-portal +3. Please keep a note of Application ID, secret that you had created in last step ( you would be using this in PS sample) +4. Assign the service principal Schema registry contributor role (To know more about how to assign a role: https://docs.microsoft.com/en-us/azure/role-based-access-control/role-assignments-portal?tabs=current ), so that it could make changes to schema groups. Here is quick read about Schema registry contributor role and permissions it posseses : https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles#schema-registry-contributor-preview +5. Finally, update the following placeholders in sample code with information associated to your environment: +- Application_ID: Application ID of AD Application +- Application_Secret: Secret value associated to Application +- resource: needs to be set as "https://eventhubs.azure.net", since we are authenticating against data endpoint +- Tenant_Id: Tenant ID of Associated AD tenant +- Namespace_Name: Name of Event Hubs namespace +- SchemaGroup_Name: Name of schema groups under event hub namespace +- Schema_Name: Name of schema to be added under Schema group + +# Script Overview + +This PS sample can be spliced into two REST calls that are being made. + +1. Once we have the AD application details as talked above, we would make first API call to fetch the authorization token needed to successfully make schema addition call. +2. After fetching and formatting the bearer token, we trigger another API call to add schema under schema groups. Once this code runs successfully, you should be able to find the newly added schema under schema groups on Azure Portal. + +# Additional Reference: + +Here are some additional reference links that could be helpful: + +- Schema Registry in Event Hubs - https://docs.microsoft.com/en-us/azure/event-hubs/schema-registry-overview +- Event hubs Overview- https://docs.microsoft.com/en-us/azure/event-hubs/ + +# Conclusion + +We would love to hear about your feedback about this sample. We would be happy to see any pull requests if in case you are interested to contribute to our community. + +Stay tuned for future updates from Azure Messaging team. Happy scripting! + diff --git a/samples/README.md b/samples/README.md index 9e340fb12..fe936aa22 100644 --- a/samples/README.md +++ b/samples/README.md @@ -1,25 +1,34 @@ # Azure Event Hubs samples -## .NET +**Note:** This repository holds samples for the legacy libraries for Azure Event Hubs for .NET and Java developers. We highly recommend you to upgrade to the newer packages. -### Microsoft.Azure.EventHubs -Any samples within the `Azure.Microsoft.EventHubs` folder target the newer .NET Standard library. +.NET legacy packages -#### SampleSender +- Microsoft.Azure.EventHubs (**legacy**) +- Microsoft.Azure.EventHubs.Processor (**legacy**) +- WindowsAzure.ServiceBus (**legacy**) -[This sample](./DotNet/Microsoft.Azure.EventHubs/SampleSender/readme.md) shows how to write a .NET Core console application that sends a set of messages to an Event Hub. +Java legacy packages -#### SampleEphReceiver +- com.microsoft.azure:azure-eventhubs (**legacy**) +- com.microsoft.azure:azure-eventhubs-eph (**legacy**) -[This sample](./DotNet/Microsoft.Azure.EventHubs/SampleEphReceiver/readme.md) shows how to write a .NET Core console application that receives messages from an Event Hub using the **EventProcessorHost**. +Samples for **latest** .NET packages for Azure Event Hubs -## Java +- [Azure.Messaging.EventHubs](https://docs.microsoft.com/samples/azure/azure-sdk-for-net/azuremessagingeventhubs-samples/) +- [Azure.Messaging.EventHubs.Processor](https://docs.microsoft.com/samples/azure/azure-sdk-for-net/azuremessagingeventhubsprocessor-samples/) -[Go here](./Java) for the Java language samples. +Samples for **latest** Java packages for Azure Event Hubs + +- [com.azure:azure-messaging-eventhubs](https://github.com/Azure/azure-sdk-for-java/tree/master/sdk/eventhubs/azure-messaging-eventhubs/src/samples) +- [com.azure:azure-messaging-eventhubs-checkpointstore-blob](https://github.com/Azure/azure-sdk-for-java/tree/master/sdk/eventhubs/azure-messaging-eventhubs-checkpointstore-blob/src/samples) ## Miscellaneous +### Manage Event Hubs resources + +The samples under the folder `Management` in this repository show how to manage your Event Hubs resources via CLI, the .NET package `Microsoft.Azure.Management.EventHub` and PowerShell + ### proton-c-sender-dotnet-framework-receiver [This sample](./Miscellaneous/proton-c-sender-dotnet-framework-receiver/README.md) shows how to use Azure Event Hubs with clients that use different protocols. This scenario sends using an Apache Proton C++ client, and receives using the .NET Framework client. - diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/DWDumper/App.config b/samples/e2e/EventHubsCaptureEventGridDemo/DWDumper/App.config deleted file mode 100644 index 390b41521..000000000 --- a/samples/e2e/EventHubsCaptureEventGridDemo/DWDumper/App.config +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - - - - - - - - \ No newline at end of file diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/DWDumper/DWDumper.csproj b/samples/e2e/EventHubsCaptureEventGridDemo/DWDumper/DWDumper.csproj deleted file mode 100644 index b947b0d03..000000000 --- a/samples/e2e/EventHubsCaptureEventGridDemo/DWDumper/DWDumper.csproj +++ /dev/null @@ -1,82 +0,0 @@ - - - - - Debug - AnyCPU - {C9E1D3C8-0C0D-45EB-A3B5-15F95EFC01E3} - Exe - DWDumper - DWDumper - v4.6.1 - 512 - true - - - AnyCPU - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - - - AnyCPU - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - - ..\packages\Apache.Avro.1.7.7.2\lib\Avro.dll - - - ..\packages\log4net.1.2.10\lib\2.0\log4net.dll - - - ..\packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll - - - ..\packages\Microsoft.Data.Edm.5.8.2\lib\net40\Microsoft.Data.Edm.dll - - - ..\packages\Microsoft.Data.OData.5.8.2\lib\net40\Microsoft.Data.OData.dll - - - ..\packages\Microsoft.Data.Services.Client.5.8.2\lib\net40\Microsoft.Data.Services.Client.dll - - - ..\packages\WindowsAzure.Storage.8.3.0\lib\net45\Microsoft.WindowsAzure.Storage.dll - - - ..\packages\Newtonsoft.Json.6.0.8\lib\net45\Newtonsoft.Json.dll - True - - - - - ..\packages\System.Spatial.5.8.2\lib\net40\System.Spatial.dll - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/DWDumper/Program.cs b/samples/e2e/EventHubsCaptureEventGridDemo/DWDumper/Program.cs deleted file mode 100644 index 2b9e73718..000000000 --- a/samples/e2e/EventHubsCaptureEventGridDemo/DWDumper/Program.cs +++ /dev/null @@ -1,113 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Data; -using System.Data.SqlClient; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Avro.File; -using Avro.Generic; -using Newtonsoft.Json; -using Microsoft.WindowsAzure.Storage; - -namespace DWDumper -{ - /// - /// A simple test program to dump a single Avro blob file created by EventHubs Capture into a SQL data warehouse (DW). - /// This is useful for testing connections with your SQL DW before integrating this DW dumping code with Azure Functions - /// - class Program - { - private const string StorageConnectionString = "[provide your storage connection string]"; - private const string EventHubsCaptureAvroBlobUri = "[provide the blob path to a single blob file just to test if it can be parsed and dumped to the DW]"; - private const string SqlDwConnection = "[provide the SQL DW connection string]"; - - private static int Main(string[] args) - { - var p = new Program(); - p.Dump(); - - return 0; - } - - public void Dump() - { - // Get the blob reference - var storageAccount = CloudStorageAccount.Parse(StorageConnectionString); - var blobClient = storageAccount.CreateCloudBlobClient(); - var blob = blobClient.GetBlobReferenceFromServer(new Uri(EventHubsCaptureAvroBlobUri)); - - using (var dataTable = GetWindTurbineMetricsTable()) - { - // Parse the Avro File - using (var avroReader = DataFileReader.OpenReader(blob.OpenRead())) - { - while (avroReader.HasNext()) - { - GenericRecord r = avroReader.Next(); - - byte[] body = (byte[]) r["Body"]; - var windTurbineMeasure = DeserializeToWindTurbineMeasure(body); - - // Add the row to in memory table - AddWindTurbineMetricToTable(dataTable, windTurbineMeasure); - } - } - - if (dataTable.Rows.Count > 0) - { - BatchInsert(dataTable); - } - } - } - - private void BatchInsert(DataTable table) - { - // Write the data to SQL DW using SqlBulkCopy - using (var sqlDwConnection = new SqlConnection(SqlDwConnection)) - { - sqlDwConnection.Open(); - - using (var bulkCopy = new SqlBulkCopy(sqlDwConnection)) - { - bulkCopy.BulkCopyTimeout = 30; - bulkCopy.DestinationTableName = "dbo.Fact_WindTurbineMetrics"; - bulkCopy.WriteToServer(table); - } - } - } - - private WindTurbineMeasure DeserializeToWindTurbineMeasure(byte[] body) - { - string payload = Encoding.ASCII.GetString(body); - return JsonConvert.DeserializeObject(payload); - } - - private DataTable GetWindTurbineMetricsTable() - { - var dt = new DataTable(); - dt.Columns.AddRange - ( - new DataColumn[5] - { - new DataColumn("DeviceId", typeof(string)), - new DataColumn("MeasureTime", typeof(DateTime)), - new DataColumn("GeneratedPower", typeof(float)), - new DataColumn("WindSpeed", typeof(float)), - new DataColumn("TurbineSpeed", typeof(float)) - } - ); - - return dt; - } - - private void AddWindTurbineMetricToTable(DataTable table, WindTurbineMeasure wtm) - { - table.Rows.Add(wtm.DeviceId, wtm.MeasureTime, wtm.GeneratedPower, wtm.WindSpeed, wtm.TurbineSpeed); - Console.WriteLine( - "DeviceId: {0}, MeasureTime: {1}, GeneratedPower: {2}, WindSpeed: {3}, TurbineSpeed: {4}", - wtm.DeviceId, wtm.MeasureTime, wtm.GeneratedPower, wtm.WindSpeed, wtm.TurbineSpeed); - } - } - -} diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/DWDumper/WindTurbineMeasure.cs b/samples/e2e/EventHubsCaptureEventGridDemo/DWDumper/WindTurbineMeasure.cs deleted file mode 100644 index 0ab7b9217..000000000 --- a/samples/e2e/EventHubsCaptureEventGridDemo/DWDumper/WindTurbineMeasure.cs +++ /dev/null @@ -1,14 +0,0 @@ -using System; - -namespace DWDumper -{ - // TODO, move this class to a Contracts assembly that is shared across different projects - class WindTurbineMeasure - { - public string DeviceId { get; set; } - public DateTime MeasureTime { get; set; } - public float GeneratedPower { get; set; } - public float WindSpeed { get; set; } - public float TurbineSpeed { get; set; } - } -} \ No newline at end of file diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/DWDumper/packages.config b/samples/e2e/EventHubsCaptureEventGridDemo/DWDumper/packages.config deleted file mode 100644 index 935667da9..000000000 --- a/samples/e2e/EventHubsCaptureEventGridDemo/DWDumper/packages.config +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/EventHubsCaptureEventGridDemo.sln b/samples/e2e/EventHubsCaptureEventGridDemo/EventHubsCaptureEventGridDemo.sln index 2d7d0cecf..bbdbb344a 100644 --- a/samples/e2e/EventHubsCaptureEventGridDemo/EventHubsCaptureEventGridDemo.sln +++ b/samples/e2e/EventHubsCaptureEventGridDemo/EventHubsCaptureEventGridDemo.sln @@ -1,37 +1,31 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.26730.12 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WindTurbineDataGenerator", "WindTurbineDataGenerator\WindTurbineDataGenerator.csproj", "{0C87A54A-AD21-46D5-8B75-4247DA32237E}" -EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DWDumper", "DWDumper\DWDumper.csproj", "{C9E1D3C8-0C0D-45EB-A3B5-15F95EFC01E3}" -EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FunctionEGDWDumper", "FunctionEGDWDumper\FunctionEGDWDumper.csproj", "{27D28A3D-4157-4ED2-A1C4-5525507CCA04}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {0C87A54A-AD21-46D5-8B75-4247DA32237E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {0C87A54A-AD21-46D5-8B75-4247DA32237E}.Debug|Any CPU.Build.0 = Debug|Any CPU - {0C87A54A-AD21-46D5-8B75-4247DA32237E}.Release|Any CPU.ActiveCfg = Release|Any CPU - {0C87A54A-AD21-46D5-8B75-4247DA32237E}.Release|Any CPU.Build.0 = Release|Any CPU - {C9E1D3C8-0C0D-45EB-A3B5-15F95EFC01E3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {C9E1D3C8-0C0D-45EB-A3B5-15F95EFC01E3}.Debug|Any CPU.Build.0 = Debug|Any CPU - {C9E1D3C8-0C0D-45EB-A3B5-15F95EFC01E3}.Release|Any CPU.ActiveCfg = Release|Any CPU - {C9E1D3C8-0C0D-45EB-A3B5-15F95EFC01E3}.Release|Any CPU.Build.0 = Release|Any CPU - {27D28A3D-4157-4ED2-A1C4-5525507CCA04}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {27D28A3D-4157-4ED2-A1C4-5525507CCA04}.Debug|Any CPU.Build.0 = Debug|Any CPU - {27D28A3D-4157-4ED2-A1C4-5525507CCA04}.Release|Any CPU.ActiveCfg = Release|Any CPU - {27D28A3D-4157-4ED2-A1C4-5525507CCA04}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {0BDB7260-AF54-438B-B99E-95E7136C1486} - EndGlobalSection -EndGlobal + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.1.32407.343 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WindTurbineDataGenerator", "WindTurbineDataGenerator\WindTurbineDataGenerator.csproj", "{0C87A54A-AD21-46D5-8B75-4247DA32237E}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FunctionEGDWDumper", "FunctionEGDWDumper\FunctionEGDWDumper.csproj", "{231010C2-FC1F-4393-ABD6-4209523A652B}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {0C87A54A-AD21-46D5-8B75-4247DA32237E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0C87A54A-AD21-46D5-8B75-4247DA32237E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0C87A54A-AD21-46D5-8B75-4247DA32237E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0C87A54A-AD21-46D5-8B75-4247DA32237E}.Release|Any CPU.Build.0 = Release|Any CPU + {231010C2-FC1F-4393-ABD6-4209523A652B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {231010C2-FC1F-4393-ABD6-4209523A652B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {231010C2-FC1F-4393-ABD6-4209523A652B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {231010C2-FC1F-4393-ABD6-4209523A652B}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {CABDAC3A-BADD-423D-B33B-983C7CBD0F46} + EndGlobalSection +EndGlobal diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/EventHubsDataMigration.json b/samples/e2e/EventHubsCaptureEventGridDemo/EventHubsDataMigration.json new file mode 100644 index 000000000..6cd1a68ca --- /dev/null +++ b/samples/e2e/EventHubsCaptureEventGridDemo/EventHubsDataMigration.json @@ -0,0 +1,231 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "eventHubNamespaceName": { + "type": "string", + "metadata": { + "description": "The name of the EventHub namespace" + } + }, + "eventHubName": { + "type": "string", + "metadata": { + "description": "The name of the Event Hub" + } + }, + "sqlServerName": { + "type": "string", + "metadata": { + "description": "The administrator username of the SQL Server" + } + }, + "sqlServerUserName": { + "type": "string", + "metadata": { + "description": "Name of the SQL Server" + } + }, + "sqlServerPassword": { + "type": "securestring", + "metadata": { + "description": "The administrator password of the SQL Server" + } + }, + "sqlServerDatabaseName": { + "type": "string", + "metadata": { + "description": "The name of the SQL Server database" + } + }, + "storageName": { + "type": "string", + "metadata": { + "description": "The name of the storage account" + } + }, + "functionAppName": { + "type": "string", + "metadata": { + "description": "The name of the function app" + } + } + }, + "variables": { + "blobContainerName": "windturbinecapture", + "functionAppPlanName": "[concat(parameters('functionAppName'),'Plan')]", + "storageAccountid": "[concat(resourceGroup().id,'/providers/','Microsoft.Storage/storageAccounts/', parameters('storageName'))]" + }, + "resources": [ + { + "apiVersion": "2017-04-01", + "name": "[parameters('eventHubNamespaceName')]", + "type": "Microsoft.EventHub/namespaces", + "location": "[resourceGroup().location]", + "sku": { + "name": "Standard" + }, + "properties": { + "isAutoInflateEnabled": "true", + "maximumThroughputUnits": "7" + }, + "dependsOn": [ + "[resourceId('Microsoft.Storage/storageAccounts', parameters('storageName'))]" + ], + "resources": [ + { + "apiVersion": "2017-04-01", + "name": "[parameters('eventHubName')]", + "type": "EventHubs", + "dependsOn": [ + "[concat('Microsoft.EventHub/namespaces/', parameters('eventHubNamespaceName'))]" + ], + "properties": { + "messageRetentionInDays": "1", + "partitionCount": "2", + "captureDescription": { + "enabled": "true", + "encoding": "Avro", + "intervalInSeconds": "60", + "sizeLimitInBytes": "314572800", + "destination": { + "name": "EventHubArchive.AzureBlockBlob", + "properties": { + "storageAccountResourceId": "[resourceId('Microsoft.Storage/storageAccounts', parameters('storageName'))]", + "blobContainer": "[variables('blobContainerName')]", + "archiveNameFormat": "{Namespace}/{EventHub}/{PartitionId}/{Year}/{Month}/{Day}/{Hour}/{Minute}/{Second}" + } + } + } + } + } + ] + }, + { + "type": "Microsoft.Sql/servers", + "name": "[parameters('sqlServerName')]", + "apiVersion": "2014-04-01", + "location": "[resourceGroup().location]", + "scale": null, + "properties": { + "administratorLogin": "[parameters('sqlServerUserName')]", + "administratorLoginPassword": "[parameters('sqlServerPassword')]", + "version": "12.0" + }, + "resources": [ + { + "name": "[parameters('sqlServerDatabaseName')]", + "type": "databases", + "apiVersion": "2017-10-01-preview", + "location": "[resourceGroup().location]", + "sku": { + "name": "DW100c", + "tier": "DataWarehouse" + }, + "properties": { + "collation": "SQL_Latin1_General_CP1_CI_AS" + }, + "dependsOn": [ + "[resourceId('Microsoft.Sql/servers', parameters('sqlServerName'))]" + ] + }, + { + "type": "firewallRules", + "apiVersion": "2014-04-01", + "dependsOn": [ + "[parameters('sqlServerName')]" + ], + "location": "[resourceGroup().location]", + "name": "AllowAllAzureIps", + "properties": { + "endIpAddress": "0.0.0.0", + "startIpAddress": "0.0.0.0" + } + } + ] + }, + { + "type": "Microsoft.Storage/storageAccounts", + "sku": { + "name": "Standard_LRS", + "tier": "Standard" + }, + "kind": "Storage", + "name": "[parameters('storageName')]", + "apiVersion": "2016-01-01", + "location": "[resourceGroup().location]", + "tags": {}, + "scale": null, + "properties": {}, + "dependsOn": [] + }, + { + "type": "Microsoft.Web/serverfarms", + "apiVersion": "2015-08-01", + "name": "[variables('functionAppPlanName')]", + "location": "[resourceGroup().location]", + "kind": "functionapp", + "sku": { + "name": "Y1", + "tier": "Dynamic", + "size": "Y1", + "family": "Y", + "capacity": 0 + }, + "properties": { + "name": "[variables('functionAppPlanName')]", + "numberOfWorkers": 0 + } + }, + { + "apiVersion": "2016-08-01", + "type": "Microsoft.Web/sites", + "name": "[parameters('functionAppName')]", + "location": "[resourceGroup().location]", + "kind": "functionapp", + "dependsOn": [ + "[resourceId('Microsoft.Web/serverfarms', variables('functionAppPlanName'))]", + "[resourceId('Microsoft.Storage/storageAccounts', parameters('storageName'))]" + ], + "properties": { + "serverFarmId": "[resourceId('Microsoft.Web/serverfarms', variables('functionAppPlanName'))]", + "siteConfig": { + "appSettings": [ + { + "name": "AzureWebJobsDashboard", + "value": "[concat('DefaultEndpointsProtocol=https;AccountName=', parameters('storageName'), ';AccountKey=', listKeys(variables('storageAccountid'),'2015-05-01-preview').key1)]" + }, + { + "name": "AzureWebJobsStorage", + "value": "[concat('DefaultEndpointsProtocol=https;AccountName=', parameters('storageName'), ';AccountKey=', listKeys(variables('storageAccountid'),'2015-05-01-preview').key1)]" + }, + { + "name": "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING", + "value": "[concat('DefaultEndpointsProtocol=https;AccountName=', parameters('storageName'), ';AccountKey=', listKeys(variables('storageAccountid'),'2015-05-01-preview').key1)]" + }, + { + "name": "WEBSITE_CONTENTSHARE", + "value": "[toLower(parameters('functionAppName'))]" + }, + { + "name": "StorageAccountName", + "value": "[parameters('storageName')]" + }, + { + "name": "StorageAccessKey", + "value": "[listKeys(variables('storageAccountId'), '2015-05-01-preview').key1]" + }, + { + "name": "SqlDwConnection", + "value": "[concat('Server=tcp:',parameters('sqlServerName'),'.database.windows.net,1433;Database=', parameters('sqlServerDatabaseName'), ';Trusted_Connection=False;User ID=',parameters('sqlServerUserName'),'@',parameters('sqlServerName'),';Password=',parameters('sqlServerPassword'),';Connection Timeout=30;Encrypt=True')]" + }, + { + "name": "FUNCTIONS_EXTENSION_VERSION", + "value": "~3" + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/FunctionEGDWDumper/.gitignore b/samples/e2e/EventHubsCaptureEventGridDemo/FunctionEGDWDumper/.gitignore new file mode 100644 index 000000000..ff5b00c50 --- /dev/null +++ b/samples/e2e/EventHubsCaptureEventGridDemo/FunctionEGDWDumper/.gitignore @@ -0,0 +1,264 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. + +# Azure Functions localsettings file +local.settings.json + +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# DNX +project.lock.json +project.fragment.lock.json +artifacts/ + +*_i.c +*_p.c +*_i.h +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# TODO: Comment the next line if you want to checkin your web deploy settings +# but database connection strings (with potential passwords) will be unencrypted +#*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/packages/* +# except build/, which is used as an MSBuild target. +!**/packages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/packages/repositories.config +# NuGet v3's project.json files produces more ignoreable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +node_modules/ +orleans.codegen.cs + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm + +# SQL Server files +*.mdf +*.ldf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# CodeRush +.cr/ + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc \ No newline at end of file diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/FunctionEGDWDumper/EventGridEvent.cs b/samples/e2e/EventHubsCaptureEventGridDemo/FunctionEGDWDumper/Data.cs similarity index 100% rename from samples/e2e/EventHubsCaptureEventGridDemo/FunctionEGDWDumper/EventGridEvent.cs rename to samples/e2e/EventHubsCaptureEventGridDemo/FunctionEGDWDumper/Data.cs diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/FunctionEGDWDumper/Function1.cs b/samples/e2e/EventHubsCaptureEventGridDemo/FunctionEGDWDumper/Function1.cs index 41944c22f..324e92aa0 100644 --- a/samples/e2e/EventHubsCaptureEventGridDemo/FunctionEGDWDumper/Function1.cs +++ b/samples/e2e/EventHubsCaptureEventGridDemo/FunctionEGDWDumper/Function1.cs @@ -1,153 +1,152 @@ -// This is the default URL for triggering event grid function in the local environment. -// http://localhost:7071/admin/extensions/EventGridExtensionConfig?functionName={functionname} - -using System; -using System.Data; -using System.Data.SqlClient; -using System.Globalization; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Text; -using System.Threading.Tasks; -using Avro.File; -using Avro.Generic; -using Microsoft.Azure.WebJobs; -using Microsoft.Azure.WebJobs.Extensions.EventGrid; -using Microsoft.Azure.WebJobs.Extensions.Http; -using Microsoft.Azure.WebJobs.Host; -using Microsoft.WindowsAzure.Storage; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; - -namespace FunctionEGDWDumper -{ - - public static class Function1 - { - private static readonly string StorageConnectionString = Environment.GetEnvironmentVariable("StorageConnectionString"); - private static readonly string SqlDwConnection = Environment.GetEnvironmentVariable("SqlDwConnection"); - - /// - /// Use the accompanying .sql script to create this table in the data warehouse - /// - private const string TableName = "dbo.Fact_WindTurbineMetrics"; - - [FunctionName("EventGridTriggerMigrateData")] - public static void Run([EventGridTrigger]JObject eventGridEvent, TraceWriter log) - { - log.Info("C# EventGrid trigger function processed a request."); - log.Info(eventGridEvent.ToString(Formatting.Indented)); - - try - { - // Copy to a static Album instance - EventGridEHEvent ehEvent = eventGridEvent.ToObject(); - - // Get the URL from the event that points to the Capture file - var uri = new Uri(ehEvent.data.fileUrl); - - // Get data from the file and migrate to data warehouse - Dump(uri); - } - catch (Exception e) - { - string s = string.Format(CultureInfo.InvariantCulture, - "Error processing request. Exception: {0}, Request: {1}", e, eventGridEvent.ToString()); - log.Error(s); - } - } - - /// - /// Dumps the data from the Avro blob to the data warehouse (DW). - /// Before running this, ensure that the DW has the required table created. - /// - private static void Dump(Uri fileUri) - { - // Get the blob reference - var storageAccount = CloudStorageAccount.Parse(StorageConnectionString); - var blobClient = storageAccount.CreateCloudBlobClient(); - var blob = blobClient.GetBlobReferenceFromServer(fileUri); - - using (var dataTable = GetWindTurbineMetricsTable()) - { - // Parse the Avro File - using (var avroReader = DataFileReader.OpenReader(blob.OpenRead())) - { - while (avroReader.HasNext()) - { - GenericRecord r = avroReader.Next(); - - byte[] body = (byte[])r["Body"]; - var windTurbineMeasure = DeserializeToWindTurbineMeasure(body); - - // Add the row to in memory table - AddWindTurbineMetricToTable(dataTable, windTurbineMeasure); - } - } - - if (dataTable.Rows.Count > 0) - { - BatchInsert(dataTable); - } - } - } - - /// - /// Open connection to data warehouse. Write the parsed data to the table. - /// - private static void BatchInsert(DataTable table) - { - // Write the data to SQL DW using SqlBulkCopy - using (var sqlDwConnection = new SqlConnection(SqlDwConnection)) - { - sqlDwConnection.Open(); - - using (var bulkCopy = new SqlBulkCopy(sqlDwConnection)) - { - bulkCopy.BulkCopyTimeout = 30; - bulkCopy.DestinationTableName = TableName; - bulkCopy.WriteToServer(table); - } - } - } - - /// - /// Deserialize data and return object with expected properties. - /// - private static WindTurbineMeasure DeserializeToWindTurbineMeasure(byte[] body) - { - string payload = Encoding.ASCII.GetString(body); - return JsonConvert.DeserializeObject(payload); - } - - /// - /// Define the in-memory table to store the data. The columns match the columns in the .sql script. - /// - private static DataTable GetWindTurbineMetricsTable() - { - var dt = new DataTable(); - dt.Columns.AddRange - ( - new DataColumn[5] - { - new DataColumn("DeviceId", typeof(string)), - new DataColumn("MeasureTime", typeof(DateTime)), - new DataColumn("GeneratedPower", typeof(float)), - new DataColumn("WindSpeed", typeof(float)), - new DataColumn("TurbineSpeed", typeof(float)) - } - ); - - return dt; - } - - /// - /// For each parsed record, add a row to the in-memory table. - /// - private static void AddWindTurbineMetricToTable(DataTable table, WindTurbineMeasure wtm) - { - table.Rows.Add(wtm.DeviceId, wtm.MeasureTime, wtm.GeneratedPower, wtm.WindSpeed, wtm.TurbineSpeed); - } - } -} +// Default URL for triggering event grid function in the local environment. +// http://localhost:7071/runtime/webhooks/EventGrid?functionName={functionname} +using System; +using Microsoft.Azure.WebJobs; +using Microsoft.Azure.WebJobs.Extensions.EventGrid; +using Microsoft.Extensions.Logging; + +using System.Data; +using System.Data.SqlClient; +using System.Globalization; +using System.IO; +using System.Text; +using Avro.File; +using Avro.Generic; +using Azure.Storage.Blobs; +using Newtonsoft.Json; + +using Azure.Messaging.EventGrid; +using Azure; +using Azure.Storage; + +namespace FunctionEGDWDumper +{ + public static class Function1 + { + private static readonly string StorageAccountName = Environment.GetEnvironmentVariable("StorageAccountName"); + private static readonly string StorageAccessKey = Environment.GetEnvironmentVariable("StorageAccessKey"); + private static readonly string SqlDwConnection = Environment.GetEnvironmentVariable("SqlDwConnection"); + + /// + /// Use the accompanying .sql script to create this table in the data warehouse + /// + private const string TableName = "dbo.Fact_WindTurbineMetrics"; + + [FunctionName("MyEventGridTriggerMigrateData")] + public static void Run([EventGridTrigger]EventGridEvent eventGridEvent, ILogger log) + { + log.LogInformation("C# EventGrid trigger function processed a request."); + log.LogInformation(eventGridEvent.Data.ToString()); + + try + { + // Get the URL from the event that points to the Capture file + Data data = eventGridEvent.Data.ToObjectFromJson(); + var uri = new Uri(data.fileUrl); + log.LogInformation($"file URL: {data.fileUrl}"); + + // Get data from the file and migrate to data warehouse + Dump(uri, log); + } + catch (Exception e) + { + string s = string.Format(CultureInfo.InvariantCulture, + "Error processing request. Exception: {0}, Request: {1}", e, eventGridEvent.ToString()); + log.LogError(s); + } + } + + /// + /// Dumps the data from the Avro blob to the data warehouse (DW). + /// Before running this, ensure that the DW has the required table created. + /// + private static async void Dump(Uri fileUri, ILogger log) + { + // Get the blob reference + BlobClient blob = new BlobClient(fileUri, new StorageSharedKeyCredential(StorageAccountName, StorageAccessKey)); + + using (var dataTable = GetWindTurbineMetricsTable()) + { + // Parse the Avro File + Stream blobStream = await blob.OpenReadAsync(null); + using (var avroReader = DataFileReader.OpenReader(blobStream)) + { + while (avroReader.HasNext()) + { + GenericRecord r = avroReader.Next(); + + byte[] body = (byte[])r["Body"]; + var windTurbineMeasure = DeserializeToWindTurbineMeasure(body); + + // Add the row to in memory table + AddWindTurbineMetricToTable(dataTable, windTurbineMeasure); + } + } + + if (dataTable.Rows.Count > 0) + { + log.LogInformation("Batch insert into the dedicated SQL pool."); + BatchInsert(dataTable, log); + } + } + } + + /// + /// Open connection to data warehouse. Write the parsed data to the table. + /// + private static void BatchInsert(DataTable table, ILogger log) + { + // Write the data to SQL DW using SqlBulkCopy + using (var sqlDwConnection = new SqlConnection(SqlDwConnection)) + { + sqlDwConnection.Open(); + + log.LogInformation("Bulk copying data."); + using (var bulkCopy = new SqlBulkCopy(sqlDwConnection)) + { + bulkCopy.BulkCopyTimeout = 30; + bulkCopy.DestinationTableName = TableName; + bulkCopy.WriteToServer(table); + } + } + } + + /// + /// Deserialize data and return object with expected properties. + /// + private static WindTurbineMeasure DeserializeToWindTurbineMeasure(byte[] body) + { + string payload = Encoding.ASCII.GetString(body); + return JsonConvert.DeserializeObject(payload); + } + + /// + /// Define the in-memory table to store the data. The columns match the columns in the .sql script. + /// + private static DataTable GetWindTurbineMetricsTable() + { + var dt = new DataTable(); + dt.Columns.AddRange + ( + new DataColumn[5] + { + new DataColumn("DeviceId", typeof(string)), + new DataColumn("MeasureTime", typeof(DateTime)), + new DataColumn("GeneratedPower", typeof(float)), + new DataColumn("WindSpeed", typeof(float)), + new DataColumn("TurbineSpeed", typeof(float)) + } + ); + + return dt; + } + + /// + /// For each parsed record, add a row to the in-memory table. + /// + private static void AddWindTurbineMetricToTable(DataTable table, WindTurbineMeasure wtm) + { + table.Rows.Add(wtm.DeviceId, wtm.MeasureTime, wtm.GeneratedPower, wtm.WindSpeed, wtm.TurbineSpeed); + } + } +} diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/FunctionEGDWDumper/FunctionEGDWDumper.csproj b/samples/e2e/EventHubsCaptureEventGridDemo/FunctionEGDWDumper/FunctionEGDWDumper.csproj index 84dfa5bc1..24b52c22f 100644 --- a/samples/e2e/EventHubsCaptureEventGridDemo/FunctionEGDWDumper/FunctionEGDWDumper.csproj +++ b/samples/e2e/EventHubsCaptureEventGridDemo/FunctionEGDWDumper/FunctionEGDWDumper.csproj @@ -1,23 +1,23 @@ - - - net461 - - - - - - - - - - - - - PreserveNewest - - - PreserveNewest - Never - - - + + + netcoreapp3.1 + v4 + + + + + + + + + + + + PreserveNewest + + + PreserveNewest + Never + + + diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/FunctionEGDWDumper/host.json b/samples/e2e/EventHubsCaptureEventGridDemo/FunctionEGDWDumper/host.json index 7a73a41bf..809a3f20b 100644 --- a/samples/e2e/EventHubsCaptureEventGridDemo/FunctionEGDWDumper/host.json +++ b/samples/e2e/EventHubsCaptureEventGridDemo/FunctionEGDWDumper/host.json @@ -1,2 +1,11 @@ -{ +{ + "version": "2.0", + "logging": { + "applicationInsights": { + "samplingSettings": { + "isEnabled": true, + "excludedTypes": "Request" + } + } + } } \ No newline at end of file diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/FunctionEGDWDumper/local.settings.json b/samples/e2e/EventHubsCaptureEventGridDemo/FunctionEGDWDumper/local.settings.json index 8f901f116..4fce9ff39 100644 --- a/samples/e2e/EventHubsCaptureEventGridDemo/FunctionEGDWDumper/local.settings.json +++ b/samples/e2e/EventHubsCaptureEventGridDemo/FunctionEGDWDumper/local.settings.json @@ -1,7 +1,7 @@ { - "IsEncrypted": false, - "Values": { - "AzureWebJobsStorage": "", - "AzureWebJobsDashboard": "" - } + "IsEncrypted": false, + "Values": { + "AzureWebJobsStorage": "UseDevelopmentStorage=true", + "FUNCTIONS_WORKER_RUNTIME": "dotnet" + } } \ No newline at end of file diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/Readme.md b/samples/e2e/EventHubsCaptureEventGridDemo/Readme.md deleted file mode 100644 index b74ec2631..000000000 --- a/samples/e2e/EventHubsCaptureEventGridDemo/Readme.md +++ /dev/null @@ -1,143 +0,0 @@ -# Migrate Captured Event Hubs data to a SQL Data Warehouse using Event Grid and Azure Function - -Event Hubs [Capture](https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-capture-overview) is the easiest way to automatically deliver streamed data in Event Hubs to an Azure Blob storage or Azure Data Lake store. You can subsequently process and deliver the data to any other storage destinations of your choice, such as SQL Data Warehouse or Cosmos DB. In this tutorial, we will demonstrate how you can pipe Captured data from your event hub into a SQL Database Warehouse by using an [Event Grid](https://docs.microsoft.com/azure/event-grid/overview) triggered Azure Function. - -### What is covered in this tutorial: - -![Visual Studio](./media/EventGridIntegrationOverview.PNG) - -* First, we create an Azure Event Hub with Capture enabled and set an Azure blob storage as the destination. Data generated by WindTurbineGenerator will be streamed into the event hub and automatically Captured into Azure Storage as Avro files. -* Next, we create an Azure Event Grid subscription with the Event Hubs namespace as its source and the Azure Function endpoint as its destination. -* Whenever a new Avro file is delivered to the Storage blob by Event Hubs Capture, Event Grid notifies the Azure Function with the blob URI. The Function then does the required processing to migrate the data from the Storage blob to a SQL Database data warehouse. - -There are no worker services involved in polling for these Avro files, which eliminates management overhead and significantly lower COGS, especially in a cloud-scale production environment! - -This sample solution contains files that do the following: -1. *WindTurbineDataGenerator* – A simple publisher that sends wind turbine data to a Capture-enabled event hub -1. *FunctionDWDumper* – An Azure Function that receives an Event Grid notification an Avro file is Captured to the Azure Storage blob. It receives the blob’s URI path, reads its contents and pushes this data to a SQL Data Warehouse. - -# Prerequisites -* [Visual studio 2017 Version 15.3.2 or greater](https://www.visualstudio.com/vs/) -* While installing, ensure that you install the following workloads: .NET desktop development, Azure development, ASP.NET and web development, Node.js development, Python development - -![Visual Studio](./media/EventCaptureGridDemo1.png) - -# Detailed steps -### Overview: -1. Deploy the infrastructure for this solution -2. Create a table in SQL Data Warehouse -3. Publish code to the Functions App -4. Create an Event Grid subscription from the Functions app, with your Event Hubs Namespace as the source and your Function endpoint as the destination. -5. Run WindTurbineDataGenerator.exe to generate data streams to the Event Hub. -6. Observe the Captured data that has been migrated to your SQL Data Warehouse table by the Azure Function - -## 1. Deploy the infrastructure -Deploy the infrastructure needed for this tutorial by using this [Azure Resource Manager template](https://raw.githubusercontent.com/Azure/azure-docs-json-samples/master/event-grid/EventHubsDataMigration.json). This creates the following resources: -- Event Hub with Capture enabled -- Storage account for the files from Capture -- Azure app service plan for hosting the Functions app -- Function app for processing Captured event files (Function code to be added in Step 3) -- SQL Server for hosting the Data Warehouse -- SQL Data Warehouse for storing the migrated data (SQL table to be added in Step 2) -To deploy the template using Azure CLI, use: - -```azurecli-interactive -az group create -l westcentralus -n rgDataMigrationSample - -az group deployment create \ - --resource-group rgDataMigrationSample \ - --template-uri https://raw.githubusercontent.com/Azure/azure-docs-json-samples/master/event-grid/EventHubsDataMigration.json \ - --parameters eventHubNamespaceName= eventHubName=hubdatamigration sqlServerName= sqlServerUserName= sqlServerPassword= sqlServerDatabaseName= storageName= functionAppName= -``` -To deploy the template using PowerShell, use: - -```powershell -New-AzureRmResourceGroup -Name rgDataMigration -Location westcentralus - -New-AzureRmResourceGroupDeployment -ResourceGroupName rgDataMigration -TemplateUri https://raw.githubusercontent.com/Azure/azure-docs-json-samples/master/event-grid/EventHubsDataMigration.json -eventHubNamespaceName -eventHubName hubdatamigration -sqlServerName -sqlServerUserName -sqlServerDatabaseName -storageName -functionAppName -``` - -## 2. Create a table in SQL Data Warehouse -Create a table in your Data Warehouse by running the *CreateDataWarehouseTable.sql* script using Visual Studio or the Query Editor in the portal. - -## 3. Publish code to the Functions App - -1. Open the project solution *EventHubsCaptureEventGridDemo.sln* in Visual Studio 2017 (15.3.2 or greater). - -1. In Solution Explorer, right-click *FunctionEGDWDumper*, and select **Publish**. - - ![Publish function app](./media/publish-function-app.png) - -1. Select **Azure Function App** and **Select Existing**. Select **Publish**. - - ![Target function app](./media/pick-target.png) - -1. Select the function app that you deployed through the template. Select **OK**. - - ![Select function app](./media/select-function-app.png) - -1. When Visual Studio has configured the profile, select **Publish**. - - ![Select publish](./media/select-publish.png) - -After publishing the function, you are ready to subscribe to the Capture event! - - -## 4. Create an Event Grid subscription from the Functions app - -1. Go to the [Azure portal](https://portal.azure.com/). Select your resource group and function app. - - ![View function app](./media/view-function-app.png) - -1. Select the function. - - ![Select function](./media/select-function.png) - -1. Select **Add Event Grid subscription**. - - ![Add subscription](./media/add-event-grid-subscription.png) - -1. Give the event grid subscription a name. Use **Event Hubs Namespaces** as the event type. Provide values to select your instance of the Event Hubs namespace. Leave the subscriber endpoint as the provided value. Select **Create**. - - ![Create subscription](./media/set-subscription-values.png) - -## 5. Run WindTurbineDataGenerator.exe to generate data -You have now set up your Event Hub, SQL data warehouse, Azure Function App, and Event Grid subscription. Upon completing the simple configuration below, you can run WindTurbineDataGenerator.exe to generate data streams to the Event Hub. - -1. In the portal, select your event hub namespace. Select **Connection Strings**. - - ![Select connection strings](./media/event-hub-connection.png) - -2. Select **RootManageSharedAccessKey** - - ![Select key](./media/show-root-key.png) - -3. Copy **Connection string - primary Key** - - ![Copy key](./media/copy-key.png) - -4. Go back to your Visual Studio project. In the *WindTurbineDataGenerator* project, open *program.cs*. - -5. Replace the two constant values. Use the copied value for **EventHubConnectionString**. Use **hubdatamigration** the event hub name. - - ```cs - private const string EventHubConnectionString = "Endpoint=sb://demomigrationnamespace.servicebus.windows.net/..."; - private const string EventHubName = "hubdatamigration"; - ``` - -6. Build the solution, then run the WindTurbineGenerator.exe application. - -## 6. Observe the Captured data migrate to your SQL Data Warehouse table -After a couple of minutes, query the table in your data warehouse. You will observe that data generated by the WindTurbineDataGenerator has been streamed to your Event Hub, Captured into an Azure Storage container, and then migrated into the SQL data table by Azure Function. - -## Next steps -You can use powerful data visualization tools with your data warehouse to achieve your Actionable insights. - -This article shows how to use [Power BI with SQL Data Warehouse](https://docs.microsoft.com/azure/sql-data-warehouse/sql-data-warehouse-integrate-power-bi) - -Now you are all set to plug in the UI you need to get valuable business insights for your management. - -# Conclusion -We look forward to your feedback after you give this tutorial a try! We also love to see pull requests so if you would like to contribute to our community. - -Stay tuned for more samples from the Azure Messaging team, and till next time! diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/WindTurbineDataGenerator/App.config b/samples/e2e/EventHubsCaptureEventGridDemo/WindTurbineDataGenerator/App.config index 731f6de6c..3b0c70459 100644 --- a/samples/e2e/EventHubsCaptureEventGridDemo/WindTurbineDataGenerator/App.config +++ b/samples/e2e/EventHubsCaptureEventGridDemo/WindTurbineDataGenerator/App.config @@ -1,6 +1,42 @@ - - - - - - \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/WindTurbineDataGenerator/Program.cs b/samples/e2e/EventHubsCaptureEventGridDemo/WindTurbineDataGenerator/Program.cs index e97740b68..b4062035c 100644 --- a/samples/e2e/EventHubsCaptureEventGridDemo/WindTurbineDataGenerator/Program.cs +++ b/samples/e2e/EventHubsCaptureEventGridDemo/WindTurbineDataGenerator/Program.cs @@ -6,22 +6,24 @@ using System.Threading; using System.Threading.Tasks; using Newtonsoft.Json; -using Microsoft.Azure.EventHubs; +using Azure.Messaging.EventHubs; +using Azure.Messaging.EventHubs.Producer; namespace WindTurbineDataGenerator { internal class Program { private const string EventHubConnectionString = - "[provide the EH connection string]"; + ""; - private const string EventHubName = "[provide the EH name]"; + private const string EventHubName = "hubdatamigration"; private static int Main(string[] args) { Console.WriteLine("Starting wind turbine generator. Press to exit"); // Start generation of events + var cts = new CancellationTokenSource(); var t0 = StartEventGenerationAsync(cts.Token); @@ -39,30 +41,30 @@ private static async Task StartEventGenerationAsync(CancellationToken cancellati { var random = new Random((int)DateTimeOffset.UtcNow.Ticks); - var connectionStringBuilder = new EventHubsConnectionStringBuilder(EventHubConnectionString) - { - EntityPath = EventHubName - }; - - EventHubClient client = EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString()); + // create an Event Hubs Producer client using the namespace connection string and the event hub name + EventHubProducerClient producerClient = new EventHubProducerClient(EventHubConnectionString, EventHubName); while (!cancellationToken.IsCancellationRequested) { try { // Simulate sending data from 100 weather sensors - var devicesData = new List(); - + // prepare a batch of events to send to the event hub. + EventDataBatch eventBatch = await producerClient.CreateBatchAsync(); for (int i = 0; i < 100; i++) { int scaleFactor = random.Next(0, 25); var windTurbineMeasure = GenerateTurbineMeasure("Turbine_" + i, scaleFactor); EventData evData = SerializeWindTurbineToEventData(windTurbineMeasure); - devicesData.Add(evData); + // add the event to the batch + if (eventBatch.TryAdd(evData) == false) + break; } Console.Write("."); - await client.SendAsync(devicesData); + + // send the batch of events to the event hub + await producerClient.SendAsync(eventBatch); } catch (Exception ex) { @@ -72,7 +74,7 @@ private static async Task StartEventGenerationAsync(CancellationToken cancellati await Task.Delay(1000, cancellationToken); } - } + } private static WindTurbineMeasure GenerateTurbineMeasure(string turbineId, int scaleFactor) { diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/WindTurbineDataGenerator/WindTurbineDataGenerator.csproj b/samples/e2e/EventHubsCaptureEventGridDemo/WindTurbineDataGenerator/WindTurbineDataGenerator.csproj index 8ddc90db0..9969d2337 100644 --- a/samples/e2e/EventHubsCaptureEventGridDemo/WindTurbineDataGenerator/WindTurbineDataGenerator.csproj +++ b/samples/e2e/EventHubsCaptureEventGridDemo/WindTurbineDataGenerator/WindTurbineDataGenerator.csproj @@ -1,63 +1,157 @@ - - - - - Debug - AnyCPU - {0C87A54A-AD21-46D5-8B75-4247DA32237E} - Exe - WindTurbineDataGenerator - WindTurbineDataGenerator - v4.6.1 - 512 - true - - - AnyCPU - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - - - AnyCPU - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - - ..\packages\Microsoft.Azure.Amqp.2.0.4\lib\net45\Microsoft.Azure.Amqp.dll - - - ..\packages\Microsoft.Azure.EventHubs.1.0.2\lib\net451\Microsoft.Azure.EventHubs.dll - - - ..\packages\Newtonsoft.Json.10.0.3\lib\net45\Newtonsoft.Json.dll - - - - - - - - - - - - - - - - - - - - + + + + + Debug + AnyCPU + {0C87A54A-AD21-46D5-8B75-4247DA32237E} + Exe + WindTurbineDataGenerator + WindTurbineDataGenerator + v4.8 + 512 + true + + + + AnyCPU + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + AnyCPU + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + ..\packages\Azure.Core.1.20.0\lib\net461\Azure.Core.dll + + + ..\packages\Azure.Core.Amqp.1.2.0\lib\netstandard2.0\Azure.Core.Amqp.dll + + + ..\packages\Azure.Messaging.EventHubs.5.6.2\lib\netstandard2.0\Azure.Messaging.EventHubs.dll + + + ..\packages\Microsoft.Azure.Amqp.2.5.6\lib\net45\Microsoft.Azure.Amqp.dll + + + ..\packages\Microsoft.Bcl.AsyncInterfaces.1.0.0\lib\net461\Microsoft.Bcl.AsyncInterfaces.dll + + + ..\packages\Newtonsoft.Json.13.0.1\lib\net45\Newtonsoft.Json.dll + + + + ..\packages\System.Buffers.4.5.1\lib\net461\System.Buffers.dll + + + + + ..\packages\System.Diagnostics.DiagnosticSource.4.6.0\lib\net46\System.Diagnostics.DiagnosticSource.dll + + + ..\packages\System.IO.4.1.0\lib\net462\System.IO.dll + True + True + + + ..\packages\System.Memory.4.5.4\lib\net461\System.Memory.dll + + + ..\packages\System.Memory.Data.1.0.2\lib\net461\System.Memory.Data.dll + + + ..\packages\System.Net.WebSockets.4.0.0\lib\net46\System.Net.WebSockets.dll + True + True + + + ..\packages\System.Net.WebSockets.Client.4.0.2\lib\net46\System.Net.WebSockets.Client.dll + True + True + + + + ..\packages\System.Numerics.Vectors.4.5.0\lib\net46\System.Numerics.Vectors.dll + + + ..\packages\System.Reflection.TypeExtensions.4.5.1\lib\net461\System.Reflection.TypeExtensions.dll + + + ..\packages\System.Runtime.4.1.0\lib\net462\System.Runtime.dll + True + True + + + ..\packages\System.Runtime.CompilerServices.Unsafe.4.6.0\lib\netstandard2.0\System.Runtime.CompilerServices.Unsafe.dll + + + + ..\packages\System.Runtime.Serialization.Primitives.4.3.0\lib\net46\System.Runtime.Serialization.Primitives.dll + True + True + + + ..\packages\System.Security.Cryptography.Algorithms.4.2.0\lib\net461\System.Security.Cryptography.Algorithms.dll + True + True + + + ..\packages\System.Security.Cryptography.Encoding.4.0.0\lib\net46\System.Security.Cryptography.Encoding.dll + True + True + + + ..\packages\System.Security.Cryptography.Primitives.4.0.0\lib\net46\System.Security.Cryptography.Primitives.dll + True + True + + + ..\packages\System.Security.Cryptography.X509Certificates.4.1.0\lib\net461\System.Security.Cryptography.X509Certificates.dll + True + True + + + ..\packages\System.Text.Encodings.Web.4.7.2\lib\net461\System.Text.Encodings.Web.dll + + + ..\packages\System.Text.Json.4.6.0\lib\net461\System.Text.Json.dll + + + ..\packages\System.Threading.Channels.4.6.0\lib\netstandard2.0\System.Threading.Channels.dll + + + ..\packages\System.Threading.Tasks.Extensions.4.5.4\lib\net461\System.Threading.Tasks.Extensions.dll + + + ..\packages\System.ValueTuple.4.5.0\lib\net461\System.ValueTuple.dll + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/WindTurbineDataGenerator/packages.config b/samples/e2e/EventHubsCaptureEventGridDemo/WindTurbineDataGenerator/packages.config index 440e8d5c6..1131fe2cd 100644 --- a/samples/e2e/EventHubsCaptureEventGridDemo/WindTurbineDataGenerator/packages.config +++ b/samples/e2e/EventHubsCaptureEventGridDemo/WindTurbineDataGenerator/packages.config @@ -1,6 +1,30 @@ - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/EventCaptureGridDemo1.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/EventCaptureGridDemo1.png deleted file mode 100644 index 67412d540..000000000 Binary files a/samples/e2e/EventHubsCaptureEventGridDemo/media/EventCaptureGridDemo1.png and /dev/null differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/EventCaptureGridDemo2.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/EventCaptureGridDemo2.png deleted file mode 100644 index 9cb1c75df..000000000 Binary files a/samples/e2e/EventHubsCaptureEventGridDemo/media/EventCaptureGridDemo2.png and /dev/null differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/EventCaptureGridDemo3.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/EventCaptureGridDemo3.png deleted file mode 100644 index 13fa44323..000000000 Binary files a/samples/e2e/EventHubsCaptureEventGridDemo/media/EventCaptureGridDemo3.png and /dev/null differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/EventCaptureGridDemo4.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/EventCaptureGridDemo4.png deleted file mode 100644 index 9079b343f..000000000 Binary files a/samples/e2e/EventHubsCaptureEventGridDemo/media/EventCaptureGridDemo4.png and /dev/null differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/EventCaptureGridDemo5.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/EventCaptureGridDemo5.png deleted file mode 100644 index 13f7328d1..000000000 Binary files a/samples/e2e/EventHubsCaptureEventGridDemo/media/EventCaptureGridDemo5.png and /dev/null differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/EventGridIntegrationOverview.PNG b/samples/e2e/EventHubsCaptureEventGridDemo/media/EventGridIntegrationOverview.PNG deleted file mode 100644 index 1b12e73ed..000000000 Binary files a/samples/e2e/EventHubsCaptureEventGridDemo/media/EventGridIntegrationOverview.PNG and /dev/null differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/add-event-grid-subscription.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/add-event-grid-subscription.png deleted file mode 100644 index 468d29a6f..000000000 Binary files a/samples/e2e/EventHubsCaptureEventGridDemo/media/add-event-grid-subscription.png and /dev/null differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/add-event-subscription.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/add-event-subscription.png deleted file mode 100644 index 945a415b6..000000000 Binary files a/samples/e2e/EventHubsCaptureEventGridDemo/media/add-event-subscription.png and /dev/null differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/copy-key.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/copy-key.png deleted file mode 100644 index 12cee1432..000000000 Binary files a/samples/e2e/EventHubsCaptureEventGridDemo/media/copy-key.png and /dev/null differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/copy-url.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/copy-url.png deleted file mode 100644 index b807de316..000000000 Binary files a/samples/e2e/EventHubsCaptureEventGridDemo/media/copy-url.png and /dev/null differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/azure-portal.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/azure-portal.png new file mode 100644 index 000000000..ca771c328 Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/azure-portal.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/cloud-shell-initialized.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/cloud-shell-initialized.png new file mode 100644 index 000000000..17829f44d Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/cloud-shell-initialized.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/confirm-event-subscription.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/confirm-event-subscription.png new file mode 100644 index 000000000..ebb178ff3 Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/confirm-event-subscription.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/confirm-function-creation.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/confirm-function-creation.png new file mode 100644 index 000000000..5ad6057a3 Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/confirm-function-creation.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/create-storage-cloud-shell.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/create-storage-cloud-shell.png new file mode 100644 index 000000000..74ab7ad58 Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/create-storage-cloud-shell.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/dependency-storage-changes-summary.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/dependency-storage-changes-summary.png new file mode 100644 index 000000000..2967d8e81 Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/dependency-storage-changes-summary.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/dependency-storage-connection-string.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/dependency-storage-connection-string.png new file mode 100644 index 000000000..eb4edf3bf Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/dependency-storage-connection-string.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/event-hub-add-subscription-link.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/event-hub-add-subscription-link.png new file mode 100644 index 000000000..d9c46dda3 Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/event-hub-add-subscription-link.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/event-hub-namespace-shared-access-policies.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/event-hub-namespace-shared-access-policies.png new file mode 100644 index 000000000..487f87a66 Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/event-hub-namespace-shared-access-policies.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/event-subscription-select-function.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/event-subscription-select-function.png new file mode 100644 index 000000000..cb62feb83 Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/event-subscription-select-function.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/function-invocations.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/function-invocations.png new file mode 100644 index 000000000..b7f874924 Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/function-invocations.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/function-runtime-version.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/function-runtime-version.png new file mode 100644 index 000000000..0a2b669a4 Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/function-runtime-version.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/get-publish-profile.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/get-publish-profile.png new file mode 100644 index 000000000..1afea4f1b Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/get-publish-profile.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/import-profile.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/import-profile.png new file mode 100644 index 000000000..dd8458bf7 Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/import-profile.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/launch-cloud-shell.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/launch-cloud-shell.png new file mode 100644 index 000000000..88839872f Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/launch-cloud-shell.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/overview.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/overview.png similarity index 100% rename from samples/e2e/EventHubsCaptureEventGridDemo/media/overview.png rename to samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/overview.png diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/publish-select-function-app.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/publish-select-function-app.png new file mode 100644 index 000000000..698813155 Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/publish-select-function-app.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/publish-storage-configure-link.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/publish-storage-configure-link.png new file mode 100644 index 000000000..1859b7e7f Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/publish-storage-configure-link.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/query-results.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/query-results.png new file mode 100644 index 000000000..b0a436df4 Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/query-results.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/resources-in-resource-group.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/resources-in-resource-group.png new file mode 100644 index 000000000..d5c87b24b Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/resources-in-resource-group.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/run-sql-query.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/run-sql-query.png new file mode 100644 index 000000000..77ae95cc7 Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/run-sql-query.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/select-dependency-storage.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/select-dependency-storage.png new file mode 100644 index 000000000..41fdc74be Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/select-dependency-storage.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/select-function-app.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/select-function-app.png new file mode 100644 index 000000000..e041b4fdf Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/select-function-app.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/select-publish.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/select-publish.png new file mode 100644 index 000000000..428bdc3c1 Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/select-publish.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/select-resource-group.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/select-resource-group.png new file mode 100644 index 000000000..d8b41ae42 Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/select-resource-group.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/sql-data-warehouse-page.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/sql-data-warehouse-page.png new file mode 100644 index 000000000..2667025a6 Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/sql-data-warehouse-page.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/sql-server-authentication.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/sql-server-authentication.png new file mode 100644 index 000000000..4427098c7 Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/sql-server-authentication.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/start-publish-button.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/start-publish-button.png new file mode 100644 index 000000000..20f0799e3 Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/start-publish-button.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/storage-captured-file.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/storage-captured-file.png new file mode 100644 index 000000000..a703cda92 Binary files /dev/null and b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-grid-event-hubs-functions-synapse-analytics/storage-captured-file.png differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-hub-connection.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/event-hub-connection.png deleted file mode 100644 index 08360628c..000000000 Binary files a/samples/e2e/EventHubsCaptureEventGridDemo/media/event-hub-connection.png and /dev/null differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/get-function-url.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/get-function-url.png deleted file mode 100644 index a37cee2bd..000000000 Binary files a/samples/e2e/EventHubsCaptureEventGridDemo/media/get-function-url.png and /dev/null differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/pick-target.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/pick-target.png deleted file mode 100644 index 058aff7f8..000000000 Binary files a/samples/e2e/EventHubsCaptureEventGridDemo/media/pick-target.png and /dev/null differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/provide-values.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/provide-values.png deleted file mode 100644 index 76fd5139f..000000000 Binary files a/samples/e2e/EventHubsCaptureEventGridDemo/media/provide-values.png and /dev/null differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/publish-function-app.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/publish-function-app.png deleted file mode 100644 index 1a2a9b033..000000000 Binary files a/samples/e2e/EventHubsCaptureEventGridDemo/media/publish-function-app.png and /dev/null differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/select-event-grid.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/select-event-grid.png deleted file mode 100644 index c7526f0cd..000000000 Binary files a/samples/e2e/EventHubsCaptureEventGridDemo/media/select-event-grid.png and /dev/null differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/select-function-app.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/select-function-app.png deleted file mode 100644 index 0116e466d..000000000 Binary files a/samples/e2e/EventHubsCaptureEventGridDemo/media/select-function-app.png and /dev/null differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/select-function.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/select-function.png deleted file mode 100644 index 7eaa5a3ae..000000000 Binary files a/samples/e2e/EventHubsCaptureEventGridDemo/media/select-function.png and /dev/null differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/select-publish.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/select-publish.png deleted file mode 100644 index 5471bf104..000000000 Binary files a/samples/e2e/EventHubsCaptureEventGridDemo/media/select-publish.png and /dev/null differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/set-subscription-values.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/set-subscription-values.png deleted file mode 100644 index 44c95a864..000000000 Binary files a/samples/e2e/EventHubsCaptureEventGridDemo/media/set-subscription-values.png and /dev/null differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/show-root-key.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/show-root-key.png deleted file mode 100644 index a47c57ae3..000000000 Binary files a/samples/e2e/EventHubsCaptureEventGridDemo/media/show-root-key.png and /dev/null differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/media/view-function-app.png b/samples/e2e/EventHubsCaptureEventGridDemo/media/view-function-app.png deleted file mode 100644 index 3f2947aa3..000000000 Binary files a/samples/e2e/EventHubsCaptureEventGridDemo/media/view-function-app.png and /dev/null differ diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/readme.md b/samples/e2e/EventHubsCaptureEventGridDemo/readme.md new file mode 100644 index 000000000..f00d17265 --- /dev/null +++ b/samples/e2e/EventHubsCaptureEventGridDemo/readme.md @@ -0,0 +1,308 @@ +--- +title: 'Tutorial: Migrate event data to Azure Synapse Analytics - Azure Event Hubs' +description: Describes how to use Azure Event Grid and Functions to migrate Event Hubs captured data to Azure Synapse Analytics. +services: event-hubs +ms.date: 04/29/2022 +ms.topic: tutorial +ms.custom: devx-track-csharp +--- + +# Tutorial: Migrate captured Event Hubs Avro data to Azure Synapse Analytics using Event Grid and Azure Functions +Azure Event Hubs Capture enables you to automatically capture the streaming data in Event Hubs in an Azure Blob storage or Azure Data Lake Storage. This tutorial shows you how to migrate captured Event Hubs data from Storage to Azure Synapse Analytics by using an Azure function that's triggered by Event Grid. + +:::image type="content" source="media/event-grid-event-hubs-functions-synapse-analytics/overview.png" alt-text="Application overview"::: + +This diagram depicts the workflow of the solution you build in this tutorial: + +1. Data sent to an Azure event hub is captured in an Azure blob storage. +2. When the data capture is complete, an event is generated and sent to Azure Event Grid. +3. Azure Event Grid forwards this event data to an Azure function app. +4. The function app uses the blob URL in the event data to retrieve the blob from the storage. +5. The function app migrates the blob data to an Azure Synapse Analytics. + +In this article, you take the following steps: + +> [!div class="checklist"] +> - Deploy the required infrastructure for the tutorial +> - Publish code to a Functions App +> - Create an Event Grid subscription +> - Stream sample data into Event Hubs +> - Verify captured data in Azure Synapse Analytics + +## Prerequisites +To complete this tutorial, you must have: + +- An Azure subscription. If you don't have an Azure subscription, create a [free account](https://azure.microsoft.com/free/) before you begin. +- [Visual studio](https://www.visualstudio.com/vs/) with workloads for: .NET desktop development, Azure development, ASP.NET and web development, Node.js development, and Python development. +- Download the [EventHubsCaptureEventGridDemo sample project](https://github.com/Azure/azure-event-hubs/tree/master/samples/e2e/EventHubsCaptureEventGridDemo) to your computer. + - WindTurbineDataGenerator – A simple publisher that sends sample wind turbine data to a capture-enabled event hub + - FunctionDWDumper – An Azure Function that receives a notification from Azure Event Grid when an Avro file is captured to the Azure Storage blob. It receives the blob’s URI path, reads its contents, and pushes this data to Azure Synapse Analytics (dedicated SQL pool). + +## Deploy the infrastructure +In this step, you deploy the required infrastructure with a [Resource Manager template](https://github.com/Azure/azure-docs-json-samples/blob/master/event-grid/EventHubsDataMigration.json). When you deploy the template, the following resources are created: + +* Event hub with the Capture feature enabled. +* Storage account for the captured files. +* App service plan for hosting the function app +* Function app for processing the event +* SQL Server for hosting the data warehouse +* Azure Synapse Analytics (dedicated SQL pool) for storing the migrated data + +### Use Azure CLI to deploy the infrastructure + +1. Sign in to the [Azure portal](https://portal.azure.com). +2. Select **Cloud Shell** button at the top. + + :::image type="content" source="media/event-grid-event-hubs-functions-synapse-analytics/azure-portal.png" alt-text="Screenshot of Azure portal showing the selection of Cloud Shell button."::: +3. You see the Cloud Shell opened at the bottom of the browser. + 1. If you're using the Cloud Shell for the first time: + 1. If you see an option to select between **Bash** and **PowerShell**, select **Bash**. + + :::image type="content" source="media/event-grid-event-hubs-functions-synapse-analytics/launch-cloud-shell.png" alt-text="Screenshot of Cloud Shell with Bash selected."::: + + 1. Create a storage account by selecting **Create storage**. Azure Cloud Shell requires an Azure storage account to store some files. + + :::image type="content" source="media/event-grid-event-hubs-functions-synapse-analytics/create-storage-cloud-shell.png" alt-text="Screenshot showing the creation of storage for Cloud Shell."::: + 3. Wait until the Cloud Shell is initialized. + + :::image type="content" source="media/event-grid-event-hubs-functions-synapse-analytics/cloud-shell-initialized.png" alt-text="Screenshot showing the Cloud Shell initialized."::: +4. In the Cloud Shell, select **Bash** as shown in the above image, if it isn't already selected. +1. Create an Azure resource group by running the following CLI command: + 1. Copy and paste the following command into the Cloud Shell window. Change the resource group name and location if you want. + + ```azurecli + az group create -l eastus -n rgDataMigration + ``` + 2. Press **ENTER**. + + Here's an example: + + ```azurecli + user@Azure:~$ az group create -l eastus -n rgDataMigration + { + "id": "/subscriptions/00000000-0000-0000-0000-0000000000000/resourceGroups/rgDataMigration", + "location": "eastus", + "managedBy": null, + "name": "rgDataMigration", + "properties": { + "provisioningState": "Succeeded" + }, + "tags": null + } + ``` +2. Deploy all the resources mentioned in the previous section (event hub, storage account, functions app, Azure Synapse Analytics) by running the following CLI command: + 1. Copy and paste the command into the Cloud Shell window. Alternatively, you may want to copy/paste into an editor of your choice, set values, and then copy the command to the Cloud Shell. + + > [!IMPORTANT] + > Specify values for the following entities before running the command: + > - Name of the resource group you created earlier. + > - Name for the event hub namespace. + > - Name for the event hub. You can leave the value as it is (hubdatamigration). + > - Name for the SQL server. + > - Name of the SQL user and password. + > - Name for the database. + > - Name of the storage account. + > - Name for the function app. + + + ```azurecli + az deployment group create \ + --resource-group rgDataMigration \ + --template-uri https://raw.githubusercontent.com/Azure/azure-docs-json-samples/master/event-grid/EventHubsDataMigration.json \ + --parameters eventHubNamespaceName= eventHubName=hubdatamigration sqlServerName= sqlServerUserName= sqlServerPassword= sqlServerDatabaseName= storageName= functionAppName= + ``` + 3. Press **ENTER** in the Cloud Shell window to run the command. This process may take a while since you're creating a bunch of resources. In the result of the command, ensure that there have been no failures. +1. Close the Cloud Shell by selecting the **Cloud Shell** button in the portal (or) **X** button in the top-right corner of the Cloud Shell window. + +### Verify that the resources are created + +1. In the Azure portal, select **Resource groups** on the left menu. +2. Filter the list of resource groups by entering the name of your resource group in the search box. +3. Select your resource group in the list. + + :::image type="content" source="media/event-grid-event-hubs-functions-synapse-analytics/select-resource-group.png" alt-text="Screenshot showing the selection of your resource group."::: +4. Confirm that you see the following resources in the resource group: + + :::image type="content" source="media/event-grid-event-hubs-functions-synapse-analytics/resources-in-resource-group.png" alt-text="Screenshot showing resources in the resource group." lightbox="media/event-grid-event-hubs-functions-synapse-analytics/resources-in-resource-group.png"::: + +### Create a table in Azure Synapse Analytics +In this section, you create a table in the dedicated SQL pool you created earlier. + +1. In the list of resources in the resource group, select your **dedicated SQL pool**. +2. On the **Dedicated SQL pool** page, in the **Common Tasks** section on the left menu, select **Query editor (preview)**. + + :::image type="content" source="media/event-grid-event-hubs-functions-synapse-analytics/sql-data-warehouse-page.png" alt-text="Screenshot showing the selection of Query Editor on a Dedicated SQL pool page in the Azure portal."::: +2. Enter the name of **user** and **password** for the SQL server, and select **OK**. If you see a message about allowing your client to access the SQL server, follow these steps: + 1. Select the link: **Set server firewall**. + 2. On the **Firewall settings** page, select **Add client IP** on the toolbar, and then select **Save** on the toolbar. + 3. Select **OK** on the success message. + 4. Navigate back to the **Dedicated SQL pool** page, and select **Query editor (preview)** on the left menu. + 5. Enter **user** and **password**, and then select **OK**. +1. In the query window, copy and run the following SQL script: + + ```sql + CREATE TABLE [dbo].[Fact_WindTurbineMetrics] ( + [DeviceId] nvarchar(50) COLLATE SQL_Latin1_General_CP1_CI_AS NULL, + [MeasureTime] datetime NULL, + [GeneratedPower] float NULL, + [WindSpeed] float NULL, + [TurbineSpeed] float NULL + ) + WITH (CLUSTERED COLUMNSTORE INDEX, DISTRIBUTION = ROUND_ROBIN); + ``` + + :::image type="content" source="media/event-grid-event-hubs-functions-synapse-analytics/run-sql-query.png" alt-text="Screenshot showing the query editor."::: +5. Keep this tab or window open so that you can verify that the data is created at the end of the tutorial. + +## Publish the Azure Functions app +First, get the publish profile for the Functions app from the Azure portal. Then, use the publish profile to publish the Azure Functions project or app from Visual Studio. + +### Get the publish profile + +1. On the **Resource Group** page, select the **Azure Functions app** in the list of resources. + + :::image type="content" source="media/event-grid-event-hubs-functions-synapse-analytics/select-function-app.png" alt-text="Screenshot showing the selection of the function app in the list of resources for a resource group."::: +1. On the **Function App** page for your app, select **Get publish profile** on the command bar. + + :::image type="content" source="media/event-grid-event-hubs-functions-synapse-analytics/get-publish-profile.png" alt-text="Screenshot showing the selection of the **Get Publish Profile** button on the command bar of the function app page."::: +1. Download and save the file into the **FunctionEGDDumper** subfolder of the **EventHubsCaptureEventGridDemo** folder. + +### Use the publish profile to publish the Functions app + +1. Launch Visual Studio. +2. Open the **EventHubsCaptureEventGridDemo.sln** solution that you downloaded from the [GitHub](https://github.com/Azure/azure-event-hubs/tree/master/samples/e2e/EventHubsCaptureEventGridDemo) as part of the prerequisites. You can find it in the `/samples/e2e/EventHubsCaptureEventGridDemo` folder. +3. In Solution Explorer, right-click **FunctionEGDWDumper** project, and select **Publish**. +4. In the following screen, select **Start** or **Add a publish profile**. +5. In the **Publish** dialog box, select **Import Profile** for **Target**, and select **Next**. + + :::image type="content" source="media/event-grid-event-hubs-functions-synapse-analytics/import-profile.png" alt-text="Screenshot showing the selection **Import Profile** on the **Publish** dialog box."::: +1. On the **Import profile** tab, select the publish settings file that you saved earlier in the **FunctionEGDWDumper** folder, and then select **Finish**. +1. When Visual Studio has configured the profile, select **Publish**. Confirm that the publishing succeeded. +2. In the web browser that has the **Azure Function** page open, select **Functions** on the left menu. Confirm that the **EventGridTriggerMigrateData** function shows up in the list. If you don't see it, try publishing from Visual Studio again, and then refresh the page in the portal. + + :::image type="content" source="media/event-grid-event-hubs-functions-synapse-analytics/confirm-function-creation.png" alt-text="Screenshot showing the confirmation of function creation."::: + +After publishing the function, you're ready to subscribe to the event. + +## Subscribe to the event + +1. In a new tab or new window of a web browser, navigate to the [Azure portal](https://portal.azure.com). +2. In the Azure portal, select **Resource groups** on the left menu. +3. Filter the list of resource groups by entering the name of your resource group in the search box. +4. Select your resource group in the list. +1. Select the **Event Hubs namespace** from the list of resources. +1. On the **Event Hubs Namespace** page, select **Events** on the left menu, and then select **+ Event Subscription** on the toolbar. + + :::image type="content" source="media/event-grid-event-hubs-functions-synapse-analytics/event-hub-add-subscription-link.png" alt-text="Add event subscription link on the Events page for an Event Hubs namespace"::: +1. On the **Create Event Subscription** page, follow these steps: + 1. Enter a name for the **event subscription**. + 1. Enter a name for the **system topic**. A system topic provides an endpoint for the sender to send events. For more information, see [System topics](../system-topics.md) + 1. For **Endpoint Type**, select **Azure Function**. + 1. For **Endpoint**, select the link. + 1. On the **Select Azure Function** page, follow these steps if they aren't automatically filled. + 1. Select the Azure subscription that has the Azure function. + 1. Select the resource group for the function. + 1. Select the function app. + 1. Select the deployment slot. + 1. Select the function **EventGridTriggerMigrateData**. + 1. On the **Select Azure Function** page, select **Confirm Selection**. + 1. Then, back on the **Create Event Subscription** page, select **Create**. + + :::image type="content" source="media/event-grid-event-hubs-functions-synapse-analytics/event-subscription-select-function.png" alt-text="Create an event subscription using the function" lightbox="media/event-grid-event-hubs-functions-synapse-analytics/event-subscription-select-function.png"::: +1. Verify that the event subscription is created. Switch to the **Event Subscriptions** tab on the **Events** page for the Event Hubs namespace. + + :::image type="content" source="media/event-grid-event-hubs-functions-synapse-analytics/confirm-event-subscription.png" alt-text="Confirm event subscription" lightbox="media/event-grid-event-hubs-functions-synapse-analytics/confirm-event-subscription.png"::: +1. Select the App Service plan (not the App Service) in the list of resources in the resource group. + +## Run the app to generate data +You've finished setting up your event hub, dedicate SQL pool (formerly SQL Data Warehouse), Azure function app, and event subscription. Before running an application that generates data for event hub, you need to configure a few values. + +1. In the Azure portal, navigate to your resource group as you did earlier. +2. Select the Event Hubs namespace. +3. In the **Event Hubs Namespace** page, select **Shared access policies** on the left menu. +4. Select **RootManageSharedAccessKey** in the list of policies. + + :::image type="content" source="media/event-grid-event-hubs-functions-synapse-analytics/event-hub-namespace-shared-access-policies.png" alt-text="Shared access policies page for an Event Hubs namespace"::: +1. Select the copy button next to the **Connection string-primary key** text box. +1. Go back to your Visual Studio solution. +1. Right-click **WindTurbineDataGenerator** project, and select **Set as Startup project**. +1. In the WindTurbineDataGenerator project, open **program.cs**. +1. Replace `` with the connection string you copied from the portal. +1. If you've used a different name for the event hub other than `hubdatamigration`, replace `` with the name of the event hub. + + ```cs + private const string EventHubConnectionString = "Endpoint=sb://demomigrationnamespace.servicebus.windows.net/..."; + private const string EventHubName = "hubdatamigration"; + ``` +6. Build the solution. Run the **WindTurbineGenerator.exe** application. +7. After a couple of minutes, in the other browser tab where you have the query window open, query the table in your data warehouse for the migrated data. + + ```sql + select * from [dbo].[Fact_WindTurbineMetrics] + ``` + + ![Query results](media/event-grid-event-hubs-functions-synapse-analytics/query-results.png) + +## Monitor the solution +This section helps you with monitoring or troubleshooting the solution. + +### View captured data in the storage account +1. Navigate to the resource group and select the storage account used for capturing event data. +1. On the **Storage account** page, select **Storage Explorer (preview**) on the left menu. +1. Expand **BLOB CONTAINERS**, and select **windturbinecapture**. +1. Open the folder named same as your **Event Hubs namespace** in the right pane. +1. Open the folder named same as your event hub (**hubdatamigration**). +1. Drill through the folders and you see the AVRO files. Here's an example: + + :::image type="content" source="media/event-grid-event-hubs-functions-synapse-analytics/storage-captured-file.png" alt-text="Captured file in the storage" lightbox="media/event-grid-event-hubs-functions-synapse-analytics/storage-captured-file.png"::: + + +### Verify that the Event Grid trigger invoked the function +1. Navigate to the resource group and select the function app. +1. Select **Functions** on the left menu. +1. Select the **EventGridTriggerMigrateData** function from the list. +1. On the **Function** page, select **Monitor** on the left menu. +1. Select **Configure** to configure application insights to capture invocation logs. +1. Create a new **Application Insights** resource or use an existing resource. +1. Navigate back to the **Monitor** page for the function. +1. Confirm that the client application (**WindTurbineDataGenerator**) that's sending the events is still running. If not, run the app. +1. Wait for a few minutes (5 minutes or more) and select the **Refresh** button to see function invocations. + + :::image type="content" source="media/event-grid-event-hubs-functions-synapse-analytics/function-invocations.png" alt-text="Function invocations"::: +1. Select an invocation to see details. + + Event Grid distributes event data to the subscribers. The following example shows event data generated when data streaming through an event hub is captured in a blob. In particular, notice the `fileUrl` property in the `data` object points to the blob in the storage. The function app uses this URL to retrieve the blob file with captured data. + + ```json + { + "topic": "/subscriptions//resourcegroups/rgDataMigration/providers/Microsoft.EventHub/namespaces/spehubns1207", + "subject": "hubdatamigration", + "eventType": "Microsoft.EventHub.CaptureFileCreated", + "id": "4538f1a5-02d8-4b40-9f20-36301ac976ba", + "data": { + "fileUrl": "https://spehubstorage1207.blob.core.windows.net/windturbinecapture/spehubns1207/hubdatamigration/0/2020/12/07/21/49/12.avro", + "fileType": "AzureBlockBlob", + "partitionId": "0", + "sizeInBytes": 473444, + "eventCount": 2800, + "firstSequenceNumber": 55500, + "lastSequenceNumber": 58299, + "firstEnqueueTime": "2020-12-07T21:49:12.556Z", + "lastEnqueueTime": "2020-12-07T21:50:11.534Z" + }, + "dataVersion": "1", + "metadataVersion": "1", + "eventTime": "2020-12-07T21:50:12.7065524Z" + } + ``` + +### Verify that the data is stored in the dedicated SQL pool +In the browser tab where you have the query window open, query the table in your dedicated SQL pool for the migrated data. + +![Query results](media/event-grid-event-hubs-functions-synapse-analytics/query-results.png) + + +## Next steps +You can use powerful data visualization tools with your data warehouse to achieve actionable insights. + +This article shows how to use [Power BI with Azure Synapse Analytics](/power-bi/connect-data/service-azure-sql-data-warehouse-with-direct-connect) diff --git a/samples/e2e/EventHubsCaptureEventGridDemo/scripts/CreateDataWarehouseTable.sql b/samples/e2e/EventHubsCaptureEventGridDemo/scripts/CreateDataWarehouseTable.sql deleted file mode 100644 index 163f9eea3..000000000 --- a/samples/e2e/EventHubsCaptureEventGridDemo/scripts/CreateDataWarehouseTable.sql +++ /dev/null @@ -1,8 +0,0 @@ -CREATE TABLE [dbo].[Fact_WindTurbineMetrics] ( - [DeviceId] nvarchar(50) COLLATE SQL_Latin1_General_CP1_CI_AS NULL, - [MeasureTime] datetime NULL, - [GeneratedPower] float NULL, - [WindSpeed] float NULL, - [TurbineSpeed] float NULL -) -WITH (CLUSTERED COLUMNSTORE INDEX, DISTRIBUTION = ROUND_ROBIN); \ No newline at end of file