Merge pull request #2095 from microsoft/dpaul-HcDiag

Add back pressure detection in Health Checker

Diagnostics/HealthChecker/Analyzer/Invoke-AnalyzerExchangeInformation.ps1

@@ -466,6 +466,34 @@ function Invoke-AnalyzerExchangeInformation {
         }
     }
 
+    if ($null -ne $exchangeInformation.EdgeTransportResourceThrottling) {
+        try {
+            # SystemMemory does not block mail flow.
+            $resourceThrottling = ([xml]$exchangeInformation.EdgeTransportResourceThrottling).Diagnostics.Components.ResourceThrottling.ResourceTracker.ResourceMeter |
+                Where-Object { $_.Resource -ne "SystemMemory" -and $_.CurrentResourceUse -ne "Low" }
+        } catch {
+            Invoke-CatchActions
+        }
+
+        if ($null -ne $resourceThrottling) {
+            $resourceThrottlingList = @($resourceThrottling.Resource |
+                    ForEach-Object {
+                        $index = $_.IndexOf("[")
+                        if ($index -eq -1) {
+                            $_
+                        } else {
+                            $_.Substring(0, $index)
+                        }
+                    })
+            $params = $baseParams + @{
+                Name             = "Transport Back Pressure"
+                Details          = "--ERROR-- The following resources are causing back pressure: $([string]::Join(", ", $resourceThrottlingList))"
+                DisplayWriteType = "Red"
+            }
+            Add-AnalyzedResultInformation @params
+        }
+    }
+
     Write-Verbose "Working on Exchange Server Maintenance"
     $serverMaintenance = $exchangeInformation.ServerMaintenance
     $getMailboxServer = $exchangeInformation.GetMailboxServer

Diagnostics/HealthChecker/DataCollection/ExchangeInformation/Get-ExchangeInformation.ps1

@@ -4,6 +4,7 @@
 . $PSScriptRoot\..\..\..\..\Shared\Get-ExtendedProtectionConfiguration.ps1
 . $PSScriptRoot\..\..\..\..\Shared\ErrorMonitorFunctions.ps1
 . $PSScriptRoot\..\..\..\..\Shared\Get-ExchangeBuildVersionInformation.ps1
+. $PSScriptRoot\..\..\..\..\Shared\Get-ExchangeDiagnosticInformation.ps1
 . $PSScriptRoot\..\..\..\..\Shared\Get-ExchangeSettingOverride.ps1
 . $PSScriptRoot\..\..\..\..\Shared\Get-ExSetupFileVersionInfo.ps1
 . $PSScriptRoot\..\..\..\..\Shared\Get-FileContentInformation.ps1
@@ -186,6 +187,14 @@ function Get-ExchangeInformation {
             VersionInformation = $versionInformation
         }
         $aes256CbcDetails = Get-ExchangeAES256CBCDetails @aes256CbcParams
+
+        Write-Verbose "Getting Exchange Diagnostic Information"
+        $params = @{
+            Server    = $Server
+            Process   = "EdgeTransport"
+            Component = "ResourceThrottling"
+        }
+        $edgeTransportResourceThrottling = Get-ExchangeDiagnosticInformation @params
     } end {
 
         Write-Verbose "Exiting: Get-ExchangeInformation"
@@ -202,6 +211,7 @@ function Get-ExchangeInformation {
             ServerMaintenance                        = $serverMaintenance
             ExchangeCertificates                     = [array]$exchangeCertificates
             ExchangeEmergencyMitigationServiceResult = $eemsEndpointResults
+            EdgeTransportResourceThrottling          = $edgeTransportResourceThrottling # If we want to checkout other diagnosticInfo, we should create a new object here.
             ApplicationConfigFileStatus              = $applicationConfigFileStatus
             DependentServices                        = $dependentServices
             IISSettings                              = $iisSettings

Diagnostics/HealthChecker/Tests/HealthChecker.E19.Scenarios.Tests.ps1

@@ -44,7 +44,10 @@ Describe "Testing Health Checker by Mock Data Imports" {
             Mock Get-WebConfigFile -ParameterFilter { $PSPath -eq "IIS:\Sites\Exchange Back End/ecp" } -MockWith { return [PSCustomObject]@{ FullName = "$Script:MockDataCollectionRoot\Exchange\IIS\ClientAccess\ecp\web.config" } }
             Mock Get-WebConfigFile -ParameterFilter { $PSPath -eq "IIS:\Sites\Default Web Site/ecp" } -MockWith { return [PSCustomObject]@{ FullName = "$Script:MockDataCollectionRoot\Exchange\IIS\DefaultWebSite_web.config" } }
             Mock Invoke-ScriptBlockHandler -ParameterFilter { $ScriptBlockDescription -eq "Getting applicationHost.config" } -MockWith { return Get-Content "$Script:MockDataCollectionRoot\Exchange\IIS\applicationHost1.config" -Raw -Encoding UTF8 }
-            Mock Get-ExchangeDiagnosticInfo { return Import-Clixml "$Script:MockDataCollectionRoot\Exchange\GetExchangeDiagnosticInfo1.xml" }
+            Mock Get-ExchangeDiagnosticInfo -ParameterFilter { $Process -eq "Microsoft.Exchange.Directory.TopologyService" -and $Component -eq "VariantConfiguration" -and $Argument -eq "Overrides" } `
+                -MockWith { return Import-Clixml "$Script:MockDataCollectionRoot\Exchange\GetExchangeDiagnosticInfo_ADTopVariantConfiguration1.xml" }
+            Mock Get-ExchangeDiagnosticInfo -ParameterFilter { $Process -eq "EdgeTransport" -and $Component -eq "ResourceThrottling" } `
+                -MockWith { return Import-Clixml "$Script:MockDataCollectionRoot\Exchange\GetExchangeDiagnosticInfo_EdgeTransportResourceThrottling1.xml" }
             Mock Get-IISModules { return Import-Clixml "$Script:MockDataCollectionRoot\Exchange\GetIISModulesNoTokenCacheModule.xml" }
             Mock Get-Service {
                 param(
@@ -62,6 +65,7 @@ Describe "Testing Health Checker by Mock Data Imports" {
             SetActiveDisplayGrouping "Exchange Information"
             TestObjectMatch "Setting Overrides Detected" $true
             TestObjectMatch "Extended Protection Enabled (Any VDir)" $true
+            TestObjectMatch "Transport Back Pressure" "--ERROR-- The following resources are causing back pressure: DatabaseUsedSpace" -WriteType "Red"
         }
 
         It "Dependent Services" {
@@ -211,7 +215,8 @@ Describe "Testing Health Checker by Mock Data Imports" {
     Context "Checking Scenarios 2" {
         BeforeAll {
             Mock Get-SettingOverride { return Import-Clixml "$Script:MockDataCollectionRoot\Exchange\GetSettingOverride1.xml" }
-            Mock Get-ExchangeDiagnosticInfo { return Import-Clixml "$Script:MockDataCollectionRoot\Exchange\GetExchangeDiagnosticInfo1.xml" }
+            Mock Get-ExchangeDiagnosticInfo -ParameterFilter { $Process -eq "Microsoft.Exchange.Directory.TopologyService" -and $Component -eq "VariantConfiguration" -and $Argument -eq "Overrides" } `
+                -MockWith { return Import-Clixml "$Script:MockDataCollectionRoot\Exchange\GetExchangeDiagnosticInfo_ADTopVariantConfiguration1.xml" }
             Mock Get-RemoteRegistryValue -ParameterFilter { $GetValue -eq "KeepAliveTime" } -MockWith { return 1800000 }
             Mock Get-RemoteRegistryValue -ParameterFilter { $GetValue -eq "DisableGranularReplication" } -MockWith { return 1 }
             Mock Get-RemoteRegistryValue -ParameterFilter { $GetValue -eq "DisableAsyncNotification" } -MockWith { return 1 }

Diagnostics/HealthChecker/Tests/HealthChecker.MockedCalls.Tests.ps1

@@ -31,7 +31,8 @@ Describe "Testing Health Checker by Mock Data Imports" {
             Mock Get-OrganizationConfig { return Import-Clixml "$Script:MockDataCollectionRoot\Exchange\GetOrganizationConfig.xml" }
             Mock Get-InternalTransportCertificateFromServer { return Import-Clixml "$Script:MockDataCollectionRoot\Exchange\GetInternalTransportCertificateFromServer.xml" }
             Mock Get-HybridConfiguration { return $null }
-            Mock Get-ExchangeDiagnosticInfo { return Import-Clixml "$Script:MockDataCollectionRoot\Exchange\GetExchangeDiagnosticInfo.xml" }
+            Mock Get-ExchangeDiagnosticInfo -ParameterFilter { $Process -eq "Microsoft.Exchange.Directory.TopologyService" -and $Component -eq "VariantConfiguration" -and $Argument -eq "Overrides" } `
+                -MockWith { return Import-Clixml "$Script:MockDataCollectionRoot\Exchange\GetExchangeDiagnosticInfo_ADTopVariantConfiguration.xml" }
             # do not need to match the function. Only needed really to test the Assert-MockCalled
             Mock Get-Service { return Import-Clixml "$Script:MockDataCollectionRoot\Exchange\GetServiceMitigation.xml" }
             Mock Get-SettingOverride { return Import-Clixml "$Script:MockDataCollectionRoot\Exchange\GetSettingOverride.xml" }
@@ -94,7 +95,7 @@ Describe "Testing Health Checker by Mock Data Imports" {
             Assert-MockCalled Get-ReceiveConnector -Exactly 1
             Assert-MockCalled Get-SendConnector -Exactly 1
             Assert-MockCalled Get-IISModules -Exactly 1
-            Assert-MockCalled Get-ExchangeDiagnosticInfo -Exactly 1
+            Assert-MockCalled Get-ExchangeDiagnosticInfo -Exactly 2
             Assert-MockCalled Get-ExchangeADSplitPermissionsEnabled -Exactly 1
             Assert-MockCalled Get-DynamicDistributionGroup -Exactly 1
             Assert-MockCalled Get-ActiveSyncVirtualDirectory -Exactly 1

Diagnostics/HealthChecker/Tests/HealthCheckerTest.CommonMocks.NotPublished.ps1

@@ -229,6 +229,13 @@ Mock Get-ExSetupFileVersionInfo {
 # Do nothing
 Mock Invoke-CatchActions { }
 
+function Get-ExchangeDiagnosticInfo { param($Argument, $Component, $Process, $Server) }
+
+Mock Get-ExchangeDiagnosticInfo -ParameterFilter { $Process -eq "Microsoft.Exchange.Directory.TopologyService" -and $Component -eq "VariantConfiguration" -and $Argument -eq "Overrides" } `
+    -MockWith { return Import-Clixml "$Script:MockDataCollectionRoot\Exchange\GetExchangeDiagnosticInfo_ADTopVariantConfiguration.xml" }
+Mock Get-ExchangeDiagnosticInfo -ParameterFilter { $Process -eq "EdgeTransport" -and $Component -eq "ResourceThrottling" } `
+    -MockWith { return Import-Clixml "$Script:MockDataCollectionRoot\Exchange\GetExchangeDiagnosticInfo_EdgeTransportResourceThrottling.xml" }
+
 # Need to use function instead of Mock for Exchange cmdlets
 function Get-ExchangeServer {
     return Import-Clixml "$Script:MockDataCollectionRoot\Exchange\GetExchangeServer.xml"
@@ -322,7 +329,3 @@ function Get-ExchangeProtocolContainer {
 function Get-ExchangeWebSitesFromAd {
     return Import-Clixml "$Script:MockDataCollectionRoot\Exchange\GetExchangeWebSitesFromAd.xml"
 }
-
-function Get-ExchangeDiagnosticInfo {
-    return Import-Clixml "$Script:MockDataCollectionRoot\Exchange\GetExchangeDiagnosticInfo.xml"
-}

Shared/Get-ExchangeDiagnosticInformation.ps1

@@ -0,0 +1,42 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+. $PSScriptRoot\Invoke-CatchActionError.ps1
+function Get-ExchangeDiagnosticInformation {
+    [CmdletBinding()]
+    param(
+        [Parameter(Mandatory = $true)]
+        [string]$Server,
+
+        [Parameter(Mandatory = $true)]
+        [string]$Process,
+
+        [Parameter(Mandatory = $true)]
+        [string]$Component,
+
+        [string]$Argument,
+
+        [Parameter(Mandatory = $false)]
+        [ScriptBlock]$CatchActionFunction
+    )
+    process {
+        try {
+            Write-Verbose "Calling: $($MyInvocation.MyCommand)"
+            $params = @{
+                Process     = $Process
+                Component   = $Component
+                Server      = $Server
+                ErrorAction = "Stop"
+            }
+
+            if (-not ([string]::IsNullOrEmpty($Argument))) {
+                $params.Add("Argument", $Argument)
+            }
+
+            return (Get-ExchangeDiagnosticInfo @params)
+        } catch {
+            Write-Verbose "Failed to execute $($MyInvocation.MyCommand). Inner Exception: $_"
+            Invoke-CatchActionError $CatchActionFunction
+        }
+    }
+}

Shared/Get-ExchangeSettingOverride.ps1

@@ -1,6 +1,7 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
+. $PSScriptRoot\Get-ExchangeDiagnosticInformation.ps1
 . $PSScriptRoot\Invoke-CatchActionError.ps1
 
 function Get-ExchangeSettingOverride {
@@ -21,33 +22,38 @@ function Get-ExchangeSettingOverride {
     process {
         try {
             $params = @{
-                Process     = "Microsoft.Exchange.Directory.TopologyService"
-                Component   = "VariantConfiguration"
-                Argument    = "Overrides"
-                Server      = $Server
-                ErrorAction = "Stop"
+                Process             = "Microsoft.Exchange.Directory.TopologyService"
+                Component           = "VariantConfiguration"
+                Argument            = "Overrides"
+                Server              = $Server
+                CatchActionFunction = $CatchActionFunction
             }
-            $diagnosticInfo = Get-ExchangeDiagnosticInfo @params
-            Write-Verbose "Successfully got the Exchange Diagnostic Information"
-            $xml = [xml]$diagnosticInfo.Result
-            $overrides = $xml.Diagnostics.Components.VariantConfiguration.Overrides
-            $updatedTime = $overrides.Updated
-            $settingOverrides = $overrides.SettingOverride
+            $diagnosticInfo = Get-ExchangeDiagnosticInformation @params
 
-            foreach ($override in $settingOverrides) {
-                Write-Verbose "Working on $($override.Name)"
-                $simpleSettingOverrides.Add([PSCustomObject]@{
-                        Name          = $override.Name
-                        ModifiedBy    = $override.ModifiedBy
-                        Reason        = $override.Reason
-                        ComponentName = $override.ComponentName
-                        SectionName   = $override.SectionName
-                        Status        = $override.Status
-                        Parameters    = $override.Parameters.Parameter
-                    })
+            if ($null -ne $diagnosticInfo) {
+                Write-Verbose "Successfully got the Exchange Diagnostic Information"
+                $xml = [xml]$diagnosticInfo.Result
+                $overrides = $xml.Diagnostics.Components.VariantConfiguration.Overrides
+                $updatedTime = $overrides.Updated
+                $settingOverrides = $overrides.SettingOverride
+
+                foreach ($override in $settingOverrides) {
+                    Write-Verbose "Working on $($override.Name)"
+                    $simpleSettingOverrides.Add([PSCustomObject]@{
+                            Name          = $override.Name
+                            ModifiedBy    = $override.ModifiedBy
+                            Reason        = $override.Reason
+                            ComponentName = $override.ComponentName
+                            SectionName   = $override.SectionName
+                            Status        = $override.Status
+                            Parameters    = $override.Parameters.Parameter
+                        })
+                }
+            } else {
+                Write-Verbose "Failed to get Exchange Diagnostic Information"
             }
         } catch {
-            Write-Verbose "Failed to get the Exchange setting override"
+            Write-Verbose "Failed to get the Exchange setting override. Inner Exception: $_"
             Invoke-CatchActionError $CatchActionFunction
         }
     }