-
Notifications
You must be signed in to change notification settings - Fork 0
/
TroubleshootError.ps1
347 lines (310 loc) · 13.9 KB
/
TroubleshootError.ps1
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
#
# ClassifyError.ps1
#
<#
.DESCRIPTION
Classifies the error type that a user is facing with their AKS cluster
.PARAMETER SubscriptionId
Subscription Id that the AKS cluster is in
.PARAMETER ResourceGroupName
Resource Group name where the AKS cluster is in
.PARAMETER AKSClusterName
AKS Cluster name
#>
param(
[Parameter(mandatory=$true)]
[string]$SubscriptionId,
[Parameter(mandatory=$true)]
[string]$ResourceGroupName,
[Parameter(mandatory=$true)]
[string]$AKSClusterName
)
$ErrorActionPreference = "Stop";
Start-Transcript -path .\TroubleshootDump.txt -Force
$DocumentationLink = "https://github.com/Microsoft/OMS-docker/blob/troubleshooting_doc/Troubleshoot/README.md"
$OptOutLink = "https://docs.microsoft.com/en-us/azure/monitoring/monitoring-container-health#how-to-stop-monitoring-with-container-health"
$OptInLink = "https://docs.microsoft.com/en-us/azure/monitoring/monitoring-container-health#enable-container-health-monitoring-for-a-new-cluster"
$message = "This script will try to install the latest versions of the following Modules : `
AzureRM.Resources, AzureRM.OperationalInsights and AzureRM.profile using the command`
`'Install-Module {Insert Module Name} -Repository PSGallery -Force -AllowClobber -ErrorAction Stop -WarningAction Stop'
`It might not be able to troubleshoot if you don't have the latest version of these Modules"
$question = "Do you want to Install the modules and run the script or just run the script?"
$choices = New-Object Collections.ObjectModel.Collection[Management.Automation.Host.ChoiceDescription]
$choices.Add((New-Object Management.Automation.Host.ChoiceDescription -ArgumentList '&Yes, Install and run'))
$choices.Add((New-Object Management.Automation.Host.ChoiceDescription -ArgumentList '&Continue but *donot* install'))
$choices.Add((New-Object Management.Automation.Host.ChoiceDescription -ArgumentList '&Quit'))
$decision = $Host.UI.PromptForChoice($message, $question, $choices, 0)
switch ($decision) {
0 {
try {
Write-Host("Installing AzureRM.profile...")
Install-Module AzureRM.profile -Repository PSGallery -Force -AllowClobber -ErrorAction Stop
} catch {
Write-Host("Close other powershell logins and try installing the latest modules for AzureRM.profile in a new powershell window: eg. 'Install-Module AzureRM.profile -Repository PSGallery -Force'") -ForegroundColor Red
exit
}
try {
Write-Host("Installing AzureRM.Resources...")
Install-Module AzureRM.Resources -Repository PSGallery -Force -AllowClobber -ErrorAction Stop
} catch {
Write-Host("Close other powershell logins and try installing the latest modules for AzureRM.Resoureces in a new powershell window: eg. 'Install-Module AzureRM.Resoureces -Repository PSGallery -Force'") -ForegroundColor Red
exit
}
try {
Write-Host("Installing AzureRM.OperationalInsights...")
Install-Module AzureRM.OperationalInsights -Repository PSGallery -Force -AllowClobber -ErrorAction Stop
} catch {
Write-Host("Close other powershell logins and try installing the latest modules for AzureRM.OperationalInsights in a new powershell window: eg. 'Install-Module AzureRM.OperationalInsights -Repository PSGallery -Force'") -ForegroundColor Red
exit
}
}
1 {
try {
Import-Module AzureRM.profile -ErrorAction Stop
} catch {
Write-Host("Couldnot import AzureRM.profile...") -ForegroundColor Red
}
try {
Import-Module AzureRM.Resources
} catch {
Write-Host("Couldnot import AzureRM.Resources...") -ForegroundColor Red
}
try {
Import-Module AzureRM.OperationalInsights
} catch {
Write-Host("Couldnot import AzureRM.OperationalInsights...") -ForegroundColor Red
}
Write-Host("Running troubleshooting script...")
Write-Host("")
}
2 {
Write-Host("")
Stop-Transcript
exit
}
}
try {
Write-Host("")
Write-Host("Trying to get current AzureRM login context...")
$account = Get-AzureRmContext -ErrorAction Stop
Write-Host("Successfully fetched current AzureRM context...") -ForegroundColor Green
Write-Host("")
} catch {
Write-Host("")
Write-Host("Could not fetch AzureRMContext..." ) -ForegroundColor Red
Write-Host("")
}
#
# Subscription existance and access check
#
if ($account.Account -eq $null) {
try {
Write-Host("Please login...")
Login-AzureRmAccount -subscriptionid $SubscriptionId
} catch {
Write-Host("")
Write-Host("Couldnot select subscription with ID : " + $SubscriptionId + ". Please make sure the ID you entered is correct and you have access to the cluster" ) -ForegroundColor Red
Write-Host("")
Stop-Transcript
exit
}
} else {
if ($account.Subscription.Id -eq $SubscriptionId) {
Write-Host("Subscription: $SubscriptionId is already selected. Account details: ")
$account
} else {
try {
Write-Host("Current Subscription:")
$account
Write-Host("Changing to subscription: $SubscriptionId")
Select-AzureRmSubscription -SubscriptionId $SubscriptionId
} catch {
Write-Host("")
Write-Host("Couldnot select subscription with ID : " + $SubscriptionId + ". Please make sure the ID you entered is correct and you have access to the cluster" ) -ForegroundColor Red
Write-Host("")
Stop-Transcript
exit
}
}
}
#
# Resource group existance and access check
#
Write-Host("Checking resource group details...")
Get-AzureRmResourceGroup -Name $ResourceGroupName -ErrorVariable notPresent -ErrorAction SilentlyContinue
if ($notPresent) {
Write-Host("")
Write-Host("Couldnot find RG. Please make sure that the resource group name: '" + $ResourceGroupName + "'is correct and you have access to the cluster") -ForegroundColor Red
Write-Host("")
Stop-Transcript
exit
}
Write-Host("Successfully checked resourece groups details...") -ForegroundColor Green
Write-Host("Checking AKS Cluster details details...")
try {
$ResourceDetailsArray = Get-AzureRmResource -ResourceGroupName $ResourceGroupName -Name $AKSClusterName -ExpandProperties -ErrorAction Stop -WarningAction Stop
} catch {
Write-Host("")
Write-Host("Could not fetch cluster details: Please make sure that the AKS Cluster name: '" + $AKSClusterName + "' is correct and you have access to the cluster") -ForegroundColor Red
Write-Host("")
Stop-Transcript
exit
}
if ($ResourceDetailsArray -eq $null) {
Write-Host("")
Write-Host("Could not fetch cluster details: Please make sure that the AKS Cluster name: '" + $AKSClusterName + "' is correct and you have access to the cluster") -ForegroundColor Red
Write-Host("")
Stop-Transcript
exit
} else {
Write-Host("Successfully checked AKS Cluster details details...") -ForegroundColor Green
Write-Host("")
foreach ($ResourceDetail in $ResourceDetailsArray) {
if ($ResourceDetail.ResourceType -eq "Microsoft.ContainerService/managedClusters") {
$LogAnalyticsWorkspaceResourceID = $ResourceDetail.Properties.addonProfiles.omsagent.config.logAnalyticsWorkspaceResourceID
break
}
}
}
if ($LogAnalyticsWorkspaceResourceID -eq $null) {
Write-Host("")
Write-Host("No log analytics workspace associated with the cluster: Please see how to onboard your cluster to Container health from the following documentation: " + $OptInLink) -ForegroundColor Red
Write-Host("")
Stop-Transcript
exit
} else {
try {
if($SubscriptionId -eq $LogAnalyticsWorkspaceResourceID.split("/")[2]) {
#Nothing to do here
} else {
Write-Host("Changing to workspace's subscription")
Select-AzureRmSubscription -SubscriptionId $LogAnalyticsWorkspaceResourceID.split("/")[2]
}
} catch {
Write-Host("")
Write-Host("Couldnot select subscription with ID : " + $SubscriptionId + ". Please make sure the ID you entered is correct and you have access to this workspace" ) -ForegroundColor Red
Write-Host("")
Stop-Transcript
exit
}
#
# Check WS subscription exists and access
#
try {
Write-Host("Checking workspace subscription details...")
Get-AzureRmSubscription -SubscriptionId $LogAnalyticsWorkspaceResourceID.split("/")[2] -ErrorAction Stop
} catch {
Write-Host("")
Write-Host("The subscription containing the workspace (" + $LogAnalyticsWorkspaceResourceID +") looks like it was deleted or you do NOT have access to this workspace") -ForegroundColor Red
Write-Host("Please try to opt out of monitoring and opt-in using the following links:") -ForegroundColor Red
Write-Host("Opt-out - " + $OptOutLink) -ForegroundColor Red
Write-Host("Opt-in - " + $OptInLink) -ForegroundColor Red
Write-Host("")
Stop-Transcript
exit
}
Write-Host("Successfully fetched workspace subcription details...") -ForegroundColor Green
Write-Host("")
#
# Check WS Resourecegroup exists and access
#
Write-Host("Checking workspace's resource group details...")
Get-AzureRmResourceGroup -Name $LogAnalyticsWorkspaceResourceID.split("/")[4] -ErrorVariable notPresent -ErrorAction SilentlyContinue
if ($notPresent) {
Write-Host("")
Write-Host("Couldnot find RG. Please make sure that the resource group name: '" + $ResourceGroupName + "'is correct and you have access to the workspace") -ForegroundColor Red
Write-Host("Please try to opt out of monitoring and opt-in using the following links:") -ForegroundColor Red
Write-Host("Opt-out - " + $OptOutLink) -ForegroundColor Red
Write-Host("Opt-in - " + $OptInLink) -ForegroundColor Red
Stop-Transcript
exit
}
Write-Host("Successfully fetched workspace resource group...") -ForegroundColor Green
Write-Host("")
#
# Check WS exits and access
#
try {
Write-Host("Checking workspace name's details...")
$WorkspaceInformation = Get-AzureRmOperationalInsightsWorkspace -ResourceGroupName $LogAnalyticsWorkspaceResourceID.split("/")[4] -Name $LogAnalyticsWorkspaceResourceID.split("/")[8] -ErrorAction Stop
Write-Host("Successfully fetched workspace name...") -ForegroundColor Green
Write-Host("")
} catch {
Write-Host("")
Write-Host("Could not fetch details for the workspace : '" + $LogAnalyticsWorkspaceResourceID.split("/")[8] + "'. Please make sure that it hasn't been deleted and you have access to it.") -ForegroundColor Red
Write-Host("Please try to opt out of monitoring and opt-in using the following links:") -ForegroundColor Red
Write-Host("Opt-out - " + $OptOutLink) -ForegroundColor Red
Write-Host("Opt-in - " + $OptInLink) -ForegroundColor Red
Write-Host("")
Stop-Transcript
exit
}
$WorkspaceLocation = $WorkspaceInformation.Location
if ($WorkspaceLocation -eq $null) {
Write-Host("")
Write-Host("Cannot fetch workspace location. Please try again...") -ForegroundColor Red
Write-Host("")
Stop-Transcript
exit
}
try {
$WorkspaceIPDetails = Get-AzureRmOperationalInsightsIntelligencePacks -ResourceGroupName $LogAnalyticsWorkspaceResourceID.split("/")[4] -WorkspaceName $LogAnalyticsWorkspaceResourceID.split("/")[8] -ErrorAction Stop
Write-Host("Successfully fetched workspace IP details...") -ForegroundColor Green
Write-Host("")
} catch {
Write-Host("")
Write-Host("Failed to get the list of solutions onboarded to the workspace. Please make sure that it hasn't been deleted and you have access to it.") -ForegroundColor Red
Write-Host("")
Stop-Transcript
exit
}
try {
$ContainerInsightsIndex = $WorkspaceIPDetails.Name.IndexOf("ContainerInsights");
Write-Host("Successfully got ContainerInsights index ") -ForegroundColor Green
Write-Host("")
} catch {
Write-Host("Failed to get ContainerInsights solution details from the workspace") -ForegroundColor Red
Write-Host("")
Stop-Transcript
exit
}
$isSolutionOnboarded = $WorkspaceIPDetails.Enabled[$ContainerInsightsIndex]
if ($isSolutionOnboarded) {
Write-Host("Everything looks good according to this script. Please contact us by emailing [email protected] for help") -ForegroundColor Green
} else {
#
# Check contributor access to WS
#
$message = "Detected that there is a workspace associated with this cluster, but workspace - '" + $LogAnalyticsWorkspaceResourceID.split("/")[8] + "' in subscription '" + $LogAnalyticsWorkspaceResourceID.split("/")[2] + "' IS NOT ONBOARDED with container health solution.";
$question = " Do you want to onboard the solution to the workspace?"
$choices = New-Object Collections.ObjectModel.Collection[Management.Automation.Host.ChoiceDescription]
$choices.Add((New-Object Management.Automation.Host.ChoiceDescription -ArgumentList '&Yes'))
$choices.Add((New-Object Management.Automation.Host.ChoiceDescription -ArgumentList '&No'))
$decision = $Host.UI.PromptForChoice($message, $question, $choices, 0)
if ($decision -eq 0) {
Write-Host("Deploying template to onboard Container Insights : Please wait...")
$CurrentDir = (Get-Item -Path ".\" -Verbose).FullName
$TemplateFile = $CurrentDir + "\ContainerInsightsSolution.json"
$DeploymentName = "ContainerHealthOnboarding-Solution-" + ((Get-Date).ToUniversalTime()).ToString('MMdd-HHmm')
$Parameters = @{}
$Parameters.Add("workspaceResourceId", $LogAnalyticsWorkspaceResourceID)
$Parameters.Add("workspaceRegion", $WorkspaceLocation)
$Parameters
try {
New-AzureRmResourceGroupDeployment -Name $DeploymentName `
-ResourceGroupName $LogAnalyticsWorkspaceResourceID.split("/")[4] `
-TemplateFile $TemplateFile `
-TemplateParameterObject $Parameters -ErrorAction Stop`
Write-Host("")
Write-Host("Template deployment was successful. You will be able to see data flowing into your cluster in 10-15 mins.") -ForegroundColor Green
Write-Host("")
} catch {
Write-Host("Template deployment failed : Please contact us by emailing [email protected] for help") -ForegroundColor Red
}
} else {
Write-Host("The Container Insights solution isn't onboarded to your cluster. Please contact us by emailing [email protected]") -ForegroundColor Red
}
}
}
Write-Host("")
Stop-Transcript