From a2b898c3eb380fbfefc74164ef639109cf38e8f6 Mon Sep 17 00:00:00 2001 From: Artur Khantimirov Date: Fri, 17 Jul 2026 16:29:24 +1200 Subject: [PATCH 1/2] fix: validate Windows node reset task result Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2ba2b88c-27a6-4f9b-9fd5-bf757a9d31c2 --- parts/windows/kuberneteswindowssetup.ps1 | 16 +------ parts/windows/windowscsehelper.ps1 | 40 ++++++++++++++++ parts/windows/windowscsehelper.tests.ps1 | 61 ++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 15 deletions(-) diff --git a/parts/windows/kuberneteswindowssetup.ps1 b/parts/windows/kuberneteswindowssetup.ps1 index 486422eb411..1ef97a72459 100644 --- a/parts/windows/kuberneteswindowssetup.ps1 +++ b/parts/windows/kuberneteswindowssetup.ps1 @@ -651,21 +651,7 @@ function NodePrep { Postpone-RestartComputer } else { Logs-To-Event -TaskName "AKS.WindowsCSE.StartScheduledTask" -TaskMessage "Setup Complete, start NodeResetScriptTask to register Windows node without reboot" - Start-ScheduledTask -TaskName "k8s-restart-job" - - $timeout = 180 ## seconds - $timer = [Diagnostics.Stopwatch]::StartNew() - while ((Get-ScheduledTask -TaskName 'k8s-restart-job').State -ne 'Ready') { - # The task `k8s-restart-job` needs ~8 seconds. - if ($timer.Elapsed.TotalSeconds -gt $timeout) { - Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds" - } - - Write-Log -Message "Waiting on NodeResetScriptTask..." - Start-Sleep -Seconds 3 - } - $timer.Stop() - Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask" + Start-NodeResetScriptTask } Write-Log "NodePrep completed successfully" Logs-To-Event -TaskName "AKS.WindowsCSE.NodePrep" -TaskMessage "NodePrep completed successfully" diff --git a/parts/windows/windowscsehelper.ps1 b/parts/windows/windowscsehelper.ps1 index 2fbac973d0e..449c3f47054 100644 --- a/parts/windows/windowscsehelper.ps1 +++ b/parts/windows/windowscsehelper.ps1 @@ -300,6 +300,46 @@ function Set-ExitCode exit $ExitCode } +function Start-NodeResetScriptTask +{ + Param( + [Parameter(Mandatory=$false)][int] + $TimeoutSeconds = 180 + ) + + $taskName = "k8s-restart-job" + $previousRunTime = (Get-ScheduledTaskInfo -TaskName $taskName).LastRunTime + Start-ScheduledTask -TaskName $taskName + + $timer = [Diagnostics.Stopwatch]::StartNew() + do { + $task = Get-ScheduledTask -TaskName $taskName + $taskInfo = Get-ScheduledTaskInfo -TaskName $taskName + if ($task.State -eq "Ready" -and $taskInfo.LastRunTime -ne $previousRunTime) { + break + } + + if ($timer.Elapsed.TotalSeconds -gt $TimeoutSeconds) { + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask is not finished after [$($timer.Elapsed.TotalSeconds)] seconds" + } + + Write-Log -Message "Waiting on NodeResetScriptTask..." + Start-Sleep -Seconds 3 + } while ($true) + $timer.Stop() + + if ($taskInfo.LastTaskResult -ne 0) { + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask failed with result [$($taskInfo.LastTaskResult)]" + } + + $kubeletService = Get-Service -Name "kubelet" -ErrorAction SilentlyContinue + if ($null -eq $kubeletService -or $kubeletService.Status -ne "Running") { + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "kubelet service is not running after NodeResetScriptTask completed" + } + + Write-Log -Message "We waited [$($timer.Elapsed.TotalSeconds)] seconds on NodeResetScriptTask" +} + function Postpone-RestartComputer { Logs-To-Event -TaskName "AKS.WindowsCSE.PostponeRestartComputer" -TaskMessage "Start to create an one-time task to restart the VM" diff --git a/parts/windows/windowscsehelper.tests.ps1 b/parts/windows/windowscsehelper.tests.ps1 index 6dc486430e0..3867a521352 100644 --- a/parts/windows/windowscsehelper.tests.ps1 +++ b/parts/windows/windowscsehelper.tests.ps1 @@ -589,3 +589,64 @@ Describe "Update-BaseUrl" { $result | Should -Be "https://packages.aks.azure.com/path/to/resource" } } + +Describe "Start-NodeResetScriptTask" { + BeforeEach { + $script:taskInfoCallCount = 0 + Mock Start-ScheduledTask -MockWith {} + Mock Get-ScheduledTask -MockWith { return [pscustomobject]@{ State = "Ready" } } + Mock Get-ScheduledTaskInfo -MockWith { + $script:taskInfoCallCount++ + if ($script:taskInfoCallCount -eq 1) { + return [pscustomobject]@{ LastRunTime = [datetime]"2026-01-01"; LastTaskResult = 0 } + } + return [pscustomobject]@{ LastRunTime = [datetime]"2026-01-02"; LastTaskResult = 0 } + } + Mock Get-Service -MockWith { return [pscustomobject]@{ Status = "Running" } } + Mock Start-Sleep -MockWith {} + Mock Write-Log -MockWith {} + Mock Set-ExitCode -MockWith { + Param($ExitCode, $ErrorMessage) + throw $ErrorMessage + } + } + + It "waits for a new successful run and running kubelet" { + Start-NodeResetScriptTask + + Assert-MockCalled -CommandName Start-ScheduledTask -Exactly -Times 1 + Assert-MockCalled -CommandName Set-ExitCode -Exactly -Times 0 + } + + It "does not accept Ready before the new run starts" { + Mock Get-ScheduledTaskInfo -MockWith { + $script:taskInfoCallCount++ + if ($script:taskInfoCallCount -le 2) { + return [pscustomobject]@{ LastRunTime = [datetime]"2026-01-01"; LastTaskResult = 0 } + } + return [pscustomobject]@{ LastRunTime = [datetime]"2026-01-02"; LastTaskResult = 0 } + } + + Start-NodeResetScriptTask + + Assert-MockCalled -CommandName Start-Sleep -Exactly -Times 1 + } + + It "fails when the task result is nonzero" { + Mock Get-ScheduledTaskInfo -MockWith { + $script:taskInfoCallCount++ + if ($script:taskInfoCallCount -eq 1) { + return [pscustomobject]@{ LastRunTime = [datetime]"2026-01-01"; LastTaskResult = 0 } + } + return [pscustomobject]@{ LastRunTime = [datetime]"2026-01-02"; LastTaskResult = 1 } + } + + { Start-NodeResetScriptTask } | Should -Throw "*failed with result [1]*" + } + + It "fails when kubelet is not running" { + Mock Get-Service -MockWith { return [pscustomobject]@{ Status = "Stopped" } } + + { Start-NodeResetScriptTask } | Should -Throw "*kubelet service is not running*" + } +} From 7322815bf643889bee27d6489fe358756e8931ea Mon Sep 17 00:00:00 2001 From: Artur Khantimirov Date: Mon, 20 Jul 2026 10:02:03 +1200 Subject: [PATCH 2/2] fix: avoid Windows node reset task race Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 2ba2b88c-27a6-4f9b-9fd5-bf757a9d31c2 --- parts/windows/windowscsehelper.ps1 | 10 +++++++--- parts/windows/windowscsehelper.tests.ps1 | 19 ++++++++++++++++++- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/parts/windows/windowscsehelper.ps1 b/parts/windows/windowscsehelper.ps1 index 449c3f47054..2d54c6f8d5e 100644 --- a/parts/windows/windowscsehelper.ps1 +++ b/parts/windows/windowscsehelper.ps1 @@ -308,15 +308,19 @@ function Start-NodeResetScriptTask ) $taskName = "k8s-restart-job" + $taskRunningResult = 0x00041301 $previousRunTime = (Get-ScheduledTaskInfo -TaskName $taskName).LastRunTime Start-ScheduledTask -TaskName $taskName $timer = [Diagnostics.Stopwatch]::StartNew() do { - $task = Get-ScheduledTask -TaskName $taskName $taskInfo = Get-ScheduledTaskInfo -TaskName $taskName + $task = Get-ScheduledTask -TaskName $taskName if ($task.State -eq "Ready" -and $taskInfo.LastRunTime -ne $previousRunTime) { - break + $taskInfo = Get-ScheduledTaskInfo -TaskName $taskName + if ($taskInfo.LastRunTime -ne $previousRunTime -and $taskInfo.LastTaskResult -ne $taskRunningResult) { + break + } } if ($timer.Elapsed.TotalSeconds -gt $TimeoutSeconds) { @@ -329,7 +333,7 @@ function Start-NodeResetScriptTask $timer.Stop() if ($taskInfo.LastTaskResult -ne 0) { - Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask failed with result [$($taskInfo.LastTaskResult)]" + Set-ExitCode -ExitCode $global:WINDOWS_CSE_ERROR_START_NODE_RESET_SCRIPT_TASK -ErrorMessage "NodeResetScriptTask failed with result $($taskInfo.LastTaskResult)" } $kubeletService = Get-Service -Name "kubelet" -ErrorAction SilentlyContinue diff --git a/parts/windows/windowscsehelper.tests.ps1 b/parts/windows/windowscsehelper.tests.ps1 index 3867a521352..4ac98b7b643 100644 --- a/parts/windows/windowscsehelper.tests.ps1 +++ b/parts/windows/windowscsehelper.tests.ps1 @@ -632,6 +632,23 @@ Describe "Start-NodeResetScriptTask" { Assert-MockCalled -CommandName Start-Sleep -Exactly -Times 1 } + It "does not accept stale Ready while the task is running" { + Mock Get-ScheduledTaskInfo -MockWith { + $script:taskInfoCallCount++ + if ($script:taskInfoCallCount -eq 1) { + return [pscustomobject]@{ LastRunTime = [datetime]"2026-01-01"; LastTaskResult = 0 } + } + if ($script:taskInfoCallCount -le 3) { + return [pscustomobject]@{ LastRunTime = [datetime]"2026-01-02"; LastTaskResult = 0x00041301 } + } + return [pscustomobject]@{ LastRunTime = [datetime]"2026-01-02"; LastTaskResult = 0 } + } + + Start-NodeResetScriptTask + + Assert-MockCalled -CommandName Start-Sleep -Exactly -Times 1 + } + It "fails when the task result is nonzero" { Mock Get-ScheduledTaskInfo -MockWith { $script:taskInfoCallCount++ @@ -641,7 +658,7 @@ Describe "Start-NodeResetScriptTask" { return [pscustomobject]@{ LastRunTime = [datetime]"2026-01-02"; LastTaskResult = 1 } } - { Start-NodeResetScriptTask } | Should -Throw "*failed with result [1]*" + { Start-NodeResetScriptTask } | Should -Throw "*failed with result 1*" } It "fails when kubelet is not running" {