diff --git a/changelogs/fragments/winrm-quota.yml b/changelogs/fragments/winrm-quota.yml new file mode 100644 index 00000000000..2a84f3315dc --- /dev/null +++ b/changelogs/fragments/winrm-quota.yml @@ -0,0 +1,3 @@ +bugfixes: + - winrm - Add retry after exceeding commands per user quota that can occur in loops and action plugins running + multiple commands. diff --git a/lib/ansible/plugins/connection/winrm.py b/lib/ansible/plugins/connection/winrm.py index c6a4683d9da..1d50ad891da 100644 --- a/lib/ansible/plugins/connection/winrm.py +++ b/lib/ansible/plugins/connection/winrm.py @@ -207,6 +207,14 @@ except ImportError as e: HAS_WINRM = False WINRM_IMPORT_ERR = e +try: + from winrm.exceptions import WSManFaultError +except ImportError: + # This was added in pywinrm 0.5.0, we just use our no-op exception for + # older versions which won't be able to handle this scenario. + class WSManFaultError(Exception): # type: ignore[no-redef] + pass + try: import xmltodict HAS_XMLTODICT = True @@ -633,7 +641,11 @@ class Connection(ConnectionBase): command_id = None try: stdin_push_failed = False - command_id = self.protocol.run_command(self.shell_id, to_bytes(command), map(to_bytes, args), console_mode_stdin=(stdin_iterator is None)) + command_id = self._winrm_run_command( + to_bytes(command), + tuple(map(to_bytes, args)), + console_mode_stdin=(stdin_iterator is None), + ) try: if stdin_iterator: @@ -697,6 +709,39 @@ class Connection(ConnectionBase): display.warning("Failed to cleanup running WinRM command, resources might still be in use on the target server") + def _winrm_run_command( + self, + command: bytes, + args: tuple[bytes, ...], + console_mode_stdin: bool = False, + ) -> str: + """Starts a command with handling when the WSMan quota is exceeded.""" + try: + return self.protocol.run_command( + self.shell_id, + command, + args, + console_mode_stdin=console_mode_stdin, + ) + except WSManFaultError as fault_error: + if fault_error.wmierror_code != 0x803381A6: + raise + + # 0x803381A6 == ERROR_WSMAN_QUOTA_MAX_OPERATIONS + # WinRS does not decrement the operation count for commands, + # only way to avoid this is to re-create the shell. This is + # important for action plugins that might be running multiple + # processes in the same connection. + display.vvvvv("Shell operation quota exceeded, re-creating shell", host=self._winrm_host) + self.close() + self._connect() + return self.protocol.run_command( + self.shell_id, + command, + args, + console_mode_stdin=console_mode_stdin, + ) + def _connect(self) -> Connection: if not HAS_WINRM: diff --git a/test/integration/targets/connection_winrm/aliases b/test/integration/targets/connection_winrm/aliases index af3f193fb0e..59dba602728 100644 --- a/test/integration/targets/connection_winrm/aliases +++ b/test/integration/targets/connection_winrm/aliases @@ -1,3 +1,4 @@ +destructive windows shippable/windows/group1 shippable/windows/smoketest diff --git a/test/integration/targets/connection_winrm/tests.yml b/test/integration/targets/connection_winrm/tests.yml index cf109a8c6cd..3a117fe7ee8 100644 --- a/test/integration/targets/connection_winrm/tests.yml +++ b/test/integration/targets/connection_winrm/tests.yml @@ -41,3 +41,20 @@ - assert: that: - timeout_cmd.msg == 'The win_shell action failed to execute in the expected time frame (5) and was terminated' + + - name: get WinRM quota value + win_shell: (Get-Item WSMan:\localhost\Service\MaxConcurrentOperationsPerUser).Value + changed_when: false + register: winrm_quota + + - block: + - name: set WinRM quota to lower value + win_shell: Set-Item WSMan:\localhost\Service\MaxConcurrentOperationsPerUser 3 + + - name: run ping with loop to exceed quota + win_ping: + loop: '{{ range(0, 4) }}' + + always: + - name: reset WinRM quota value + win_shell: Set-Item WSMan:\localhost\Service\MaxConcurrentOperationsPerUser {{ winrm_quota.stdout | trim }} diff --git a/test/lib/ansible_test/_data/requirements/constraints.txt b/test/lib/ansible_test/_data/requirements/constraints.txt index e1ad2da664a..40b84a1b1d4 100644 --- a/test/lib/ansible_test/_data/requirements/constraints.txt +++ b/test/lib/ansible_test/_data/requirements/constraints.txt @@ -1,7 +1,7 @@ # do not add a cryptography or pyopenssl constraint to this file, they require special handling, see get_cryptography_requirements in python_requirements.py # do not add a coverage constraint to this file, it is handled internally by ansible-test pypsrp < 1.0.0 # in case the next major version is too big of a change -pywinrm >= 0.4.3 # support for Python 3.11 +pywinrm >= 0.5.0 # support for WSManFaultError and type annotation pytest >= 4.5.0 # pytest 4.5.0 added support for --strict-markers ntlm-auth >= 1.3.0 # message encryption support using cryptography requests-ntlm >= 1.1.0 # message encryption support