diff --git a/changelogs/fragments/pamd-make-idempotence-fix.yaml b/changelogs/fragments/pamd-make-idempotence-fix.yaml
new file mode 100644
index 00000000000..2ab8aa971b4
--- /dev/null
+++ b/changelogs/fragments/pamd-make-idempotence-fix.yaml
@@ -0,0 +1,2 @@
+bugfixes:
+  - Backport of https://github.com/ansible/ansible/pull/54105, pamd - fix idempotence issue when removing rules
diff --git a/lib/ansible/modules/system/pamd.py b/lib/ansible/modules/system/pamd.py
index bc4192e4227..c1e838c3be1 100644
--- a/lib/ansible/modules/system/pamd.py
+++ b/lib/ansible/modules/system/pamd.py
@@ -480,7 +480,7 @@ class PamdService(object):
                 else:
                     self._head = current_line.next
                     current_line.next.prev = None
-            changed += 1
+                changed += 1
 
             current_line = current_line.next
         return changed
diff --git a/test/units/modules/system/test_pamd.py b/test/units/modules/system/test_pamd.py
index 6a7141c69dd..637564a3d17 100644
--- a/test/units/modules/system/test_pamd.py
+++ b/test/units/modules/system/test_pamd.py
@@ -349,5 +349,7 @@ session    required pam_unix.so"""
 
     def test_remove_rule(self):
         self.assertTrue(self.pamd.remove('account', 'required', 'pam_unix.so'))
+        # Second run should not change anything
+        self.assertFalse(self.pamd.remove('account', 'required', 'pam_unix.so'))
         test_rule = PamdRule('account', 'required', 'pam_unix.so')
         self.assertNotIn(str(test_rule), str(self.pamd))