diff --git a/moto/s3/models.py b/moto/s3/models.py
index 237975724af..409b24174cb 100644
--- a/moto/s3/models.py
+++ b/moto/s3/models.py
@@ -1827,8 +1827,6 @@ def put_object(
         if bucket.is_versioned:
             keys = existing_keys + [new_key]
         else:
-            for key in existing_keys:
-                key.dispose()
             keys = [new_key]
         bucket.keys.setlist(key_name, keys)
 
diff --git a/moto/s3/utils.py b/moto/s3/utils.py
index 11c53d59ecb..d6bd84fb557 100644
--- a/moto/s3/utils.py
+++ b/moto/s3/utils.py
@@ -148,6 +148,12 @@ def setlist(self, key, list_):
         elif not isinstance(list_, list):
             list_ = [list_]
 
+        for existing_version in self.getlist(key, []):
+            # Dispose of any FakeKeys that we will not keep
+            # We should only have FakeKeys here - but we're checking hasattr to be sure
+            if existing_version not in list_ and hasattr(existing_version, "dispose"):
+                existing_version.dispose()
+
         super().__setitem__(key, list_)
 
     def _iteritems(self):
diff --git a/tests/test_s3/test_s3_file_handles.py b/tests/test_s3/test_s3_file_handles.py
index b98fb2cef92..cb43665008a 100644
--- a/tests/test_s3/test_s3_file_handles.py
+++ b/tests/test_s3/test_s3_file_handles.py
@@ -251,6 +251,17 @@ def test_overwrite_file(self):
         with mock_s3():
             pass
 
+    @verify_zero_warnings
+    def test_delete_object_with_version(self):
+        with mock_s3():
+            self.s3.create_bucket(Bucket="foo")
+            self.s3.put_bucket_versioning(
+                Bucket="foo",
+                VersioningConfiguration={"Status": "Enabled", "MFADelete": "Disabled"},
+            )
+            version = self.s3.put_object(Bucket="foo", Key="b", Body="s")["VersionId"]
+            self.s3.delete_object(Bucket="foo", Key="b", VersionId=version)
+
 
 def test_verify_key_can_be_copied_after_disposing():
     # https://github.com/getmoto/moto/issues/5588