Don't error on removal of an ignored homekit_controller config entry (h…

…ome-assistant#30083)

* Don't error on ignored entries

* Don't ever call async_remove_entry or async_unload_entry for an unignored ignore config entry

homeassistant/config_entries.py

@@ -246,6 +246,10 @@ async def async_unload(
 
         Returns if unload is possible and was successful.
         """
+        if self.source == SOURCE_IGNORE:
+            self.state = ENTRY_STATE_NOT_LOADED
+            return True
+
         if integration is None:
             integration = await loader.async_get_integration(hass, self.domain)
 
@@ -292,6 +296,9 @@ async def async_unload(
 
     async def async_remove(self, hass: HomeAssistant) -> None:
         """Invoke remove callback on component."""
+        if self.source == SOURCE_IGNORE:
+            return
+
         integration = await loader.async_get_integration(hass, self.domain)
         component = integration.get_component()
         if not hasattr(component, "async_remove_entry"):