Test landing page is reachable without internet connection (#3499)

* Test landing page is reachable without internet connection

Add test that checks user is able to access the landing page even when HAOS has
no internet connection. We still need some sort of outgoing connectivity, so
outgoing connection attempts don't end up with "network is unreachable". To
simulate this, restricted network is created for the QEMU instance used in the
test, and when everything is started, unresponsive default gateway is added.

This intents to test regression that was fixed by
home-assistant/supervisor#5204, Supervisor 2024.7.0+ is thus needed for this
test to pass.

* Bump requirements for tests

tests/conftest.py

@@ -9,6 +9,18 @@
 logger = logging.getLogger(__name__)
 
 
+@pytest.fixture(scope="function")
+def without_internet(strategy):
+    default_nic = strategy.qemu.nic
+    if strategy.status.name == "shell":
+        strategy.transition("off")
+    strategy.qemu.nic = "user,net=192.168.76.0/24,dhcpstart=192.168.76.10,restrict=yes"
+    strategy.transition("shell")
+    yield
+    strategy.transition("off")
+    strategy.qemu.nic = default_nic
+
+
 @pytest.fixture(autouse=True, scope="module")
 def restart_qemu(strategy):
     """Use fresh QEMU instance for each module."""

tests/qemu_shell_strategy.py

@@ -56,8 +56,7 @@ def transition(self, status, *, step):  # pylint: disable=redefined-outer-name
             step.skip("nothing to do")
             return  # nothing to do
         elif status == Status.off:
-            self.target.activate(self.qemu)
-            self.qemu.off()
+            self.target.deactivate(self.qemu)
             self.target.deactivate(self.shell)
         elif status == Status.shell:
             self.target.activate(self.qemu)

tests/requirements.txt

@@ -1,4 +1,4 @@
-labgrid==23.0.3
+labgrid==23.0.6
 pytest==7.2.2
-pytest-dependency==0.5.1
-pytest-timeout==2.2.0
+pytest-dependency==0.6.0
+pytest-timeout==2.3.1

tests/smoke_test/test_offline.py

@@ -0,0 +1,65 @@
+import logging
+from time import sleep
+
+import pytest
+from labgrid.driver import ExecutionError
+
+_LOGGER = logging.getLogger(__name__)
+
+
+def _check_connectivity(shell, *, connected):
+    for target in ["home-assistant.io", "1.1.1.1"]:
+        try:
+            output = shell.run_check(f"ping {target}")
+            if f"{target} is alive!" in output:
+                if connected:
+                    return True
+                else:
+                    raise AssertionError(f"expecting disconnected but {target} is alive")
+        except ExecutionError as exc:
+            if not connected:
+                stdout = "\n".join(exc.stdout)
+                assert ("Network is unreachable" in stdout
+                        or "bad address" in stdout
+                        or "No response" in stdout)
+
+    if connected:
+        raise AssertionError(f"expecting connected but all targets are down")
+
+
+@pytest.mark.timeout(300)  # takes quite a while also because of 90s NTP sync timeout
+@pytest.mark.usefixtures("without_internet")
+def test_ha_runs_offline(shell):
+    def check_container_running(container_name):
+        out = shell.run_check(
+            f"docker container inspect -f '{{{{.State.Status}}}}' {container_name} || true"
+        )
+        return "running" in out
+
+    # wait for supervisor to create network
+    while True:
+        if check_container_running("hassio_supervisor"):
+            nm_conns = shell.run_check('nmcli con show')
+            if "Supervisor" in " ".join(nm_conns):
+                break
+        sleep(1)
+
+    # To simulate situation where HAOS is not connected to internet, we need to add
+    # default gateway to the supervisor connection. So we add a default route to
+    # a non-existing IP address in the VM's subnet. Maybe there is a better way?
+    shell.run_check('nmcli con modify "Supervisor enp0s3" ipv4.addresses "192.168.76.10/24" '
+                    '&& nmcli con modify "Supervisor enp0s3" ipv4.gateway 192.168.76.1 '
+                    '&& nmcli device reapply enp0s3')
+
+    _check_connectivity(shell, connected=False)
+
+    for _ in range(60):
+        if check_container_running("homeassistant") and check_container_running("hassio_cli"):
+            break
+        sleep(1)
+    else:
+        shell.run_check("docker logs hassio_supervisor")
+        raise AssertionError("homeassistant or hassio_cli not running after 60s")
+
+    web_index = shell.run_check("curl http://localhost:8123")
+    assert "</html>" in " ".join(web_index)