Merge pull request #2089 from mwhudson/bridge-kernel

support falling back to a bridge kernel

examples/answers/bridge.yaml

@@ -0,0 +1,44 @@
+#source-catalog: examples/sources/bridge.yaml
+Source:
+  source: ubuntu-server
+  search_drivers: true
+Welcome:
+  lang: en_US
+Refresh:
+  update: no
+Keyboard:
+  layout: us
+Zdev:
+  accept-default: yes
+Network:
+  accept-default: yes
+Proxy:
+  proxy: ""
+Mirror:
+  country-code: us
+Filesystem:
+  guided: yes
+  guided-index: 0
+Identity:
+  realname: Ubuntu
+  username: ubuntu
+  hostname: ubuntu-server
+  # ubuntu
+  password: '$6$wdAcoXrU039hKYPd$508Qvbe7ObUnxoj15DRCkzC3qO7edjH0VV7BPNRDYK4QR8ofJaEEF2heacn0QgD.f8pO8SNp83XNdWG6tocBM1'
+SSH:
+  install_server: true
+  pwauth: false
+  authorized_keys:
+    - |
+      ssh-rsa AAAAAAAAAAAAAAAAAAAAAAAAA # ssh-import-id lp:subiquity
+UbuntuPro:
+  token: ""
+SnapList:
+  snaps:
+    hello:
+      channel: stable
+      classic: false
+InstallProgress:
+  reboot: yes
+Drivers:
+  install: yes

examples/sources/bridge.yaml

@@ -0,0 +1,30 @@
+version: 1
+sources:
+  - description:
+      en: This version has been customized to have a small runtime footprint in environments
+        where humans are not expected to log in.
+    id: ubuntu-server-minimal
+    locale_support: none
+    name:
+      en: Ubuntu Server (minimized)
+    path: ubuntu-server-minimal.squashfs
+    size: 530485248
+    type: fsimage
+    variant: server
+  - default: true
+    description:
+      en: The default install contains a curated set of packages that provide a comfortable
+        experience for operating your server.
+    id: ubuntu-server
+    locale_support: locale-only
+    name:
+      en: Ubuntu Server
+    path: ubuntu-server-minimal.ubuntu-server.squashfs
+    size: 1066115072
+    type: fsimage-layered
+    variant: server
+kernel:
+  default: linux-generic
+  bridge: linux-generic-brg-22.04
+  bridge_reasons:
+    - nvidia

scripts/runtests.sh

@@ -48,6 +48,10 @@ validate () {
     case $testname in
         answers-core-desktop|answers-uc24)
             ;;
+        answers-bridge)
+            python3 scripts/check-yaml-fields.py $tmpdir/var/log/installer/curtin-install/subiquity-curthooks.conf \
+                    kernel.package="linux-generic-brg-22.04"
+            ;;
         *)
             python3 scripts/validate-autoinstall-user-data.py --legacy --check-link < $tmpdir/var/log/installer/autoinstall-user-data
             # After the lunar release and the introduction of mirror testing, it

subiquity/models/filesystem.py

@@ -2431,10 +2431,13 @@ def add_zpool(
         self._actions.append(zpool)
         return zpool
 
+    def uses_zfs(self):
+        return self._one(type="zpool") is not None
+
     async def live_packages(self) -> Tuple[Set, Set]:
         before = set()
         during = set()
-        if self._one(type="zpool") is not None:
+        if self.uses_zfs():
             before.add("zfsutils-linux")
         if self.reset_partition is not None:
             during.add("efibootmgr")

subiquity/models/source.py

@@ -13,6 +13,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+import enum
 import logging
 import os
 import typing
@@ -56,6 +57,34 @@ def __attrs_post_init__(self):
             )
 
 
+# It is possible that on release day the default (i.e. generic) kernel
+# will not support ZFS or NVidia drivers. In this case, if the target
+# system and the user's choices combine to want to use these features,
+# we install a "bridge" kernel instead (when as the generic/default
+# kernel does support these features, the bridge kernel metapackage
+# will be switched to referencing the default kernel metapackage). All
+# the details about this are stored in the source catalog on the ISO.
+
+
+class BridgeKernelReason(enum.Enum):
+    NVIDIA = "nvidia"
+    ZFS = "zfs"
+
+
+@attr.s(auto_attribs=True, kw_only=True)
+class KernelInfo:
+    default: str
+    bridge: typing.Optional[str] = None
+    bridge_reasons: typing.List[BridgeKernelReason] = attr.Factory(list)
+
+
+@attr.s(auto_attribs=True, kw_only=True)
+class SourceCatalog:
+    version: int
+    sources: typing.List[CatalogEntry]
+    kernel: KernelInfo
+
+
 legacy_server_entry = CatalogEntry(
     variant="server",
     id="synthesized",
@@ -71,32 +100,43 @@ def __attrs_post_init__(self):
     },
 )
 
+_serializer = Serializer(ignore_unknown_fields=True, serialize_enums_by="value")
+
 
 class SourceModel:
     def __init__(self):
         self._dir = "/cdrom/casper"
         self.current = legacy_server_entry
-        self.sources = [self.current]
+        self.catalog = SourceCatalog(
+            version=1,
+            sources=[self.current],
+            kernel=KernelInfo(default="linux-generic"),
+        )
         self.lang = None
         self.search_drivers = False
 
     def load_from_file(self, fp):
         self._dir = os.path.dirname(fp.name)
-        self.sources = []
         self.current = None
-        self.sources = Serializer(ignore_unknown_fields=True).deserialize(
-            typing.List[CatalogEntry], yaml.safe_load(fp)
-        )
-        for entry in self.sources:
+        content = yaml.safe_load(fp)
+        if isinstance(content, list):
+            self.catalog = SourceCatalog(
+                version=1,
+                sources=_serializer.deserialize(typing.List[CatalogEntry], content),
+                kernel=KernelInfo(default="linux-generic"),
+            )
+        else:
+            self.catalog = _serializer.deserialize(SourceCatalog, content)
+        for entry in self.catalog.sources:
             if entry.default:
                 self.current = entry
-        log.debug("loaded %d sources from %r", len(self.sources), fp.name)
+        log.debug("loaded %d sources from %r", len(self.catalog.sources), fp.name)
         if self.current is None:
-            self.current = self.sources[0]
+            self.current = self.catalog.sources[0]
 
     def get_matching_source(self, id_: str) -> CatalogEntry:
         """Return a source object that has the ID requested."""
-        for source in self.sources:
+        for source in self.catalog.sources:
             if source.id == id_:
                 return source
         raise KeyError

subiquity/models/tests/test_source.py

@@ -116,7 +116,7 @@ def test_canary(self):
         with open("examples/sources/install-canary.yaml") as fp:
             model = SourceModel()
             model.load_from_file(fp)
-        self.assertEqual(2, len(model.sources))
+        self.assertEqual(2, len(model.catalog.sources))
 
         minimal = model.get_matching_source("ubuntu-desktop-minimal")
         self.assertIsNotNone(minimal.variations)

subiquity/server/controllers/drivers.py

@@ -53,6 +53,7 @@ def __init__(self, app) -> None:
 
         self._list_drivers_task: Optional[asyncio.Task] = None
         self.list_drivers_done_event = asyncio.Event()
+        self.configured_event = asyncio.Event()
 
         # None means that the list has not (yet) been retrieved whereas an
         # empty list means that no drivers are available.
@@ -73,6 +74,13 @@ def start(self):
         self.app.hub.subscribe(
             (InstallerChannels.CONFIGURED, "source"), self.restart_querying_drivers_list
         )
+        self.app.hub.subscribe(
+            (InstallerChannels.CONFIGURED, "drivers"),
+            self.configured_event.set,
+        )
+        self._send_drivers_decided_task = asyncio.create_task(
+            self._send_drivers_decided()
+        )
 
     def restart_querying_drivers_list(self):
         """Start querying the list of available drivers. This method can be
@@ -118,6 +126,15 @@ async def _list_drivers(self, context):
         self.list_drivers_done_event.set()
         log.debug("Available drivers to install: %s", self.drivers)
 
+    async def _send_drivers_decided(self):
+        await self.list_drivers_done_event.wait()
+        if self.drivers:
+            # If there are drivers, we need to wait until all
+            # postinstall models are configured before we can be sure
+            # if the user will change their mind or not.
+            await self.app.base_model.wait_postinstall()
+        await self.app.hub.abroadcast(InstallerChannels.DRIVERS_DECIDED)
+
     async def GET(self, wait: bool = False) -> DriversResponse:
         local_only = not self.app.base_model.network.has_network
         if wait:

subiquity/server/controllers/install.py

@@ -82,6 +82,10 @@ class InstallController(SubiquityController):
     def __init__(self, app):
         super().__init__(app)
         self.model = app.base_model
+        self.bridge_kernel_decided = asyncio.Event()
+        self.app.hub.subscribe(
+            InstallerChannels.BRIDGE_KERNEL_DECIDED, self.bridge_kernel_decided.set
+        )
 
         self.tb_extractor = TracebackExtractor()
 
@@ -446,6 +450,8 @@ async def run_curtin_step(name, stages, step_config, source=None):
             if self.supports_apt():
                 await self.pre_curthooks_oem_configuration(context=context)
 
+            await self.bridge_kernel_decided.wait()
+
             await run_curtin_step(
                 name="curthooks",
                 stages=["curthooks"],

subiquity/server/controllers/kernel.py

@@ -16,8 +16,10 @@
 import logging
 import os
 
+from subiquity.models.source import BridgeKernelReason
 from subiquity.server.controller import NonInteractiveController
 from subiquity.server.kernel import flavor_to_pkgname
+from subiquity.server.types import InstallerChannels
 
 log = logging.getLogger("subiquity.server.controllers.kernel")
 
@@ -59,14 +61,63 @@ def start(self):
                 with open(mp_file) as fp:
                     kernel_package = fp.read().strip()
                 self.model.metapkg_name = kernel_package
+                self.default_metapkg_name = self.model.metapkg_name
                 # built-in kernel requirements are not considered
                 # explicitly_requested
                 self.model.explicitly_requested = False
                 log.debug(f"Using kernel {kernel_package} due to {mp_file}")
                 break
         else:
-            log.debug("Using default kernel linux-generic")
-            self.model.metapkg_name = "linux-generic"
+            # no default kernel found in etc or run, use default from
+            # source catalog.
+            self.app.hub.subscribe(
+                (InstallerChannels.CONFIGURED, "source"), self._set_source
+            )
+        self.needs_bridge = {}
+        self.app.hub.subscribe(
+            InstallerChannels.INSTALL_CONFIRMED,
+            self._confirmed,
+        )
+        self.app.hub.subscribe(
+            InstallerChannels.DRIVERS_DECIDED,
+            self._drivers_decided,
+        )
+
+    async def _set_source(self):
+        self.model.metapkg_name = self.app.base_model.source.catalog.kernel.default
+        self.default_metapkg_name = self.model.metapkg_name
+
+    def _maybe_set_bridge_kernel(self, reason, value):
+        if reason in self.needs_bridge:
+            return
+        reasons = self.app.base_model.source.catalog.kernel.bridge_reasons
+        if reason not in reasons:
+            value = False
+        self.needs_bridge[reason] = value
+        if len(self.needs_bridge) < len(BridgeKernelReason):
+            return
+        log.debug("bridge kernel decided %s", self.needs_bridge)
+        if any(self.needs_bridge.values()):
+            self.model.metapkg_name = self.app.base_model.source.catalog.kernel.bridge
+        else:
+            self.model.metapkg_name = self.default_metapkg_name
+        self.app.hub.broadcast(InstallerChannels.BRIDGE_KERNEL_DECIDED)
+
+    def _confirmed(self):
+        fs_model = self.app.base_model.filesystem
+        if not self.app.base_model.source.catalog.kernel.bridge_reasons:
+            self.app.hub.broadcast(InstallerChannels.BRIDGE_KERNEL_DECIDED)
+        self._maybe_set_bridge_kernel(BridgeKernelReason.ZFS, fs_model.uses_zfs())
+        if not self.app.base_model.source.search_drivers:
+            self._maybe_set_bridge_kernel(BridgeKernelReason.NVIDIA, False)
+
+    def _drivers_decided(self):
+        drivers_controller = self.app.controllers.Drivers
+        self._maybe_set_bridge_kernel(
+            BridgeKernelReason.NVIDIA,
+            drivers_controller.model.do_install
+            and any("nvidia" in driver for driver in drivers_controller.drivers),
+        )
 
     def load_autoinstall_data(self, data):
         if data is None:

subiquity/server/controllers/source.py

@@ -114,6 +114,8 @@ def start(self):
         self.app.hub.subscribe(
             (InstallerChannels.CONFIGURED, "locale"), self._set_locale
         )
+        if self.model.catalog.version != 1:
+            raise Exception("unknown source catalog version")
 
     def _set_locale(self):
         current = self.app.base_model.locale.selected_language
@@ -128,7 +130,7 @@ async def GET(self) -> SourceSelectionAndSetting:
             search_drivers = True
 
         return SourceSelectionAndSetting(
-            [convert_source(source, cur_lang) for source in self.model.sources],
+            [convert_source(source, cur_lang) for source in self.model.catalog.sources],
             self.model.current.id,
             search_drivers=search_drivers,
         )