PCI host bridge support

resources/chroot.sh

@@ -11,7 +11,7 @@ PS4='+\t '
 
 cp -ruv $rootfs/* /
 
-packages="udev systemd-sysv openssh-server iproute2 curl socat python3-minimal iperf3 iputils-ping fio kmod tmux hwloc-nox vim-tiny trace-cmd linuxptp strace"
+packages="udev systemd-sysv openssh-server iproute2 curl socat python3-minimal iperf3 iputils-ping fio kmod tmux hwloc-nox vim-tiny trace-cmd linuxptp strace pciutils"
 
 # msr-tools is only supported on x86-64.
 arch=$(uname -m)

resources/guest_configs/pcie.config

@@ -0,0 +1,8 @@
+CONFIG_BLK_MQ_PCI=y
+CONFIG_PCI=y
+CONFIG_PCI_MMCONFIG=y
+CONFIG_PCI_MSI=y
+CONFIG_PCIEPORTBUS=y
+CONFIG_VIRTIO_PCI=y
+CONFIG_PCI_HOST_COMMON=y
+CONFIG_PCI_HOST_GENERIC=y

resources/overlay/etc/systemd/system/fcnet.service

@@ -1,5 +1,6 @@
 [Service]
 Type=oneshot
+ExecStartPre=/usr/bin/udevadm settle
 ExecStart=/usr/local/bin/fcnet-setup.sh
 [Install]
 WantedBy=sshd.service

resources/rebuild.sh

@@ -223,15 +223,16 @@ function build_al_kernels {
     clone_amazon_linux_repo
 
     CI_CONFIG="$PWD/guest_configs/ci.config"
+    PCIE_CONFIG="$PWD/guest_configs/pcie.config"
 
     if [[ "$KERNEL_VERSION" == @(all|5.10) ]]; then
-        build_al_kernel $PWD/guest_configs/microvm-kernel-ci-$ARCH-5.10.config "$CI_CONFIG"
+        build_al_kernel $PWD/guest_configs/microvm-kernel-ci-$ARCH-5.10.config "$CI_CONFIG" "$PCIE_CONFIG"
     fi
     if [[ $ARCH == "x86_64" && "$KERNEL_VERSION" == @(all|5.10-no-acpi) ]]; then
-        build_al_kernel $PWD/guest_configs/microvm-kernel-ci-$ARCH-5.10-no-acpi.config "$CI_CONFIG"
+        build_al_kernel $PWD/guest_configs/microvm-kernel-ci-$ARCH-5.10-no-acpi.config "$CI_CONFIG" "$PCIE_CONFIG"
     fi
     if [[ "$KERNEL_VERSION" == @(all|6.1) ]]; then
-        build_al_kernel $PWD/guest_configs/microvm-kernel-ci-$ARCH-6.1.config "$CI_CONFIG"
+        build_al_kernel $PWD/guest_configs/microvm-kernel-ci-$ARCH-6.1.config "$CI_CONFIG" "$PCIE_CONFIG"
     fi
 
     # Build debug kernels
@@ -240,11 +241,11 @@ function build_al_kernels {
     OUTPUT_DIR=$OUTPUT_DIR/debug
     mkdir -pv $OUTPUT_DIR
     if [[ "$KERNEL_VERSION" == @(all|5.10) ]]; then
-        build_al_kernel "$PWD/guest_configs/microvm-kernel-ci-$ARCH-5.10.config" "$CI_CONFIG" "$FTRACE_CONFIG" "$DEBUG_CONFIG"
+        build_al_kernel "$PWD/guest_configs/microvm-kernel-ci-$ARCH-5.10.config" "$CI_CONFIG" "$PCIE_CONFIG" "$FTRACE_CONFIG" "$DEBUG_CONFIG"
         vmlinux_split_debuginfo $OUTPUT_DIR/vmlinux-5.10.*
     fi
     if [[ "$KERNEL_VERSION" == @(all|6.1) ]]; then
-        build_al_kernel "$PWD/guest_configs/microvm-kernel-ci-$ARCH-6.1.config" "$CI_CONFIG" "$FTRACE_CONFIG" "$DEBUG_CONFIG"
+        build_al_kernel "$PWD/guest_configs/microvm-kernel-ci-$ARCH-6.1.config" "$CI_CONFIG" "$PCIE_CONFIG" "$FTRACE_CONFIG" "$DEBUG_CONFIG"
         vmlinux_split_debuginfo $OUTPUT_DIR/vmlinux-6.1.*
     fi
 }

src/acpi-tables/src/lib.rs

@@ -10,13 +10,15 @@ pub mod aml;
 pub mod dsdt;
 pub mod fadt;
 pub mod madt;
+pub mod mcfg;
 pub mod rsdp;
 pub mod xsdt;
 
 pub use aml::Aml;
 pub use dsdt::Dsdt;
 pub use fadt::Fadt;
 pub use madt::Madt;
+pub use mcfg::Mcfg;
 pub use rsdp::Rsdp;
 pub use xsdt::Xsdt;
 use zerocopy::little_endian::{U32, U64};
@@ -89,7 +91,7 @@ pub struct SdtHeader {
     pub oem_table_id: [u8; 8],
     pub oem_revision: U32,
     pub creator_id: [u8; 4],
-    pub creator_revison: U32,
+    pub creator_revision: U32,
 }
 
 impl SdtHeader {
@@ -110,7 +112,7 @@ impl SdtHeader {
             oem_table_id,
             oem_revision: U32::new(oem_revision),
             creator_id: FC_ACPI_CREATOR_ID,
-            creator_revison: U32::new(FC_ACPI_CREATOR_REVISION),
+            creator_revision: U32::new(FC_ACPI_CREATOR_REVISION),
         }
     }
 }

src/acpi-tables/src/mcfg.rs

@@ -0,0 +1,77 @@
+// Copyright © 2019 Intel Corporation
+// Copyright © 2023 Rivos, Inc.
+// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+use std::mem::size_of;
+
+use vm_memory::{Bytes, GuestAddress, GuestMemory};
+use zerocopy::{Immutable, IntoBytes};
+
+use crate::{Result, Sdt, SdtHeader, checksum};
+
+#[allow(dead_code)]
+#[repr(C, packed)]
+#[derive(Default, Debug, IntoBytes, Clone, Copy, Immutable)]
+struct PciRangeEntry {
+    pub base_address: u64,
+    pub segment: u16,
+    pub start: u8,
+    pub end: u8,
+    _reserved: u32,
+}
+
+#[allow(dead_code)]
+#[repr(C, packed)]
+#[derive(Clone, Copy, Debug, Default, IntoBytes, Immutable)]
+pub struct Mcfg {
+    header: SdtHeader,
+    _reserved: u64,
+    pci_range_entry: PciRangeEntry,
+}
+
+impl Mcfg {
+    pub fn new(
+        oem_id: [u8; 6],
+        oem_table_id: [u8; 8],
+        oem_revision: u32,
+        pci_mmio_config_addr: u64,
+    ) -> Self {
+        let header = SdtHeader::new(
+            *b"MCFG",
+            size_of::<Mcfg>().try_into().unwrap(),
+            1,
+            oem_id,
+            oem_table_id,
+            oem_revision,
+        );
+
+        let mut mcfg = Mcfg {
+            header,
+            pci_range_entry: PciRangeEntry {
+                base_address: pci_mmio_config_addr,
+                segment: 0,
+                start: 0,
+                end: 0,
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+
+        mcfg.header.checksum = checksum(&[mcfg.as_bytes()]);
+
+        mcfg
+    }
+}
+
+impl Sdt for Mcfg {
+    fn len(&self) -> usize {
+        self.as_bytes().len()
+    }
+
+    fn write_to_guest<M: GuestMemory>(&mut self, mem: &M, address: GuestAddress) -> Result<()> {
+        mem.write_slice(self.as_bytes(), address)?;
+        Ok(())
+    }
+}

src/firecracker/src/api_server_adapter.rs

@@ -143,6 +143,7 @@
     instance_info: InstanceInfo,
     process_time_reporter: ProcessTimeReporter,
     boot_timer_enabled: bool,
+    pci_enabled: bool,
     api_payload_limit: usize,
     mmds_size_limit: usize,
     metadata_json: Option<&str>,
@@ -212,6 +213,7 @@
             json,
             instance_info,
             boot_timer_enabled,
+            pci_enabled,
             mmds_size_limit,
             metadata_json,
         )
@@ -224,6 +226,7 @@
             &to_api,
             &api_event_fd,
             boot_timer_enabled,
+            pci_enabled,
             mmds_size_limit,
             metadata_json,
         )

src/firecracker/src/main.rs

@@ -260,6 +260,11 @@
                 Argument::new("mmds-size-limit")
                     .takes_value(true)
                     .help("Mmds data store limit, in bytes."),
+            )
+            .arg(
+                Argument::new("enable-pci")
+                    .takes_value(false)
+                    .help("Enables PCIe support."),
             );
 
     arg_parser.parse_from_cmdline()?;
@@ -369,6 +374,7 @@
         .map(|x| x.expect("Unable to open or read from the mmds content file"));
 
     let boot_timer_enabled = arguments.flag_present("boot-timer");
+    let pci_enabled = arguments.flag_present("enable-pci");
     let api_enabled = !arguments.flag_present("no-api");
     let api_payload_limit = arg_parser
         .arguments()
@@ -422,6 +428,7 @@
             instance_info,
             process_time_reporter,
             boot_timer_enabled,
+            pci_enabled,
             api_payload_limit,
             mmds_size_limit,
             metadata_json.as_deref(),
@@ -437,6 +444,7 @@
             vmm_config_json,
             instance_info,
             boot_timer_enabled,
+            pci_enabled,
             mmds_size_limit,
             metadata_json.as_deref(),
         )
@@ -449,7 +457,7 @@
 /// the default the jailer would set).
 ///
 /// We do this resizing because the kernel default is 64, with a reallocation happening whenever
-/// the tabel fills up. This was happening for some larger microVMs, and reallocating the
+/// the table fills up. This was happening for some larger microVMs, and reallocating the
 /// fdtable while a lot of file descriptors are active (due to being eventfds/timerfds registered
 /// to epoll) incurs a penalty of 30ms-70ms on the snapshot restore path.
 fn resize_fdtable() -> Result<(), ResizeFdTableError> {
@@ -554,19 +562,22 @@
 }
 
 // Configure and start a microVM as described by the command-line JSON.
+#[allow(clippy::too_many_arguments)]
 fn build_microvm_from_json(
     seccomp_filters: &BpfThreadMap,
     event_manager: &mut EventManager,
     config_json: String,
     instance_info: InstanceInfo,
     boot_timer_enabled: bool,
+    pci_enabled: bool,
     mmds_size_limit: usize,
     metadata_json: Option<&str>,
 ) -> Result<(VmResources, Arc<Mutex<vmm::Vmm>>), BuildFromJsonError> {
     let mut vm_resources =
         VmResources::from_json(&config_json, &instance_info, mmds_size_limit, metadata_json)
             .map_err(BuildFromJsonError::ParseFromJson)?;
     vm_resources.boot_timer = boot_timer_enabled;
+    vm_resources.pci_enabled = pci_enabled;
     let vmm = vmm::builder::build_and_boot_microvm(
         &instance_info,
         &vm_resources,
@@ -593,6 +604,7 @@
     config_json: Option<String>,
     instance_info: InstanceInfo,
     bool_timer_enabled: bool,
+    pci_enabled: bool,
     mmds_size_limit: usize,
     metadata_json: Option<&str>,
 ) -> Result<(), RunWithoutApiError> {
@@ -610,6 +622,7 @@
         config_json.unwrap(),
         instance_info,
         bool_timer_enabled,
+        pci_enabled,
         mmds_size_limit,
         metadata_json,
     )

src/pci/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+authors = ["Samuel Ortiz <sameo@linux.intel.com>"]
+edition = "2021"
+name = "pci"
+version = "0.1.0"
+license = "Apache-2.0 AND BSD-3-Clause"
+
+[features]
+default = []
+
+[dependencies]
+byteorder = "1.5.0"
+libc = "0.2.172"
+log = "0.4.27"
+serde = { version = "1.0.219", features = ["derive"] }
+thiserror = "2.0.12"
+vm-allocator = { git = "https://github.com/rust-vmm/vm-allocator" }
+vm-device = { path = "../vm-device" }
+vm-memory = { version = "0.16.1", features = [
+  "backend-mmap",
+  "backend-bitmap",
+] }