Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion dep/sddf
Submodule sddf updated from 16236a to 17cd4e
2 changes: 1 addition & 1 deletion examples/virtio/meta.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ def generate(sdf_file: str, output_dir: str, dtb: DeviceTree, client_dtb: Device
for pd in pds:
sdf.add_pd(pd)

client0.add_virtio_mmio_net(guest_net_node, net_system, client0_net_copier)
client0.add_virtio_mmio_net(guest_net_node, net_system, copier=client0_net_copier, vswitch=None)

# Block subsystem
blk_driver = ProtectionDomain("blk_driver", "blk_driver.elf", priority=200)
Expand Down
6 changes: 3 additions & 3 deletions examples/virtio/virtio.mk
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,9 @@ qemu: $(IMAGE_FILE) blk_storage
-m size=2G \
-nographic \
-global virtio-mmio.force-legacy=false \
-drive file=blk_storage,format=raw,if=none,id=hd \
$(QEMU_BLK_ARGS) \
$(QEMU_NET_ARGS) \
-drive file=blk_storage,format=raw,if=none,id=drive0 \
-device virtio-blk-device,drive=drive0,id=virtblk0,num-queues=1,bus=virtio-mmio-bus.1 \
-device virtio-net-device,netdev=netdev0,bus=virtio-mmio-bus.0 \
-netdev user,id=netdev0,hostfwd=tcp::1236-:1236,hostfwd=tcp::1237-:1237,hostfwd=udp::1235-:1235 \

clean::
Expand Down
2 changes: 1 addition & 1 deletion examples/virtio_pci/client_vmm.c
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ void init(void)
return;
}

success = virtio_pci_ecam_init(0x10000000, 0x10000000, 0x100000);
success = virtio_pci_ecam_init(0x10000000, 0x100000000, 0x100000);
assert(success);
success = virtio_pci_register_memory_resource(0x20100000, 0x20100000, 0xFF00000);
assert(success);
Expand Down
2 changes: 1 addition & 1 deletion examples/virtio_pci/meta.py
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ def generate(sdf_file: str, output_dir: str, dtb: DeviceTree, client_dtb: Device
############ VIRTIO PCI ############
config_space = MemoryRegion(sdf, name="ecam", size=0x100000)
sdf.add_mr(config_space)
vmm_client0.add_map(Map(config_space, vaddr=0x10000000, perms="rw"))
vmm_client0.add_map(Map(config_space, vaddr=0x100000000, perms="rw"))

memory_resource = MemoryRegion(sdf, name="memory_resource", size=0x10000)
sdf.add_mr(memory_resource)
Expand Down
6 changes: 3 additions & 3 deletions examples/virtio_pci/virtio_pci.mk
Original file line number Diff line number Diff line change
Expand Up @@ -171,9 +171,9 @@ qemu: $(IMAGE_FILE) blk_storage
-m size=2G \
-nographic \
-global virtio-mmio.force-legacy=false \
-drive file=blk_storage,format=raw,if=none,id=hd \
$(QEMU_BLK_ARGS) \
$(QEMU_NET_ARGS) \
-drive file=blk_storage,format=raw,if=none,id=drive0 \
-device virtio-blk-device,drive=drive0,id=virtblk0,num-queues=1,bus=virtio-mmio-bus.1 \
-device virtio-net-device,netdev=netdev0,bus=virtio-mmio-bus.0 \
-netdev user,id=netdev0,hostfwd=tcp::1236-:1236,hostfwd=tcp::1237-:1237,hostfwd=udp::1235-:1235 \

clean::
Expand Down
62 changes: 62 additions & 0 deletions examples/virtio_vswitch/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
#
# Copyright 2024, UNSW
#
# SPDX-License-Identifier: BSD-2-Clause
#

BUILD_DIR ?= build
export MICROKIT_CONFIG ?= debug

ifeq ($(strip $(MICROKIT_SDK)),)
$(error MICROKIT_SDK must be specified)
endif
override MICROKIT_SDK := $(abspath $(MICROKIT_SDK))

# Both QEMU and Maaxboard uses the same guest device tree as the guest does not
# need to access any physical device. The only difference is that QEMU is GICv2 while
# the Maaxboard is GICv3. We apply a GIC overlay for the correct version on the board
# at build time.
# TODO: we should probably determine this better if we have lots of supported platforms.
ifeq ($(strip $(MICROKIT_BOARD)), maaxboard)
export GIC_DT_OVERLAY := gic_v3_overlay.dts
else ifeq ($(strip $(MICROKIT_BOARD)), qemu_virt_aarch64)
export GIC_DT_OVERLAY := gic_v2_overlay.dts

export BLK_NUM_PART = 1
export BLK_SIZE = 512
# 16MiB of disk space
export BLK_MEM ?= 16777216
endif

# Allow to user to specify a custom partition
PARTITION :=
ifdef PARTITION
export PARTITION_ARG := --partition $(PARTITION)
endif

# All platforms use the same Linux and initrd images.
export LINUX := a3f4bf9e2eb24fa8fc0d3d8cd02e4d8097062e8b-linux
export INITRD := b6a276df6a0e39f76bc8950e975daa2888ad83df-rootfs.cpio.gz

BUILD_DIR ?= build
export BUILD_DIR := $(abspath ${BUILD_DIR})
export VIRTIO_EXAMPLE := $(abspath .)

export CC_USERLEVEL := zig cc
export MICROKIT_TOOL ?= $(MICROKIT_SDK)/bin/microkit
export SDDF := $(abspath ../../dep/sddf)
export LIBVMM := $(abspath ../../)

IMAGE_FILE := $(BUILD_DIR)/loader.img
REPORT_FILE := $(BUILD_DIR)/report.txt

all: $(IMAGE_FILE)

qemu $(IMAGE_FILE) $(REPORT_FILE) clean clobber: $(BUILD_DIR)/Makefile FORCE
$(MAKE) -C $(BUILD_DIR) MICROKIT_SDK=$(MICROKIT_SDK) $(notdir $@)

$(BUILD_DIR)/Makefile: virtio.mk
mkdir -p $(BUILD_DIR)
cp virtio.mk $@

FORCE:
188 changes: 188 additions & 0 deletions examples/virtio_vswitch/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
<!--
Copyright 2024, UNSW
SPDX-License-Identifier: CC-BY-SA-4.0
-->

# Using multiple virtIO devices with a Linux guest

This example shows off the virtIO support that libvmm provides using the
[seL4 Device Driver Framework (sDDF)](https://github.com/au-ts/sddf) to talk to
the actual hardware.

This example makes use of the following virtIO devices emulated by libvmm:

* console
* block
* network

All of the virtIO devices are emulated with their corresponding native drivers
from sDDF.

The example currently works on the following platforms:

* QEMU virt AArch64
* Avnet MaaXBoard

### Metaprogram

Unlike the other examples, this one uses a metaprogram (`meta.py`) with
the [sdfgen](https://github.com/au-ts/microkit_sdf_gen) tooling to generate the
System Description File (SDF) and other necessary artefacts. Previously,
SDFs were written manually, along with C headers for sDDF-specific configurations,
but this approach was tedious and error-prone. Wit this tooling, we can describe
the system at a higher level, automating the generation of system-specific data.

## Dependencies

In addition to the dependencies outlined in the top-level README, the following
dependencies are needed:
* mkfs.fat
* sdfgen (for generating the System Description File with a metaprogram).

### Linux

On apt based Linux distributions run the following commands:
```sh
sudo apt-get install dosfstools
pip3 install sdfgen==0.26.0
```

If you get error: `externally-managed-environment` when installing via pip, instead run:
```sh
pip3 install --break-system-packages sdfgen==0.26.0
```

### macOS

On macOS, you can install the dependencies via Homebrew:
```sh
brew install dosfstools
pip3 install sdfgen==0.26.0
```

If you get error: `externally-managed-environment` when installing via pip, instead run:
```sh
pip3 install --break-system-packages sdfgen==0.26.0
```

### Nix

There is a Nix flake available in the repository, so you can get a development shell via:
```sh
nix develop
```

Note that this will set the `MICROKIT_SDK` environment variable to the SDK path, you do not
need to download the Microkit SDK manually.

## Building

```sh
make MICROKIT_BOARD=<BOARD> MICROKIT_SDK=/path/to/sdk
```

Where `<BOARD>` is one of:

* `qemu_virt_aarch64`
* `maaxboard`

Other configuration options can be passed to the Makefile such as `CONFIG`
and `BUILD_DIR`, see the Makefile for details.

By default the build system fetches the Linux kernel and initrd images from
Trustworthy Systems' website. To use your own images, specify `LINUX` and/or
`INITRD`. For example:

```sh
make MICROKIT_BOARD=qemu_virt_aarch64 MICROKIT_SDK=/path/to/sdk LINUX=/path/to/linux/arch/<arch>/boot/Image INITRD=/path/to/initrd
```

If you would like to simulate the QEMU board you can run the following command:
```sh
make MICROKIT_BOARD=qemu_virt_aarch64 MICROKIT_SDK=/path/to/sdk qemu
```

This will build the example code as well as run the QEMU command to simulate a
system running the whole system.

## Running

### virtIO console

This example makes use of the virtIO console device so that the guest has access
to the serial device on the platform. The virtIO console support in libvmm talks to
the sDDF serial sub-system which contains a driver for input/output to the physical
serial device.

### virtIO block

The guest also doubles as a client in the block system that talks virtIO to a native
block device. The requests from the guest are multiplexed through the additional block
virtualiser component.

When you boot the example, the native block driver will boot first. When it is ready, the
client VM will boot. After the client VM boots, it will attempt to mount the
virtIO block device `/dev/vda` into `/mnt`.

The kernel logs from linux will show the virtIO drive initialising, for example:
```
[ 5.381885] virtio_blk virtio1: [vda] 2040 512-byte logical blocks (1.04 MB/1020 KiB)
```

When you reboot the example, the client VM may display a warning indicating that the
FAT filesystem on the vda device was not cleanly unmounted, which could lead to potential
data corruption:
```
[ 12.292600] FAT-fs (vda): Volume was not properly unmounted. Some data may be corrupt. Please run fsck.
```
To prevent this, always shut down the system properly by running `poweroff` after use,
instead of forcefully terminating the VM.

The system expects the storage device to contain an MBR partition table that contains
one partition. Each partition is allocated to a single client. Partitions must have a
starting block number that is a multiple of sDDF block's transfer size of 4096 bytes
divided by the disk's logical size. Partitions that do not follow this restriction
are unsupported.

By default on QEMU virt AArch64, we mount the first partition of the disk image,
on Avnet MaaXBoard we mount the third partition of the SD Card. You can change the partition mounted
by passing `PARTITION=n` when executing the Makefile.

### virtIO net

In addition to virtIO console and block, the guest can also talk with the native
sDDF network driver via virtIO for in-guest networking. Packets in and out of
the guest are multiplexed through the network virtualiser components.

When the guest starts, it will automatically bring up the network device
and obtain an IP address via DHCP. This is done with the `net_client_init`
script that is packaged into the root file system.

To test the guest network, you can try to ping Google DNS with:
```
# ping 8.8.8.8
PING 8.8.8.8 (8.8.8.8): 56 data bytes
64 bytes from 8.8.8.8: seq=0 ttl=255 time=18.560 ms
64 bytes from 8.8.8.8: seq=1 ttl=255 time=8.859 ms
64 bytes from 8.8.8.8: seq=2 ttl=255 time=5.361 ms
64 bytes from 8.8.8.8: seq=3 ttl=255 time=6.902 ms
64 bytes from 8.8.8.8: seq=4 ttl=255 time=9.198 ms
^C
--- 8.8.8.8 ping statistics ---
5 packets transmitted, 5 packets received, 0% packet loss
round-trip min/avg/max = 5.361/9.776/18.560 ms
```

The guest has a DNS resolver so you can also ping a URL.

### QEMU set up

When running on QEMU, read and writes go to an emulated ramdisk instead of to your
local storage device. The ramdisk file supplied to QEMU is formatted during build
time to contain a FAT filesystem for both partitions.

### Hardware set up

When running on one of the supported hardware platforms, the system expects to
read and write from the SD card. You will need to format the SD card prior to
booting.
Loading