This removes the old host bridge + taps glue, and instead connects the apps to their net provider's router instance. Signed-off-by: Yureka Lilian <yureka@cyberchaos.dev> --- host/rootfs/default.nix | 4 +- host/rootfs/file-list.mk | 2 + .../data/service/spectrum-router/down | 0 .../template/data/service/spectrum-router/run | 9 +++ host/rootfs/image/usr/bin/run-vmm | 21 +++--- host/rootfs/image/usr/bin/vm-import | 13 ---- tools/start-vmm/ch.rs | 66 +---------------- tools/start-vmm/lib.rs | 73 ++++++++++++------- tools/start-vmm/meson.build | 9 +-- tools/start-vmm/net-util.c | 39 ---------- tools/start-vmm/net-util.h | 6 -- tools/start-vmm/net.c | 55 -------------- tools/start-vmm/net.rs | 11 --- tools/start-vmm/tests/meson.build | 5 -- .../start-vmm/tests/tap_open-name-too-long.c | 20 ----- tools/start-vmm/tests/tap_open.c | 28 ------- 16 files changed, 77 insertions(+), 284 deletions(-) create mode 100644 host/rootfs/image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/down create mode 100755 host/rootfs/image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/run delete mode 100644 tools/start-vmm/net-util.c delete mode 100644 tools/start-vmm/net-util.h delete mode 100644 tools/start-vmm/net.c delete mode 100644 tools/start-vmm/tests/tap_open-name-too-long.c delete mode 100644 tools/start-vmm/tests/tap_open.c diff --git a/host/rootfs/default.nix b/host/rootfs/default.nix index b441a51..a759568 100644 --- a/host/rootfs/default.nix +++ b/host/rootfs/default.nix @@ -8,7 +8,7 @@ import ../../lib/call-package.nix ( }: pkgsMusl.callPackage ( -{ spectrum-host-tools +{ spectrum-host-tools, spectrum-router , lib, stdenvNoCC, nixos, runCommand, writeClosure, erofs-utils, s6-rc , busybox, cloud-hypervisor, cosmic-files, crosvm, cryptsetup , dejavu_fonts, dbus, execline, foot, fuse3, iproute2, inotify-tools @@ -27,7 +27,7 @@ let packages = [ btrfs-progs cloud-hypervisor cosmic-files crosvm cryptsetup dbus execline fuse3 inotify-tools iproute2 jq kmod mdevd s6 s6-linux-init - s6-rc socat spectrum-host-tools util-linuxMinimal virtiofsd + s6-rc socat spectrum-host-tools spectrum-router util-linuxMinimal virtiofsd xdg-desktop-portal-spectrum-host (foot.override { allowPgo = false; }) diff --git a/host/rootfs/file-list.mk b/host/rootfs/file-list.mk index 56f693e..bfe3940 100644 --- a/host/rootfs/file-list.mk +++ b/host/rootfs/file-list.mk @@ -27,6 +27,8 @@ FILES = \ image/etc/s6-linux-init/run-image/service/vm-services/run \ image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/dbus/notification-fd \ image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/dbus/run \ + image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/down \ + image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/run \ image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/vhost-user-fs/notification-fd \ image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/vhost-user-fs/run \ image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/vhost-user-gpu/notification-fd \ diff --git a/host/rootfs/image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/down b/host/rootfs/image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/down new file mode 100644 index 0000000..e69de29 diff --git a/host/rootfs/image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/run b/host/rootfs/image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/run new file mode 100755 index 0000000..7b3e3db --- /dev/null +++ b/host/rootfs/image/etc/s6-linux-init/run-image/service/vm-services/template/data/service/spectrum-router/run @@ -0,0 +1,9 @@ +#!/bin/execlineb -P +# SPDX-License-Identifier: EUPL-1.2+ +# SPDX-FileCopyrightText: 2025 Yureka Lilian <yureka@cyberchaos.dev> + +importas -i VM VM + +export RUST_LOG spectrum-router=debug,info +spectrum-router --app-listen-path ${VM}/router-app.sock --driver-listen-path ${VM}/router-driver.sock + diff --git a/host/rootfs/image/usr/bin/run-vmm b/host/rootfs/image/usr/bin/run-vmm index 5649674..ba8b59c 100755 --- a/host/rootfs/image/usr/bin/run-vmm +++ b/host/rootfs/image/usr/bin/run-vmm @@ -20,6 +20,7 @@ background -d { assign-devices } + # Find any net-provider relationships this VM is involved in cd /run/vm/by-id elglob -0 providers */config/providers/net/* forx -pE path { $providers } @@ -31,23 +32,21 @@ background -d { backtick -E link_path { readlink /run/vm/by-name/${router} } basename -- $link_path } - + # This VM may be either the driver or the client if -n { if { test $client_id != $1 } test $router_id != $1 } - backtick -E mac { - pipeline { ip -j link show client-${client_id} } - pipeline { jq -r ".[].ifindex" } - awk "{ - printf \"02:01:%02X:%02X:%02X:%02X\", $0 / 256 ^ 3 % 256, - $0 / 256 ^ 2 % 256, $0 / 256 % 256, $0 % 256 - }" + if { + s6-svc -U /run/service/vm-services/instance/${router_id}/data/service/spectrum-router } - - ch-remote --api-socket /run/vm/by-id/${router_id}/vmm add-net - id=router-${client_id},tap=router-${client_id},mac=${mac} + if { + s6-svwait -U /run/service/vmm/instance/${router_id} + } + # Adding the interface is re-entrant and may be called multiple times. Thus, accept failures. + redirfd -w 2 /dev/null + ch-remote --api-socket /run/vm/by-id/${router_id}/vmm add-net id=router,vhost_user=on,socket=/run/vm/by-id/${router_id}/router-driver.sock,mac=02:01:00:00:00:01 } unexport ! fdmove -c 3 0 diff --git a/host/rootfs/image/usr/bin/vm-import b/host/rootfs/image/usr/bin/vm-import index de88f08..c1d1bbc 100755 --- a/host/rootfs/image/usr/bin/vm-import +++ b/host/rootfs/image/usr/bin/vm-import @@ -14,19 +14,6 @@ if { ln -s -- ${dir} /run/vm/by-name/${1}.${name} } if { ln -s -- ${2}/${name} ${dir}/config } if { ln -s -- /run/service/vmm/instance/${id} ${dir}/service } -if { - if -t { elglob -0d " " providers ${name}/providers/net test -n $providers } - - if { ip link add br-${id} type bridge } - if { ip link set br-${id} up } - - if { ip tuntap add client-${id} mode tap } - if { ip link set client-${id} master br-${id} up } - - if { ip tuntap add router-${id} mode tap } - ip link set router-${id} master br-${id} up -} - if { create-vm-dependencies $id } s6-instance-create -- /run/service/vmm $id diff --git a/tools/start-vmm/ch.rs b/tools/start-vmm/ch.rs index abe1742..4764dd5 100644 --- a/tools/start-vmm/ch.rs +++ b/tools/start-vmm/ch.rs @@ -1,26 +1,19 @@ // SPDX-License-Identifier: EUPL-1.2+ // SPDX-FileCopyrightText: 2022-2024 Alyssa Ross <hi@alyssa.is> +// SPDX-FileCopyrightText: 2025 Yureka Lilian <yureka@cyberchaos.dev> -use std::convert::TryFrom; use std::ffi::OsStr; use std::fs::File; use std::io::Write; -use std::mem::take; -use std::num::NonZeroI32; use std::os::unix::prelude::*; use std::path::Path; use std::process::{Command, Stdio}; -use std::string::FromUtf8Error; use miniserde::{Serialize, json}; use crate::net::MacAddress; use crate::s6::notify_readiness; -// Trivially safe. -const EPERM: NonZeroI32 = NonZeroI32::new(1).unwrap(); -const EPROTO: NonZeroI32 = NonZeroI32::new(71).unwrap(); - #[derive(Serialize)] pub struct ConsoleConfig { pub mode: &'static str, @@ -46,7 +39,8 @@ pub struct GpuConfig { #[derive(Serialize)] pub struct NetConfig { - pub fd: RawFd, + pub vhost_user: bool, + pub vhost_socket: String, pub id: String, pub mac: MacAddress, } @@ -99,11 +93,7 @@ fn command(vm_dir: &Path, s: impl AsRef<OsStr>) -> Command { command } -pub fn create_vm(vm_dir: &Path, ready_fd: File, mut config: VmConfig) -> Result<(), String> { - // Net devices can't be created from file descriptors in vm.create. - // https://github.com/cloud-hypervisor/cloud-hypervisor/issues/5523 - let nets = take(&mut config.net); - +pub fn create_vm(vm_dir: &Path, ready_fd: File, config: VmConfig) -> Result<(), String> { let mut ch_remote = command(vm_dir, "create") .args(["--", "-"]) .stdin(Stdio::piped()) @@ -128,53 +118,5 @@ pub fn create_vm(vm_dir: &Path, ready_fd: File, mut config: VmConfig) -> Result< notify_readiness(ready_fd)?; - for net in nets { - add_net(vm_dir, &net).map_err(|e| format!("failed to add net: {e}"))?; - } - Ok(()) } - -pub fn add_net(vm_dir: &Path, net: &NetConfig) -> Result<(), NonZeroI32> { - let mut ch_remote = command(vm_dir, "add-net") - .arg(format!("fd={},id={},mac={}", net.fd, net.id, net.mac)) - .stdout(Stdio::piped()) - .spawn() - .or(Err(EPERM))?; - - if let Ok(ch_remote_status) = ch_remote.wait() - && ch_remote_status.success() - { - return Ok(()); - } - - Err(EPROTO) -} - -#[repr(C)] -pub struct NetConfigC { - pub fd: RawFd, - pub id: [u8; 18], - pub mac: MacAddress, -} - -impl<'a> TryFrom<&'a NetConfigC> for NetConfig { - type Error = FromUtf8Error; - - fn try_from(c: &'a NetConfigC) -> Result<NetConfig, Self::Error> { - let nul_index = c.id.iter().position(|&c| c == 0).unwrap_or(c.id.len()); - Ok(NetConfig { - fd: c.fd, - id: String::from_utf8(c.id[..nul_index].to_vec())?, - mac: c.mac, - }) - } -} - -impl TryFrom<NetConfigC> for NetConfig { - type Error = FromUtf8Error; - - fn try_from(c: NetConfigC) -> Result<NetConfig, Self::Error> { - Self::try_from(&c) - } -} diff --git a/tools/start-vmm/lib.rs b/tools/start-vmm/lib.rs index 0422d85..b44e037 100644 --- a/tools/start-vmm/lib.rs +++ b/tools/start-vmm/lib.rs @@ -1,23 +1,24 @@ // SPDX-License-Identifier: EUPL-1.2+ // SPDX-FileCopyrightText: 2022-2024 Alyssa Ross <hi@alyssa.is> +// SPDX-FileCopyrightText: 2025 Yureka Lilian <yureka@cyberchaos.dev> mod ch; mod net; mod s6; use std::borrow::Cow; -use std::convert::TryInto; use std::env::args_os; use std::ffi::OsStr; use std::fs::File; -use std::io::{self, ErrorKind}; +use std::hash::{Hash, Hasher}; +use std::io::ErrorKind; use std::path::Path; use ch::{ - ConsoleConfig, DiskConfig, FsConfig, GpuConfig, LandlockConfig, MemoryConfig, PayloadConfig, - VmConfig, VsockConfig, + ConsoleConfig, DiskConfig, FsConfig, GpuConfig, LandlockConfig, MemoryConfig, NetConfig, + PayloadConfig, VmConfig, VsockConfig, }; -use net::net_setup; +use net::MacAddress; pub fn prog_name() -> String { args_os() @@ -40,8 +41,6 @@ pub fn vm_config(vm_dir: &Path) -> Result<VmConfig, String> { return Err(format!("VM name may not contain a colon: {vm_name:?}")); } - let name_bytes = vm_name.as_bytes(); - let config_dir = vm_dir.join("config"); let blk_dir = config_dir.join("blk"); let kernel_path = config_dir.join("vmlinux"); @@ -97,24 +96,48 @@ pub fn vm_config(vm_dir: &Path) -> Result<VmConfig, String> { shared: true, }, net: match net_providers_dir.read_dir() { - Ok(_) => { - // SAFETY: we check the result. - let net = unsafe { - net_setup( - name_bytes.as_ptr().cast(), - name_bytes - .len() - .try_into() - .map_err(|e| format!("VM name too long: {e}"))?, - ) - }; - if net.fd == -1 { - let e = io::Error::last_os_error(); - return Err(format!("setting up networking failed: {e}")); - } - - vec![net.try_into().unwrap()] - } + Ok(entries) => entries + .into_iter() + .map(|result| { + Ok(result + .map_err(|e| format!("examining directory entry: {e}"))? + .path()) + }) + .map(|result: Result<_, String>| { + let provider_name = result? + .file_name() + .ok_or("unable to get net provider name".to_string())? + .to_str() + .unwrap() + .to_string(); + + if provider_name.contains(',') { + return Err(format!( + "illegal ',' character in net provider name {provider_name:?}" + )); + } + + let mut hasher = std::hash::DefaultHasher::new(); + vm_name.hash(&mut hasher); + let id_hashed = hasher.finish(); + + let mac = MacAddress::new([ + 0x02, // IEEE 802c administratively assigned + 0x00, // Spectrum client + (id_hashed >> 24) as u8, + (id_hashed >> 16) as u8, + (id_hashed >> 8) as u8, + id_hashed as u8, + ]); + + Ok(NetConfig { + vhost_user: true, + vhost_socket: format!("/run/vm/by-name/{provider_name}/router-app.sock"), + id: provider_name, + mac, + }) + }) + .collect::<Result<_, _>>()?, Err(e) if e.kind() == ErrorKind::NotFound => Default::default(), Err(e) => return Err(format!("reading directory {net_providers_dir:?}: {e}")), }, diff --git a/tools/start-vmm/meson.build b/tools/start-vmm/meson.build index d07c5a0..4c79f5d 100644 --- a/tools/start-vmm/meson.build +++ b/tools/start-vmm/meson.build @@ -1,12 +1,8 @@ # SPDX-License-Identifier: EUPL-1.2+ # SPDX-FileCopyrightText: 2022-2024 Alyssa Ross <hi@alyssa.is> -c_lib = static_library('start-vmm', 'net.c', 'net-util.c', - c_args : '-D_GNU_SOURCE') - rust_lib = static_library('start_vmm', 'lib.rs', - dependencies : miniserde_dep, - link_with : c_lib) + dependencies : miniserde_dep) rust_lib_dep = declare_dependency( dependencies : miniserde_dep, @@ -20,8 +16,7 @@ executable('start-vmm', 'start-vmm.rs', if get_option('tests') test_exe = executable('start-vmm-test', 'lib.rs', dependencies : miniserde_dep, - rust_args : ['--test'], - link_with : c_lib) + rust_args : ['--test']) test('Rust unit tests', test_exe, protocol : 'rust') subdir('tests') diff --git a/tools/start-vmm/net-util.c b/tools/start-vmm/net-util.c deleted file mode 100644 index 49003e9..0000000 --- a/tools/start-vmm/net-util.c +++ /dev/null @@ -1,39 +0,0 @@ -// SPDX-License-Identifier: EUPL-1.2+ -// SPDX-FileCopyrightText: 2022, 2024 Alyssa Ross <hi@alyssa.is> - -#include "net-util.h" - -#include <errno.h> -#include <fcntl.h> -#include <string.h> -#include <unistd.h> - -#include <sys/ioctl.h> - -#include <linux/if_tun.h> - -int tap_open(char name[static IFNAMSIZ], int flags) -{ - struct ifreq ifr; - int fd, e; - - if (strnlen(name, IFNAMSIZ) == IFNAMSIZ) { - errno = ENAMETOOLONG; - return -1; - } - - strncpy(ifr.ifr_name, name, IFNAMSIZ - 1); - ifr.ifr_flags = IFF_TAP|flags; - - if ((fd = open("/dev/net/tun", O_RDWR)) == -1) - return -1; - if (ioctl(fd, TUNSETIFF, &ifr) == -1) { - e = errno; - close(fd); - errno = e; - return -1; - } - - strncpy(name, ifr.ifr_name, IFNAMSIZ); - return fd; -} diff --git a/tools/start-vmm/net-util.h b/tools/start-vmm/net-util.h deleted file mode 100644 index 8f55206..0000000 --- a/tools/start-vmm/net-util.h +++ /dev/null @@ -1,6 +0,0 @@ -// SPDX-License-Identifier: EUPL-1.2+ -// SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is> - -#include <net/if.h> - -int tap_open(char name[static IFNAMSIZ], int flags); diff --git a/tools/start-vmm/net.c b/tools/start-vmm/net.c deleted file mode 100644 index 78fe7f6..0000000 --- a/tools/start-vmm/net.c +++ /dev/null @@ -1,55 +0,0 @@ -// SPDX-License-Identifier: EUPL-1.2+ -// SPDX-FileCopyrightText: 2022-2024 Alyssa Ross <hi@alyssa.is> - -#include "ch.h" -#include "net-util.h" - -#include <assert.h> -#include <errno.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> -#include <unistd.h> - -#include <arpa/inet.h> - -#include <linux/if_tun.h> - -static int get_tap_name(char tap_name[static IFNAMSIZ], - const char tap_prefix[static 1], - const char name[static 1], int name_len) -{ - int r = snprintf(tap_name, IFNAMSIZ, "%s-%*s", tap_prefix, name_len, name); - if (r >= IFNAMSIZ) - errno = ENAMETOOLONG; - return r < 0 || r >= IFNAMSIZ ? -1 : 0; -} - -struct net_config net_setup(const char name[static 1], int name_len) -{ - int e; - unsigned int client_index; - struct net_config r = { .fd = -1, .mac = { 0 } }; - - if ((get_tap_name(r.id, "client", name, name_len)) == -1) - return r; - - if (!(client_index = htonl(if_nametoindex(r.id)))) - return r; - - if ((r.fd = tap_open(r.id, IFF_NO_PI|IFF_VNET_HDR)) == -1) - goto fail_close; - - r.mac[0] = 0x02; // IEEE 802c administratively assigned - r.mac[1] = 0x00; // Spectrum client - memcpy(&r.mac[2], &client_index, 4); - - return r; - -fail_close: - e = errno; - close(r.fd); - errno = e; - r.fd = -1; - return r; -} diff --git a/tools/start-vmm/net.rs b/tools/start-vmm/net.rs index ebfef7a..7b237df 100644 --- a/tools/start-vmm/net.rs +++ b/tools/start-vmm/net.rs @@ -2,14 +2,11 @@ // SPDX-FileCopyrightText: 2022-2025 Alyssa Ross <hi@alyssa.is> use std::borrow::Cow; -use std::ffi::{c_char, c_int}; use std::fmt::{self, Display, Formatter}; use miniserde::Serialize; use miniserde::ser::Fragment; -use crate::ch::NetConfigC; - #[repr(transparent)] #[derive(Copy, Clone)] pub struct MacAddress([u8; 6]); @@ -36,14 +33,6 @@ impl Serialize for MacAddress { } } -// SAFETY: declaration is compatible with C. -unsafe extern "C" { - /// # Safety - /// - /// The rest of the result is only valid if the returned fd is not -1. - pub fn net_setup(name: *const c_char, len: c_int) -> NetConfigC; -} - #[cfg(test)] mod tests { use super::*; diff --git a/tools/start-vmm/tests/meson.build b/tools/start-vmm/tests/meson.build index bfdfc46..5538822 100644 --- a/tools/start-vmm/tests/meson.build +++ b/tools/start-vmm/tests/meson.build @@ -4,11 +4,6 @@ rust_helper = static_library('test_helper', 'helper.rs', dependencies : rust_lib_dep) -test('tap_open', executable('tap_open', 'tap_open.c', '../net-util.c', - c_args : '-D_GNU_SOURCE')) -test('tap_open (name too long)', executable('tap_open-name-too-long', - 'tap_open-name-too-long.c', '../net-util.c', c_args : '-D_GNU_SOURCE')) - test('vm_command-basic', executable('vm_command-basic', 'vm_command-basic.rs', dependencies : rust_lib_dep, diff --git a/tools/start-vmm/tests/tap_open-name-too-long.c b/tools/start-vmm/tests/tap_open-name-too-long.c deleted file mode 100644 index ba4ebd6..0000000 --- a/tools/start-vmm/tests/tap_open-name-too-long.c +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: EUPL-1.2+ -// SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is> - -#include "../net-util.h" - -#include <assert.h> -#include <errno.h> -#include <net/if.h> -#include <string.h> - -int main(void) -{ - char name[IFNAMSIZ]; - int fd; - - memset(name, 'a', sizeof name); - fd = tap_open(name, 0); - assert(fd == -1); - assert(errno == ENAMETOOLONG); -} diff --git a/tools/start-vmm/tests/tap_open.c b/tools/start-vmm/tests/tap_open.c deleted file mode 100644 index bf5d00c..0000000 --- a/tools/start-vmm/tests/tap_open.c +++ /dev/null @@ -1,28 +0,0 @@ -// SPDX-License-Identifier: EUPL-1.2+ -// SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is> - -#include "../net-util.h" - -#include <assert.h> -#include <errno.h> -#include <sched.h> -#include <string.h> - -#include <sys/ioctl.h> - -#include <linux/if_tun.h> - -int main(void) -{ - char name[IFNAMSIZ] = "tap%d"; - struct ifreq ifr; - int fd; - - unshare(CLONE_NEWUSER|CLONE_NEWNET); - - fd = tap_open(name, 0); - if (fd == -1 && (errno == EPERM || errno == ENOENT)) - return 77; - assert(!ioctl(fd, (unsigned)TUNGETIFF, &ifr)); - assert(!strcmp(name, ifr.ifr_name)); -} -- 2.51.2