The xdp-forwarder's purpose is implementing the functionality needed within the net-vm (a VM running the Linux drivers for any physical interfaces on the spectrum system). In the future, the net-vm will load the included XDP programs on the passed-through physical interfaces as well as the downstream virtio interface going into the router (recognized by its special MAC address). The net-vm needs to multiplex between the physical interfaces, as there might be several interfaces in the same IOMMU-group. For this, the XDP program loaded on the physical interfaces (`prog_physical.o`) applies a VLAN tag corresponding to the interface id and redirects the packets to the router interface (identified by the `router_iface` bpf map). In the other direction the XDP program loaded on the router interface (`prog_router.o`) removes one layer of VLAN tagging and redirects the packets to the interface read from the VLAN tag. The helper program `set_router_iface` is used to update the `router_iface` bpf map to point to the interface passed as argument to the program. Co-authored-by: Demi Marie Obenour <demiobenour@gmail.com> Signed-off-by: Yureka Lilian <yureka@cyberchaos.dev> --- pkgs/default.nix | 4 + release/checks/pkg-tests.nix | 1 + tools/default.nix | 14 +- tools/meson.build | 4 + tools/meson_options.txt | 3 + tools/xdp-forwarder/include/parsing_helpers.h | 274 ++++++++++++++++++ tools/xdp-forwarder/include/rewrite_helpers.h | 146 ++++++++++ tools/xdp-forwarder/meson.build | 48 +++ tools/xdp-forwarder/prog_physical.c | 39 +++ tools/xdp-forwarder/prog_router.c | 43 +++ tools/xdp-forwarder/set_router_iface.c | 30 ++ 11 files changed, 603 insertions(+), 3 deletions(-) create mode 100644 tools/xdp-forwarder/include/parsing_helpers.h create mode 100644 tools/xdp-forwarder/include/rewrite_helpers.h create mode 100644 tools/xdp-forwarder/meson.build create mode 100644 tools/xdp-forwarder/prog_physical.c create mode 100644 tools/xdp-forwarder/prog_router.c create mode 100644 tools/xdp-forwarder/set_router_iface.c diff --git a/pkgs/default.nix b/pkgs/default.nix index 2472218..df3cfdc 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -42,6 +42,10 @@ let appSupport = false; hostSupport = true; }; + spectrum-driver-tools = self.callSpectrumPackage ../tools { + appSupport = false; + driverSupport = true; + }; xdg-desktop-portal-spectrum-host = self.callSpectrumPackage ../tools/xdg-desktop-portal-spectrum-host {}; diff --git a/release/checks/pkg-tests.nix b/release/checks/pkg-tests.nix index d7be42b..b1a048f 100644 --- a/release/checks/pkg-tests.nix +++ b/release/checks/pkg-tests.nix @@ -14,5 +14,6 @@ import ../../lib/call-package.nix ( tools = lib.recurseIntoAttrs (callSpectrumPackage ../../tools { appSupport = true; hostSupport = true; + driverSupport = true; }).tests; }) (_: {}) diff --git a/tools/default.nix b/tools/default.nix index 201afae..6132c61 100644 --- a/tools/default.nix +++ b/tools/default.nix @@ -1,13 +1,17 @@ # SPDX-License-Identifier: MIT # SPDX-FileCopyrightText: 2022-2025 Alyssa Ross <hi@alyssa.is> +# SPDX-FileCopyrightText: 2025 Yureka Lilian <yureka@cyberchaos.dev> import ../lib/call-package.nix ( { src, lib, stdenv, fetchCrate, fetchurl, runCommand, buildPackages , meson, ninja, pkg-config, rustc , clang-tools, clippy , dbus +# clang 19 (current nixpkgs default) is too old to support -fwrapv-pointer +, clang_21, libbpf , appSupport ? true , hostSupport ? false +, driverSupport ? false }: let @@ -70,15 +74,18 @@ stdenv.mkDerivation (finalAttrs: { ./lsvm ./start-vmm ./subprojects + ] ++ lib.optionals driverSupport [ + ./xdp-forwarder ])); }; sourceRoot = "source/tools"; depsBuildBuild = lib.optionals hostSupport [ buildPackages.stdenv.cc ]; nativeBuildInputs = [ meson ninja ] - ++ lib.optionals appSupport [ pkg-config ] - ++ lib.optionals hostSupport [ rustc ]; - buildInputs = lib.optionals appSupport [ dbus ]; + ++ lib.optionals (appSupport || driverSupport) [ pkg-config ] + ++ lib.optionals hostSupport [ rustc ] + ++ lib.optionals driverSupport [ clang_21 ]; + buildInputs = lib.optionals appSupport [ dbus ] ++ lib.optionals driverSupport [ libbpf ]; postPatch = lib.optionals hostSupport (lib.concatMapStringsSep "\n" (crate: '' mkdir -p subprojects/packagecache @@ -88,6 +95,7 @@ stdenv.mkDerivation (finalAttrs: { mesonFlags = [ (lib.mesonBool "app" appSupport) (lib.mesonBool "host" hostSupport) + (lib.mesonBool "driver" driverSupport) "-Dhostfsrootdir=/run/virtiofs/virtiofs0" "-Dtests=false" "-Dunwind=false" diff --git a/tools/meson.build b/tools/meson.build index e8b0cf2..059baeb 100644 --- a/tools/meson.build +++ b/tools/meson.build @@ -26,3 +26,7 @@ endif if get_option('app') subdir('xdg-desktop-portal-spectrum') endif + +if get_option('driver') + subdir('xdp-forwarder') +endif diff --git a/tools/meson_options.txt b/tools/meson_options.txt index fb520ae..7b46739 100644 --- a/tools/meson_options.txt +++ b/tools/meson_options.txt @@ -7,6 +7,9 @@ option('host', type : 'boolean', value : false, option('app', type : 'boolean', description : 'Build tools for Spectrum app VMs') +option('driver', type : 'boolean', value : false, + description : 'Build tools for Spectrum driver VMs') + option('hostfsrootdir', type : 'string', value : '/run/host', description : 'Path where the virtio-fs provided by the host will be mounted') diff --git a/tools/xdp-forwarder/include/parsing_helpers.h b/tools/xdp-forwarder/include/parsing_helpers.h new file mode 100644 index 0000000..da09934 --- /dev/null +++ b/tools/xdp-forwarder/include/parsing_helpers.h @@ -0,0 +1,274 @@ +/* SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) */ +/* SPDX-FileCopyrightText: 2021 The xdp-tutorial Authors */ +/* Vendored from https://github.com/xdp-project/xdp-tutorial/blob/d3d3eed6ea9a63d1302bfa8b5a8... */ +/* + * This file contains parsing functions that are used in the packetXX XDP + * programs. The functions are marked as __always_inline, and fully defined in + * this header file to be included in the BPF program. + * + * Each helper parses a packet header, including doing bounds checking, and + * returns the type of its contents if successful, and -1 otherwise. + * + * For Ethernet and IP headers, the content type is the type of the payload + * (h_proto for Ethernet, nexthdr for IPv6), for ICMP it is the ICMP type field. + * All return values are in host byte order. + * + * The versions of the functions included here are slightly expanded versions of + * the functions in the packet01 lesson. For instance, the Ethernet header + * parsing has support for parsing VLAN tags. + */ + +#ifndef __PARSING_HELPERS_H +#define __PARSING_HELPERS_H + +#include <stddef.h> +#include <linux/if_ether.h> +#include <linux/if_packet.h> +#include <linux/ip.h> +#include <linux/ipv6.h> +#include <linux/icmp.h> +#include <linux/icmpv6.h> +#include <linux/udp.h> +#include <linux/tcp.h> + +/* Header cursor to keep track of current parsing position */ +struct hdr_cursor { + void *pos; +}; + +/* + * struct vlan_hdr - vlan header + * @h_vlan_TCI: priority and VLAN ID + * @h_vlan_encapsulated_proto: packet type ID or len + */ +struct vlan_hdr { + __be16 h_vlan_TCI; + __be16 h_vlan_encapsulated_proto; +}; + +/* + * Struct icmphdr_common represents the common part of the icmphdr and icmp6hdr + * structures. + */ +struct icmphdr_common { + __u8 type; + __u8 code; + __sum16 cksum; +}; + +/* Allow users of header file to redefine VLAN max depth */ +#ifndef VLAN_MAX_DEPTH +#define VLAN_MAX_DEPTH 2 +#endif + +#define VLAN_VID_MASK 0x0fff /* VLAN Identifier */ +/* Struct for collecting VLANs after parsing via parse_ethhdr_vlan */ +struct collect_vlans { + __u16 id[VLAN_MAX_DEPTH]; +}; + +static __always_inline int proto_is_vlan(__u16 h_proto) +{ + return !!(h_proto == bpf_htons(ETH_P_8021Q) || + h_proto == bpf_htons(ETH_P_8021AD)); +} + +/* Notice, parse_ethhdr() will skip VLAN tags, by advancing nh->pos and returns + * next header EtherType, BUT the ethhdr pointer supplied still points to the + * Ethernet header. Thus, caller can look at eth->h_proto to see if this was a + * VLAN tagged packet. + */ +static __always_inline int parse_ethhdr_vlan(struct hdr_cursor *nh, + void *data_end, + struct ethhdr **ethhdr, + struct collect_vlans *vlans) +{ + struct ethhdr *eth = nh->pos; + int hdrsize = sizeof(*eth); + struct vlan_hdr *vlh; + __u16 h_proto; + int i; + + /* Byte-count bounds check; check if current pointer + size of header + * is after data_end. + */ + if (nh->pos + hdrsize > data_end) + return -1; + + nh->pos += hdrsize; + *ethhdr = eth; + vlh = nh->pos; + h_proto = eth->h_proto; + + /* Use loop unrolling to avoid the verifier restriction on loops; + * support up to VLAN_MAX_DEPTH layers of VLAN encapsulation. + */ + #pragma unroll + for (i = 0; i < VLAN_MAX_DEPTH; i++) { + if (!proto_is_vlan(h_proto)) + break; + + if (vlh + 1 > data_end) + break; + + h_proto = vlh->h_vlan_encapsulated_proto; + if (vlans) /* collect VLAN ids */ + vlans->id[i] = + (bpf_ntohs(vlh->h_vlan_TCI) & VLAN_VID_MASK); + + vlh++; + } + + nh->pos = vlh; + return h_proto; /* network-byte-order */ +} + +static __always_inline int parse_ethhdr(struct hdr_cursor *nh, + void *data_end, + struct ethhdr **ethhdr) +{ + /* Expect compiler removes the code that collects VLAN ids */ + return parse_ethhdr_vlan(nh, data_end, ethhdr, NULL); +} + +static __always_inline int parse_ip6hdr(struct hdr_cursor *nh, + void *data_end, + struct ipv6hdr **ip6hdr) +{ + struct ipv6hdr *ip6h = nh->pos; + + /* Pointer-arithmetic bounds check; pointer +1 points to after end of + * thing being pointed to. We will be using this style in the remainder + * of the tutorial. + */ + if (ip6h + 1 > data_end) + return -1; + + nh->pos = ip6h + 1; + *ip6hdr = ip6h; + + return ip6h->nexthdr; +} + +static __always_inline int parse_iphdr(struct hdr_cursor *nh, + void *data_end, + struct iphdr **iphdr) +{ + struct iphdr *iph = nh->pos; + int hdrsize; + + if (iph + 1 > data_end) + return -1; + + hdrsize = iph->ihl * 4; + /* Sanity check packet field is valid */ + if(hdrsize < sizeof(*iph)) + return -1; + + /* Variable-length IPv4 header, need to use byte-based arithmetic */ + if (nh->pos + hdrsize > data_end) + return -1; + + nh->pos += hdrsize; + *iphdr = iph; + + return iph->protocol; +} + +static __always_inline int parse_icmp6hdr(struct hdr_cursor *nh, + void *data_end, + struct icmp6hdr **icmp6hdr) +{ + struct icmp6hdr *icmp6h = nh->pos; + + if (icmp6h + 1 > data_end) + return -1; + + nh->pos = icmp6h + 1; + *icmp6hdr = icmp6h; + + return icmp6h->icmp6_type; +} + +static __always_inline int parse_icmphdr(struct hdr_cursor *nh, + void *data_end, + struct icmphdr **icmphdr) +{ + struct icmphdr *icmph = nh->pos; + + if (icmph + 1 > data_end) + return -1; + + nh->pos = icmph + 1; + *icmphdr = icmph; + + return icmph->type; +} + +static __always_inline int parse_icmphdr_common(struct hdr_cursor *nh, + void *data_end, + struct icmphdr_common **icmphdr) +{ + struct icmphdr_common *h = nh->pos; + + if (h + 1 > data_end) + return -1; + + nh->pos = h + 1; + *icmphdr = h; + + return h->type; +} + +/* + * parse_udphdr: parse the udp header and return the length of the udp payload + */ +static __always_inline int parse_udphdr(struct hdr_cursor *nh, + void *data_end, + struct udphdr **udphdr) +{ + int len; + struct udphdr *h = nh->pos; + + if (h + 1 > data_end) + return -1; + + nh->pos = h + 1; + *udphdr = h; + + len = bpf_ntohs(h->len) - sizeof(struct udphdr); + if (len < 0) + return -1; + + return len; +} + +/* + * parse_tcphdr: parse and return the length of the tcp header + */ +static __always_inline int parse_tcphdr(struct hdr_cursor *nh, + void *data_end, + struct tcphdr **tcphdr) +{ + int len; + struct tcphdr *h = nh->pos; + + if (h + 1 > data_end) + return -1; + + len = h->doff * 4; + /* Sanity check packet field is valid */ + if(len < sizeof(*h)) + return -1; + + /* Variable-length TCP header, need to use byte-based arithmetic */ + if (nh->pos + len > data_end) + return -1; + + nh->pos += len; + *tcphdr = h; + + return len; +} + +#endif /* __PARSING_HELPERS_H */ diff --git a/tools/xdp-forwarder/include/rewrite_helpers.h b/tools/xdp-forwarder/include/rewrite_helpers.h new file mode 100644 index 0000000..71aa23e --- /dev/null +++ b/tools/xdp-forwarder/include/rewrite_helpers.h @@ -0,0 +1,146 @@ +/* SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-Clause) */ +/* SPDX-FileCopyrightText: 2019 The xdp-tutorial Authors */ +/* Vendored from https://github.com/xdp-project/xdp-tutorial/blob/d3d3eed6ea9a63d1302bfa8b5a8... */ +/* + * This file contains functions that are used in the packetXX XDP programs to + * manipulate on packets data. The functions are marked as __always_inline, and + * fully defined in this header file to be included in the BPF program. + */ + +#ifndef __REWRITE_HELPERS_H +#define __REWRITE_HELPERS_H + +#include <linux/bpf.h> +#include <linux/ip.h> +#include <linux/ipv6.h> +#include <linux/if_ether.h> + +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_endian.h> + +/* Pops the outermost VLAN tag off the packet. Returns the popped VLAN ID on + * success or negative errno on failure. + */ +static __always_inline int vlan_tag_pop(struct xdp_md *ctx, struct ethhdr *eth) +{ + void *data_end = (void *)(long)ctx->data_end; + struct ethhdr eth_cpy; + struct vlan_hdr *vlh; + __be16 h_proto; + int vlid; + + if (!proto_is_vlan(eth->h_proto)) + return -1; + + /* Careful with the parenthesis here */ + vlh = (void *)(eth + 1); + + /* Still need to do bounds checking */ + if (vlh + 1 > data_end) + return -1; + + /* Save vlan ID for returning, h_proto for updating Ethernet header */ + vlid = bpf_ntohs(vlh->h_vlan_TCI); + h_proto = vlh->h_vlan_encapsulated_proto; + + /* Make a copy of the outer Ethernet header before we cut it off */ + __builtin_memcpy(ð_cpy, eth, sizeof(eth_cpy)); + + /* Actually adjust the head pointer */ + if (bpf_xdp_adjust_head(ctx, (int)sizeof(*vlh))) + return -1; + + /* Need to re-evaluate data *and* data_end and do new bounds checking + * after adjusting head + */ + eth = (void *)(long)ctx->data; + data_end = (void *)(long)ctx->data_end; + if (eth + 1 > data_end) + return -1; + + /* Copy back the old Ethernet header and update the proto type */ + __builtin_memcpy(eth, ð_cpy, sizeof(*eth)); + eth->h_proto = h_proto; + + return vlid; +} + +/* Pushes a new VLAN tag after the Ethernet header. Returns 0 on success, + * -1 on failure. + */ +static __always_inline int vlan_tag_push(struct xdp_md *ctx, + struct ethhdr *eth, int vlid) +{ + void *data_end = (void *)(long)ctx->data_end; + struct ethhdr eth_cpy; + struct vlan_hdr *vlh; + + /* First copy the original Ethernet header */ + __builtin_memcpy(ð_cpy, eth, sizeof(eth_cpy)); + + /* Then add space in front of the packet */ + if (bpf_xdp_adjust_head(ctx, 0 - (int)sizeof(*vlh))) + return -1; + + /* Need to re-evaluate data_end and data after head adjustment, and + * bounds check, even though we know there is enough space (as we + * increased it). + */ + data_end = (void *)(long)ctx->data_end; + eth = (void *)(long)ctx->data; + + if (eth + 1 > data_end) + return -1; + + /* Copy back Ethernet header in the right place, populate VLAN tag with + * ID and proto, and set outer Ethernet header to VLAN type. + */ + __builtin_memcpy(eth, ð_cpy, sizeof(*eth)); + + vlh = (void *)(eth + 1); + + if (vlh + 1 > data_end) + return -1; + + vlh->h_vlan_TCI = bpf_htons(vlid); + vlh->h_vlan_encapsulated_proto = eth->h_proto; + + eth->h_proto = bpf_htons(ETH_P_8021Q); + return 0; +} + +/* + * Swaps destination and source MAC addresses inside an Ethernet header + */ +static __always_inline void swap_src_dst_mac(struct ethhdr *eth) +{ + __u8 h_tmp[ETH_ALEN]; + + __builtin_memcpy(h_tmp, eth->h_source, ETH_ALEN); + __builtin_memcpy(eth->h_source, eth->h_dest, ETH_ALEN); + __builtin_memcpy(eth->h_dest, h_tmp, ETH_ALEN); +} + +/* + * Swaps destination and source IPv6 addresses inside an IPv6 header + */ +static __always_inline void swap_src_dst_ipv6(struct ipv6hdr *ipv6) +{ + struct in6_addr tmp = ipv6->saddr; + + ipv6->saddr = ipv6->daddr; + ipv6->daddr = tmp; +} + +/* + * Swaps destination and source IPv4 addresses inside an IPv4 header + */ +static __always_inline void swap_src_dst_ipv4(struct iphdr *iphdr) +{ + __be32 tmp = iphdr->saddr; + + iphdr->saddr = iphdr->daddr; + iphdr->daddr = tmp; +} + +#endif /* __REWRITE_HELPERS_H */ diff --git a/tools/xdp-forwarder/meson.build b/tools/xdp-forwarder/meson.build new file mode 100644 index 0000000..9b70ce3 --- /dev/null +++ b/tools/xdp-forwarder/meson.build @@ -0,0 +1,48 @@ +# SPDX-License-Identifier: EUPL-1.2+ +# SPDX-FileCopyrightText: 2025 Yureka Lilian <yureka@cyberchaos.dev> +# SPDX-FileCopyrightText: 2025 Demi Marie Obenour <demiobenour@gmail.com> + +libbpf = dependency('libbpf', version : '1.6.2') + +executable('set-router-iface', 'set_router_iface.c', + dependencies : libbpf, + install : true) + +clang = find_program('clang', native: true) + +bpf_o_cmd = [ + clang.full_path(), + '-fno-stack-protector', + '-fno-strict-aliasing', + '-fwrapv', '-fwrapv-pointer', + '-Wall', + '-Wextra', + '-O2', + '-target', 'bpf', + '-I', meson.current_source_dir() + '/include', + '-g', + '-c', + '-o', '@OUTPUT@', + '-MD', + '-MP', + '-MF', '@DEPFILE@', + '--', + '@INPUT@', +] + +prog_router_o = custom_target( + input : 'prog_router.c', + output : 'prog_router.o', + depfile : 'prog_router.o.dep', + command : bpf_o_cmd, + install: true, + install_dir: 'lib/xdp') + +prog_physical_o = custom_target( + input : 'prog_physical.c', + output : 'prog_physical.o', + depfile : 'prog_physical.o.dep', + command : bpf_o_cmd, + install: true, + install_dir: 'lib/xdp') + diff --git a/tools/xdp-forwarder/prog_physical.c b/tools/xdp-forwarder/prog_physical.c new file mode 100644 index 0000000..2b70654 --- /dev/null +++ b/tools/xdp-forwarder/prog_physical.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: EUPL-1.2+ +// SPDX-FileCopyrightText: 2025 Yureka Lilian <yureka@cyberchaos.dev> + +#define VLAN_MAX_DEPTH 1 + +#include <linux/bpf.h> +#include <bpf/bpf_endian.h> +#include "parsing_helpers.h" +#include "rewrite_helpers.h" + +struct { + __uint(type, BPF_MAP_TYPE_DEVMAP); + __type(key, int); + __type(value, int); + __uint(max_entries, 1); + __uint(pinning, LIBBPF_PIN_BY_NAME); +} router_iface SEC(".maps"); + +SEC("xdp") +int physical(struct xdp_md *ctx) +{ + void *data_end = (void *)(long)ctx->data_end; + void *data = (void *)(long)ctx->data; + + struct hdr_cursor nh; + nh.pos = data; + + struct ethhdr *eth; + if (parse_ethhdr(&nh, data_end, ð) < 0) + return XDP_DROP; + + if (ctx->ingress_ifindex < 1 || ctx->ingress_ifindex > VLAN_VID_MASK) + return XDP_DROP; + + if (vlan_tag_push(ctx, eth, ctx->ingress_ifindex) < 0) + return XDP_DROP; + + return bpf_redirect_map(&router_iface, 0, 0); +} diff --git a/tools/xdp-forwarder/prog_router.c b/tools/xdp-forwarder/prog_router.c new file mode 100644 index 0000000..6704b2c --- /dev/null +++ b/tools/xdp-forwarder/prog_router.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: EUPL-1.2+ +// SPDX-FileCopyrightText: 2025 Yureka Lilian <yureka@cyberchaos.dev> + +#define VLAN_MAX_DEPTH 1 + +#include <linux/bpf.h> +#include <bpf/bpf_endian.h> +#include "parsing_helpers.h" +#include "rewrite_helpers.h" + +// The map is actually not used by this program, but just included +// to keep the reference-counted pin alive before any physical interfaces +// are added. +struct { + __uint(type, BPF_MAP_TYPE_DEVMAP); + __type(key, int); + __type(value, int); + __uint(max_entries, 1); + __uint(pinning, LIBBPF_PIN_BY_NAME); +} router_iface SEC(".maps"); + + +SEC("xdp") +int router(struct xdp_md *ctx) +{ + void *data_end = (void *)(long)ctx->data_end; + void *data = (void *)(long)ctx->data; + + struct hdr_cursor nh; + nh.pos = data; + + struct ethhdr *eth; + int r; + if ((r = parse_ethhdr(&nh, data_end, ð)) < 0) + return XDP_DROP; + + int vlid = vlan_tag_pop(ctx, eth); + if (vlid < 0) { + return XDP_DROP; + } + + return bpf_redirect(vlid, 0); +} diff --git a/tools/xdp-forwarder/set_router_iface.c b/tools/xdp-forwarder/set_router_iface.c new file mode 100644 index 0000000..7e6c7a2 --- /dev/null +++ b/tools/xdp-forwarder/set_router_iface.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: EUPL-1.2+ +// SPDX-FileCopyrightText: 2025 Yureka Lilian <yureka@cyberchaos.dev> + +#include <stdio.h> +#include <stdlib.h> +#include <net/if.h> +#include <bpf/bpf.h> +#include <err.h> + +int main(int argc, char **argv) +{ + if (argc < 2) { + err(EXIT_FAILURE, "missing interface name"); + } + + int router_idx = if_nametoindex(argv[1]); + if (router_idx <= 0) { + err(EXIT_FAILURE, "error getting router interface"); + } + + int map_fd = bpf_obj_get("/sys/fs/bpf/router_iface"); + if (map_fd < 0) { + err(EXIT_FAILURE, "failed to open bpf map"); + } + + int id = 0; + if (bpf_map_update_elem(map_fd, &id, &router_idx, 0) < 0) { + err(EXIT_FAILURE, "failed to update bpf map"); + } +} -- 2.51.0

Reviewed-by: Alyssa Ross <hi@alyssa.is> Signed-off-by: Yureka Lilian <yureka@cyberchaos.dev> --- Documentation/about/architecture.adoc | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/Documentation/about/architecture.adoc b/Documentation/about/architecture.adoc index e32ab27..2b86616 100644 --- a/Documentation/about/architecture.adoc +++ b/Documentation/about/architecture.adoc @@ -68,3 +68,23 @@ nix-build img/live --no-out-link | xargs -o nix-tree See the https://diode.zone/w/8DBDQ6HQUe5UUdLkpDuL35[video] of Spectrum live image interactive analysis with nix-tree. + +== Networking + +The net-vm's purpose is running the Linux drivers for any physical +interfaces on the spectrum system. + +A net-vm (there could be multiple, one per IOMMU-group) will load the +xdp-forwarder XDP programs on the passed-through physical interfaces as well +as the downstream virtio interface going into the router (recognized by +its special MAC address) using mdev events. + +The net-vm needs to multiplex between the physical interfaces, as there +might be several interfaces in the same IOMMU-group. + +For this, the xdp-forwarder applies a +VLAN tag corresponding to the interface id, and redirects the packets to +the router interface (identified by the router_iface bpf map). +In the other direction the XDP program loaded on the router interface +removes one layer of VLAN tagging, and redirects the packets to the +interface read from the VLAN tag. -- 2.51.0

Signed-off-by: Yureka Lilian <yureka@cyberchaos.dev> Reviewed-by: Alyssa Ross <hi@alyssa.is> --- vm/sys/net/Makefile | 8 +----- vm/sys/net/default.nix | 21 ++++++++------- vm/sys/net/image/etc/fstab | 2 ++ vm/sys/net/image/etc/mdev/iface | 27 ++++++------------- vm/sys/net/image/etc/nftables.conf | 8 ------ vm/sys/net/image/etc/s6-rc/connman/type | 1 - .../net/image/etc/s6-rc/connman/type.license | 2 -- .../net/image/etc/s6-rc/mdevd-coldplug/type | 1 - .../net/image/etc/s6-rc/nftables/type.license | 2 -- vm/sys/net/image/etc/s6-rc/nftables/up | 6 ----- 10 files changed, 23 insertions(+), 55 deletions(-) delete mode 100644 vm/sys/net/image/etc/nftables.conf delete mode 100644 vm/sys/net/image/etc/s6-rc/connman/type delete mode 100644 vm/sys/net/image/etc/s6-rc/connman/type.license delete mode 100644 vm/sys/net/image/etc/s6-rc/mdevd-coldplug/type delete mode 100644 vm/sys/net/image/etc/s6-rc/nftables/type.license delete mode 100644 vm/sys/net/image/etc/s6-rc/nftables/up diff --git a/vm/sys/net/Makefile b/vm/sys/net/Makefile index 0193cc8..e279e88 100644 --- a/vm/sys/net/Makefile +++ b/vm/sys/net/Makefile @@ -34,12 +34,11 @@ VM_FILES = \ image/etc/init \ image/etc/mdev.conf \ image/etc/mdev/iface \ - image/etc/nftables.conf \ image/etc/passwd \ image/etc/s6-linux-init/run-image/service/getty-hvc0/run \ image/etc/s6-linux-init/scripts/rc.init \ image/etc/sysctl.conf -VM_DIRS = dev etc/s6-linux-init/env run proc sys var/lib/connman +VM_DIRS = dev etc/s6-linux-init/env proc run sys # These are separate because they need to be included, but putting # them as make dependencies would confuse make. @@ -59,9 +58,6 @@ build/rootfs.erofs: ../../../scripts/make-erofs.sh $(PACKAGES_FILE) $(VM_FILES) ) | ../../../scripts/make-erofs.sh $@ VM_S6_RC_FILES = \ - image/etc/s6-rc/connman/dependencies.d/dbus \ - image/etc/s6-rc/connman/run \ - image/etc/s6-rc/connman/type \ image/etc/s6-rc/dbus/notification-fd \ image/etc/s6-rc/dbus/run \ image/etc/s6-rc/dbus/type \ @@ -71,8 +67,6 @@ VM_S6_RC_FILES = \ image/etc/s6-rc/mdevd/notification-fd \ image/etc/s6-rc/mdevd/run \ image/etc/s6-rc/mdevd/type \ - image/etc/s6-rc/nftables/type \ - image/etc/s6-rc/nftables/up \ image/etc/s6-rc/ok-all/contents \ image/etc/s6-rc/ok-all/type \ image/etc/s6-rc/sysctl/type \ diff --git a/vm/sys/net/default.nix b/vm/sys/net/default.nix index 2953c3a..8159247 100644 --- a/vm/sys/net/default.nix +++ b/vm/sys/net/default.nix @@ -1,13 +1,14 @@ # SPDX-License-Identifier: MIT # SPDX-FileCopyrightText: 2021-2023 Alyssa Ross <hi@alyssa.is> +# SPDX-FileCopyrightText: 2025 Yureka Lilian <yureka@cyberchaos.dev> -import ../../../lib/call-package.nix ({ lseek, src, terminfo, pkgsMusl }: +import ../../../lib/call-package.nix ({ lseek, spectrum-driver-tools, src, terminfo, pkgsMusl }: pkgsMusl.callPackage ( { lib, stdenvNoCC, nixos, runCommand, writeClosure , erofs-utils, jq, s6-rc, util-linux, xorg -, busybox, connmanMinimal, dbus, execline, kmod, linux_latest, mdevd, nftables -, s6, s6-linux-init +, busybox, dbus, execline, kmod, linux_latest, mdevd +, s6, s6-linux-init, xdp-tools }: let @@ -50,10 +51,8 @@ let ]; }); - connman = connmanMinimal; - packages = [ - connman dbus execline kmod mdevd s6 s6-linux-init s6-rc + dbus execline kmod mdevd s6 s6-linux-init s6-rc xdp-tools (busybox.override { extraConfig = '' @@ -66,13 +65,16 @@ let CONFIG_RMMOD n ''; }) - - (nftables.override { withCli = false; }) ]; # Packages that should be fully linked into /usr, # (not just their bin/* files). - usrPackages = [ connman dbus firmware kernel.modules terminfo ]; + usrPackages = [ + dbus firmware kernel terminfo + + # for xdp-forwarder + spectrum-driver-tools + ]; packagesSysroot = runCommand "packages-sysroot" { inherit packages; @@ -94,6 +96,7 @@ let system.stateVersion = lib.trivial.release; }); + in stdenvNoCC.mkDerivation { diff --git a/vm/sys/net/image/etc/fstab b/vm/sys/net/image/etc/fstab index 6a82ecc..5a1bbf4 100644 --- a/vm/sys/net/image/etc/fstab +++ b/vm/sys/net/image/etc/fstab @@ -1,6 +1,8 @@ # SPDX-License-Identifier: CC0-1.0 # SPDX-FileCopyrightText: 2020-2021 Alyssa Ross <hi@alyssa.is> +# SPDX-FileCopyrightText: 2025 Yureka Lilian <yureka@cyberchaos.dev> proc /proc proc defaults 0 0 devpts /dev/pts devpts defaults,gid=4,mode=620 0 0 tmpfs /dev/shm tmpfs defaults 0 0 sysfs /sys sysfs defaults 0 0 +bpffs /sys/fs/bpf bpf defaults 0 0 diff --git a/vm/sys/net/image/etc/mdev/iface b/vm/sys/net/image/etc/mdev/iface index 2306575..ff4bf53 100755 --- a/vm/sys/net/image/etc/mdev/iface +++ b/vm/sys/net/image/etc/mdev/iface @@ -1,36 +1,25 @@ #!/bin/execlineb -P # SPDX-License-Identifier: EUPL-1.2+ # SPDX-FileCopyrightText: 2020-2021 Alyssa Ross <hi@alyssa.is> +# SPDX-FileCopyrightText: 2025 Yureka Lilian <yureka@cyberchaos.dev> importas -Si INTERFACE ifte { - # This interface is connected to another VM. - - # The other VM's IP is encoded in the NIC-specific portion of the - # interface's MAC address. - backtick -E CLIENT_IP { - awk -F: "{printf \"100.64.%d.%d\\n\", \"0x\" $5, \"0x\" $6}" - /sys/class/net/${INTERFACE}/address - } - - if { ip address add 169.254.0.1/32 dev $INTERFACE } - if { ip link set $INTERFACE up } - ip route add $CLIENT_IP dev $INTERFACE + # This interface is connected to the router + if { xdp-loader load $INTERFACE /usr/lib/xdp/prog_router.o -m skb -p /sys/fs/bpf } + if { ip link set $INTERFACE promisc on } + if { set-router-iface $INTERFACE } + ip link set $INTERFACE up } { if { test $INTERFACE != lo } # This is a physical connection to a network device. - background { s6-rc -bu change connman } - if { s6-rc -bu change nftables } - if { - forx -pE module { nft_counter nft_masq } - modprobe $module - } - nft add rule ip nat postrouting oifname $INTERFACE counter masquerade + if { xdp-loader load $INTERFACE /usr/lib/xdp/prog_physical.o -m skb -p /sys/fs/bpf } + ip link set $INTERFACE up } grep -iq ^02:01: /sys/class/net/${INTERFACE}/address diff --git a/vm/sys/net/image/etc/nftables.conf b/vm/sys/net/image/etc/nftables.conf deleted file mode 100644 index 296d92c..0000000 --- a/vm/sys/net/image/etc/nftables.conf +++ /dev/null @@ -1,8 +0,0 @@ -# SPDX-License-Identifier: EUPL-1.2+ -# SPDX-FileCopyrightText: 2021 Alyssa Ross <hi@alyssa.is> - -table nat { - chain postrouting { - type nat hook postrouting priority 100; - } -} diff --git a/vm/sys/net/image/etc/s6-rc/connman/type b/vm/sys/net/image/etc/s6-rc/connman/type deleted file mode 100644 index 5883cff..0000000 --- a/vm/sys/net/image/etc/s6-rc/connman/type +++ /dev/null @@ -1 +0,0 @@ -longrun diff --git a/vm/sys/net/image/etc/s6-rc/connman/type.license b/vm/sys/net/image/etc/s6-rc/connman/type.license deleted file mode 100644 index 2b3b032..0000000 --- a/vm/sys/net/image/etc/s6-rc/connman/type.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-License-Identifier: CC0-1.0 -SPDX-FileCopyrightText: 2020 Alyssa Ross <hi@alyssa.is> diff --git a/vm/sys/net/image/etc/s6-rc/mdevd-coldplug/type b/vm/sys/net/image/etc/s6-rc/mdevd-coldplug/type deleted file mode 100644 index bdd22a1..0000000 --- a/vm/sys/net/image/etc/s6-rc/mdevd-coldplug/type +++ /dev/null @@ -1 +0,0 @@ -oneshot diff --git a/vm/sys/net/image/etc/s6-rc/nftables/type.license b/vm/sys/net/image/etc/s6-rc/nftables/type.license deleted file mode 100644 index c49c11b..0000000 --- a/vm/sys/net/image/etc/s6-rc/nftables/type.license +++ /dev/null @@ -1,2 +0,0 @@ -SPDX-License-Identifier: CC0-1.0 -SPDX-FileCopyrightText: 2021 Alyssa Ross <hi@alyssa.is> diff --git a/vm/sys/net/image/etc/s6-rc/nftables/up b/vm/sys/net/image/etc/s6-rc/nftables/up deleted file mode 100644 index 7d5f141..0000000 --- a/vm/sys/net/image/etc/s6-rc/nftables/up +++ /dev/null @@ -1,6 +0,0 @@ -# SPDX-License-Identifier: EUPL-1.2+ -# SPDX-FileCopyrightText: 2021 Alyssa Ross <hi@alyssa.is> - -if { modprobe nft_chain_nat } - -nft -f /etc/nftables.conf -- 2.51.0