This is required for any use of vhost-user. Since any VM might have a vhost-user device hotplugged, this needs to be on from the start for all VMs. Signed-off-by: Alyssa Ross <hi@alyssa.is> --- host/start-vm/lib.rs | 2 +- host/start-vm/tests/vm_command-basic.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/host/start-vm/lib.rs b/host/start-vm/lib.rs index ef79091..9a92ab7 100644 --- a/host/start-vm/lib.rs +++ b/host/start-vm/lib.rs @@ -44,7 +44,7 @@ pub fn vm_command(dir: PathBuf, config_root: &Path) -> Result<Command, String> { command.arg("cloud-hypervisor"); command.args(&["--api-socket", "env/cloud-hypervisor.sock"]); command.args(&["--cmdline", "console=ttyS0 root=PARTLABEL=root"]); - command.args(&["--memory", "size=128M"]); + command.args(&["--memory", "size=128M,shared=on"]); command.args(&["--console", "pty"]); command.arg("--kernel"); command.arg(config_dir.join("vmlinux")); diff --git a/host/start-vm/tests/vm_command-basic.rs b/host/start-vm/tests/vm_command-basic.rs index b9585f9..4145b94 100644 --- a/host/start-vm/tests/vm_command-basic.rs +++ b/host/start-vm/tests/vm_command-basic.rs @@ -37,7 +37,7 @@ fn main() -> std::io::Result<()> { OsStr::new("--cmdline"), OsStr::new("console=ttyS0 root=PARTLABEL=root"), OsStr::new("--memory"), - OsStr::new("size=128M"), + OsStr::new("size=128M,shared=on"), OsStr::new("--console"), OsStr::new("pty"), OsStr::new("--kernel"), -- 2.37.1

Signed-off-by: Alyssa Ross <hi@alyssa.is> --- host/start-vm/lib.rs | 26 +++++++++++++ host/start-vm/tests/meson.build | 2 + host/start-vm/tests/vm_command-shared-dir.rs | 41 ++++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 host/start-vm/tests/vm_command-shared-dir.rs diff --git a/host/start-vm/lib.rs b/host/start-vm/lib.rs index 9a92ab7..3959566 100644 --- a/host/start-vm/lib.rs +++ b/host/start-vm/lib.rs @@ -112,6 +112,32 @@ pub fn vm_command(dir: PathBuf, config_root: &Path) -> Result<Command, String> { return Err("no block devices specified".to_string()); } + let shared_dirs_dir = config_dir.join("shared-dirs"); + match shared_dirs_dir.read_dir().map(Iterator::peekable) { + Ok(mut entries) => { + if entries.peek().is_some() { + command.arg("--fs"); + } + + for result in entries { + let entry = result + .map_err(|e| format!("examining directory entry: {}", e))? + .file_name(); + + let mut arg = OsString::from("tag="); + arg.push(&entry); + arg.push(",socket=../"); + arg.push(vm_name); + arg.push("-fs-"); + arg.push(&entry); + arg.push("/env/virtiofsd.sock"); + command.arg(arg); + } + } + Err(e) if e.kind() == ErrorKind::NotFound => {} + Err(e) => return Err(format!("reading directory {:?}: {}", shared_dirs_dir, e)), + } + command.arg("--serial").arg({ let mut serial = OsString::from("file=/run/"); serial.push(&vm_name); diff --git a/host/start-vm/tests/meson.build b/host/start-vm/tests/meson.build index 00bd33d..857414b 100644 --- a/host/start-vm/tests/meson.build +++ b/host/start-vm/tests/meson.build @@ -31,3 +31,5 @@ test('tap_open (name too long)', executable('tap_open-name-too-long', test('vm_command-basic', executable('vm_command-basic', 'vm_command-basic.rs', link_with : [rust_lib, rust_helper])) +test('vm_command-shared-dir', executable('vm_command-shared-dir', + 'vm_command-shared-dir.rs', link_with : [rust_lib, rust_helper])) diff --git a/host/start-vm/tests/vm_command-shared-dir.rs b/host/start-vm/tests/vm_command-shared-dir.rs new file mode 100644 index 0000000..2b13663 --- /dev/null +++ b/host/start-vm/tests/vm_command-shared-dir.rs @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: EUPL-1.2+ +// SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is> + +use std::fs::{create_dir, create_dir_all, File}; +use std::os::unix::fs::symlink; + +use start_vm::vm_command; +use test_helper::TempDir; + +fn contains_seq<H: PartialEq<N>, N>(haystack: &[H], needle: &[N]) -> bool { + let start_indexes = 0..=(haystack.len() - needle.len()); + let mut candidates = start_indexes.map(|i| &haystack[i..][..needle.len()]); + candidates.any(|c| c == needle) +} + +fn main() -> std::io::Result<()> { + let tmp_dir = TempDir::new()?; + + let service_dir = tmp_dir.path().join("testvm"); + let vm_config = tmp_dir.path().join("svc/data/testvm"); + + create_dir(&service_dir)?; + create_dir_all(&vm_config)?; + File::create(vm_config.join("vmlinux"))?; + create_dir(vm_config.join("blk"))?; + symlink("/dev/null", vm_config.join("blk/root.img"))?; + + create_dir(vm_config.join("shared-dirs"))?; + create_dir(vm_config.join("shared-dirs/root"))?; + symlink("/", vm_config.join("shared-dirs/root/dir"))?; + + let expected_args = [ + "--fs", + "tag=root,socket=../testvm-fs-root/env/virtiofsd.sock", + ]; + + let command = vm_command(service_dir, &tmp_dir.path().join("svc/data")).unwrap(); + let args: Box<[_]> = command.get_args().collect(); + assert!(contains_seq(&args, &expected_args)); + Ok(()) +} -- 2.37.1

The approach of having a template service for each virtiofsd comes from the work I need on Wayland support for Unikie, hence the Unikie copyright in etc/s6-rc/ext-rc-init/up. Signed-off-by: Alyssa Ross <hi@alyssa.is> --- host/rootfs/Makefile | 3 +++ host/rootfs/default.nix | 4 ++-- host/rootfs/etc/s6-rc/ext-rc-init/up | 11 +++++++++++ host/rootfs/etc/template/fs/notification-fd | 1 + host/rootfs/etc/template/fs/notification-fd.license | 2 ++ host/rootfs/etc/template/fs/run | 11 +++++++++++ host/rootfs/etc/template/fs/type | 1 + host/rootfs/etc/template/fs/type.license | 2 ++ 8 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 host/rootfs/etc/template/fs/notification-fd create mode 100644 host/rootfs/etc/template/fs/notification-fd.license create mode 100755 host/rootfs/etc/template/fs/run create mode 100644 host/rootfs/etc/template/fs/type create mode 100644 host/rootfs/etc/template/fs/type.license diff --git a/host/rootfs/Makefile b/host/rootfs/Makefile index 31f76d2..249af6d 100644 --- a/host/rootfs/Makefile +++ b/host/rootfs/Makefile @@ -28,6 +28,9 @@ FILES = \ etc/mdev/wait \ etc/parse-devname \ etc/passwd \ + etc/template/fs/notification-fd \ + etc/template/fs/run \ + etc/template/fs/type \ etc/s6-linux-init/run-image/service/getty-tty1/run \ etc/s6-linux-init/run-image/service/getty-tty2/run \ etc/s6-linux-init/run-image/service/getty-tty3/run \ diff --git a/host/rootfs/default.nix b/host/rootfs/default.nix index 4788628..7932845 100644 --- a/host/rootfs/default.nix +++ b/host/rootfs/default.nix @@ -7,7 +7,7 @@ pkgs.pkgsStatic.callPackage ( { lib, stdenvNoCC, nixos, runCommand, writeReferencesToFile, s6-rc, tar2ext4 , busybox, cloud-hypervisor, cryptsetup, execline, jq, kmod -, mdevd, s6, s6-linux-init, socat, util-linuxMinimal, xorg +, mdevd, s6, s6-linux-init, socat, util-linuxMinimal, virtiofsd, xorg }: let @@ -45,7 +45,7 @@ let packages = [ cloud-hypervisor execline jq kmod mdevd s6 s6-linux-init s6-rc socat - start-vm + start-vm virtiofsd (cryptsetup.override { programs = { diff --git a/host/rootfs/etc/s6-rc/ext-rc-init/up b/host/rootfs/etc/s6-rc/ext-rc-init/up index 1aec7fb..53ab127 100644 --- a/host/rootfs/etc/s6-rc/ext-rc-init/up +++ b/host/rootfs/etc/s6-rc/ext-rc-init/up @@ -1,5 +1,6 @@ # SPDX-License-Identifier: EUPL-1.2+ # SPDX-FileCopyrightText: 2021-2022 Alyssa Ross <hi@alyssa.is> +# SPDX-FileCopyrightText: 2022 Unikie if { mkdir -p /run/s6-rc.ext.src } @@ -15,6 +16,16 @@ if { if { redirfd -w 1 ${name}/notification-fd echo 3 } if { ln -s -- /bin/start-vm ${name}/run } + if { + elglob -0 paths ${dir}/shared-dirs/* + forx -pE path { $paths } + backtick -E fsname { basename -- $path } + if { cp -R -- /etc/template/fs ${name}-fs-${fsname} } + if { mkdir -- ${name}-fs-${fsname}/data ${name}-fs-${fsname}/env } + if { cp -P -- ${path}/dir ${name}-fs-${fsname}/data/shared-dir } + touch -- ${name}/dependencies.d/${name}-fs-${fsname} + } + elglob -0 paths /ext/svc/data/${name}/providers/net/* forx -pE path { $paths } backtick -E dep { basename -- $path } diff --git a/host/rootfs/etc/template/fs/notification-fd b/host/rootfs/etc/template/fs/notification-fd new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/host/rootfs/etc/template/fs/notification-fd @@ -0,0 +1 @@ +3 diff --git a/host/rootfs/etc/template/fs/notification-fd.license b/host/rootfs/etc/template/fs/notification-fd.license new file mode 100644 index 0000000..5a40633 --- /dev/null +++ b/host/rootfs/etc/template/fs/notification-fd.license @@ -0,0 +1,2 @@ +SPDX-License-Identifier: CC0-1.0 +SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is> diff --git a/host/rootfs/etc/template/fs/run b/host/rootfs/etc/template/fs/run new file mode 100755 index 0000000..5ca214a --- /dev/null +++ b/host/rootfs/etc/template/fs/run @@ -0,0 +1,11 @@ +#!/bin/execlineb -P +# SPDX-License-Identifier: EUPL-1.2+ +# SPDX-FileCopyrightText: 2021-2022 Alyssa Ross <hi@alyssa.is> + +s6-ipcserver-socketbinder -a 0700 -B env/virtiofsd.sock + +if { fdmove 1 3 echo } +fdclose 3 + +export TMPDIR /run +virtiofsd --fd 0 --shared-dir data/shared-dir diff --git a/host/rootfs/etc/template/fs/type b/host/rootfs/etc/template/fs/type new file mode 100644 index 0000000..5883cff --- /dev/null +++ b/host/rootfs/etc/template/fs/type @@ -0,0 +1 @@ +longrun diff --git a/host/rootfs/etc/template/fs/type.license b/host/rootfs/etc/template/fs/type.license new file mode 100644 index 0000000..5a40633 --- /dev/null +++ b/host/rootfs/etc/template/fs/type.license @@ -0,0 +1,2 @@ +SPDX-License-Identifier: CC0-1.0 +SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is> -- 2.37.1

Signed-off-by: Alyssa Ross <hi@alyssa.is> --- Documentation/_sass/custom/custom.scss | 22 ++++++++++++++++++++++ Documentation/creating-vms.adoc | 8 ++++++++ 2 files changed, 30 insertions(+) create mode 100644 Documentation/_sass/custom/custom.scss diff --git a/Documentation/_sass/custom/custom.scss b/Documentation/_sass/custom/custom.scss new file mode 100644 index 0000000..eb50c81 --- /dev/null +++ b/Documentation/_sass/custom/custom.scss @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is> +// SPDX-License-Identifier: MIT + +.main-content { + // Just the Docs puts titles next to definitions, rather than on + // the line before. This makes the definitions too horizontally + // compressed. + dl { + display: block; + } + dt { + text-align: left; + font-weight: bold; + + &::after { + content: none; + } + } + dd { + margin-left: 2ch; + } +} diff --git a/Documentation/creating-vms.adoc b/Documentation/creating-vms.adoc index 6d4fde0..efae489 100644 --- a/Documentation/creating-vms.adoc +++ b/Documentation/creating-vms.adoc @@ -19,6 +19,14 @@ blk:: A directory containing disk images (with file names ending in ".img") that will be provided to the guest as a virtio-blk device. Order is not guaranteed. At least one image is *required*. +shared-dirs:: A directory whose *subdirectories* each describe a +directory to be shared with a VM. The name of each subdirectory is +used as the virtiofs tag, and the contents are as follows: + + dir::: A symbolic link pointing to the directory to be shared with + the VM. The directory must be on the xref:user-partition.adoc[user + data partition]. *Required.* + providers/net:: A directory containing a file named for each VM that should provide networking to this VM. The contents of these files are ignored. -- 2.37.1

Signed-off-by: Alyssa Ross <hi@alyssa.is> --- vm-lib/make-vm.nix | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/vm-lib/make-vm.nix b/vm-lib/make-vm.nix index 2c50ca5..61e0b2d 100644 --- a/vm-lib/make-vm.nix +++ b/vm-lib/make-vm.nix @@ -13,11 +13,12 @@ pkgs.pkgsStatic.callPackage ( { lib, runCommand, writeReferencesToFile, e2fsprogs, tar2ext4 }: -{ run, providers ? {} }: +{ run, providers ? {}, sharedDirs ? {} }: let inherit (lib) - any attrValues concatLists concatStrings hasInfix mapAttrsToList; + any attrValues concatLists concatStrings concatStringsSep hasInfix + mapAttrsToList; in assert !(any (hasInfix "\n") (concatLists (attrValues providers))); @@ -29,7 +30,7 @@ runCommand "spectrum-vm" { (mapAttrsToList (kind: map (vm: "${kind}/${vm}\n")) providers)); passAsFile = [ "providerDirs" ]; } '' - mkdir -p "$out"/{blk,providers} + mkdir -p "$out"/{blk,providers,shared-dirs} mkdir root cd root @@ -40,11 +41,22 @@ runCommand "spectrum-vm" { tar2ext4 -i ../run.tar -o "$out/blk/run.img" e2label "$out/blk/run.img" ext - pushd "$out/providers" + pushd "$out" + + pushd providers xargs -rd '\n' dirname -- < "$providerDirsPath" | xargs -rd '\n' mkdir -p -- xargs -rd '\n' touch -- < "$providerDirsPath" popd + pushd shared-dirs + ${concatStringsSep "\n" (mapAttrsToList (key: { path }: '' + mkdir ${lib.escapeShellArg key} + ln -s ${lib.escapeShellArgs [ path "${key}/dir" ]} + '') sharedDirs)} + popd + + popd + ln -s /usr/img/appvm/blk/root.img "$out/blk" ln -s /usr/img/appvm/vmlinux "$out" '' -- 2.37.1

Signed-off-by: Alyssa Ross <hi@alyssa.is> --- .gitignore | 5 ++++- img/app/Makefile | 19 ++++++++++++++++--- img/app/shell.nix | 6 +++++- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index a97f309..802df73 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,10 @@ # SPDX-License-Identifier: CC0-1.0 -# SPDX-FileCopyrightText: 2021 Alyssa Ross <hi@alyssa.is> +# SPDX-FileCopyrightText: 2021-2022 Alyssa Ross <hi@alyssa.is> /config.nix build/ result result-* + +# s6-ipcserver-socketbinder creates lockfiles for sockets. +*.sock.lock diff --git a/img/app/Makefile b/img/app/Makefile index c5a4684..7d8e1c1 100644 --- a/img/app/Makefile +++ b/img/app/Makefile @@ -5,7 +5,10 @@ # Nixpkgs. If you don't have qemu-kvm, you'll need to set e.g. # QEMU_KVM = qemu-system-x86_64 -enable-kvm. QEMU_KVM = qemu-kvm +BACKGROUND = background CLOUD_HYPERVISOR = cloud-hypervisor +S6_IPCSERVER_SOCKETBINDER = s6-ipcserver-socketbinder +VIRTIOFSD = virtiofsd prefix = /usr/local imgdir = $(prefix)/img @@ -94,25 +97,35 @@ build/etc/s6-rc: $(VM_S6_RC_FILES) s6-rc-compile $@ $$dir; \ exit=$$?; rm -r $$dir; exit $$exit -run-qemu: build/host/appvm/blk/root.img +start-virtiofsd: + $(S6_IPCSERVER_SOCKETBINDER) -B virtiofsd.sock \ + $(BACKGROUND) $(VIRTIOFSD) --fd 0 --shared-dir . "" +.PHONY: start-virtiofsd + +run-qemu: build/host/appvm/blk/root.img start-virtiofsd $(QEMU_KVM) -m 128 -cpu host -machine q35,kernel=$(KERNEL) -vga none \ -drive file=build/host/appvm/blk/root.img,if=virtio,format=raw,readonly=on \ -drive file=$(RUN_IMG),if=virtio,format=raw,readonly=on \ -append "console=ttyS0 root=PARTLABEL=root" \ -netdev user,id=net0 \ -device virtio-net,netdev=net0,mac=0A:B3:EC:00:00:00 \ + -chardev socket,id=virtiofsd,path=virtiofsd.sock \ + -device vhost-user-fs-pci,chardev=virtiofsd,tag=virtiofs0 \ + -object memory-backend-memfd,id=mem,size=128M,share=on \ + -numa node,memdev=mem \ -chardev vc,id=virtiocon0 \ -device virtio-serial-pci \ -device virtconsole,chardev=virtiocon0 .PHONY: run-qemu -run-cloud-hypervisor: build/host/appvm/blk/root.img +run-cloud-hypervisor: build/host/appvm/blk/root.img start-virtiofsd $(CLOUD_HYPERVISOR) \ --api-socket path=vmm.sock \ - --memory size=128M \ + --memory size=128M,shared=on \ --disk path=build/host/appvm/blk/root.img,readonly=on \ path=$(RUN_IMG),readonly=on \ --net tap=tap0,mac=0A:B3:EC:00:00:00 \ + --fs tag=virtiofs0,socket=virtiofsd.sock \ --kernel $(KERNEL) \ --cmdline "console=ttyS0 root=PARTLABEL=root" \ --console tty \ diff --git a/img/app/shell.nix b/img/app/shell.nix index 83dcd76..60ca465 100644 --- a/img/app/shell.nix +++ b/img/app/shell.nix @@ -12,7 +12,11 @@ with config.pkgs; { nativeBuildInputs = nativeBuildInputs ++ [ - cloud-hypervisor jq qemu_kvm reuse + # Both QEMU and virtiofsd come with a virtiofsd executable, + # so we have to list virtiofsd first. + virtiofsd + + cloud-hypervisor execline jq qemu_kvm reuse s6 ]; KERNEL = "${passthru.kernel.dev}/vmlinux"; -- 2.37.1

Not all VMs will even have networking! So it makes more sense to put waiting for the network in the application-specific part. The "listen" and "wait" scripts are copied from the host system. Signed-off-by: Alyssa Ross <hi@alyssa.is> --- img/app/Makefile | 2 ++ img/app/etc/mdev/iface | 4 ++-- img/app/etc/mdev/listen | 12 ++++++++++++ img/app/etc/mdev/wait | 15 +++++++++++++++ img/app/etc/s6-rc/ok-all/contents | 1 + vm/app/catgirl.nix | 1 + vm/app/lynx.nix | 1 + 7 files changed, 34 insertions(+), 2 deletions(-) create mode 100755 img/app/etc/mdev/listen create mode 100755 img/app/etc/mdev/wait diff --git a/img/app/Makefile b/img/app/Makefile index 7d8e1c1..9aae612 100644 --- a/img/app/Makefile +++ b/img/app/Makefile @@ -50,6 +50,8 @@ VM_FILES = \ etc/init \ etc/mdev.conf \ etc/mdev/iface \ + etc/mdev/listen \ + etc/mdev/wait \ etc/passwd \ etc/resolv.conf \ etc/s6-linux-init/scripts/rc.init diff --git a/img/app/etc/mdev/iface b/img/app/etc/mdev/iface index d8ceda5..93a12eb 100755 --- a/img/app/etc/mdev/iface +++ b/img/app/etc/mdev/iface @@ -1,6 +1,6 @@ #!/bin/execlineb -P # SPDX-License-Identifier: EUPL-1.2+ -# SPDX-FileCopyrightText: 2020-2021 Alyssa Ross <hi@alyssa.is> +# SPDX-FileCopyrightText: 2020-2022 Alyssa Ross <hi@alyssa.is> importas -i INTERFACE INTERFACE @@ -33,4 +33,4 @@ foreground { } } -s6-rc -u change app +/etc/mdev/listen network-online diff --git a/img/app/etc/mdev/listen b/img/app/etc/mdev/listen new file mode 100755 index 0000000..6bc6f2c --- /dev/null +++ b/img/app/etc/mdev/listen @@ -0,0 +1,12 @@ +#!/bin/execlineb -s1 +# SPDX-License-Identifier: EUPL-1.2+ +# SPDX-FileCopyrightText: 2021 Alyssa Ross <hi@alyssa.is> + +foreground { + redirfd -w 2 /dev/null + foreground { mkdir /run/wait } + mkfifo /run/wait/${1} +} + +redirfd -w 1 /run/wait/${1} +echo diff --git a/img/app/etc/mdev/wait b/img/app/etc/mdev/wait new file mode 100755 index 0000000..3b85de8 --- /dev/null +++ b/img/app/etc/mdev/wait @@ -0,0 +1,15 @@ +#!/bin/execlineb -s1 +# SPDX-License-Identifier: EUPL-1.2+ +# SPDX-FileCopyrightText: 2021 Alyssa Ross <hi@alyssa.is> + +foreground { + redirfd -w 2 /dev/null + foreground { mkdir /run/wait } + mkfifo /run/wait/${1} +} + +foreground { + redirfd -w 1 /dev/null + head -1 /run/wait/${1} +} +rm /run/wait/${1} diff --git a/img/app/etc/s6-rc/ok-all/contents b/img/app/etc/s6-rc/ok-all/contents index c4ea84f..92f3ef1 100644 --- a/img/app/etc/s6-rc/ok-all/contents +++ b/img/app/etc/s6-rc/ok-all/contents @@ -2,3 +2,4 @@ # SPDX-FileCopyrightText: 2021 Alyssa Ross <hi@alyssa.is> # mdevd-coldplug +app diff --git a/vm/app/catgirl.nix b/vm/app/catgirl.nix index a4c05e3..54181c5 100644 --- a/vm/app/catgirl.nix +++ b/vm/app/catgirl.nix @@ -9,6 +9,7 @@ import ../make-vm.nix { inherit config; } { { writeScript, catgirl }: writeScript "run-catgirl" '' #!/bin/execlineb -P + if { /etc/mdev/wait network-online } foreground { printf "IRC nick (to join #spectrum): " } backtick -E nick { head -1 } ${catgirl}/bin/catgirl -h irc.libera.chat -j "#spectrum" -n $nick diff --git a/vm/app/lynx.nix b/vm/app/lynx.nix index 00d449e..af25df8 100644 --- a/vm/app/lynx.nix +++ b/vm/app/lynx.nix @@ -9,6 +9,7 @@ import ../make-vm.nix { inherit config; } { { writeScript, lynx }: writeScript "run-lynx" '' #!/bin/execlineb -P + if { /etc/mdev/wait network-online } ${lynx}/bin/lynx https://spectrum-os.org '' ) { }; -- 2.37.1

Signed-off-by: Alyssa Ross <hi@alyssa.is> --- img/app/Makefile | 1 + img/app/etc/mdev.conf | 3 ++- img/app/etc/mdev/virtiofs | 10 ++++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) create mode 100755 img/app/etc/mdev/virtiofs diff --git a/img/app/Makefile b/img/app/Makefile index 9aae612..259e819 100644 --- a/img/app/Makefile +++ b/img/app/Makefile @@ -51,6 +51,7 @@ VM_FILES = \ etc/mdev.conf \ etc/mdev/iface \ etc/mdev/listen \ + etc/mdev/virtiofs \ etc/mdev/wait \ etc/passwd \ etc/resolv.conf \ diff --git a/img/app/etc/mdev.conf b/img/app/etc/mdev.conf index f114719..e370da8 100644 --- a/img/app/etc/mdev.conf +++ b/img/app/etc/mdev.conf @@ -1,5 +1,6 @@ # SPDX-License-Identifier: EUPL-1.2+ -# SPDX-FileCopyrightText: 2021 Alyssa Ross <hi@alyssa.is> +# SPDX-FileCopyrightText: 2021-2022 Alyssa Ross <hi@alyssa.is> -$MODALIAS=.* 0:0 660 +importas -iu MODALIAS MODALIAS modprobe -q $MODALIAS $INTERFACE=.* 0:0 660 ! +/etc/mdev/iface +$MODALIAS=virtio:d0000001Av.* 0:0 660 ! +/etc/mdev/virtiofs diff --git a/img/app/etc/mdev/virtiofs b/img/app/etc/mdev/virtiofs new file mode 100755 index 0000000..a028002 --- /dev/null +++ b/img/app/etc/mdev/virtiofs @@ -0,0 +1,10 @@ +#!/bin/execlineb -P +# SPDX-License-Identifier: EUPL-1.2+ +# SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is> + +# There's currently no kernel mechanism to let us enumerate available +# virtiofs filesystems, so we only supported a single, hardcoded +# filesystem tag for now. +if { mkdir -p /run/virtiofs/virtiofs0 } +if { mount -t virtiofs virtiofs0 /run/virtiofs/virtiofs0 } +/etc/mdev/listen virtiofs0 -- 2.37.1

mg is a text editor with a built-in filesystem browser, so it will be useful for testing virtiofs. Signed-off-by: Alyssa Ross <hi@alyssa.is> --- host/initramfs/extfs.nix | 4 +++- vm/app/mg.nix | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 vm/app/mg.nix diff --git a/host/initramfs/extfs.nix b/host/initramfs/extfs.nix index dec4017..0ee337b 100644 --- a/host/initramfs/extfs.nix +++ b/host/initramfs/extfs.nix @@ -11,18 +11,20 @@ let appvm-catgirl = import ../../vm/app/catgirl.nix { inherit config; }; appvm-lynx = import ../../vm/app/lynx.nix { inherit config; }; + appvm-mg = import ../../vm/app/mg.nix { inherit config; }; in runCommand "ext.ext4" { nativeBuildInputs = [ tar2ext4 ]; } '' - mkdir -p svc/data/appvm-{catgirl,lynx} + mkdir -p svc/data/appvm-{catgirl,lynx,mg} tar -C ${netvm} -c data | tar -C svc -x chmod +w svc/data tar -C ${appvm-catgirl} -c . | tar -C svc/data/appvm-catgirl -x tar -C ${appvm-lynx} -c . | tar -C svc/data/appvm-lynx -x + tar -C ${appvm-mg} -c . | tar -C svc/data/appvm-mg -x tar -cf ext.tar svc tar2ext4 -i ext.tar -o $out diff --git a/vm/app/mg.nix b/vm/app/mg.nix new file mode 100644 index 0000000..391b518 --- /dev/null +++ b/vm/app/mg.nix @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is> + +{ config ? import ../../../nix/eval-config.nix {} }: + +import ../make-vm.nix { inherit config; } { + providers.net = [ "netvm" ]; + sharedDirs.virtiofs0.path = "/ext"; + run = "${config.pkgs.pkgsStatic.mg}/bin/mg"; +} -- 2.37.1

Signed-off-by: Alyssa Ross <hi@alyssa.is> --- vm/app/mg.nix | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/vm/app/mg.nix b/vm/app/mg.nix index 391b518..fdfef8f 100644 --- a/vm/app/mg.nix +++ b/vm/app/mg.nix @@ -6,5 +6,12 @@ import ../make-vm.nix { inherit config; } { providers.net = [ "netvm" ]; sharedDirs.virtiofs0.path = "/ext"; - run = "${config.pkgs.pkgsStatic.mg}/bin/mg"; + run = config.pkgs.pkgsStatic.callPackage ( + { writeScript, mg }: + writeScript "run-mg" '' + #!/bin/execlineb -P + if { /etc/mdev/wait virtiofs0 } + ${mg}/bin/mg /run/virtiofs/virtiofs0 + '' + ) { }; } -- 2.37.1

Broadly, it makes sense to do things in s6-rc rather than mdev commands where possible, since mdev commands are not run in parallel. That's unlikely to make a big difference in this case, but we might want to do more things before mounting as well (e.g. growing the filesystem) and I think it makes sense to do that here. Signed-off-by: Alyssa Ross <hi@alyssa.is> --- host/rootfs/etc/mdev/block/add | 1 - host/rootfs/etc/s6-rc/ext/up | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/host/rootfs/etc/mdev/block/add b/host/rootfs/etc/mdev/block/add index 75e0c31..3e24355 100755 --- a/host/rootfs/etc/mdev/block/add +++ b/host/rootfs/etc/mdev/block/add @@ -35,5 +35,4 @@ if { } if { ln -s $devname ext } -if { mount ext /ext } /etc/mdev/listen ext diff --git a/host/rootfs/etc/s6-rc/ext/up b/host/rootfs/etc/s6-rc/ext/up index d9e0426..4e0b9fc 100644 --- a/host/rootfs/etc/s6-rc/ext/up +++ b/host/rootfs/etc/s6-rc/ext/up @@ -1,4 +1,5 @@ # SPDX-License-Identifier: EUPL-1.2+ # SPDX-FileCopyrightText: 2021-2022 Alyssa Ross <hi@alyssa.is> -/etc/mdev/wait ext +if { /etc/mdev/wait ext } +mount /dev/ext /ext -- 2.37.1

In future, we probably want to make sure we only do this once. I haven't decided what the best way to do that would be. Since we're likely going to switch to btrfs anyway for the user partition, it's probably not worth thinking about too much. Signed-off-by: Alyssa Ross <hi@alyssa.is> --- host/rootfs/default.nix | 10 +++++++--- host/rootfs/etc/s6-rc/ext/up | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/host/rootfs/default.nix b/host/rootfs/default.nix index 7932845..0b08603 100644 --- a/host/rootfs/default.nix +++ b/host/rootfs/default.nix @@ -6,7 +6,7 @@ pkgs.pkgsStatic.callPackage ( { lib, stdenvNoCC, nixos, runCommand, writeReferencesToFile, s6-rc, tar2ext4 -, busybox, cloud-hypervisor, cryptsetup, execline, jq, kmod +, busybox, cloud-hypervisor, cryptsetup, execline, e2fsprogs, jq, kmod , mdevd, s6, s6-linux-init, socat, util-linuxMinimal, virtiofsd, xorg }: @@ -44,8 +44,8 @@ let foot = pkgsGui.foot.override { allowPgo = false; }; packages = [ - cloud-hypervisor execline jq kmod mdevd s6 s6-linux-init s6-rc socat - start-vm virtiofsd + cloud-hypervisor e2fsprogs execline jq kmod mdevd s6 s6-linux-init s6-rc + socat start-vm virtiofsd (cryptsetup.override { programs = { @@ -57,11 +57,15 @@ let (busybox.override { extraConfig = '' + CONFIG_CHATTR n CONFIG_DEPMOD n CONFIG_FINDFS n CONFIG_INIT n CONFIG_INSMOD n + CONFIG_LSATTR n CONFIG_LSMOD n + CONFIG_MKE2FS n + CONFIG_MKFS_EXT2 n CONFIG_MODINFO n CONFIG_MODPROBE n CONFIG_RMMOD n diff --git a/host/rootfs/etc/s6-rc/ext/up b/host/rootfs/etc/s6-rc/ext/up index 4e0b9fc..c189471 100644 --- a/host/rootfs/etc/s6-rc/ext/up +++ b/host/rootfs/etc/s6-rc/ext/up @@ -2,4 +2,5 @@ # SPDX-FileCopyrightText: 2021-2022 Alyssa Ross <hi@alyssa.is> if { /etc/mdev/wait ext } +if { resize2fs /dev/ext } mount /dev/ext /ext -- 2.37.1

So that we can test things that write to disk. Unlinking the file prior to starting the VM ensures that it won't stick around once it's no longer needed. I disable make's printing of the commands in favour of the shell's, because the cp can take some time on filesystems that don't support reflinks, and it's nice to see which command is currently running rather than all of them at once as make would show. Signed-off-by: Alyssa Ross <hi@alyssa.is> --- host/rootfs/Makefile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/host/rootfs/Makefile b/host/rootfs/Makefile index 249af6d..bad52a2 100644 --- a/host/rootfs/Makefile +++ b/host/rootfs/Makefile @@ -144,14 +144,20 @@ build/live.img: $(SCRIPTS)/format-uuid.sh $(SCRIPTS)/make-gpt.sh build/rootfs.ve mv $@.tmp $@ run: build/live.img $(EXT_FS) build/rootfs.verity.roothash - $(QEMU_KVM) -cpu host -m 2G \ + @set -x && \ + ext="$$(mktemp build/spectrum-rootfs-extfs.XXXXXXXXXX.img)" && \ + cp $(EXT_FS) "$$ext" && \ + exec 3<>"$$ext" && \ + rm -f "$$ext" && \ + truncate -s +10G /proc/self/fd/3 && \ + exec $(QEMU_KVM) -cpu host -m 2G \ -machine q35,kernel=$(KERNEL),kernel-irqchip=split,initrd=$(INITRAMFS) \ -display gtk,gl=on \ -qmp unix:vmm.sock,server,nowait \ -monitor vc \ -parallel none \ -drive file=build/live.img,if=virtio,format=raw,readonly=on \ - -drive file=$(EXT_FS),if=virtio,format=raw,readonly=on \ + -drive file=/proc/self/fd/3,if=virtio,format=raw \ -append "console=ttyS0 roothash=$$(< build/rootfs.verity.roothash) ext=/dev/vdb intel_iommu=on" \ -device intel-iommu,intremap=on \ -device virtio-vga-gl \ -- 2.37.1

tar2ext4 is designed for making read-only, minimally-sized ext4 images. That's great for the Spectrum host system's root filesystem, but it doesn't make sense for the user data partition, because even if we mark it read/write after running tar2ext4, there will be very few inodes available, so it won't be very useful for storing data. To fix this, we use the normal tools for creating an ext4 filesystem. We create a sparse image of an extremely large size to ensure there's enough space available for metadata and things for the partition to grow to fit any disk a user might want to to use Spectrum on, then shrink the image down to its minimum size. Doing it this way assumes that the build environment will always support sparse files. If this turns out not to be the case, we can use mkfs.ext4's -E resize= option to set the maximum expected disk size, and use a smaller size for the initial image size. I haven't done that here to avoid having to pick that smaller number. Now that the filesystem is no longer marked as read-only, we need to fsck it before every mount, or it won't mount any more after the first time. Signed-off-by: Alyssa Ross <hi@alyssa.is> --- host/initramfs/extfs.nix | 16 +++++++++++----- host/rootfs/etc/s6-rc/ext/up | 1 + 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/host/initramfs/extfs.nix b/host/initramfs/extfs.nix index 0ee337b..a510c02 100644 --- a/host/initramfs/extfs.nix +++ b/host/initramfs/extfs.nix @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: 2021-2022 Alyssa Ross <hi@alyssa.is> # SPDX-License-Identifier: MIT -{ config, runCommand, tar2ext4 }: +{ config, runCommand, e2fsprogs, tar2ext4 }: let netvm = import ../../vm/sys/net { @@ -15,9 +15,10 @@ let in runCommand "ext.ext4" { - nativeBuildInputs = [ tar2ext4 ]; + nativeBuildInputs = [ e2fsprogs ]; } '' - mkdir -p svc/data/appvm-{catgirl,lynx,mg} + mkdir -p root/svc/data/appvm-{catgirl,lynx,mg,nix} + cd root tar -C ${netvm} -c data | tar -C svc -x chmod +w svc/data @@ -26,6 +27,11 @@ runCommand "ext.ext4" { tar -C ${appvm-lynx} -c . | tar -C svc/data/appvm-lynx -x tar -C ${appvm-mg} -c . | tar -C svc/data/appvm-mg -x - tar -cf ext.tar svc - tar2ext4 -i ext.tar -o $out + mkfs.ext4 -d . $out 16T + resize2fs -M $out + + # The generated image will have all files owned by the uid and gid + # mkfs.ext4 was run as, so we need to normalize ownership to root. + find -exec echo $'set_inode_field {} uid 0\nset_inode_field {} gid 0' ';' | + debugfs -wf - $out '' diff --git a/host/rootfs/etc/s6-rc/ext/up b/host/rootfs/etc/s6-rc/ext/up index c189471..78b2637 100644 --- a/host/rootfs/etc/s6-rc/ext/up +++ b/host/rootfs/etc/s6-rc/ext/up @@ -2,5 +2,6 @@ # SPDX-FileCopyrightText: 2021-2022 Alyssa Ross <hi@alyssa.is> if { /etc/mdev/wait ext } +foreground { fsck.ext4 -fp /dev/ext } if { resize2fs /dev/ext } mount /dev/ext /ext -- 2.37.1

Nix profile generations are always symlinks to store paths, i.e. paths starting with /nix. On the Spectrum host, these paths would resolve to the host's /nix, which isn't going to contain the user's VM definitions. So to enable managing VM definitions with Nix profiles, the host needs to resolve VM definition symlinks specially. It needs to consider the root directory to be somewhere on the user data partition, so that the user can place their Nix store at /nix/store. This special chroot-like symlink resolution is only done to open the VM definition directory. Configuration within VM definition directories is opened as normal. It would probably be better to use a subdirectory of /ext (e.g. /ext/vm-root) as the root directory for resolving symlinks, to keep the root directory of the user data partition tidy, and to make it clearer that the Nix store used for VM definitions is not a global Nix store for all things on the user data partition. I haven't done that here yet, because it would require handling the additional case of the root directory not existing. Signed-off-by: Alyssa Ross <hi@alyssa.is> --- host/start-vm/fs.c | 17 +++++ host/start-vm/fs.rs | 68 +++++++++++++++++++ host/start-vm/lib.rs | 32 +++++---- host/start-vm/meson.build | 2 +- host/start-vm/start-vm.rs | 15 ++-- host/start-vm/tests/meson.build | 2 + host/start-vm/tests/vm_command-basic.rs | 4 +- .../tests/vm_command-config-symlink.rs | 30 ++++++++ host/start-vm/tests/vm_command-shared-dir.rs | 4 +- 9 files changed, 152 insertions(+), 22 deletions(-) create mode 100644 host/start-vm/fs.c create mode 100644 host/start-vm/fs.rs create mode 100644 host/start-vm/tests/vm_command-config-symlink.rs diff --git a/host/start-vm/fs.c b/host/start-vm/fs.c new file mode 100644 index 0000000..d8ec1a1 --- /dev/null +++ b/host/start-vm/fs.c @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: EUPL-1.2+ +// SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is> + +#include <fcntl.h> +#include <unistd.h> + +#include <sys/syscall.h> + +#include <linux/openat2.h> + +int openat_path_resolve_in_root(int dirfd, const char *pathname) +{ + struct open_how how = { 0 }; + how.flags = O_PATH | O_CLOEXEC; + how.resolve = RESOLVE_IN_ROOT | RESOLVE_NO_MAGICLINKS | RESOLVE_NO_XDEV; + return syscall(SYS_openat2, dirfd, pathname, &how, sizeof how); +} diff --git a/host/start-vm/fs.rs b/host/start-vm/fs.rs new file mode 100644 index 0000000..4bc6ada --- /dev/null +++ b/host/start-vm/fs.rs @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: EUPL-1.2+ +// SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is> + +use std::ffi::CString; +use std::fs::{read_link, File}; +use std::io::{Error, Result}; +use std::os::raw::c_char; +use std::os::unix::prelude::*; +use std::path::{Path, PathBuf}; + +extern "C" { + fn openat_path_resolve_in_root(dirfd: BorrowedFd, pathname: *const c_char) -> Option<OwnedFd>; +} + +// We can't make a const Path yet. +// https://github.com/rust-lang/rust/issues/102619 +const PROC_FD_PATH: &str = "/proc/self/fd"; + +pub trait OwnedFdExt: AsFd + AsRawFd + Sized { + /// Returns the path in proc(5) that corresponds to this file descriptor. + fn fd_path(&self) -> PathBuf { + let mut path = PathBuf::from(PROC_FD_PATH); + path.push(self.as_raw_fd().to_string()); + path + } + + /// Returns the path to this file, according to the magic link in proc(5). + fn path(&self) -> Result<PathBuf> { + read_link(self.fd_path()) + } +} + +impl OwnedFdExt for OwnedFd {} + +/// An `O_PATH` file descriptor. +#[derive(Debug)] +#[repr(transparent)] +pub struct Root(OwnedFd); + +fn c_string(path: impl Into<PathBuf>) -> CString { + let bytes = path.into().into_os_string().into_vec(); + CString::new(bytes).unwrap() +} + +impl Root { + /// Panics if `path` contains NUL bytes. + pub fn open(path: impl AsRef<Path>) -> Result<Self> { + Ok(Self(OwnedFd::from(File::open(path)?))) + } + + /// Resolves `path` as if the directory represented by `self` was the root directory + /// (including for symlink resolution). The returned file descriptor will not have + /// the `O_CLOEXEC` flag set. + /// + /// Panics if `path` contains NUL bytes. + pub fn resolve_no_cloexec(&self, path: impl Into<PathBuf>) -> Result<OwnedFd> { + let c_path = c_string(path); + + unsafe { openat_path_resolve_in_root(self.as_fd(), c_path.as_ptr()) } + .ok_or_else(Error::last_os_error) + } +} + +impl AsFd for Root { + fn as_fd(&self) -> BorrowedFd { + self.0.as_fd() + } +} diff --git a/host/start-vm/lib.rs b/host/start-vm/lib.rs index 3959566..f167efe 100644 --- a/host/start-vm/lib.rs +++ b/host/start-vm/lib.rs @@ -2,16 +2,19 @@ // SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is> mod ch; +pub mod fs; mod net; use std::borrow::Cow; use std::env::args_os; use std::ffi::{CString, OsStr, OsString}; +use std::fs::read_dir; use std::io::{self, ErrorKind}; use std::os::unix::prelude::*; use std::path::{Path, PathBuf}; use std::process::Command; +use fs::{OwnedFdExt, Root}; use net::{format_mac, net_setup, NetConfig}; pub fn prog_name() -> String { @@ -25,7 +28,7 @@ pub fn prog_name() -> String { .into_owned() } -pub fn vm_command(dir: PathBuf, config_root: &Path) -> Result<Command, String> { +pub fn vm_command(dir: PathBuf, root: &Root) -> Result<Command, String> { let dir = dir.into_os_string().into_vec(); let dir = PathBuf::from(OsString::from_vec(dir)); @@ -37,7 +40,10 @@ pub fn vm_command(dir: PathBuf, config_root: &Path) -> Result<Command, String> { return Err(format!("VM name may not contain a comma: {:?}", vm_name)); } - let config_dir = config_root.join(vm_name); + let config = root + .resolve_no_cloexec(Path::new("svc/data").join(vm_name)) + .and_then(|fd| fd.path()) + .map_err(|e| format!("resolving configuration directory for {:?}: {}", vm_name, e))?; let mut command = Command::new("s6-notifyoncheck"); command.args(&["-dc", "test -S env/cloud-hypervisor.sock"]); @@ -46,11 +52,11 @@ pub fn vm_command(dir: PathBuf, config_root: &Path) -> Result<Command, String> { command.args(&["--cmdline", "console=ttyS0 root=PARTLABEL=root"]); command.args(&["--memory", "size=128M,shared=on"]); command.args(&["--console", "pty"]); - command.arg("--kernel"); - command.arg(config_dir.join("vmlinux")); - let net_providers_dir = config_dir.join("providers/net"); - match net_providers_dir.read_dir() { + command.arg("--kernel"); + command.arg(config.join("vmlinux")); + + match read_dir(config.join("providers/net")) { Ok(entries) => { for r in entries { let entry = r @@ -78,13 +84,12 @@ pub fn vm_command(dir: PathBuf, config_root: &Path) -> Result<Command, String> { } } Err(e) if e.kind() == ErrorKind::NotFound => {} - Err(e) => return Err(format!("reading directory {:?}: {}", net_providers_dir, e)), + Err(e) => return Err(format!("reading providers/net: {}", e)), } command.arg("--disk"); - let blk_dir = config_dir.join("blk"); - match blk_dir.read_dir() { + match read_dir(config.join("blk")) { Ok(entries) => { for result in entries { let entry = result @@ -105,15 +110,14 @@ pub fn vm_command(dir: PathBuf, config_root: &Path) -> Result<Command, String> { command.arg(arg); } } - Err(e) => return Err(format!("reading directory {:?}: {}", blk_dir, e)), + Err(e) => return Err(format!("reading blk: {}", e)), } if command.get_args().last() == Some(OsStr::new("--disk")) { return Err("no block devices specified".to_string()); } - let shared_dirs_dir = config_dir.join("shared-dirs"); - match shared_dirs_dir.read_dir().map(Iterator::peekable) { + match read_dir(config.join("shared-dirs")).map(Iterator::peekable) { Ok(mut entries) => { if entries.peek().is_some() { command.arg("--fs"); @@ -135,7 +139,7 @@ pub fn vm_command(dir: PathBuf, config_root: &Path) -> Result<Command, String> { } } Err(e) if e.kind() == ErrorKind::NotFound => {} - Err(e) => return Err(format!("reading directory {:?}: {}", shared_dirs_dir, e)), + Err(e) => return Err(format!("reading shared-dirs: {}", e)), } command.arg("--serial").arg({ @@ -154,7 +158,7 @@ mod tests { #[test] fn test_vm_name_comma() { - assert!(vm_command("/v,m".into(), Path::new("/")) + assert!(vm_command("/v,m".into(), &Root::open("/").unwrap()) .unwrap_err() .contains("comma")); } diff --git a/host/start-vm/meson.build b/host/start-vm/meson.build index 74b88f0..de1defc 100644 --- a/host/start-vm/meson.build +++ b/host/start-vm/meson.build @@ -7,7 +7,7 @@ project('start-vm', 'rust', 'c', add_project_arguments('-D_GNU_SOURCE', language : 'c') add_project_arguments('-C', 'panic=abort', language : 'rust') -c_lib = static_library('start-vm', 'net.c', 'net-util.c') +c_lib = static_library('start-vm', 'fs.c', 'net.c', 'net-util.c') rust_lib = static_library('start_vm', 'lib.rs', link_with : c_lib) executable('start-vm', 'start-vm.rs', link_with : rust_lib, install : true) diff --git a/host/start-vm/start-vm.rs b/host/start-vm/start-vm.rs index 4790841..92b34b6 100644 --- a/host/start-vm/start-vm.rs +++ b/host/start-vm/start-vm.rs @@ -3,20 +3,25 @@ use std::env::current_dir; use std::os::unix::prelude::*; -use std::path::Path; use std::process::exit; +use start_vm::fs::Root; use start_vm::{prog_name, vm_command}; -const CONFIG_ROOT: &str = "/ext/svc/data"; +const ROOT: &str = "/ext"; fn run() -> String { - let dir = match current_dir().map_err(|e| format!("getting current directory: {}", e)) { + let dir = match current_dir() { Ok(dir) => dir, - Err(e) => return e, + Err(e) => return format!("getting current directory: {}", e), }; - match vm_command(dir, Path::new(CONFIG_ROOT)) { + let config_root = match Root::open(ROOT) { + Ok(fd) => fd, + Err(e) => return format!("opening {}: {}", ROOT, e), + }; + + match vm_command(dir, &config_root) { Ok(mut command) => format!("failed to exec: {}", command.exec()), Err(e) => e, } diff --git a/host/start-vm/tests/meson.build b/host/start-vm/tests/meson.build index 857414b..efd78bc 100644 --- a/host/start-vm/tests/meson.build +++ b/host/start-vm/tests/meson.build @@ -31,5 +31,7 @@ test('tap_open (name too long)', executable('tap_open-name-too-long', test('vm_command-basic', executable('vm_command-basic', 'vm_command-basic.rs', link_with : [rust_lib, rust_helper])) +test('vm_command-config-symlink', executable('vm_command-config-symlink', + 'vm_command-config-symlink.rs', link_with : [rust_lib, rust_helper])) test('vm_command-shared-dir', executable('vm_command-shared-dir', 'vm_command-shared-dir.rs', link_with : [rust_lib, rust_helper])) diff --git a/host/start-vm/tests/vm_command-basic.rs b/host/start-vm/tests/vm_command-basic.rs index 4145b94..234f6e3 100644 --- a/host/start-vm/tests/vm_command-basic.rs +++ b/host/start-vm/tests/vm_command-basic.rs @@ -4,11 +4,13 @@ use std::ffi::{OsStr, OsString}; use std::fs::{create_dir, create_dir_all, File}; +use start_vm::fs::Root; use start_vm::vm_command; use test_helper::TempDir; fn main() -> std::io::Result<()> { let tmp_dir = TempDir::new()?; + let root = Root::open(tmp_dir.path())?; let service_dir = tmp_dir.path().join("testvm"); create_dir(&service_dir)?; @@ -21,7 +23,7 @@ fn main() -> std::io::Result<()> { File::create(&kernel_path)?; File::create(&image_path)?; - let command = vm_command(service_dir, &tmp_dir.path().join("svc/data")).unwrap(); + let command = vm_command(service_dir, &root).unwrap(); assert_eq!(command.get_program(), "s6-notifyoncheck"); let mut expected_disk_arg = OsString::from("path="); diff --git a/host/start-vm/tests/vm_command-config-symlink.rs b/host/start-vm/tests/vm_command-config-symlink.rs new file mode 100644 index 0000000..e1e7112 --- /dev/null +++ b/host/start-vm/tests/vm_command-config-symlink.rs @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: EUPL-1.2+ +// SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is> + +use std::fs::{create_dir, create_dir_all, File}; +use std::os::unix::fs::symlink; + +use start_vm::fs::Root; +use start_vm::vm_command; +use test_helper::TempDir; + +fn main() -> std::io::Result<()> { + let tmp_dir = TempDir::new().unwrap(); + let root = Root::open(tmp_dir.path()).unwrap(); + + let service_dir = tmp_dir.path().join("testvm"); + create_dir(&service_dir).unwrap(); + + create_dir(tmp_dir.path().join("svc"))?; + create_dir(tmp_dir.path().join("svc/data"))?; + symlink("/config-1/testvm", tmp_dir.path().join("svc/data/testvm"))?; + + let vm_config = tmp_dir.path().join("config-1/testvm"); + create_dir_all(&vm_config)?; + File::create(vm_config.join("vmlinux"))?; + create_dir(vm_config.join("blk"))?; + symlink("/dev/null", vm_config.join("blk/root.img"))?; + + vm_command(service_dir, &root).unwrap(); + Ok(()) +} diff --git a/host/start-vm/tests/vm_command-shared-dir.rs b/host/start-vm/tests/vm_command-shared-dir.rs index 2b13663..092d55c 100644 --- a/host/start-vm/tests/vm_command-shared-dir.rs +++ b/host/start-vm/tests/vm_command-shared-dir.rs @@ -4,6 +4,7 @@ use std::fs::{create_dir, create_dir_all, File}; use std::os::unix::fs::symlink; +use start_vm::fs::Root; use start_vm::vm_command; use test_helper::TempDir; @@ -15,6 +16,7 @@ fn contains_seq<H: PartialEq<N>, N>(haystack: &[H], needle: &[N]) -> bool { fn main() -> std::io::Result<()> { let tmp_dir = TempDir::new()?; + let root = Root::open(tmp_dir.path())?; let service_dir = tmp_dir.path().join("testvm"); let vm_config = tmp_dir.path().join("svc/data/testvm"); @@ -34,7 +36,7 @@ fn main() -> std::io::Result<()> { "tag=root,socket=../testvm-fs-root/env/virtiofsd.sock", ]; - let command = vm_command(service_dir, &tmp_dir.path().join("svc/data")).unwrap(); + let command = vm_command(service_dir, &root).unwrap(); let args: Box<[_]> = command.get_args().collect(); assert!(contains_seq(&args, &expected_args)); Ok(()) -- 2.37.1

Signed-off-by: Alyssa Ross <hi@alyssa.is> --- Documentation/creating-vms.adoc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Documentation/creating-vms.adoc b/Documentation/creating-vms.adoc index efae489..8150f87 100644 --- a/Documentation/creating-vms.adoc +++ b/Documentation/creating-vms.adoc @@ -8,7 +8,9 @@ Each VM is configured using a directory under svc/data/ on the xref:user-partition.adoc[user partition]. The name of the directory -determines the name of the VM. +determines the name of the VM. If the directory is a symbolic link, +`/` will be the root directory of the user partition, rather than the +root directory of the Spectrum host system, while resolving it. The directory can contain the following files: -- 2.37.1

This is a VM that provides a shell set up to run Nix. It has network access, and full filesystem access to the user data partition. The included vm-rebuild command makes it possible to build Spectrum VMs with Nix. It's expected that vm-config/vms.nix be a Nix expression that builds a directory of Spectrum VM definitions. Code from Spectrum (i.e. vm-lib) can be placed at vm-config/spectrum and will appear as <spectrum> in the NIX_PATH. This code is not included as part of the Nix VM, because it would be bad for reproducibility if updating the host system changed the Nix expressions used to build VMs. Nix-built VMs are each individually symlinked into svc/data, so that managing VMs with Nix is not all-or-nothing. vm-rebuild will not yet remove symlinks pointing to VMs that no longer exist in the current generation, but that shouldn't be difficult to fix — just delete any broken symlinks pointing into the Nix store. Signed-off-by: Alyssa Ross <hi@alyssa.is> --- host/initramfs/extfs.nix | 2 ++ vm/app/nix/bin/vm-rebuild | 25 +++++++++++++++++++++++ vm/app/nix/default.nix | 43 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+) create mode 100755 vm/app/nix/bin/vm-rebuild create mode 100644 vm/app/nix/default.nix diff --git a/host/initramfs/extfs.nix b/host/initramfs/extfs.nix index a510c02..9f00793 100644 --- a/host/initramfs/extfs.nix +++ b/host/initramfs/extfs.nix @@ -12,6 +12,7 @@ let appvm-catgirl = import ../../vm/app/catgirl.nix { inherit config; }; appvm-lynx = import ../../vm/app/lynx.nix { inherit config; }; appvm-mg = import ../../vm/app/mg.nix { inherit config; }; + appvm-nix = import ../../vm/app/nix { inherit config; }; in runCommand "ext.ext4" { @@ -26,6 +27,7 @@ runCommand "ext.ext4" { tar -C ${appvm-catgirl} -c . | tar -C svc/data/appvm-catgirl -x tar -C ${appvm-lynx} -c . | tar -C svc/data/appvm-lynx -x tar -C ${appvm-mg} -c . | tar -C svc/data/appvm-mg -x + tar -C ${appvm-nix} -c . | tar -C svc/data/appvm-nix -x mkfs.ext4 -d . $out 16T resize2fs -M $out diff --git a/vm/app/nix/bin/vm-rebuild b/vm/app/nix/bin/vm-rebuild new file mode 100755 index 0000000..98eae10 --- /dev/null +++ b/vm/app/nix/bin/vm-rebuild @@ -0,0 +1,25 @@ +#!/bin/execlineb -S1 +# SPDX-License-Identifier: EUPL-1.2+ +# SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is> + +if -n { + if -n { test $# -eq 1 -a $1 = switch } + fdmove -c 1 2 + echo "Usage: ${0} switch" +} + +cd /run/virtiofs/virtiofs0 + +backtick -E dir { mktemp -d } +foreground { + if { nix-build -o ${dir}/system <spectrum-vms> } + if { nix-env -p nix/var/nix/profiles/vms --set ${dir}/system } + backtick -E vmsdir { resolve_in_root . nix/var/nix/profiles/vms } + cd $vmsdir + elglob -0 glob * + forx -E vm { $glob } + ln -s /nix/var/nix/profiles/vms/${vm} /run/virtiofs/virtiofs0/svc/data +} +importas -iu ? ? +background { rm -rf $dir } +exit $? diff --git a/vm/app/nix/default.nix b/vm/app/nix/default.nix new file mode 100644 index 0000000..9427ca4 --- /dev/null +++ b/vm/app/nix/default.nix @@ -0,0 +1,43 @@ +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is> + +{ config ? import ../../../../nix/eval-config.nix {} }: + +import ../../make-vm.nix { inherit config; } { + providers.net = [ "netvm" ]; + sharedDirs.virtiofs0.path = "/ext"; + run = config.pkgs.pkgsStatic.callPackage ( + { lib, runCommand, writeScript, nix }: + let + inherit (lib) concatStringsSep const hasSuffix makeBinPath; + + bin = builtins.filterSource (name: _type: + name == toString bin/. || name == toString bin/vm-rebuild) ./.; + + nixPath = [ + "nixpkgs=https://nixos.org/channels/nixos-unstable/nixexprs.tar.xz" + "spectrum=/run/virtiofs/virtiofs0/vm-config/spectrum" + "spectrum-vms=/run/virtiofs/virtiofs0/vm-config/vms.nix" + ]; + + resolve_in_root = import ../../../tools/resolve_in_root { + config = config // { pkgs = config.pkgs.pkgsStatic; }; + }; + in + writeScript "run-nix" '' + #!/bin/execlineb -P + importas -i PATH PATH + export NIX_CONFIG "build-users-group =" + export NIX_REMOTE /run/virtiofs/virtiofs0 + export NIX_PATH ${concatStringsSep ":" nixPath} + export PATH ${makeBinPath [ bin nix resolve_in_root ]}:''${PATH} + export XDG_CACHE_HOME /run/cache + + # FIXME: can be removed when we have nix#7070. + export XDG_DATA_HOME /run/data + + if { /etc/mdev/wait virtiofs0 } + /bin/sh -il + '' + ) { }; +} -- 2.37.1

This adds the Nix files that define a very simple VM that just runs the "yes" command, which repeatedly prints 'y' to the console. Only the Nix files that define the VM are added, and not the VM itself, so it will only appear when vm-rebuild is run in appvm-nix. Signed-off-by: Alyssa Ross <hi@alyssa.is> --- host/initramfs/extfs.nix | 11 ++++++++++- vm/app/nix/example.nix | 13 +++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 vm/app/nix/example.nix diff --git a/host/initramfs/extfs.nix b/host/initramfs/extfs.nix index 9f00793..4319afb 100644 --- a/host/initramfs/extfs.nix +++ b/host/initramfs/extfs.nix @@ -4,6 +4,8 @@ { config, runCommand, e2fsprogs, tar2ext4 }: let + inherit (config.pkgs) lib; + netvm = import ../../vm/sys/net { inherit config; # inherit (foot) terminfo; @@ -13,12 +15,16 @@ let appvm-lynx = import ../../vm/app/lynx.nix { inherit config; }; appvm-mg = import ../../vm/app/mg.nix { inherit config; }; appvm-nix = import ../../vm/app/nix { inherit config; }; + + vmsNix = ../../vm/app/nix/example.nix; + spectrumNix = + builtins.filterSource (name: _type: lib.hasSuffix ".nix" name) ../../vm-lib; in runCommand "ext.ext4" { nativeBuildInputs = [ e2fsprogs ]; } '' - mkdir -p root/svc/data/appvm-{catgirl,lynx,mg,nix} + mkdir -p root/svc/data/appvm-{catgirl,lynx,mg,nix} root/vm-config cd root tar -C ${netvm} -c data | tar -C svc -x @@ -29,6 +35,9 @@ runCommand "ext.ext4" { tar -C ${appvm-mg} -c . | tar -C svc/data/appvm-mg -x tar -C ${appvm-nix} -c . | tar -C svc/data/appvm-nix -x + cp ${vmsNix} vm-config/vms.nix + cp -R ${spectrumNix} vm-config/spectrum + mkfs.ext4 -d . $out 16T resize2fs -M $out diff --git a/vm/app/nix/example.nix b/vm/app/nix/example.nix new file mode 100644 index 0000000..55068d1 --- /dev/null +++ b/vm/app/nix/example.nix @@ -0,0 +1,13 @@ +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: 2022 Alyssa Ross <hi@alyssa.is> + +import <spectrum/make-vms.nix> {} ({ makeVm, pkgs }: { + appvm-yes = makeVm { + run = pkgs.pkgsStatic.callPackage ( + { writeShellScript, coreutils }: + writeShellScript "appvm-yes-run" '' + ${coreutils}/bin/yes + '' + ) {}; + }; +}) -- 2.37.1

This is the first step to being able to do things like having a static base image for an application VM, and another image containing the application to run. When we actually use multiple images we'll need to change how we boot so we know the kernel is booting from the right image — probably by PARTUUID or PARTLABEL — because we can't guarantee how the kernel orders disks inside the VM. Signed-off-by: Alyssa Ross <hi@alyssa.is> --- Documentation/creating-vms.adoc | 8 +++--- host/start-vm/lib.rs | 36 ++++++++++++++++++++----- host/start-vm/tests/vm_command-basic.rs | 4 +-- vm/app/catgirl/Makefile | 12 ++++----- vm/app/lynx/Makefile | 12 ++++----- vm/sys/net/Makefile | 12 ++++----- 6 files changed, 55 insertions(+), 29 deletions(-) diff --git a/Documentation/creating-vms.adoc b/Documentation/creating-vms.adoc index d967098..6d4fde0 100644 --- a/Documentation/creating-vms.adoc +++ b/Documentation/creating-vms.adoc @@ -15,8 +15,9 @@ The directory can contain the following files: vmlinux:: An uncompressed Linux kernel image for the VM to boot. *Required.* -rootfs.ext4:: A disk image that will be provided to the guest as a -virtio-blk device. *Required.* +blk:: A directory containing disk images (with file names ending in +".img") that will be provided to the guest as a virtio-blk device. +Order is not guaranteed. At least one image is *required*. providers/net:: A directory containing a file named for each VM that should provide networking to this VM. The contents of these files are @@ -33,7 +34,8 @@ appvm-lynx ├── providers/ │   └── net/ │   └── netvm -├── rootfs.ext4 +├── blk/ +│   └── root.img └── vmlinux* ---- diff --git a/host/start-vm/lib.rs b/host/start-vm/lib.rs index 1230a6e..5d43a3e 100644 --- a/host/start-vm/lib.rs +++ b/host/start-vm/lib.rs @@ -81,12 +81,36 @@ pub fn vm_command(dir: PathBuf, config_root: &Path) -> Result<Command, String> { Err(e) => return Err(format!("reading directory {:?}: {}", net_providers_dir, e)), } - command.arg("--disk").arg({ - let mut disk = OsString::from("path=/ext/svc/data/"); - disk.push(&vm_name); - disk.push("/rootfs.ext4,readonly=on"); - disk - }); + command.arg("--disk"); + + let blk_dir = config_dir.join("blk"); + match blk_dir.read_dir() { + Ok(entries) => { + for result in entries { + let entry = result + .map_err(|e| format!("examining directory entry: {}", e))? + .path(); + + if entry.extension() != Some(OsStr::new("img")) { + continue; + } + + if entry.as_os_str().as_bytes().contains(&b',') { + return Err(format!("illegal ',' character in path {:?}", entry)); + } + + let mut arg = OsString::from("path="); + arg.push(entry); + arg.push(",readonly=on"); + command.arg(arg); + } + } + Err(e) => return Err(format!("reading directory {:?}: {}", blk_dir, e)), + } + + if command.get_args().last() == Some(OsStr::new("--disk")) { + return Err("no block devices specified".to_string()); + } command.arg("--serial").arg({ let mut serial = OsString::from("file=/run/"); diff --git a/host/start-vm/tests/vm_command-basic.rs b/host/start-vm/tests/vm_command-basic.rs index b2edb7c..a577a71 100644 --- a/host/start-vm/tests/vm_command-basic.rs +++ b/host/start-vm/tests/vm_command-basic.rs @@ -14,7 +14,7 @@ fn main() -> std::io::Result<()> { create_dir(&service_dir)?; let kernel_path = tmp_dir.path().join("svc/data/testvm/vmlinux"); - let image_path = tmp_dir.path().join("svc/data/testvm/rootfs.ext4"); + let image_path = tmp_dir.path().join("svc/data/testvm/blk/root.img"); create_dir_all(kernel_path.parent().unwrap())?; create_dir_all(image_path.parent().unwrap())?; @@ -43,7 +43,7 @@ fn main() -> std::io::Result<()> { OsStr::new("--kernel"), kernel_path.as_os_str(), OsStr::new("--disk"), - OsStr::new("path=/ext/svc/data/testvm/rootfs.ext4,readonly=on"), + &expected_disk_arg, OsStr::new("--serial"), OsStr::new("file=/run/testvm.log"), ]; diff --git a/vm/app/catgirl/Makefile b/vm/app/catgirl/Makefile index a32826b..9016745 100644 --- a/vm/app/catgirl/Makefile +++ b/vm/app/catgirl/Makefile @@ -12,7 +12,7 @@ VMM = qemu HOST_FILES = host/data/appvm-catgirl/providers/net/netvm HOST_BUILD_FILES = \ - build/host/data/appvm-catgirl/rootfs.ext4 \ + build/host/data/appvm-catgirl/blk/root.img \ build/host/data/appvm-catgirl/vmlinux # We produce a directory, but that doesn't play nice with Make, @@ -35,7 +35,7 @@ build/host/data/appvm-catgirl/vmlinux: $(VMLINUX) # tar2ext4 will leave half a filesystem behind if it's interrupted # half way through. -build/host/data/appvm-catgirl/rootfs.ext4: build/rootfs.tar +build/host/data/appvm-catgirl/blk/root.img: build/rootfs.tar mkdir -p $$(dirname $@) tar2ext4 -i build/rootfs.tar -o $@.tmp mv $@.tmp $@ @@ -92,9 +92,9 @@ build/etc/s6-rc: $(VM_S6_RC_FILES) s6-rc-compile $@ $$dir; \ exit=$$?; rm -r $$dir; exit $$exit -run-qemu: build/host/data/appvm-catgirl/rootfs.ext4 +run-qemu: build/host/data/appvm-catgirl/blk/root.img $(QEMU_KVM) -m 128 -cpu host -machine q35,kernel=$(KERNEL) -vga none \ - -drive file=build/host/data/appvm-catgirl/rootfs.ext4,if=virtio,format=raw,readonly=on \ + -drive file=build/host/data/appvm-catgirl/blk/root.img,if=virtio,format=raw,readonly=on \ -append "console=ttyS0 root=/dev/vda" \ -netdev user,id=net0 \ -device virtio-net,netdev=net0,mac=0A:B3:EC:00:00:00 \ @@ -103,11 +103,11 @@ run-qemu: build/host/data/appvm-catgirl/rootfs.ext4 -device virtconsole,chardev=virtiocon0 .PHONY: run-qemu -run-cloud-hypervisor: build/host/data/appvm-catgirl/rootfs.ext4 +run-cloud-hypervisor: build/host/data/appvm-catgirl/blk/root.img $(CLOUD_HYPERVISOR) \ --api-socket path=vmm.sock \ --memory size=128M \ - --disk path=build/host/data/appvm-catgirl/rootfs.ext4,readonly=on \ + --disk path=build/host/data/appvm-catgirl/blk/root.img,readonly=on \ --net tap=tap0,mac=0A:B3:EC:00:00:00 \ --kernel $(KERNEL) \ --cmdline "console=ttyS0 root=/dev/vda" \ diff --git a/vm/app/lynx/Makefile b/vm/app/lynx/Makefile index c25c9ab..ff87cb8 100644 --- a/vm/app/lynx/Makefile +++ b/vm/app/lynx/Makefile @@ -12,7 +12,7 @@ VMM = qemu HOST_FILES = host/data/appvm-lynx/providers/net/netvm HOST_BUILD_FILES = \ - build/host/data/appvm-lynx/rootfs.ext4 \ + build/host/data/appvm-lynx/blk/root.img \ build/host/data/appvm-lynx/vmlinux # We produce a directory, but that doesn't play nice with Make, @@ -35,7 +35,7 @@ build/host/data/appvm-lynx/vmlinux: $(VMLINUX) # tar2ext4 will leave half a filesystem behind if it's interrupted # half way through. -build/host/data/appvm-lynx/rootfs.ext4: build/rootfs.tar +build/host/data/appvm-lynx/blk/root.img: build/rootfs.tar mkdir -p $$(dirname $@) tar2ext4 -i build/rootfs.tar -o $@.tmp mv $@.tmp $@ @@ -91,9 +91,9 @@ build/etc/s6-rc: $(VM_S6_RC_FILES) s6-rc-compile $@ $$dir; \ exit=$$?; rm -r $$dir; exit $$exit -run-qemu: build/host/data/appvm-lynx/rootfs.ext4 +run-qemu: build/host/data/appvm-lynx/blk/root.img $(QEMU_KVM) -m 128 -cpu host -machine q35,kernel=$(KERNEL) -vga none \ - -drive file=build/host/data/appvm-lynx/rootfs.ext4,if=virtio,format=raw,readonly=on \ + -drive file=build/host/data/appvm-lynx/blk/root.img,if=virtio,format=raw,readonly=on \ -append "console=ttyS0 root=/dev/vda" \ -netdev user,id=net0 \ -device virtio-net,netdev=net0,mac=0A:B3:EC:00:00:00 \ @@ -102,11 +102,11 @@ run-qemu: build/host/data/appvm-lynx/rootfs.ext4 -device virtconsole,chardev=virtiocon0 .PHONY: run-qemu -run-cloud-hypervisor: build/host/data/appvm-lynx/rootfs.ext4 +run-cloud-hypervisor: build/host/data/appvm-lynx/blk/root.img $(CLOUD_HYPERVISOR) \ --api-socket path=vmm.sock \ --memory size=128M \ - --disk path=build/host/data/appvm-lynx/rootfs.ext4,readonly=on \ + --disk path=build/host/data/appvm-lynx/blk/root.img,readonly=on \ --net tap=tap0,mac=0A:B3:EC:00:00:00 \ --kernel $(KERNEL) \ --cmdline "console=ttyS0 root=/dev/vda" \ diff --git a/vm/sys/net/Makefile b/vm/sys/net/Makefile index 7cb7a5f..0b5fe8f 100644 --- a/vm/sys/net/Makefile +++ b/vm/sys/net/Makefile @@ -10,7 +10,7 @@ CLOUD_HYPERVISOR = cloud-hypervisor VMM = qemu HOST_BUILD_FILES = \ - build/host/data/netvm/rootfs.ext4 \ + build/host/data/netvm/blk/root.img \ build/host/data/netvm/vmlinux # We produce a directory, but that doesn't play nice with Make, @@ -32,7 +32,7 @@ build/host/data/netvm/vmlinux: $(VMLINUX) # tar2ext4 will leave half a filesystem behind if it's interrupted # half way through. -build/host/data/netvm/rootfs.ext4: build/rootfs.tar +build/host/data/netvm/blk/root.img: build/rootfs.tar mkdir -p $$(dirname $@) tar2ext4 -i build/rootfs.tar -o $@.tmp mv $@.tmp $@ @@ -97,9 +97,9 @@ build/etc/s6-rc: $(VM_S6_RC_FILES) s6-rc-compile $@ $$dir; \ exit=$$?; rm -r $$dir; exit $$exit -run-qemu: build/host/data/netvm/rootfs.ext4 +run-qemu: build/host/data/netvm/blk/root.img $(QEMU_KVM) -m 128 -cpu host -machine q35,kernel=$(KERNEL) -vga none \ - -drive file=build/host/data/netvm/rootfs.ext4,if=virtio,format=raw,readonly=on \ + -drive file=build/host/data/netvm/blk/root.img,if=virtio,format=raw,readonly=on \ -append "console=ttyS0 root=/dev/vda" \ -netdev user,id=net0 \ -device e1000e,netdev=net0 \ @@ -110,11 +110,11 @@ run-qemu: build/host/data/netvm/rootfs.ext4 -device virtconsole,chardev=virtiocon0 .PHONY: run-qemu -run-cloud-hypervisor: build/host/data/netvm/rootfs.ext4 +run-cloud-hypervisor: build/host/data/netvm/blk/root.img $(CLOUD_HYPERVISOR) \ --api-socket path=vmm.sock \ --memory size=128M \ - --disk path=build/host/data/netvm/rootfs.ext4,readonly=on \ + --disk path=build/host/data/netvm/blk/root.img,readonly=on \ --net tap=tap0 tap=tap1,mac=0A:B3:EC:80:00:00 \ --kernel $(KERNEL) \ --cmdline "console=ttyS0 root=/dev/vda" \ -- 2.37.1

This will allow booting reliably from multiple disk images, without relying on order or requiring udev in an initramfs. (The only deterministic way of doing boot order on Linux without special userspace support is to set "root" on the kernel command line to either a GPT partition UUID or label.) Signed-off-by: Alyssa Ross <hi@alyssa.is> --- vm/app/catgirl/Makefile | 13 ++++++++++--- vm/app/catgirl/default.nix | 10 +++++++--- vm/app/lynx/Makefile | 13 ++++++++++--- vm/app/lynx/default.nix | 10 +++++++--- vm/sys/net/Makefile | 13 ++++++++++--- vm/sys/net/default.nix | 10 +++++++--- 6 files changed, 51 insertions(+), 18 deletions(-) diff --git a/vm/app/catgirl/Makefile b/vm/app/catgirl/Makefile index 9016745..8129c8e 100644 --- a/vm/app/catgirl/Makefile +++ b/vm/app/catgirl/Makefile @@ -8,6 +8,7 @@ QEMU_KVM = qemu-kvm CLOUD_HYPERVISOR = cloud-hypervisor VMM = qemu +SCRIPTS = ../../../scripts HOST_FILES = host/data/appvm-catgirl/providers/net/netvm @@ -33,9 +34,15 @@ build/host/data/appvm-catgirl/vmlinux: $(VMLINUX) mkdir -p $$(dirname $@) cp $(VMLINUX) $@ +build/host/data/appvm-catgirl/blk/root.img: $(SCRIPTS)/make-gpt.sh $(SCRIPTS)/sfdisk-field.awk build/rootfs.ext4 + mkdir -p $$(dirname $@) + $(SCRIPTS)/make-gpt.sh $@.tmp \ + build/rootfs.ext4:4f68bce3-e8cd-4db1-96e7-fbcaf984b709:0d2f5f77-eb9c-453a-9463-daafcb5ce2b2:root + mv $@.tmp $@ + # tar2ext4 will leave half a filesystem behind if it's interrupted # half way through. -build/host/data/appvm-catgirl/blk/root.img: build/rootfs.tar +build/rootfs.ext4: build/rootfs.tar mkdir -p $$(dirname $@) tar2ext4 -i build/rootfs.tar -o $@.tmp mv $@.tmp $@ @@ -95,7 +102,7 @@ build/etc/s6-rc: $(VM_S6_RC_FILES) run-qemu: build/host/data/appvm-catgirl/blk/root.img $(QEMU_KVM) -m 128 -cpu host -machine q35,kernel=$(KERNEL) -vga none \ -drive file=build/host/data/appvm-catgirl/blk/root.img,if=virtio,format=raw,readonly=on \ - -append "console=ttyS0 root=/dev/vda" \ + -append "console=ttyS0 root=PARTLABEL=root" \ -netdev user,id=net0 \ -device virtio-net,netdev=net0,mac=0A:B3:EC:00:00:00 \ -chardev vc,id=virtiocon0 \ @@ -110,7 +117,7 @@ run-cloud-hypervisor: build/host/data/appvm-catgirl/blk/root.img --disk path=build/host/data/appvm-catgirl/blk/root.img,readonly=on \ --net tap=tap0,mac=0A:B3:EC:00:00:00 \ --kernel $(KERNEL) \ - --cmdline "console=ttyS0 root=/dev/vda" \ + --cmdline "console=ttyS0 root=PARTLABEL=root" \ --console tty \ --serial pty .PHONY: run-cloud-hypervisor diff --git a/vm/app/catgirl/default.nix b/vm/app/catgirl/default.nix index 61f1462..d83392d 100644 --- a/vm/app/catgirl/default.nix +++ b/vm/app/catgirl/default.nix @@ -1,5 +1,5 @@ # SPDX-License-Identifier: MIT -# SPDX-FileCopyrightText: 2021 Alyssa Ross <hi@alyssa.is> +# SPDX-FileCopyrightText: 2021-2022 Alyssa Ross <hi@alyssa.is> { config ? import ../../../nix/eval-config.nix {} , terminfo ? config.pkgs.foot.terminfo @@ -8,13 +8,15 @@ config.pkgs.pkgsStatic.callPackage ( { lib, stdenvNoCC, runCommand, writeReferencesToFile, buildPackages -, s6-rc, tar2ext4 +, jq, s6-rc, tar2ext4, util-linux , busybox, cacert, catgirl, execline, kmod, mdevd, s6, s6-linux-init }: let inherit (lib) cleanSource cleanSourceWith concatMapStringsSep hasSuffix; + scripts = import ../../../scripts { inherit config; }; + packages = [ catgirl execline kmod mdevd s6 s6-linux-init s6-rc @@ -71,11 +73,13 @@ stdenvNoCC.mkDerivation { src = cleanSource ./.; }; - nativeBuildInputs = [ s6-rc tar2ext4 ]; + nativeBuildInputs = [ jq s6-rc tar2ext4 util-linux ]; PACKAGES_TAR = packagesTar; VMLINUX = "${kernel.dev}/vmlinux"; + makeFlags = [ "SCRIPTS=${scripts}" ]; + installPhase = '' mv build/svc $out ''; diff --git a/vm/app/lynx/Makefile b/vm/app/lynx/Makefile index ff87cb8..ff42501 100644 --- a/vm/app/lynx/Makefile +++ b/vm/app/lynx/Makefile @@ -8,6 +8,7 @@ QEMU_KVM = qemu-kvm CLOUD_HYPERVISOR = cloud-hypervisor VMM = qemu +SCRIPTS = ../../../scripts HOST_FILES = host/data/appvm-lynx/providers/net/netvm @@ -33,9 +34,15 @@ build/host/data/appvm-lynx/vmlinux: $(VMLINUX) mkdir -p $$(dirname $@) cp $(VMLINUX) $@ +build/host/data/appvm-lynx/blk/root.img: $(SCRIPTS)/make-gpt.sh $(SCRIPTS)/sfdisk-field.awk build/rootfs.ext4 + mkdir -p $$(dirname $@) + $(SCRIPTS)/make-gpt.sh $@.tmp \ + build/rootfs.ext4:4f68bce3-e8cd-4db1-96e7-fbcaf984b709:41e8068d-38d5-4135-ad77-0da704743940:root + mv $@.tmp $@ + # tar2ext4 will leave half a filesystem behind if it's interrupted # half way through. -build/host/data/appvm-lynx/blk/root.img: build/rootfs.tar +build/rootfs.ext4: build/rootfs.tar mkdir -p $$(dirname $@) tar2ext4 -i build/rootfs.tar -o $@.tmp mv $@.tmp $@ @@ -94,7 +101,7 @@ build/etc/s6-rc: $(VM_S6_RC_FILES) run-qemu: build/host/data/appvm-lynx/blk/root.img $(QEMU_KVM) -m 128 -cpu host -machine q35,kernel=$(KERNEL) -vga none \ -drive file=build/host/data/appvm-lynx/blk/root.img,if=virtio,format=raw,readonly=on \ - -append "console=ttyS0 root=/dev/vda" \ + -append "console=ttyS0 root=PARTLABEL=root" \ -netdev user,id=net0 \ -device virtio-net,netdev=net0,mac=0A:B3:EC:00:00:00 \ -chardev vc,id=virtiocon0 \ @@ -109,7 +116,7 @@ run-cloud-hypervisor: build/host/data/appvm-lynx/blk/root.img --disk path=build/host/data/appvm-lynx/blk/root.img,readonly=on \ --net tap=tap0,mac=0A:B3:EC:00:00:00 \ --kernel $(KERNEL) \ - --cmdline "console=ttyS0 root=/dev/vda" \ + --cmdline "console=ttyS0 root=PARTLABEL=root" \ --console tty \ --serial pty .PHONY: run-cloud-hypervisor diff --git a/vm/app/lynx/default.nix b/vm/app/lynx/default.nix index ba715ec..92635f3 100644 --- a/vm/app/lynx/default.nix +++ b/vm/app/lynx/default.nix @@ -1,5 +1,5 @@ # SPDX-License-Identifier: MIT -# SPDX-FileCopyrightText: 2021 Alyssa Ross <hi@alyssa.is> +# SPDX-FileCopyrightText: 2021-2022 Alyssa Ross <hi@alyssa.is> { config ? import ../../../nix/eval-config.nix {} , terminfo ? config.pkgs.foot.terminfo @@ -8,13 +8,15 @@ config.pkgs.pkgsStatic.callPackage ( { lib, stdenvNoCC, runCommand, writeReferencesToFile, buildPackages -, s6-rc, tar2ext4 +, jq, s6-rc, tar2ext4, util-linux , busybox, cacert, execline, kmod, lynx, mdevd, s6, s6-linux-init }: let inherit (lib) cleanSource cleanSourceWith concatMapStringsSep hasSuffix; + scripts = import ../../../scripts { inherit config; }; + packages = [ execline kmod lynx mdevd s6 s6-linux-init s6-rc @@ -71,11 +73,13 @@ stdenvNoCC.mkDerivation { src = cleanSource ./.; }; - nativeBuildInputs = [ s6-rc tar2ext4 ]; + nativeBuildInputs = [ jq s6-rc tar2ext4 util-linux ]; PACKAGES_TAR = packagesTar; VMLINUX = "${kernel.dev}/vmlinux"; + makeFlags = [ "SCRIPTS=${scripts}" ]; + installPhase = '' mv build/svc $out ''; diff --git a/vm/sys/net/Makefile b/vm/sys/net/Makefile index 0b5fe8f..2b9004b 100644 --- a/vm/sys/net/Makefile +++ b/vm/sys/net/Makefile @@ -8,6 +8,7 @@ QEMU_KVM = qemu-kvm CLOUD_HYPERVISOR = cloud-hypervisor VMM = qemu +SCRIPTS = ../../../scripts HOST_BUILD_FILES = \ build/host/data/netvm/blk/root.img \ @@ -30,9 +31,15 @@ build/host/data/netvm/vmlinux: $(VMLINUX) mkdir -p $$(dirname $@) cp $(VMLINUX) $@ +build/host/data/netvm/blk/root.img: $(SCRIPTS)/make-gpt.sh $(SCRIPTS)/sfdisk-field.awk build/rootfs.ext4 + mkdir -p $$(dirname $@) + $(SCRIPTS)/make-gpt.sh $@.tmp \ + build/rootfs.ext4:4f68bce3-e8cd-4db1-96e7-fbcaf984b709:ea21da27-0391-48da-9235-9d2ab2ca7844:root + mv $@.tmp $@ + # tar2ext4 will leave half a filesystem behind if it's interrupted # half way through. -build/host/data/netvm/blk/root.img: build/rootfs.tar +build/rootfs.ext4: build/rootfs.tar mkdir -p $$(dirname $@) tar2ext4 -i build/rootfs.tar -o $@.tmp mv $@.tmp $@ @@ -100,7 +107,7 @@ build/etc/s6-rc: $(VM_S6_RC_FILES) run-qemu: build/host/data/netvm/blk/root.img $(QEMU_KVM) -m 128 -cpu host -machine q35,kernel=$(KERNEL) -vga none \ -drive file=build/host/data/netvm/blk/root.img,if=virtio,format=raw,readonly=on \ - -append "console=ttyS0 root=/dev/vda" \ + -append "console=ttyS0 root=PARTLABEL=root" \ -netdev user,id=net0 \ -device e1000e,netdev=net0 \ -netdev user,id=net1 \ @@ -117,7 +124,7 @@ run-cloud-hypervisor: build/host/data/netvm/blk/root.img --disk path=build/host/data/netvm/blk/root.img,readonly=on \ --net tap=tap0 tap=tap1,mac=0A:B3:EC:80:00:00 \ --kernel $(KERNEL) \ - --cmdline "console=ttyS0 root=/dev/vda" \ + --cmdline "console=ttyS0 root=PARTLABEL=root" \ --console tty \ --serial pty .PHONY: run-cloud-hypervisor diff --git a/vm/sys/net/default.nix b/vm/sys/net/default.nix index dfc7c35..e9a2f05 100644 --- a/vm/sys/net/default.nix +++ b/vm/sys/net/default.nix @@ -1,5 +1,5 @@ # SPDX-License-Identifier: MIT -# SPDX-FileCopyrightText: 2021 Alyssa Ross <hi@alyssa.is> +# SPDX-FileCopyrightText: 2021-2022 Alyssa Ross <hi@alyssa.is> { config ? import ../../../nix/eval-config.nix {} , terminfo ? config.pkgs.foot.terminfo @@ -8,7 +8,7 @@ config.pkgs.pkgsStatic.callPackage ( { lib, stdenvNoCC, runCommand, writeReferencesToFile, buildPackages -, s6-rc, tar2ext4, xorg +, jq, s6-rc, tar2ext4, util-linux, xorg , busybox, connmanMinimal, dbus, execline, kmod, mdevd, nftables, s6 , s6-linux-init }: @@ -16,6 +16,8 @@ config.pkgs.pkgsStatic.callPackage ( let inherit (lib) cleanSource cleanSourceWith concatMapStringsSep hasSuffix; + scripts = import ../../../scripts { inherit config; }; + connman = connmanMinimal; packages = [ @@ -80,11 +82,13 @@ stdenvNoCC.mkDerivation { src = cleanSource ./.; }; - nativeBuildInputs = [ s6-rc tar2ext4 ]; + nativeBuildInputs = [ jq s6-rc tar2ext4 util-linux ]; PACKAGES_TAR = packagesTar; VMLINUX = "${kernel.dev}/vmlinux"; + makeFlags = [ "SCRIPTS=${scripts}" ]; + installPhase = '' mv build/svc $out ''; -- 2.37.1

I think this is more descriptive, and "img" is a name better used for VM images. Signed-off-by: Alyssa Ross <hi@alyssa.is> --- Documentation/getting-spectrum.adoc | 2 +- release.nix | 2 +- {img => release}/combined/default.nix | 0 {img => release}/combined/eosimages.nix | 0 {img => release}/combined/grub.cfg.in | 0 {img => release}/combined/run-vm.nix | 0 .../app/0001-gpt-disable-gpt-partition-attribute-55-check.patch | 0 .../app/0002-gpt-disable-partition-table-CRC-check.patch | 0 .../installer/app/0003-install-remove-Endless-OS-ad.patch | 0 .../installer/app/0004-finished-don-t-run-eos-diagnostics.patch | 0 .../0005-finished-promote-spectrum-not-the-Endless-forum.patch | 0 {img => release}/installer/app/default.nix | 0 {img => release}/installer/app/vendor-customer-support.ini | 0 {img => release}/installer/configuration.nix | 0 {img => release}/installer/default.nix | 0 {img => release}/installer/run-vm.nix | 0 {img => release}/installer/seat.rules | 0 {img => release}/live/Makefile | 0 {img => release}/live/default.nix | 0 {img => release}/live/shell.nix | 0 20 files changed, 2 insertions(+), 2 deletions(-) rename {img => release}/combined/default.nix (100%) rename {img => release}/combined/eosimages.nix (100%) rename {img => release}/combined/grub.cfg.in (100%) rename {img => release}/combined/run-vm.nix (100%) rename {img => release}/installer/app/0001-gpt-disable-gpt-partition-attribute-55-check.patch (100%) rename {img => release}/installer/app/0002-gpt-disable-partition-table-CRC-check.patch (100%) rename {img => release}/installer/app/0003-install-remove-Endless-OS-ad.patch (100%) rename {img => release}/installer/app/0004-finished-don-t-run-eos-diagnostics.patch (100%) rename {img => release}/installer/app/0005-finished-promote-spectrum-not-the-Endless-forum.patch (100%) rename {img => release}/installer/app/default.nix (100%) rename {img => release}/installer/app/vendor-customer-support.ini (100%) rename {img => release}/installer/configuration.nix (100%) rename {img => release}/installer/default.nix (100%) rename {img => release}/installer/run-vm.nix (100%) rename {img => release}/installer/seat.rules (100%) rename {img => release}/live/Makefile (100%) rename {img => release}/live/default.nix (100%) rename {img => release}/live/shell.nix (100%) diff --git a/Documentation/getting-spectrum.adoc b/Documentation/getting-spectrum.adoc index b3fa1ef..a34ac5e 100644 --- a/Documentation/getting-spectrum.adoc +++ b/Documentation/getting-spectrum.adoc @@ -39,7 +39,7 @@ still take a very long time. [source,shell] ---- git clone https://spectrum-os.org/git/spectrum -nix-build spectrum/img/combined \ +nix-build spectrum/release/combined \ -I nixpkgs=https://spectrum-os.org/git/nixpkgs/snapshot/nixpkgs-rootfs.tar.gz ---- diff --git a/release.nix b/release.nix index 91a843b..eea10a0 100644 --- a/release.nix +++ b/release.nix @@ -12,5 +12,5 @@ { doc = import ./Documentation { inherit config; }; - combined = import img/combined/run-vm.nix { inherit config; }; + combined = import release/combined/run-vm.nix { inherit config; }; } diff --git a/img/combined/default.nix b/release/combined/default.nix similarity index 100% rename from img/combined/default.nix rename to release/combined/default.nix diff --git a/img/combined/eosimages.nix b/release/combined/eosimages.nix similarity index 100% rename from img/combined/eosimages.nix rename to release/combined/eosimages.nix diff --git a/img/combined/grub.cfg.in b/release/combined/grub.cfg.in similarity index 100% rename from img/combined/grub.cfg.in rename to release/combined/grub.cfg.in diff --git a/img/combined/run-vm.nix b/release/combined/run-vm.nix similarity index 100% rename from img/combined/run-vm.nix rename to release/combined/run-vm.nix diff --git a/img/installer/app/0001-gpt-disable-gpt-partition-attribute-55-check.patch b/release/installer/app/0001-gpt-disable-gpt-partition-attribute-55-check.patch similarity index 100% rename from img/installer/app/0001-gpt-disable-gpt-partition-attribute-55-check.patch rename to release/installer/app/0001-gpt-disable-gpt-partition-attribute-55-check.patch diff --git a/img/installer/app/0002-gpt-disable-partition-table-CRC-check.patch b/release/installer/app/0002-gpt-disable-partition-table-CRC-check.patch similarity index 100% rename from img/installer/app/0002-gpt-disable-partition-table-CRC-check.patch rename to release/installer/app/0002-gpt-disable-partition-table-CRC-check.patch diff --git a/img/installer/app/0003-install-remove-Endless-OS-ad.patch b/release/installer/app/0003-install-remove-Endless-OS-ad.patch similarity index 100% rename from img/installer/app/0003-install-remove-Endless-OS-ad.patch rename to release/installer/app/0003-install-remove-Endless-OS-ad.patch diff --git a/img/installer/app/0004-finished-don-t-run-eos-diagnostics.patch b/release/installer/app/0004-finished-don-t-run-eos-diagnostics.patch similarity index 100% rename from img/installer/app/0004-finished-don-t-run-eos-diagnostics.patch rename to release/installer/app/0004-finished-don-t-run-eos-diagnostics.patch diff --git a/img/installer/app/0005-finished-promote-spectrum-not-the-Endless-forum.patch b/release/installer/app/0005-finished-promote-spectrum-not-the-Endless-forum.patch similarity index 100% rename from img/installer/app/0005-finished-promote-spectrum-not-the-Endless-forum.patch rename to release/installer/app/0005-finished-promote-spectrum-not-the-Endless-forum.patch diff --git a/img/installer/app/default.nix b/release/installer/app/default.nix similarity index 100% rename from img/installer/app/default.nix rename to release/installer/app/default.nix diff --git a/img/installer/app/vendor-customer-support.ini b/release/installer/app/vendor-customer-support.ini similarity index 100% rename from img/installer/app/vendor-customer-support.ini rename to release/installer/app/vendor-customer-support.ini diff --git a/img/installer/configuration.nix b/release/installer/configuration.nix similarity index 100% rename from img/installer/configuration.nix rename to release/installer/configuration.nix diff --git a/img/installer/default.nix b/release/installer/default.nix similarity index 100% rename from img/installer/default.nix rename to release/installer/default.nix diff --git a/img/installer/run-vm.nix b/release/installer/run-vm.nix similarity index 100% rename from img/installer/run-vm.nix rename to release/installer/run-vm.nix diff --git a/img/installer/seat.rules b/release/installer/seat.rules similarity index 100% rename from img/installer/seat.rules rename to release/installer/seat.rules diff --git a/img/live/Makefile b/release/live/Makefile similarity index 100% rename from img/live/Makefile rename to release/live/Makefile diff --git a/img/live/default.nix b/release/live/default.nix similarity index 100% rename from img/live/default.nix rename to release/live/default.nix diff --git a/img/live/shell.nix b/release/live/shell.nix similarity index 100% rename from img/live/shell.nix rename to release/live/shell.nix -- 2.37.1