- Spectrum Development - spectrum-os.org

[PATCH] Patches to waypipe package to enable static build with the cost of fancy feature
by vadim likholetov 29 Nov '22

29 Nov '22

--- .../applications/networking/remote/waypipe/default.nix | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pkgs/applications/networking/remote/waypipe/default.nix b/pkgs/applications/networking/remote/waypipe/default.nix index ec94d504b9d..bcc3770ecf7 100644 --- a/pkgs/applications/networking/remote/waypipe/default.nix +++ b/pkgs/applications/networking/remote/waypipe/default.nix @@ -20,7 +20,15 @@ stdenv.mkDerivation rec { nativeBuildInputs = [ meson ninja pkg-config scdoc ]; buildInputs = [ # Optional dependencies: - mesa lz4 zstd ffmpeg libva + # mesa lz4 zstd ffmpeg libva # TODO: one day recompile with acceleration + lz4 zstd + ]; + + mesonFlags = [ + "-Dwith_video=disabled" + "-Dwith_systemtap=false" + "-Dwith_vaapi=disabled" + "-Dwith_dmabuf=disabled" ]; meta = with lib; { -- 2.36.2

1 0

[PATCH] This patchset enables forwarding Wayland connections from appVM to the host with the help of waypipe tool. It doesn't reqire major changes in Spectrum and tries to utilise existing packages and solutions. Main idea is to create VSOCK virtual device on the start of appVM and use this virtual socket to connect host and guest. On the host side I implement vm-start-way tool that is based on vm-start, but does nesessary preparations for wayland forwarfing. The guest VM is appvm-hello-waypipe and describe
by vadim likholetov 29 Nov '22

29 Nov '22

--- host/initramfs/extfs.nix | 4 +++- host/rootfs/Makefile | 1 + host/rootfs/default.nix | 4 ++-- host/rootfs/usr/bin/vm-start-way | 10 ++++++++++ img/app/default.nix | 3 +++ vm/app/hello-waypipe.nix | 23 +++++++++++++++++++++++ 6 files changed, 42 insertions(+), 3 deletions(-) create mode 100755 host/rootfs/usr/bin/vm-start-way create mode 100644 vm/app/hello-waypipe.nix diff --git a/host/initramfs/extfs.nix b/host/initramfs/extfs.nix index 5c5850f..f49e519 100644 --- a/host/initramfs/extfs.nix +++ b/host/initramfs/extfs.nix @@ -11,12 +11,13 @@ let appvm-catgirl = import ../../vm/app/catgirl.nix { inherit config; }; appvm-lynx = import ../../vm/app/lynx.nix { inherit config; }; + appvm-hello-waypipe = import ../../vm/app/hello-waypipe.nix { inherit config; }; in runCommand "ext.ext4" { nativeBuildInputs = [ e2fsprogs ]; } '' - mkdir -p root/svc/data/appvm-{catgirl,lynx} + mkdir -p root/svc/data/appvm-{catgirl,lynx,hello-waypipe} cd root tar -C ${netvm} -c data | tar -C svc -x @@ -24,6 +25,7 @@ runCommand "ext.ext4" { tar -C ${appvm-catgirl} -c . | tar -C svc/data/appvm-catgirl -x tar -C ${appvm-lynx} -c . | tar -C svc/data/appvm-lynx -x + tar -C ${appvm-hello-waypipe} -c . | tar -C svc/data/appvm-hello-waypipe -x mkfs.ext4 -d . $out 16T resize2fs -M $out diff --git a/host/rootfs/Makefile b/host/rootfs/Makefile index 9559c06..06e3e8e 100644 --- a/host/rootfs/Makefile +++ b/host/rootfs/Makefile @@ -39,6 +39,7 @@ FILES = \ usr/bin/lsvm \ usr/bin/vm-console \ usr/bin/vm-start \ + usr/bin/vm-start-way \ usr/bin/vm-stop DIRS = dev etc/s6-linux-init/env ext run proc sys diff --git a/host/rootfs/default.nix b/host/rootfs/default.nix index 0a84f55..de4a6b6 100644 --- a/host/rootfs/default.nix +++ b/host/rootfs/default.nix @@ -7,7 +7,7 @@ pkgs.pkgsStatic.callPackage ( { lib, stdenvNoCC, nixos, runCommand, writeReferencesToFile, s6-rc, tar2ext4 , busybox, cloud-hypervisor, cryptsetup, execline, e2fsprogs, jq, kmod -, mdevd, s6, s6-linux-init, socat, util-linuxMinimal, xorg +, mdevd, s6, s6-linux-init, socat, util-linuxMinimal, xorg, waypipe }: let @@ -49,7 +49,7 @@ let packages = [ cloud-hypervisor e2fsprogs execline jq kmod mdevd s6 s6-linux-init s6-rc - socat start-vm + socat start-vm waypipe (cryptsetup.override { programs = { diff --git a/host/rootfs/usr/bin/vm-start-way b/host/rootfs/usr/bin/vm-start-way new file mode 100755 index 0000000..bda9934 --- /dev/null +++ b/host/rootfs/usr/bin/vm-start-way @@ -0,0 +1,10 @@ +#!/bin/sh +# SPDX-License-Identifier: EUPL-1.2+ + +s6-rc -bu change ext-rc +s6-rc -l /run/s6-rc.ext -u change $1 +ch-remote --api-socket /run/service/ext-${1}/env/cloud-hypervisor.sock add-vsock cid=4,socket=/run/u.${1}.socket +cd /run +nohup waypipe --socket /run/waypipe.sock client & +nohup socat unix-listen:/run/u.${1}.socket_5000,reuseaddr,fork unix-connect:/run/waypipe.sock & + diff --git a/img/app/default.nix b/img/app/default.nix index e7d5366..80f23c2 100644 --- a/img/app/default.nix +++ b/img/app/default.nix @@ -59,6 +59,9 @@ let DRM_BOCHS = yes; DRM = yes; AGP = yes; + VSOCKETS = yes; + VIRTIO_VSOCKETS = yes; + VIRTIO_VSOCKETS_COMMON = yes; }; }; in diff --git a/vm/app/hello-waypipe.nix b/vm/app/hello-waypipe.nix new file mode 100644 index 0000000..601b638 --- /dev/null +++ b/vm/app/hello-waypipe.nix @@ -0,0 +1,23 @@ +# SPDX-License-Identifier: MIT +# SPDX-FileCopyrightText: 2021-2022 Alyssa Ross <hi(a)alyssa.is> + +{ config ? import ../../../nix/eval-config.nix {} }: + +import ../make-vm.nix { inherit config; } { + providers.net = [ "netvm" ]; + run = config.pkgs.callPackage ( + { writeScript, waypipe, havoc, foot, hello-wayland, socat}: + writeScript "run-waypipe-app" '' + #!/bin/sh + mkdir /run/0 + export XDG_RUNTIME_DIR=/run/0 + ${socat}/bin/socat unix-listen:/run/waypipe.sock,reuseaddr,fork vsock-connect:2:5000 & + sleep 1 + ${waypipe}/bin/waypipe --display wayland-local --socket /run/waypipe.sock server -- sleep inf & + export WAYLAND_DISPLAY=wayland-local + ${havoc}/bin/havoc + ${hello-wayland}/bin/hello-wayland + ${foot}/bin/foot + '' + ) { }; +} -- 2.36.2

1 0

[PATCH] Documentation: add ADR for VM Sources
by Alyssa Ross 29 Nov '22

29 Nov '22

Signed-off-by: Alyssa Ross <alyssa.ross(a)unikie.com> --- Documentation/decisions/009-vm-sources.adoc | 42 +++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 Documentation/decisions/009-vm-sources.adoc diff --git a/Documentation/decisions/009-vm-sources.adoc b/Documentation/decisions/009-vm-sources.adoc new file mode 100644 index 0000000..457491d --- /dev/null +++ b/Documentation/decisions/009-vm-sources.adoc @@ -0,0 +1,42 @@ += 009 VM Sources +:page-parent: Architecture Decision Records +:page-grand_parent: About Spectrum + +// SPDX-FileCopyrightText: 2022 Unikie +// SPDX-License-Identifier: GFDL-1.3-no-invariants-or-later OR CC-BY-SA-4.0 + +== Status + +Proposed + +== Context + +It should be possible for Spectrum VMs to come from the root +filesystem (for built-in VMs), or one or more +xref:../appendices/user-partition.adoc[user partitions]. One part of +making this work will be to have the system search multiple sources of +VM definitions, to produce a single view of what VMs are available. + +== Decision + +The system will scan /usr/img/builtin, as well as the svc/data +partition on each user partition, for VM definitions. A single s6-rc +service directory will contain generated services for VMs from all +sources, so that tools like vm-start don't have to be aware of where +VMs come from. (This will continue to be distinct from the s6-rc +database used for host system services.) When a new source of VMs is +discovered, an updated database will be generated and atomically +replace the old one using +xref:https://www.skarnet.org/software/s6-rc/s6-rc-update.html[s6-rc-update]. + +Changes to how user partitions will be discovered and mounted, to +support having multiple user partitions, will be determined later. + +== Consequences + +Built-in VMs can be moved into the root filesystem, allowing the +default user partition to be empty. This will make it easier for +people making custom Spectrum images to add custom VMs without having +to build Spectrum itself themselves -- they can simply replace the +default, empty, partition with a custom one they can produce however +they like. base-commit: 95fba8ac6c872c2ba5e2ea8be5da9f2eb63374c8 -- 2.35.1

1 0

Waypipe Wayland guest to host forwarding prototype
by Vadim Likholetov 23 Nov '22

23 Nov '22

Hi, I’ve built a prototype of forwarding Wayland client connections with the help of waypipe. It is broken the same way same way as current Alyssa’s implementation. On imx8 green cat is ok, foot don’t start with no clear diagnostics. Is this prototype have interest in community? Should I submit patches of this prototype?

2 1

[PATCH] Documentation: move Appendices to Development, fixed links
by Jenni Nikolaenko 23 Nov '22

23 Nov '22

Both sections "User Partition" and "UUID Reference" are more development topics, so by Alyssa's advice they should go directly under the Development chapter. The section Appendices was removed. In addition, I replaced broken links. Signed-off-by: Jenni Nikolaenko <evgeniia.nikolaenko(a)unikie.com> --- Documentation/about/architecture.adoc | 6 +++--- Documentation/appendices/index.adoc | 13 ------------- .../{appendices => development}/user-partition.adoc | 6 +++--- .../{appendices => development}/uuid-reference.adoc | 11 +++++------ Documentation/index.adoc | 8 ++++---- Documentation/using-spectrum/creating-vms.adoc | 2 +- 6 files changed, 16 insertions(+), 30 deletions(-) delete mode 100644 Documentation/appendices/index.adoc rename Documentation/{appendices => development}/user-partition.adoc (87%) rename Documentation/{appendices => development}/uuid-reference.adoc (85%) diff --git a/Documentation/about/architecture.adoc b/Documentation/about/architecture.adoc index b311e77..38e0a22 100644 --- a/Documentation/about/architecture.adoc +++ b/Documentation/about/architecture.adoc @@ -15,8 +15,8 @@ The default set of virtual machines includes two application VMs, _appvm-catgirl_ (an IRC client) and _appvm-lynx_ (a text-based web browser); and a system VM, _netvm_ (which handles hardware network devices and provides network services to application VMs). -Refer to xref:../getting-started/creating-vms.adoc[Creating VMs] and -xref:../getting-started/running-vms.adoc[Running +Refer to xref:../using-spectrum/creating-vms.adoc[Creating VMs] and +xref:../using-spectrum/running-vms.adoc[Running VMs] for more information about using VMs in Spectrum. == Architecture Decision Records (ADRs) @@ -28,7 +28,7 @@ https://cognitect.com/blog/2011/11/15/documenting-architecture-decisions[AD…. == Spectrum Host System Compartmentalization is implemented using -https://cloud-hypervisor.org/[cloud-hypervisor] virtual machines. +https://cloudhypervisor.org/[cloud-hypervisor] virtual machines. cloud-hypervisor uses Linux's https://en.wikipedia.org/wiki/Kernel-based_Virtual_Machine[Kernel-based Virtual Machine] (KVM) to provide lightweight, hardware-accelerated VMs. diff --git a/Documentation/appendices/index.adoc b/Documentation/appendices/index.adoc deleted file mode 100644 index ebacda7..0000000 --- a/Documentation/appendices/index.adoc +++ /dev/null @@ -1,13 +0,0 @@ -= Appendices -:description: Additional information. -:page-nav_order: 5 -:page-has_children: true -:page-has_toc: false - -// SPDX-FileCopyrightText: 2022 Unikie -// SPDX-License-Identifier: GFDL-1.3-no-invariants-or-later OR CC-BY-SA-4.0 - -This section provides additional information on how to work with: - -* xref:../appendices/user-partition.adoc[user partition] -* xref:../appendices/uuid-reference.adoc[universally unique identifier (UUID)]. diff --git a/Documentation/appendices/user-partition.adoc b/Documentation/development/user-partition.adoc similarity index 87% rename from Documentation/appendices/user-partition.adoc rename to Documentation/development/user-partition.adoc index 4121340..d8fd956 100644 --- a/Documentation/appendices/user-partition.adoc +++ b/Documentation/development/user-partition.adoc @@ -1,6 +1,6 @@ -= Appendix A. User Partition -:page-parent: Appendices -:page-nav_order: 1 += User Partition +:page-parent: Development +:page-nav_order: 5 // SPDX-FileCopyrightText: 2022 Alyssa Ross <hi(a)alyssa.is> // SPDX-License-Identifier: GFDL-1.3-no-invariants-or-later OR CC-BY-SA-4.0 diff --git a/Documentation/appendices/uuid-reference.adoc b/Documentation/development/uuid-reference.adoc similarity index 85% rename from Documentation/appendices/uuid-reference.adoc rename to Documentation/development/uuid-reference.adoc index 5099ec3..81300ee 100644 --- a/Documentation/appendices/uuid-reference.adoc +++ b/Documentation/development/uuid-reference.adoc @@ -1,7 +1,6 @@ -= Appendix B. UUID Reference -:page-parent: Appendices -:toclevels: 1 -:page-nav_order: 2 += UUID Reference +:page-parent: Development +:page-nav_order: 6 // SPDX-FileCopyrightText: 2022 Alyssa Ross <hi(a)alyssa.is> // SPDX-License-Identifier: GFDL-1.3-no-invariants-or-later OR CC-BY-SA-4.0 @@ -25,13 +24,13 @@ Specification]. === `4f68bce3-e8cd-4db1-96e7-fbcaf984b709` "Root Partition (amd64/x86_64)" from the -https://systemd.io/DISCOVERABLE_PARTITIONS/[Discoverable Partitions +https://uapi-group.org/specifications/specs/discoverable_partitions_specification/[Discoverable Partitions Specification]. === `2c7357ed-ebd2-46d9-aec1-23d437ec2bf5` "Root Verity Partition (amd64/x86_64)" from the -https://systemd.io/DISCOVERABLE_PARTITIONS/[Discoverable Partitions +https://uapi-group.org/specifications/specs/discoverable_partitions_specification/[Discoverable Partitions Specification]. === `9293e1ff-cee4-4658-88be-898ec863944f` diff --git a/Documentation/index.adoc b/Documentation/index.adoc index e0a45c7..51a6e7d 100644 --- a/Documentation/index.adoc +++ b/Documentation/index.adoc @@ -15,7 +15,7 @@ lower barrier to entry and is easy to use and maintain. To learn about what Spectrum is and how it's implemented, start with the xref:about/architecture.adoc[architecture overview]. -If you want to try Spectrum, see xref:../installation/index.adoc[Build and Run] +If you want to try Spectrum, see xref:installation/index.adoc[Build and Run] to setup a development environment. @@ -25,8 +25,8 @@ Spectrum is made of free and open-source software. It is free for anyone to use, modify, and distribute. Once you are up and running, see - xref:../development/index.adoc[Development] to understand how to work with + xref:development/index.adoc[Development] to understand how to work with Spectrum's code and participate in the collaborative development process. -If you are thinking of contributing to Spectrum docs, see - xref:../development/building-documentation.adoc[Building Documentation]. +If you are thinking of contributing to the Spectrum docs, see + xref:development/building-documentation.adoc[Building Documentation]. diff --git a/Documentation/using-spectrum/creating-vms.adoc b/Documentation/using-spectrum/creating-vms.adoc index 6847249..2c57562 100644 --- a/Documentation/using-spectrum/creating-vms.adoc +++ b/Documentation/using-spectrum/creating-vms.adoc @@ -8,7 +8,7 @@ == Configuration Each VM is configured using a directory under svc/data/ on the -xref:user-partition.adoc[user partition]. The name of the directory +xref:../development/user-partition.adoc[user partition]. The name of the directory determines the name of the VM. The directory can contain the following files: -- 2.34.1

3 2

[PATCH v2 1/2] Documentation/scripts: fix shellcheck issues
by Alyssa Ross 18 Nov '22

18 Nov '22

There's an open issue about teaching shellcheck to automatically recognize nix-shell shebangs: https://github.com/koalaman/shellcheck/issues/1210 Signed-off-by: Alyssa Ross <alyssa.ross(a)unikie.com> --- v2: no changes Documentation/scripts/build.sh | 2 +- Documentation/scripts/update-gems.sh | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Documentation/scripts/build.sh b/Documentation/scripts/build.sh index 85bd5a5..e6a2cd8 100755 --- a/Documentation/scripts/build.sh +++ b/Documentation/scripts/build.sh @@ -4,7 +4,7 @@ cd "$(dirname "$0")/.." -if [ ! -w . -a ! -w .jekyll-cache ]; then +if [ ! -w . ] && [ ! -w .jekyll-cache ]; then JEKYLLFLAGS=--disable-disk-cache fi diff --git a/Documentation/scripts/update-gems.sh b/Documentation/scripts/update-gems.sh index 7ba98d8..d70f55f 100755 --- a/Documentation/scripts/update-gems.sh +++ b/Documentation/scripts/update-gems.sh @@ -1,8 +1,11 @@ #! /usr/bin/env nix-shell #! nix-shell -i bash -p bundler bundix + # SPDX-FileCopyrightText: 2022 Unikie # SPDX-License-Identifier: EUPL-1.2+ +# shellcheck shell=bash + set -euo pipefail bundle lock --update -- 2.35.1

3 4

[PATCH] Documentation: rename "UUID" to "UUID Reference"
by Alyssa Ross 18 Nov '22

18 Nov '22

The purpose of this document is specifically to provide a reference table so that developers working with Spectrum can easily look up a given UUID to find out what it means. I think "UUID Reference" makes it clearer what the reader will find in this document, as I'd expect a document just called "UUID" to be something that explained e.g. what a UUID is. (I don't think we need to explain that at all in the Spectrum documentation, as we link to Wikipedia, which explains it well, and it's background knowledge that most developers looking to work with Spectrum will likely already have.) Signed-off-by: Alyssa Ross <alyssa.ross(a)unikie.com> Cc: Jenni Nikolaenko <evgeniia.nikolaenko(a)unikie.com> --- Documentation/appendices/uuid-reference.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/appendices/uuid-reference.adoc b/Documentation/appendices/uuid-reference.adoc index 146c068..5099ec3 100644 --- a/Documentation/appendices/uuid-reference.adoc +++ b/Documentation/appendices/uuid-reference.adoc @@ -1,4 +1,4 @@ -= Appendix B. UUID += Appendix B. UUID Reference :page-parent: Appendices :toclevels: 1 :page-nav_order: 2 base-commit: 395346bd3341fd0468bc4f8a9c427440c5d05438 -- 2.35.1

3 2

[PATCH] Documentation: rename "Getting Started" to "Using Spectrum"
by Alyssa Ross 18 Nov '22

18 Nov '22

I clicked on "Getting Started" by mistake expecting to find the documentation about obtaining and installing a Spectrum image, but it actually contains documentation that explains how to use the system once it's installed. So I think it would be clearer if it were named "Using Spectrum". Signed-off-by: Alyssa Ross <alyssa.ross(a)unikie.com> Cc: Jenni Nikolaenko <evgeniia.nikolaenko(a)unikie.com> --- .../{getting-started => using-spectrum}/creating-vms.adoc | 2 +- .../{getting-started => using-spectrum}/index.adoc | 6 +++--- .../{getting-started => using-spectrum}/running-vms.adoc | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename Documentation/{getting-started => using-spectrum}/creating-vms.adoc (98%) rename Documentation/{getting-started => using-spectrum}/index.adoc (65%) rename Documentation/{getting-started => using-spectrum}/running-vms.adoc (93%) diff --git a/Documentation/getting-started/creating-vms.adoc b/Documentation/using-spectrum/creating-vms.adoc similarity index 98% rename from Documentation/getting-started/creating-vms.adoc rename to Documentation/using-spectrum/creating-vms.adoc index a2ad952..6847249 100644 --- a/Documentation/getting-started/creating-vms.adoc +++ b/Documentation/using-spectrum/creating-vms.adoc @@ -1,5 +1,5 @@ = Creating VMs -:page-parent: Getting Started +:page-parent: Using Spectrum :page-nav_order: 1 // SPDX-FileCopyrightText: 2022 Alyssa Ross <hi(a)alyssa.is> diff --git a/Documentation/getting-started/index.adoc b/Documentation/using-spectrum/index.adoc similarity index 65% rename from Documentation/getting-started/index.adoc rename to Documentation/using-spectrum/index.adoc index 10dde7f..5ee5043 100644 --- a/Documentation/getting-started/index.adoc +++ b/Documentation/using-spectrum/index.adoc @@ -1,4 +1,4 @@ -= Getting Started += Using Spectrum :description: Exploring Spectrum OS. Using (=How-To-Guides), Configuring (adding smth). :page-nav_order: 3 :page-has_children: true @@ -9,5 +9,5 @@ Ready to get started with Spectrum? Here is what you can do next: -* xref:../getting-started/creating-vms.adoc[Create your own VM] to use others applications. -* xref:../getting-started/running-vms.adoc[Start some applications]. +* xref:creating-vms.adoc[Create your own VM] to use others applications. +* xref:running-vms.adoc[Start some applications]. diff --git a/Documentation/getting-started/running-vms.adoc b/Documentation/using-spectrum/running-vms.adoc similarity index 93% rename from Documentation/getting-started/running-vms.adoc rename to Documentation/using-spectrum/running-vms.adoc index ca70835..518b00e 100644 --- a/Documentation/getting-started/running-vms.adoc +++ b/Documentation/using-spectrum/running-vms.adoc @@ -1,5 +1,5 @@ = Running VMs -:page-parent: Getting Started +:page-parent: Using Spectrum :page-nav_order: 2 // SPDX-FileCopyrightText: 2022 Alyssa Ross <hi(a)alyssa.is> base-commit: 395346bd3341fd0468bc4f8a9c427440c5d05438 -- 2.35.1

3 2

[PATCH 1/2] Documentation/scripts: fix shellcheck issues
by Alyssa Ross 16 Nov '22

16 Nov '22

There's an open issue about teaching shellcheck to automatically recognize nix-shell shebangs: https://github.com/koalaman/shellcheck/issues/1210 Signed-off-by: Alyssa Ross <alyssa.ross(a)unikie.com> --- Documentation/scripts/build.sh | 2 +- Documentation/scripts/update-gems.sh | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Documentation/scripts/build.sh b/Documentation/scripts/build.sh index 85bd5a5..e6a2cd8 100755 --- a/Documentation/scripts/build.sh +++ b/Documentation/scripts/build.sh @@ -4,7 +4,7 @@ cd "$(dirname "$0")/.." -if [ ! -w . -a ! -w .jekyll-cache ]; then +if [ ! -w . ] && [ ! -w .jekyll-cache ]; then JEKYLLFLAGS=--disable-disk-cache fi diff --git a/Documentation/scripts/update-gems.sh b/Documentation/scripts/update-gems.sh index 7ba98d8..d70f55f 100755 --- a/Documentation/scripts/update-gems.sh +++ b/Documentation/scripts/update-gems.sh @@ -1,8 +1,11 @@ #! /usr/bin/env nix-shell #! nix-shell -i bash -p bundler bundix + # SPDX-FileCopyrightText: 2022 Unikie # SPDX-License-Identifier: EUPL-1.2+ +# shellcheck shell=bash + set -euo pipefail bundle lock --update -- 2.35.1

2 3

[PATCH v2 0/6] Introduce a shared base for application VMs
by Alyssa Ross 14 Nov '22

14 Nov '22

This series was originally developed for some work I'm finishing up now for NLnet, for letting Spectrum users build VMs on the system with Nix, so it's time for it to get another round. Changes since v1: • make-vm.nix only generates the VM's configuration directory, not a whole data/$name hierarchy that needs to be merged. • vm-lib/make-vm.nix and vm/make-vm.nix are separated, so system-provided VMs can deduplicate against the base image, while user-defined VMs can't so they're independently upgradeable. v1: https://spectrum-os.org/lists/archives/spectrum-devel/20220919073659.170327… The idea here is to reduce duplication between application VMs, both in terms of source code size and output size. After this change, creating a new VM just requires writing a very small Nix file like this: { config ? import ../../../nix/eval-config.nix {} }: import ../../vm-lib/make-vm.nix { inherit config; } { name = "appvm-lynx"; providers.net = [ "netvm" ]; run = config.pkgs.pkgsStatic.callPackage ( { writeScript, lynx }: writeScript "run-lynx" '' #!/bin/execlineb -P ${lynx}/bin/lynx https://spectrum-os.org '' ) { }; } Rather than a whole big source tree as before, most of which was duplicated with every other application VM. When a VM generated this way is started, it gets two disk images. One is the shared base image, which is part of the Spectrum base system, and the other contains only the application-specific stuff: the run script, and any store path dependencies that are not already present in the base image. This means that the amount of storage required for each new application VM is substantially reduced. Of course, this isn't the only way to generate VMs. Monolithic VMs like we had before would still work, with some small adjustments for the new disk layout. I also see this fitting well into making it possible to configure extra VMs at build time. It doesn't directly help with that, but making it so that each VM doesn't need to provide everything itself will make creating external VMs easier when it does happen. In future we might want to apply a similar mechanism to service VMs, like netvm, but since we only have one of those so far, it's not clear which parts exactly would be duplicated, so I'm leaving it for now. Other future work is considering the impacts of the shared base image on guest isolation. Can guests observe whether reads of the shared base image hit the host page cache, or even an internal disk cache? At the moment I suspect that the base image doesn't have enough specialised code in it that there would be any interesting results, but it's worth thinking about if the shared image grows new functionality, whether it would be interesting to another guest to be able to observe whether those resources have previously been loaded or not. If this _does_ turn out to be a concern, it could be mitigated by simply copying the base image to temporary storage before booting a VM, and then booting the VM from the copy. Alyssa Ross (6): host/start-vm: support multiple block devices scripts/make-gpt.sh: add support for labels vm: build GPT images host/start-vm: boot using partition label release: rename from "img" img/app: extract from appvm-{lynx,catgirl} Documentation/creating-vms.adoc | 8 +- Documentation/getting-spectrum.adoc | 2 +- host/initramfs/extfs.nix | 19 +-- host/rootfs/default.nix | 11 +- host/start-vm/lib.rs | 38 +++++- host/start-vm/tests/vm_command-basic.rs | 6 +- {vm/app/lynx => img/app}/Makefile | 57 ++++---- {vm/app/catgirl => img/app}/bin | 0 {vm/app/lynx => img/app}/default.nix | 22 ++-- img/app/etc/fstab | 8 ++ {vm/app/catgirl => img/app}/etc/init | 0 {vm/app/catgirl => img/app}/etc/mdev.conf | 0 {vm/app/lynx => img/app}/etc/mdev/iface | 2 +- {vm/app/catgirl => img/app}/etc/passwd | 0 .../catgirl => img/app}/etc/passwd.license | 0 {vm/app/catgirl => img/app}/etc/resolv.conf | 0 .../app}/etc/s6-linux-init/scripts/rc.init | 1 + .../s6-rc/lynx => img/app/etc/s6-rc/app}/run | 3 +- .../catgirl => img/app/etc/s6-rc/app}/type | 0 .../app/etc/s6-rc/app}/type.license | 0 .../etc/s6-rc/mdevd-coldplug/dependencies | 0 .../app}/etc/s6-rc/mdevd-coldplug/type | 0 .../etc/s6-rc/mdevd-coldplug/type.license | 0 .../app}/etc/s6-rc/mdevd-coldplug/up | 0 .../app}/etc/s6-rc/mdevd/notification-fd | 0 .../etc/s6-rc/mdevd/notification-fd.license | 0 .../catgirl => img/app}/etc/s6-rc/mdevd/run | 0 .../catgirl => img/app}/etc/s6-rc/mdevd/type | 0 .../app}/etc/s6-rc/mdevd/type.license | 0 .../app}/etc/s6-rc/ok-all/contents | 0 .../catgirl => img/app}/etc/s6-rc/ok-all/type | 0 .../app}/etc/s6-rc/ok-all/type.license | 0 .../app}/etc/ssl/certs/ca-certificates.crt | 0 {vm/app/lynx => img/app}/shell.nix | 11 +- release.nix | 2 +- {img => release}/combined/default.nix | 0 {img => release}/combined/eosimages.nix | 0 {img => release}/combined/grub.cfg.in | 0 {img => release}/combined/run-vm.nix | 0 ...ble-gpt-partition-attribute-55-check.patch | 0 ...pt-disable-partition-table-CRC-check.patch | 0 .../0003-install-remove-Endless-OS-ad.patch | 0 ...4-finished-don-t-run-eos-diagnostics.patch | 0 ...omote-spectrum-not-the-Endless-forum.patch | 0 {img => release}/installer/app/default.nix | 0 .../installer/app/vendor-customer-support.ini | 0 {img => release}/installer/configuration.nix | 0 {img => release}/installer/default.nix | 0 {img => release}/installer/run-vm.nix | 0 {img => release}/installer/seat.rules | 0 {img => release}/live/Makefile | 0 {img => release}/live/default.nix | 0 {img => release}/live/shell.nix | 0 scripts/make-gpt.sh | 4 +- scripts/sfdisk-field.awk | 2 +- vm-lib/make-vm.nix | 51 ++++++++ vm/app/catgirl.nix | 17 +++ vm/app/catgirl/Makefile | 123 ------------------ vm/app/catgirl/default.nix | 92 ------------- vm/app/catgirl/etc/fstab | 6 - vm/app/catgirl/etc/mdev/iface | 36 ----- .../catgirl/etc/s6-linux-init/scripts/rc.init | 10 -- vm/app/catgirl/etc/s6-rc/catgirl/run | 31 ----- .../data/appvm-catgirl/providers/net/netvm | 0 vm/app/catgirl/shell.nix | 17 --- vm/app/lynx.nix | 15 +++ vm/app/lynx/bin | 1 - vm/app/lynx/etc/fstab | 6 - vm/app/lynx/etc/init | 5 - vm/app/lynx/etc/mdev.conf | 5 - vm/app/lynx/etc/passwd | 1 - vm/app/lynx/etc/passwd.license | 2 - vm/app/lynx/etc/resolv.conf | 4 - vm/app/lynx/etc/s6-rc/lynx/type | 1 - vm/app/lynx/etc/s6-rc/lynx/type.license | 2 - .../etc/s6-rc/mdevd-coldplug/dependencies | 4 - vm/app/lynx/etc/s6-rc/mdevd-coldplug/type | 1 - .../etc/s6-rc/mdevd-coldplug/type.license | 2 - vm/app/lynx/etc/s6-rc/mdevd-coldplug/up | 4 - vm/app/lynx/etc/s6-rc/mdevd/notification-fd | 1 - .../etc/s6-rc/mdevd/notification-fd.license | 2 - vm/app/lynx/etc/s6-rc/mdevd/run | 5 - vm/app/lynx/etc/s6-rc/mdevd/type | 1 - vm/app/lynx/etc/s6-rc/mdevd/type.license | 2 - vm/app/lynx/etc/s6-rc/ok-all/contents | 4 - vm/app/lynx/etc/s6-rc/ok-all/type | 1 - vm/app/lynx/etc/s6-rc/ok-all/type.license | 2 - vm/app/lynx/etc/ssl/certs/ca-certificates.crt | 1 - .../host/data/appvm-lynx/providers/net/netvm | 0 vm/make-vm.nix | 9 ++ vm/sys/net/Makefile | 23 ++-- vm/sys/net/default.nix | 10 +- 92 files changed, 236 insertions(+), 457 deletions(-) rename {vm/app/lynx => img/app}/Makefile (66%) rename {vm/app/catgirl => img/app}/bin (100%) rename {vm/app/lynx => img/app}/default.nix (77%) create mode 100644 img/app/etc/fstab rename {vm/app/catgirl => img/app}/etc/init (100%) rename {vm/app/catgirl => img/app}/etc/mdev.conf (100%) rename {vm/app/lynx => img/app}/etc/mdev/iface (98%) rename {vm/app/catgirl => img/app}/etc/passwd (100%) rename {vm/app/catgirl => img/app}/etc/passwd.license (100%) rename {vm/app/catgirl => img/app}/etc/resolv.conf (100%) rename {vm/app/lynx => img/app}/etc/s6-linux-init/scripts/rc.init (90%) rename {vm/app/lynx/etc/s6-rc/lynx => img/app/etc/s6-rc/app}/run (80%) rename {vm/app/catgirl/etc/s6-rc/catgirl => img/app/etc/s6-rc/app}/type (100%) rename {vm/app/catgirl/etc/s6-rc/catgirl => img/app/etc/s6-rc/app}/type.license (100%) rename {vm/app/catgirl => img/app}/etc/s6-rc/mdevd-coldplug/dependencies (100%) rename {vm/app/catgirl => img/app}/etc/s6-rc/mdevd-coldplug/type (100%) rename {vm/app/catgirl => img/app}/etc/s6-rc/mdevd-coldplug/type.license (100%) rename {vm/app/catgirl => img/app}/etc/s6-rc/mdevd-coldplug/up (100%) rename {vm/app/catgirl => img/app}/etc/s6-rc/mdevd/notification-fd (100%) rename {vm/app/catgirl => img/app}/etc/s6-rc/mdevd/notification-fd.license (100%) rename {vm/app/catgirl => img/app}/etc/s6-rc/mdevd/run (100%) rename {vm/app/catgirl => img/app}/etc/s6-rc/mdevd/type (100%) rename {vm/app/catgirl => img/app}/etc/s6-rc/mdevd/type.license (100%) rename {vm/app/catgirl => img/app}/etc/s6-rc/ok-all/contents (100%) rename {vm/app/catgirl => img/app}/etc/s6-rc/ok-all/type (100%) rename {vm/app/catgirl => img/app}/etc/s6-rc/ok-all/type.license (100%) rename {vm/app/catgirl => img/app}/etc/ssl/certs/ca-certificates.crt (100%) rename {vm/app/lynx => img/app}/shell.nix (51%) rename {img => release}/combined/default.nix (100%) rename {img => release}/combined/eosimages.nix (100%) rename {img => release}/combined/grub.cfg.in (100%) rename {img => release}/combined/run-vm.nix (100%) rename {img => release}/installer/app/0001-gpt-disable-gpt-partition-attribute-55-check.patch (100%) rename {img => release}/installer/app/0002-gpt-disable-partition-table-CRC-check.patch (100%) rename {img => release}/installer/app/0003-install-remove-Endless-OS-ad.patch (100%) rename {img => release}/installer/app/0004-finished-don-t-run-eos-diagnostics.patch (100%) rename {img => release}/installer/app/0005-finished-promote-spectrum-not-the-Endless-forum.patch (100%) rename {img => release}/installer/app/default.nix (100%) rename {img => release}/installer/app/vendor-customer-support.ini (100%) rename {img => release}/installer/configuration.nix (100%) rename {img => release}/installer/default.nix (100%) rename {img => release}/installer/run-vm.nix (100%) rename {img => release}/installer/seat.rules (100%) rename {img => release}/live/Makefile (100%) rename {img => release}/live/default.nix (100%) rename {img => release}/live/shell.nix (100%) create mode 100644 vm-lib/make-vm.nix create mode 100644 vm/app/catgirl.nix delete mode 100644 vm/app/catgirl/Makefile delete mode 100644 vm/app/catgirl/default.nix delete mode 100644 vm/app/catgirl/etc/fstab delete mode 100755 vm/app/catgirl/etc/mdev/iface delete mode 100755 vm/app/catgirl/etc/s6-linux-init/scripts/rc.init delete mode 100755 vm/app/catgirl/etc/s6-rc/catgirl/run delete mode 100644 vm/app/catgirl/host/data/appvm-catgirl/providers/net/netvm delete mode 100644 vm/app/catgirl/shell.nix create mode 100644 vm/app/lynx.nix delete mode 120000 vm/app/lynx/bin delete mode 100644 vm/app/lynx/etc/fstab delete mode 100755 vm/app/lynx/etc/init delete mode 100644 vm/app/lynx/etc/mdev.conf delete mode 100644 vm/app/lynx/etc/passwd delete mode 100644 vm/app/lynx/etc/passwd.license delete mode 100644 vm/app/lynx/etc/resolv.conf delete mode 100644 vm/app/lynx/etc/s6-rc/lynx/type delete mode 100644 vm/app/lynx/etc/s6-rc/lynx/type.license delete mode 100644 vm/app/lynx/etc/s6-rc/mdevd-coldplug/dependencies delete mode 100644 vm/app/lynx/etc/s6-rc/mdevd-coldplug/type delete mode 100644 vm/app/lynx/etc/s6-rc/mdevd-coldplug/type.license delete mode 100644 vm/app/lynx/etc/s6-rc/mdevd-coldplug/up delete mode 100644 vm/app/lynx/etc/s6-rc/mdevd/notification-fd delete mode 100644 vm/app/lynx/etc/s6-rc/mdevd/notification-fd.license delete mode 100644 vm/app/lynx/etc/s6-rc/mdevd/run delete mode 100644 vm/app/lynx/etc/s6-rc/mdevd/type delete mode 100644 vm/app/lynx/etc/s6-rc/mdevd/type.license delete mode 100644 vm/app/lynx/etc/s6-rc/ok-all/contents delete mode 100644 vm/app/lynx/etc/s6-rc/ok-all/type delete mode 100644 vm/app/lynx/etc/s6-rc/ok-all/type.license delete mode 120000 vm/app/lynx/etc/ssl/certs/ca-certificates.crt delete mode 100644 vm/app/lynx/host/data/appvm-lynx/providers/net/netvm create mode 100644 vm/make-vm.nix base-commit: 7a6d44e24ddcc9cba73deed25fb85038b7c3d823 -- 2.37.1

2 12