merged layer handling

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "init"
-version = "2.4.2"
+version = "2.6.0"
 edition = "2024"
 
 [profile.release]
@@ -16,7 +16,7 @@ env_logger = "0.11.3"
 eyre = { version = "0.6.12" }
 itertools = "0.14.0"
 log = "0.4.21"
-nix = { version = "0.30.1", features = ["feature", "mount", "process", "reboot", "signal"] }
+nix = { version = "0.31.1", features = ["feature", "mount", "process", "reboot", "signal"] }
 regex = "1.11.1"
 serde = { version = "1.0.198", features = ["derive"] }
 serde_json = "1.0.116"
@@ -25,8 +25,9 @@ shell-escape = "0.1.5"
 tokio = { version = "1.38.0", features = ["rt", "net", "fs", "process", "io-std", "io-util", "sync", "macros", "signal"] }
 termios = "0.3.3"
 unix_mode = "0.1.4"
-base64 = "0.22.1"
 sys-info = "0.9.1"
 dkl = { git = "https://novit.tech/direktil/dkl", version = "1.0.0" }
 openssl = "0.10.73"
-reqwest = { version = "0.12.22", features = ["native-tls"] }
+reqwest = { version = "0.13.1", features = ["native-tls"] }
+glob = "0.3.3"
+hex = "0.4.3"

Dockerfile

@@ -1,4 +1,4 @@
-from rust:1.88.0-alpine as rust
+from rust:1.95.0-alpine as rust
 
 run apk add --no-cache git musl-dev libudev-zero-dev openssl-dev cryptsetup-dev lvm2-dev clang-libs clang-dev
 
@@ -9,8 +9,7 @@ run --mount=type=cache,id=novit-rs,target=/usr/local/cargo/registry \
     RUSTFLAGS="-C target-feature=-crt-static" cargo install --path . --root /dist
 
 # ------------------------------------------------------------------------
-from alpine:3.22.0 as initrd
+from alpine:3.23.4 as system
-run apk add zstd lz4
 
 workdir /system
 
@@ -18,9 +17,9 @@ run . /etc/os-release \
  && wget -O- https://dl-cdn.alpinelinux.org/alpine/v${VERSION_ID%.*}/releases/x86_64/alpine-minirootfs-${VERSION_ID}-x86_64.tar.gz |tar zxv
 
 run apk add --no-cache --update -p . musl libgcc coreutils \
- lvm2 lvm2-extra lvm2-dmeventd udev cryptsetup \
+ iproute2 lvm2 lvm2-extra lvm2-dmeventd udev cryptsetup \
  e2fsprogs lsblk openssl openssh-server wireguard-tools-wg-quick \
- && rm -rf usr/share/apk var/cache/apk etc/motd
+ && rm -rf usr/share/apk var/cache/apk etc/motd dev/*
 
 copy etc/sshd_config etc/ssh/sshd_config
 
@@ -31,9 +30,26 @@ run mkdir -p bin run var/log; cd bin && for cmd in init-version init-connect boo
 # check viability
 run chroot . init-version
 
-run find * |cpio -H newc -oF /initrd
+# ------------------------------------------------------------------------
+from alpine:3.23.4 as initrd
+
+copy --from=system /system /system
+run cd /system && find * |cpio -H newc -oF /initrd
 
 # ------------------------------------------------------------------------
-from alpine:3.22.0
+from debian:stable-backports as initramfs
-copy --from=initrd /initrd /
+run apt update && apt install -y erofs-utils
+
+copy --from=system /system /system
+run mkfs.erofs \
+  -z lzma -C131072 -Efragments,ztailpacking \
+  -T0 --all-time --ignore-mtime \
+  /initramfs /system
+
+# ------------------------------------------------------------------------
+from alpine:3.23.4
+copy --from=initrd /initrd /initrd
 entrypoint ["base64","/initrd"]
+
+#copy --from=initramfs /initramfs /
+#entrypoint ["base64","/initramfs"]

modd.conf

@@ -4,6 +4,7 @@ modd.conf {}
   prep: cargo test
   prep: cargo build
   prep: debug/init-version
+  #prep: cargo run --bin test
 }
 
 target/debug/init Dockerfile {

src/cmd.rs

@@ -1,5 +1,4 @@
 pub mod bootstrap;
-pub mod connect_boot;
 pub mod init;
 pub mod init_input;
 pub mod version;

src/cmd/connect_boot.rs

@@ -1 +0,0 @@
-

src/cmd/init.rs

@@ -1,6 +1,7 @@
 use eyre::{format_err, Result};
 use log::{error, info, warn};
 use std::collections::BTreeSet as Set;
+use std::convert::Infallible;
 use std::os::unix::fs::symlink;
 use tokio::sync::Mutex;
 use tokio::{fs, process::Command};
@@ -53,10 +54,10 @@ pub async fn run() {
     info!("Linux version {kernel_version}");
 
     // mount basic filesystems
-    mount(None, "/proc", "proc", None).await;
+    mount(None::<&str>, "/proc", "proc", None).await;
-    mount(None, "/sys", "sysfs", None).await;
+    mount(None::<&str>, "/sys", "sysfs", None).await;
-    mount(None, "/dev", "devtmpfs", None).await;
+    mount(None::<&str>, "/dev", "devtmpfs", None).await;
-    mount(None, "/dev/pts", "devpts", Some("gid=5,mode=620")).await;
+    mount(None::<&str>, "/dev/pts", "devpts", Some("gid=5,mode=620")).await;
 
     if utils::bool_param("debug") {
         log::set_max_level(log::LevelFilter::Debug);
@@ -106,6 +107,7 @@ pub async fn run() {
     // Wireguard VPNs
     for (name, conf) in &cfg.vpns {
         retry_or_ignore(async || {
+            info!("starting VPN {name}");
             let dir = "/etc/wireguard";
             fs::create_dir_all(dir).await?;
 
@@ -142,6 +144,9 @@ pub async fn run() {
         warn!("failed to copy {INIT_LOG} to system: {e}");
     }
 
+    if let Err(e) = nix::mount::umount2("/modules", nix::mount::MntFlags::MNT_DETACH) {
+        warn!("failed to umount /modules: {e}");
+    }
     retry(async || switch_root("/system").await).await;
 }
 
@@ -172,15 +177,27 @@ async fn chmod(path: impl AsRef<Path>, mode: u32) -> std::io::Result<()> {
     fs::set_permissions(path, perms).await
 }
 
-async fn mount(src: Option<&str>, dst: &str, fstype: &str, opts: Option<&str>) {
+async fn mount<S: AsRef<Path>>(
+    src: Option<S>,
+    dst: impl AsRef<Path>,
+    fstype: &str,
+    opts: Option<&str>,
+) {
+    let src = src.as_ref().map(|s| s.as_ref());
+    let src_str = src.map(|s| s.display().to_string());
+    let src_str = src_str.as_deref();
+
+    let dst = dst.as_ref();
+    let dst_str = &dst.display().to_string();
+
     if let Err(e) = fs::create_dir_all(dst).await {
-        error!("failed to create dir {dst}: {e}");
+        error!("failed to create dir {dst_str}: {e}");
     }
 
     retry_or_ignore(async || {
         let mut is_file = false;
 
-        if let Some(src) = src {
+        if let Some(src) = src_str {
             is_file = (fs::metadata(src).await)
                 .map_err(|e| format_err!("stat {src} failed: {e}"))?
                 .is_file();
@@ -192,7 +209,7 @@ async fn mount(src: Option<&str>, dst: &str, fstype: &str, opts: Option<&str>) {
             }
         }
 
-        let mut args = vec![src.unwrap_or("none"), dst, "-t", fstype];
+        let mut args = vec![src_str.unwrap_or("none"), dst_str, "-t", fstype];
         if let Some(opts) = opts {
             args.extend(["-o", opts]);
         }
@@ -204,11 +221,17 @@ async fn mount(src: Option<&str>, dst: &str, fstype: &str, opts: Option<&str>) {
         }
 
         let (cmd_str, _) = cmd_str("mount", &args);
-        let flags = nix::mount::MsFlags::empty();
 
         info!("# {cmd_str}",);
-        nix::mount::mount(src, dst, Some(fstype), flags, opts)
+
-            .map_err(|e| format_err!("mount {dst} failed: {e}"))
+        let mount = |flags| nix::mount::mount(src, dst, Some(fstype), flags, opts);
+
+        use nix::{errno::Errno, mount::MsFlags};
+        match mount(MsFlags::empty()) {
+            Err(Errno::EACCES) => mount(MsFlags::MS_RDONLY),
+            r => r,
+        }
+        .map_err(|e| format_err!("mount {dst_str} failed: {e}"))
     })
     .await
 }
@@ -223,6 +246,25 @@ async fn start_daemon(prog: &str, args: &[&str]) {
     .await;
 }
 
+async fn try_exec_cmd(mut cmd: tokio::process::Command) -> Result<()> {
+    info!(
+        "# {} {}",
+        cmd.as_std().get_program().to_string_lossy(),
+        cmd.as_std()
+            .get_args()
+            .map(|a| a.to_string_lossy())
+            .collect::<Vec<_>>()
+            .join(" ")
+    );
+
+    let s = cmd.status().await?;
+    if s.success() {
+        Ok(())
+    } else {
+        Err(format_err!("command failed: {s}"))
+    }
+}
+
 async fn try_exec(prog: &str, args: &[&str]) -> Result<()> {
     let (cmd_str, mut cmd) = cmd_str(prog, args);
     info!("# {cmd_str}");
@@ -322,13 +364,7 @@ async fn child_reaper() {
     }
 }
 
-macro_rules! cstr {
+async fn switch_root(root: &str) -> Result<Infallible> {
-    ($s:expr) => {
-        std::ffi::CString::new($s)?.as_c_str()
-    };
-}
-
-async fn switch_root(root: &str) -> Result<()> {
     info!("killing all processes and switching root");
     dklog::LOG.close().await;
 
@@ -339,7 +375,13 @@ async fn switch_root(root: &str) -> Result<()> {
         eprintln!("failed to kill processes: {e}");
     }
 
-    nix::unistd::execv(
+    macro_rules! cstr {
+        ($s:expr) => {
+            std::ffi::CString::new($s)?.as_c_str()
+        };
+    }
+
+    Ok(nix::unistd::execv(
         cstr!("/sbin/switch_root"),
         &[
             cstr!("switch_root"),
@@ -348,8 +390,5 @@ async fn switch_root(root: &str) -> Result<()> {
             cstr!(root),
             cstr!("/sbin/init"),
         ],
-    )
+    )?)
-    .unwrap();
-
-    unreachable!();
 }

src/cmd/init/bootstrap.rs

@@ -1,25 +1,41 @@
 use eyre::{format_err, Result};
 use log::{info, warn};
+use std::path::{Path, PathBuf};
 use tokio::{
     fs,
-    io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
+    io::{AsyncBufReadExt, AsyncReadExt, AsyncWrite, AsyncWriteExt, BufReader},
 };
 
 use dkl::{
     self,
     apply::{self, chroot, set_perms},
+    base64_decode,
     bootstrap::Config,
 };
 
-use super::{exec, mount, retry, retry_or_ignore, try_exec};
+use super::{exec, mount, retry, retry_or_ignore, try_exec, try_exec_cmd};
-use crate::utils;
+use crate::{fs::walk_dir, utils};
 
 pub async fn bootstrap(cfg: Config) {
     let verifier = retry(async || Verifier::from_config(&cfg)).await;
-    let bs = cfg.bootstrap;
+    let bs = &cfg.bootstrap;
 
     mount(Some(&bs.dev), "/bootstrap", "ext4", None).await;
 
+    // VPNs
+    for vpn_conf in walk_dir("/bootstrap/vpns").await {
+        if !vpn_conf.ends_with(".conf") {
+            continue;
+        }
+
+        retry_or_ignore(async || {
+            info!("starting VPN from {vpn_conf}");
+            try_exec("wg-quick", &["up", &vpn_conf]).await
+        })
+        .await;
+    }
+
+    // prepare system
     let boot_version = utils::param("version").unwrap_or("current");
     let base_dir = &format!("/bootstrap/{boot_version}");
 
@@ -38,7 +54,7 @@ pub async fn bootstrap(cfg: Config) {
     })
     .await;
 
-    mount_system(&sys_cfg, base_dir, &verifier).await;
+    mount_system(&sys_cfg, &cfg, base_dir, &verifier).await;
 
     retry_or_ignore(async || {
         let path = "/etc/resolv.conf";
@@ -50,13 +66,11 @@ pub async fn bootstrap(cfg: Config) {
     })
     .await;
 
-    retry_or_ignore(async || apply::files(&sys_cfg.files, "/system").await).await;
+    retry_or_ignore(async || apply::files(&sys_cfg.files, "/system", false).await).await;
 
     apply_groups(&sys_cfg.groups, "/system").await;
     apply_users(&sys_cfg.users, "/system").await;
 
-    // TODO VPNs
-
     mount_filesystems(&sys_cfg.mounts, "/system").await;
 
     retry_or_ignore(async || {
@@ -77,24 +91,27 @@ impl Verifier {
             return Ok(Self { pubkey: None });
         };
 
-        use base64::{prelude::BASE64_STANDARD, Engine};
+        let pubkey = base64_decode(pubkey)?;
-        let pubkey = BASE64_STANDARD.decode(pubkey)?;
         let pubkey = Some(pubkey);
 
         return Ok(Self { pubkey });
     }
 
-    async fn verify_path(&self, path: &str) -> Result<Vec<u8>> {
+    async fn verify_path(&self, path: impl AsRef<Path>) -> Result<Vec<u8>> {
-        let data = (fs::read(path).await).map_err(|e| format_err!("failed to read {path}: {e}"))?;
+        let path = path.as_ref();
+        let p = path.display();
+
+        let data = (fs::read(path).await).map_err(|e| format_err!("failed to read {p}: {e}"))?;
 
         let Some(ref pubkey) = self.pubkey else {
             return Ok(data);
         };
 
-        info!("verifying {path}");
+        info!("verifying {p}");
 
-        let sig = &format!("{path}.sig");
+        let sig = path.with_added_extension("sig");
-        let sig = (fs::read(sig).await).map_err(|e| format_err!("failed to read {sig}: {e}"))?;
+        let sig = (fs::read(&sig).await)
+            .map_err(|e| format_err!("failed to read {}: {e}", sig.display()))?;
 
         use openssl::{hash::MessageDigest, pkey::PKey, sign::Verifier};
         let pubkey = PKey::public_key_from_der(pubkey)?;
@@ -106,7 +123,7 @@ impl Verifier {
         if sig_ok {
             Ok(data)
         } else {
-            Err(format_err!("signature verification failed for {path}"))
+            Err(format_err!("signature verification failed for {p}"))
         }
     }
 }
@@ -175,47 +192,136 @@ fn default_root_tmpfs_opts() -> Option<String> {
     Some(format!("size={fs_size}m"))
 }
 
-async fn mount_system(cfg: &dkl::Config, bs_dir: &str, verifier: &Verifier) {
+struct LayerMounter<'t> {
+    bs_dir: &'t str,
+    layers_dir: &'t str,
+    verifier: &'t Verifier,
+    lower_dir: String,
+}
+
+impl LayerMounter<'_> {
+    fn src_path(&self, name: &str) -> PathBuf {
+        let mut p = PathBuf::from(self.bs_dir);
+        p.push(name);
+        if name != "merged" {
+            p.add_extension("fs");
+        }
+        p
+    }
+
+    async fn exists(&self, name: &str) -> bool {
+        retry(async || Ok(fs::try_exists(self.src_path(name)).await?)).await
+    }
+
+    async fn mount(&mut self, name: &str) {
+        self.mount_path(self.src_path(name), name, true).await
+    }
+
+    async fn mount_path(&mut self, src: impl AsRef<Path>, name: &str, verify: bool) {
+        let src = src.as_ref();
+        let tgt_dir = PathBuf::from(self.layers_dir).join(name);
+        let tgt = tgt_dir.with_added_extension("fs");
+
+        if let Err(e) = fs::create_dir_all(&tgt_dir).await {
+            warn!("mkdir -p {}: {e}", tgt_dir.display());
+        }
+
+        let mount_src = if name == "merged" {
+            retry(async || {
+                let data = self.verifier.verify_path(src).await?;
+                let data = MergedLayer::from_bytes(&data)
+                    .ok_or(format_err!("{}: invalid data", src.display()))?;
+
+                data.create(&tgt)
+                    .await
+                    .map_err(|e| format_err!("write {}: {e}", tgt.display()))?;
+
+                let dm_name = &format!("system");
+
+                let mut cmd = tokio::process::Command::new("veritysetup");
+                cmd.arg("open")
+                    .arg(format!("--hash-offset={}", data.hash_offset()))
+                    .arg(&tgt)
+                    .arg(dm_name)
+                    .arg(&tgt)
+                    .arg(data.root_hash_hex());
+
+                try_exec_cmd(cmd).await?;
+
+                Ok(PathBuf::from("/dev/mapper").join(dm_name))
+            })
+            .await
+        } else {
+            retry(async || {
+                let src = if verify {
+                    self.verifier.verify_path(src).await?
+                } else {
+                    fs::read(src).await?
+                };
+                fs::write(&tgt, &src).await?;
+
+                Ok(tgt.clone())
+            })
+            .await
+        };
+
+        retry(async || {
+            let mut buf = [0u8; 1028];
+            fs::File::open(&mount_src)
+                .await
+                .map_err(|e| format_err!("open {}: {e}", mount_src.display()))?
+                .read_exact(&mut buf)
+                .await
+                .map_err(|e| format_err!("read {}: {e}", mount_src.display()))?;
+
+            let fstype = if buf[1024..1028] == 0xE0F5E1E2u32.to_le_bytes() {
+                "erofs"
+            } else {
+                "squashfs"
+            };
+
+            mount(Some(&mount_src), &tgt_dir, fstype, None).await;
+            Ok(())
+        })
+        .await;
+
+        if !self.lower_dir.is_empty() {
+            self.lower_dir.push(':');
+        }
+        self.lower_dir.push_str(&tgt_dir.to_string_lossy());
+    }
+}
+
+async fn mount_system(cfg: &dkl::Config, bs_cfg: &Config, bs_dir: &str, verifier: &Verifier) {
     let opts = match utils::param("root-opts") {
         Some(s) => Some(s.to_string()),
         None => default_root_tmpfs_opts(),
     };
 
     let mem_dir = "/mem";
-    mount(None, mem_dir, "tmpfs", opts.as_deref()).await;
+    mount(None::<&str>, mem_dir, "tmpfs", opts.as_deref()).await;
 
-    let layers_dir = &format!("{mem_dir}/layers");
+    let mut mounter = LayerMounter {
-    let mut lower_dir = String::new();
+        bs_dir,
+        layers_dir: &format!("{mem_dir}/layers"),
+        verifier,
+        lower_dir: String::new(),
+    };
 
-    for layer in &cfg.layers {
+    if mounter.exists("merged").await {
-        let src = retry(async || {
+        mounter.mount("merged").await;
-            if layer == "modules" {
+    } else {
-                let src = "/modules.sqfs";
+        for layer in &cfg.layers {
-                (fs::read(src).await).map_err(|e| format_err!("read {src} failed: {e}"))
+            if layer == "modules" && bs_cfg.modules.is_some() {
-            } else {
+                continue; // take modules from initrd
-                verifier.verify_path(&format!("{bs_dir}/{layer}.fs")).await
             }
-        })
-        .await;
 
-        let tgt = &format!("{mem_dir}/{layer}.fs");
+            mounter.mount(layer).await;
-        retry(async || {
-            info!("copying layer {layer}");
-
-            let mut out = (fs::File::create(tgt).await)
-                .map_err(|e| format_err!("create {tgt} failed: {e}"))?;
-            (out.write_all(&src).await).map_err(|e| format_err!("write failed: {e}"))?;
-            (out.flush().await).map_err(|e| format_err!("write failed: {e}"))
-        })
-        .await;
-
-        let layer_dir = &format!("{layers_dir}/{layer}");
-        mount(Some(tgt), layer_dir, "squashfs", None).await;
-
-        if !lower_dir.is_empty() {
-            lower_dir.push(':');
         }
-        lower_dir.push_str(&layer_dir);
+    }
+
+    if let Some(ref modules) = bs_cfg.modules {
+        mounter.mount_path(modules, "modules", false).await;
     }
 
     let upper_dir = &format!("{mem_dir}/upper");
@@ -228,8 +334,9 @@ async fn mount_system(cfg: &dkl::Config, bs_dir: &str, verifier: &Verifier) {
     })
     .await;
 
+    let lower_dir = &mounter.lower_dir;
     let opts = format!("lowerdir={lower_dir},upperdir={upper_dir},workdir={work_dir}");
-    mount(None, "/system", "overlay", Some(&opts)).await;
+    mount(None::<&str>, "/system", "overlay", Some(&opts)).await;
 
     // make root rshared (default in systemd, required by Kubernetes 1.10+)
     // equivalent to "mount --make-rshared /"
@@ -243,6 +350,53 @@ async fn mount_system(cfg: &dkl::Config, bs_dir: &str, verifier: &Verifier) {
     .await;
 }
 
+struct MergedLayer<'t> {
+    #[allow(unused)]
+    root_hash_sig: &'t [u8],
+    root_hash: &'t [u8],
+    data: &'t [u8],
+    hash: &'t [u8],
+}
+
+impl<'t> MergedLayer<'t> {
+    fn from_bytes(mut src: &'t [u8]) -> Option<Self> {
+        let mut next = || {
+            let (len, rem) = src.split_at_checked(8)?;
+            let len = u64::from_be_bytes(len.try_into().ok()?);
+            let (data, rem) = rem.split_at_checked(len as usize)?;
+            src = rem;
+            Some(data)
+        };
+
+        Some(Self {
+            root_hash_sig: next()?,
+            root_hash: next()?,
+            data: next()?,
+            hash: next()?,
+        })
+    }
+
+    async fn create(&self, path: impl AsRef<Path>) -> std::io::Result<()> {
+        let mut out = fs::File::create(path).await?;
+        self.write_to(&mut out).await?;
+        out.shutdown().await
+    }
+
+    async fn write_to(&self, mut out: impl AsyncWrite + Unpin) -> std::io::Result<()> {
+        out.write_all(self.data).await?;
+        out.write_all(self.hash).await?;
+        Ok(())
+    }
+
+    fn hash_offset(&self) -> usize {
+        self.data.len()
+    }
+
+    fn root_hash_hex(&self) -> String {
+        hex::encode(self.root_hash)
+    }
+}
+
 async fn apply_groups(groups: &[dkl::Group], root: &str) {
     for group in groups {
         let mut args = vec![root, "groupadd", "-r"];

src/cmd/init/dmcrypt.rs

@@ -1,4 +1,4 @@
-use eyre::{Result, format_err};
+use eyre::{format_err, Result};
 use log::{error, info, warn};
 use std::collections::BTreeSet as Set;
 use std::process::Stdio;
@@ -6,7 +6,7 @@ use tokio::io::AsyncWriteExt;
 use tokio::process::Command;
 use tokio::sync::Mutex;
 
-use super::{USED_DEVS, retry_or_ignore};
+use super::{retry_or_ignore, USED_DEVS};
 use crate::blockdev::{is_uninitialized, uninitialize};
 use crate::fs::walk_dir;
 use crate::input;
@@ -29,7 +29,7 @@ pub async fn setup(devs: &[CryptDev]) {
         let all_devs = walk_dir("/dev").await;
 
         for dev in devs {
-            let mut mappings = find_dev(dev, &all_devs);
+            let mut mappings = find_dev(dev, &all_devs).await?;
             mappings.retain(|(_, dev_path)| !used_devs.contains(dev_path));
 
             if mappings.is_empty() && !dev.optional() && !done.contains(&dev.name) {
@@ -56,27 +56,72 @@ pub async fn setup(devs: &[CryptDev]) {
     .await;
 }
 
-static PREV_PW: Mutex<String> = Mutex::const_new(String::new());
+struct PrevPw {
+    pw: String,
+    reuse: bool,
+}
+
+impl PrevPw {
+    fn is_set(&self) -> bool {
+        !self.pw.is_empty()
+    }
+
+    fn can_reuse(&self) -> bool {
+        self.reuse && self.is_set()
+    }
+
+    fn invalidate(&mut self) {
+        self.pw = String::new();
+        self.reuse = false;
+    }
+
+    async fn input(&mut self, prompt: impl std::fmt::Display) -> String {
+        if self.can_reuse() {
+            info!("reusing password");
+            self.pw.clone()
+        } else if self.is_set() {
+            let pw =
+                input::read_password(format!("{prompt} (\"\" reuse, \"*\" auto-reuse)? ")).await;
+
+            match pw.as_str() {
+                "" => self.pw.clone(),
+                "*" => {
+                    self.reuse = true;
+                    self.pw.clone()
+                }
+                _ => {
+                    self.pw = pw.clone();
+                    pw
+                }
+            }
+        } else {
+            let pw = loop {
+                let pw = input::read_password(format!("{prompt}? ")).await;
+                if pw.is_empty() {
+                    error!("empty password provided!");
+                    continue;
+                }
+                break pw;
+            };
+
+            self.pw = pw.clone();
+            pw
+        }
+    }
+}
+
+static PREV_PW: Mutex<PrevPw> = Mutex::const_new(PrevPw {
+    pw: String::new(),
+    reuse: false,
+});
 
 async fn crypt_open(crypt_dev: &str, dev_path: &str) -> Result<()> {
     'open_loop: loop {
         let mut prev_pw = PREV_PW.lock().await;
-        let prompt = if prev_pw.is_empty() {
-            format!("crypt password for {crypt_dev}? ")
-        } else {
-            format!("crypt password for {crypt_dev} (enter = reuse previous)? ")
-        };
 
-        let mut pw = input::read_password(prompt).await;
+        let pw = prev_pw
-        if pw.is_empty() {
+            .input(format!("crypt password for {crypt_dev}"))
-            pw = prev_pw.clone();
+            .await;
-        }
-        if pw.is_empty() {
-            error!("empty password provided!");
-            continue;
-        }
-
-        *prev_pw = pw.clone();
 
         if cryptsetup(&pw, ["open", dev_path, crypt_dev]).await? {
             return Ok(());
@@ -107,15 +152,17 @@ async fn crypt_open(crypt_dev: &str, dev_path: &str) -> Result<()> {
                 }
                 _ => unreachable!(),
             }
-        } else {
+        }
-            // device looks initialized, don't allow format
-            warn!("{dev_path} looks initialized, formatting not allowed from init");
 
-            match input::read_choice(["[r]etry", "[i]gnore"]).await {
+        // device looks initialized, don't allow format
-                'r' => continue 'open_loop,
+        warn!("{dev_path} looks initialized, formatting not allowed from init");
-                'i' => return Ok(()),
+
-                _ => unreachable!(),
+        prev_pw.invalidate();
-            }
+
+        match input::read_choice(["[r]etry", "[i]gnore"]).await {
+            'r' => continue 'open_loop,
+            'i' => return Ok(()),
+            _ => unreachable!(),
         }
     }
 }
@@ -134,18 +181,39 @@ async fn cryptsetup<const N: usize>(pw: &str, args: [&str; N]) -> Result<bool> {
     Ok(child.wait().await?.success())
 }
 
-fn find_dev(dev: &CryptDev, all_devs: &[String]) -> Vec<(String, String)> {
+async fn find_dev(dev: &CryptDev, all_devs: &[String]) -> Result<Vec<(String, String)>> {
     let dev_name = &dev.name;
-    match dev.filter {
+    Ok(match dev.filter() {
-        DevFilter::Dev(ref path) => (all_devs.iter())
+        DevFilter::None => vec![],
+        DevFilter::Dev(path) => (all_devs.iter())
             .filter(|dev_path| dev_path == &path)
             .map(|dev_path| (dev.name.clone(), dev_path.clone()))
             .collect(),
-        DevFilter::Prefix(ref prefix) => (all_devs.iter())
+        DevFilter::Prefix(prefix) => (all_devs.iter())
             .filter_map(|path| {
                 let suffix = path.strip_prefix(prefix)?;
                 Some((format!("{dev_name}{suffix}"), path.clone()))
             })
             .collect(),
-    }
+        DevFilter::Udev(filter) => {
+            use crate::udev;
+            let devs = udev::all().await?;
+
+            let filter: udev::Filter = filter.clone().into();
+
+            (devs.iter())
+                .filter(|dev| dev.subsystem() == Some("block") && filter.matches(dev))
+                .filter_map(|dev| {
+                    let path = dev.property("DEVNAME")?.to_string();
+
+                    let mut name = dev_name.replace("${name}", dev.name()?);
+                    for (p, v) in dev.properties() {
+                        name = name.replace(&format!("${{{p}}}"), v);
+                    }
+
+                    Some((name, path))
+                })
+                .collect()
+        }
+    })
 }

src/cmd/init/lvm.rs

@@ -1,11 +1,11 @@
-use eyre::{Result, format_err};
+use eyre::{format_err, Result};
 use log::{error, info, warn};
 use tokio::process::Command;
 
-use super::{USED_DEVS, exec, retry, retry_or_ignore};
+use super::{exec, retry, retry_or_ignore, USED_DEVS};
 use crate::fs::walk_dir;
 use crate::{blockdev, lvm};
-use dkl::bootstrap::{Config, Filesystem, LvSize, LvmLV, LvmVG, TAKE_ALL};
+use dkl::bootstrap::{Config, Filesystem, LvSize, LvmLV, LvmPV, LvmVG, TAKE_ALL};
 
 pub async fn setup(cfg: &Config) {
     if cfg.lvm.is_empty() {
@@ -73,24 +73,12 @@ async fn setup_vg(vg: &LvmVG) -> Result<()> {
         info!("setting up LVM VG {vg_name} ({dev_done}/{dev_needed} devices configured)");
     }
 
-    let regexps: Vec<regex::Regex> = (vg.pvs.regexps.iter())
+    let matching_devs = find_devs(&vg.pvs).await?;
-        .filter_map(|re_str| {
-            (re_str.parse())
-                .inspect_err(|e| error!("invalid regex ignored: {re_str:?}: {e}"))
-                .ok()
-        })
-        .collect();
-
-    let mut used_devs = USED_DEVS.lock().await;
-
-    let matching_devs = (walk_dir("/dev").await.into_iter())
-        .filter(|path| !used_devs.contains(path.as_str()))
-        .filter(|path| regexps.iter().any(|re| re.is_match(path)));
 
     let devs: Vec<_> = if dev_needed == TAKE_ALL {
-        matching_devs.collect()
+        matching_devs
     } else {
-        matching_devs.take(missing_count!()).collect()
+        matching_devs.into_iter().take(missing_count!()).collect()
     };
 
     let cmd = if dev_done == 0 {
@@ -109,7 +97,7 @@ async fn setup_vg(vg: &LvmVG) -> Result<()> {
     }
 
     dev_done += devs.len();
-    used_devs.extend(devs);
+    USED_DEVS.lock().await.extend(devs);
 
     if dev_needed != TAKE_ALL && dev_done < (dev_needed as usize) {
         return Err(format_err!(
@@ -213,3 +201,33 @@ async fn install_package(pkg: &str) -> Result<()> {
         Err(format_err!("failed to install package {pkg}: {status}"))
     }
 }
+
+async fn find_devs(pvs: &LvmPV) -> Result<Vec<String>> {
+    let mut results = if let Some(ref filter) = pvs.udev {
+        use crate::udev;
+        let filter: udev::Filter = filter.clone().into();
+        (udev::all().await?.iter())
+            .filter(|dev| dev.subsystem() == Some("block") && filter.matches(dev))
+            .filter_map(|dev| dev.property("DEVNAME").map(|s| s.to_string()))
+            .collect()
+    } else if !pvs.regexps.is_empty() {
+        let regexps: Vec<regex::Regex> = (pvs.regexps.iter())
+            .filter_map(|re_str| {
+                (re_str.parse())
+                    .inspect_err(|e| error!("invalid regex ignored: {re_str:?}: {e}"))
+                    .ok()
+            })
+            .collect();
+
+        (walk_dir("/dev").await.into_iter())
+            .filter(|path| regexps.iter().any(|re| re.is_match(path)))
+            .collect()
+    } else {
+        warn!("no device filters, no matches");
+        vec![]
+    };
+
+    let used_devs = USED_DEVS.lock().await;
+    results.retain(|path| !used_devs.contains(path.as_str()));
+    Ok(results)
+}

src/cmd/init/networks.rs

@@ -3,10 +3,10 @@ use log::{info, warn};
 use std::collections::BTreeSet as Set;
 use tokio::process::Command;
 
-use super::{Result, format_err, retry_or_ignore};
+use super::{format_err, retry_or_ignore, Result};
 use crate::{
     udev,
-    utils::{NameAliases, select_n_by_regex},
+    utils::{select_n_by_regex, select_n_by_udev, NameAliases},
 };
 use dkl::bootstrap::{Config, Network};
 
@@ -26,16 +26,13 @@ pub async fn setup(cfg: &Config) {
 async fn setup_network(net: &Network, assigned: &mut Set<String>) -> Result<()> {
     info!("setting up network {}", net.name);
 
-    let netdevs = get_interfaces()?
+    let netdevs = (get_interfaces().await?)
         .filter(|dev| !assigned.contains(dev.name()))
         .collect::<Vec<_>>();
 
     for dev in &netdevs {
-        info!(
+        let names = [dev.name()].into_iter().chain(dev.aliases()).join(", ");
-            "- available network device: {}, aliases [{}]",
+        info!("- available network device: {}", names);
-            dev.name(),
-            dev.aliases().join(", ")
-        );
     }
 
     let mut cmd = Command::new("ash");
@@ -47,8 +44,19 @@ async fn setup_network(net: &Network, assigned: &mut Set<String>) -> Result<()>
     for iface in &net.interfaces {
         let var = &iface.var;
 
-        let netdevs = netdevs.iter().filter(|na| !assigned.contains(na.name()));
+        let if_names = if let Some(ref udev_filter) = iface.udev {
-        let if_names = select_n_by_regex(iface.n, &iface.regexps, netdevs);
+            select_n_by_udev(
+                iface.n,
+                "net",
+                "INTERFACE",
+                &udev_filter.clone().into(),
+                &assigned,
+            )
+            .await?
+        } else {
+            let netdevs = netdevs.iter().filter(|na| !assigned.contains(na.name()));
+            select_n_by_regex(iface.n, &iface.regexps, netdevs)
+        };
 
         if if_names.is_empty() {
             return Err(format_err!("- no interface match for {var:?}"));
@@ -71,24 +79,20 @@ async fn setup_network(net: &Network, assigned: &mut Set<String>) -> Result<()>
     Ok(())
 }
 
-fn get_interfaces() -> Result<impl Iterator<Item = NameAliases>> {
+async fn get_interfaces() -> Result<impl Iterator<Item = NameAliases>> {
-    Ok(udev::get_devices("net")?.into_iter().map(|dev| {
+    let nas: Vec<_> = (udev::all().await?.of_subsystem("net"))
-        let mut na = NameAliases::new(dev.sysname().to_string());
+        .filter_map(|dev| {
+            let name = dev.property("INTERFACE")?;
+            let mut na = NameAliases::new(name.to_string());
 
-        for (property, value) in dev.properties() {
+            for (p, v) in dev.properties() {
-            if [
+                if p.starts_with("ID_NET_NAME") {
-                "INTERFACE",
+                    na.push(v.to_string());
-                "ID_NET_NAME",
+                }
-                "ID_NET_NAME_PATH",
-                "ID_NET_NAME_MAC",
-                "ID_NET_NAME_SLOT",
-            ]
-            .contains(&property)
-            {
-                na.push(value.to_string());
             }
-        }
 
-        na
+            Some(na)
-    }))
+        })
+        .collect();
+    Ok(nas.into_iter())
 }

src/cmd/init_input.rs

@@ -2,6 +2,10 @@ use crate::input;
 
 pub async fn run() {
     tokio::spawn(async {
+        // give a bit of time for stdout
+        use tokio::time::{sleep, Duration};
+        sleep(Duration::from_millis(200)).await;
+
         if let Err(e) = input::forward_requests_from_socket().await {
             eprintln!("failed to forwards requests from socket: {e}");
             std::process::exit(1);

src/dklog.rs

@@ -37,7 +37,7 @@ impl Log {
         }
     }
 
-    pub fn subscribe(&self) -> LogWatch {
+    pub fn subscribe(&self) -> LogWatch<'_> {
         LogWatch {
             log: self,
             pos: 0,

src/udev.rs

@@ -63,3 +63,148 @@ pub fn get_devices(class: &str) -> Result<Vec<Device>> {
 
     Ok(devices)
 }
+
+pub async fn all() -> Result<Devs> {
+    let output = tokio::process::Command::new("udevadm")
+        .args(["info", "-e"])
+        .stderr(std::process::Stdio::inherit())
+        .output()
+        .await?;
+
+    if !output.status.success() {
+        return Err(eyre::format_err!("udevadm failed: {}", output.status));
+    }
+
+    Ok(Devs {
+        data: unsafe { String::from_utf8_unchecked(output.stdout) },
+    })
+}
+
+pub async fn by_path(path: &str) -> Result<Devs> {
+    let output = tokio::process::Command::new("udevadm")
+        .args(["info", path])
+        .stderr(std::process::Stdio::inherit())
+        .output()
+        .await?;
+
+    if !output.status.success() {
+        return Err(eyre::format_err!("udevadm failed: {}", output.status));
+    }
+
+    Ok(Devs {
+        data: unsafe { String::from_utf8_unchecked(output.stdout) },
+    })
+}
+
+pub struct Devs {
+    data: String,
+}
+
+impl<'t> Devs {
+    pub fn iter(&'t self) -> impl Iterator<Item = Dev<'t>> {
+        self.data
+            .split("\n\n")
+            .filter(|s| !s.is_empty())
+            .map(|s| Dev(s))
+    }
+
+    pub fn of_subsystem(&'t self, subsystem: &str) -> impl Iterator<Item = Dev<'t>> {
+        self.iter().filter(|dev| dev.subsystem() == Some(subsystem))
+    }
+}
+
+pub struct Dev<'t>(&'t str);
+
+impl<'t> Dev<'t> {
+    pub fn raw(&self) -> &str {
+        self.0
+    }
+
+    // alpine's udev prefixes we've seen:
+    // - P: Device path in /sys/
+    // - N: Kernel device node name
+    // - S: Device node symlink
+    // - L: Device node symlink priority [ignored]
+    // - E: Device property
+
+    fn by_prefix(&self, prefix: &'static str) -> impl Iterator<Item = &str> {
+        self.0.lines().filter_map(move |l| l.strip_prefix(prefix))
+    }
+
+    /// Device path in /sys/
+    pub fn path(&self) -> Option<&str> {
+        self.by_prefix("P: ").next()
+    }
+
+    /// Kernel device node name
+    pub fn name(&self) -> Option<&str> {
+        self.by_prefix("N: ").next()
+    }
+
+    /// Device node symlinks
+    pub fn symlinks(&self) -> Vec<&str> {
+        self.by_prefix("S: ").collect()
+    }
+
+    /// Device properties
+    pub fn properties(&self) -> impl Iterator<Item = (&str, &str)> {
+        self.by_prefix("E: ").filter_map(|s| s.split_once("="))
+    }
+
+    /// Device property
+    pub fn property(&self, name: &str) -> Option<&str> {
+        self.properties()
+            .filter_map(|(n, v)| (n == name).then_some(v))
+            .next()
+    }
+
+    /// Device subsystem
+    pub fn subsystem(&self) -> Option<&str> {
+        self.property("SUBSYSTEM")
+    }
+}
+
+pub enum Filter {
+    Has(String),
+    Eq(String, String),
+    Glob(String, glob::Pattern),
+    And(Vec<Filter>),
+    Or(Vec<Filter>),
+    Not(Box<Filter>),
+    False,
+}
+
+impl<'t> Filter {
+    pub fn matches(&self, dev: &Dev) -> bool {
+        match self {
+            Self::False => false,
+            Self::Has(k) => dev.property(k).is_some(),
+            Self::Eq(k, v) => dev.properties().any(|kv| kv == (k, v)),
+            Self::Glob(k, pattern) => dev
+                .properties()
+                .any(|(pk, pv)| pk == k && pattern.matches(pv)),
+            Self::And(ops) => ops.iter().all(|op| op.matches(dev)),
+            Self::Or(ops) => ops.iter().any(|op| op.matches(dev)),
+            Self::Not(op) => !op.matches(dev),
+        }
+    }
+}
+
+impl<'t> Into<Filter> for dkl::bootstrap::UdevFilter {
+    fn into(self) -> Filter {
+        match self {
+            Self::Has(p) => Filter::Has(p),
+            Self::Eq(p, v) => Filter::Eq(p, v),
+            Self::Glob(p, pattern) => match glob::Pattern::new(&pattern) {
+                Ok(pattern) => Filter::Glob(p, pattern),
+                Err(e) => {
+                    warn!("pattern {pattern:?} will never match: {e}");
+                    Filter::False
+                }
+            },
+            Self::And(ops) => Filter::And(ops.into_iter().map(Self::into).collect()),
+            Self::Or(ops) => Filter::Or(ops.into_iter().map(Self::into).collect()),
+            Self::Not(op) => Filter::Not(Box::new((*op).into())),
+        }
+    }
+}

src/utils.rs

@@ -2,6 +2,8 @@ use log::error;
 use std::collections::BTreeSet as Set;
 use std::sync::LazyLock;
 
+use crate::udev;
+
 static CMDLINE: LazyLock<String> = LazyLock::new(|| {
     std::fs::read("/proc/cmdline")
         .inspect_err(|e| error!("failed to read kernel cmdline: {e}"))
@@ -88,3 +90,24 @@ pub fn select_n_by_regex<'t>(
         nas.take(n as usize).collect()
     }
 }
+
+pub async fn select_n_by_udev<'t>(
+    n: i16,
+    subsystem: &str,
+    result_property: &str,
+    filter: &udev::Filter,
+    in_use: &Set<String>,
+) -> eyre::Result<Vec<String>> {
+    let devs = udev::all().await?;
+    let nas = devs
+        .of_subsystem(subsystem)
+        .filter(|dev| filter.matches(dev))
+        .filter_map(|dev| Some(dev.property(result_property)?.to_string()))
+        .filter(|name| !in_use.contains(name));
+
+    Ok(if n == -1 {
+        nas.collect()
+    } else {
+        nas.take(n as usize).collect()
+    })
+}

test-initrd/config.yaml

@@ -21,15 +21,13 @@ auths:
     sshKey:   ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICkpbU6sf4t0f6XAv9DuW3XH5iLM0AI5rc8PT2jwea1N
     password: bXlzZWVk:HMSxrg1cYphaPuUYUbtbl/htep/tVYYIQAuvkNMVpw0 # mypass
 
-signer_public_key: MIGbMBAGByqGSM49AgEGBSuBBAAjA4GGAAQA29glSqk7MqoUIjD+UQG+b4v59pTFkn8rYtNhOftTe7uiLUvGFsjNdzP3tW64t/c6YD2p6dtI3oQXGOVQO1vIWPEBc6Sq++BRpQ0FVna+dgNQx8/kLXN9Na0ZYbK7q0haCI7/EHWOX79JFFxJE9HJ67AOMmXwGJ2jrfa1CUnWvfCmT+E=
-
 ssh:
   listen: "[::]:22"
   user_ca: /user_ca.pub
 
 networks:
 - name: loopback
-  interfaces: [ { var: iface, n: 1, regexps: [ "^lo$" ] } ]
+  interfaces: [ { var: iface, n: 1, udev: !eq [INTERFACE, lo] } ]
   script: |
     ip a add 127.0.0.1/8 dev lo
     ip a add ::1/128 dev lo
@@ -38,31 +36,27 @@ networks:
   interfaces:
   - var: iface
     n: 1
-    regexps:
+    udev: !has ID_NET_NAME_MAC
-    - eth.*
-    - veth.*
-    - eno.*
-    - enp.*
   script: |
     ip li set $iface up
-    udhcpc -i $iface -b -t1 -T1 -A5 ||
+    ip a add 192.168.12.42/24 dev $iface
-    ip a add 2001:41d0:306:168f::1337:2eed/64 dev $iface
+    ip a add fd12:6e76:7474::1337:2eed/64 dev $iface
+    ip route add default via 192.168.12.254
+    ip route add default via fd12:6e76:7474::1 dev $iface
 
-pre_lvm_crypt:
+#pre_lvm_crypt:
-- dev: /dev/vda
+#- name: sys-${name}
-  name: sys0
+#  udev: !glob [ DEVNAME, /dev/vd* ]
-- dev: /dev/vdb
-  name: sys1
 
 lvm:
 - vg: storage
   pvs:
     n: 2
     regexps:
-    - /dev/mapper/sys[01]
+    #- ^/dev/mapper/sys-
     # to match full disks
     #- /dev/nvme[0-9]+n[0-9]+
-    #- /dev/vd[a-z]+
+    - /dev/vd[a-z]+
     #- /dev/sd[a-z]+
     #- /dev/hd[a-z]+
     # to match partitions:
@@ -78,11 +72,16 @@ lvm:
 
   lvs:
   - name: bootstrap
-    size: 2g
+    size: 1g
 
   - name: varlog
-    extents: 10%FREE
+    size: 256m
-    # size: 10g
+  - name: kubelet
+    size: 256m
+  - name: containerd
+    size: 1g
+  - name: etcd
+    size: 256m
 
   - name: podman
     extents: 10%FREE
@@ -96,11 +95,8 @@ lvm:
 #- dev: /dev/storage/bootstrap
 #- dev: /dev/storage/dls
 
+signer_public_key: 'MIGbMBAGByqGSM49AgEGBSuBBAAjA4GGAAQBe6Y3zGQUIHvVXoS5GI8irY8yoB0ozFpzn/cUykA46TkHdJ8xCEaaM1MpqMrfWgDtP/rA2KeE9HjVerLnEFD01uUAUh4/OYgCBDYJPhridVDoC78KOJpkWBj7Shl0Rp0AtETvatNPa1RRe15V7nDF/Nm75Y6O3IL29lYPQ6jqEGhR810='
 bootstrap:
-  #dev: /dev/mapper/bootstrap
   dev: /dev/storage/bootstrap
-  # TODO seed: https://direktil.novit.io/bootstraps/dls-crypt
+  seed: http://192.168.12.254:7606/public/download-set/host/m1/bootstrap.tar?set=ICIXKJJWA6U4RQESD3KQMWO3IBW6THG4FJUM2HUNFPTIODVSXGDPXTCHSFT6IOUZO6LBAG65QIGYUMIZA3TEHTPB6BXKUFONNUWKUWAJAQRE2GDEOC4RWAAAQA3DSZJXMNSDGN34NA5G2MJ2MJXW65DTORZGC4BOORQXEAAAAAACMICVFM
-  seed: http://192.168.10.254:7606/hosts/m1/bootstrap.tar
-  # TODO seed_sign_key: "..."
-  # TODO load_and_close: true