diff --git a/Dockerfile b/Dockerfile index bb88d7e..2682230 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ -from novit.tech/direktil/dkl:bbea9b9 as dkl +from novit.tech/direktil/dkl:v1.2.0 as dkl + # ------------------------------------------------------------------------ -from golang:1.25.5-trixie as build +from golang:1.26.1-trixie as build run apt-get update && apt-get install -y git @@ -29,6 +30,7 @@ env _uncache=1 run apt-get update \ && yes |apt-get install -y genisoimage gdisk dosfstools util-linux udev binutils systemd \ grub2 grub-pc-bin grub-efi-amd64-bin ca-certificates curl openssh-client qemu-utils wireguard-tools \ + erofs-utils erofsfuse cryptsetup \ && apt-get clean copy --from=dkl /bin/dkl /bin/dls /bin/ diff --git a/cmd/dkl-local-server/bootv2.go b/cmd/dkl-local-server/bootv2.go index 1c58fc8..45e562d 100644 --- a/cmd/dkl-local-server/bootv2.go +++ b/cmd/dkl-local-server/bootv2.go @@ -4,15 +4,23 @@ import ( "archive/tar" "bytes" "crypto" + "encoding/binary" + "encoding/hex" "fmt" "io" "log" "net/http" "os" + "os/exec" + "path/filepath" + "slices" + "strings" "github.com/klauspost/compress/zstd" + "novit.tech/direktil/pkg/config" "novit.tech/direktil/pkg/cpiocat" + "novit.tech/direktil/pkg/localconfig" ) func renderBootstrapConfig(w http.ResponseWriter, ctx *renderContext) (err error) { @@ -164,22 +172,8 @@ func buildBootstrap(out io.Writer, ctx *renderContext) (err error) { } // layers - for _, layer := range cfg.Layers { - if layer == "modules" { - continue // modules are in the initrd with boot v2 - } - - layerVersion := ctx.Host.Versions[layer] - if layerVersion == "" { - return fmt.Errorf("layer %q not mapped to a version", layer) - } - - outPath, err := distFetch("layers", layer, layerVersion) - if err != nil { - return err - } - - f, err := os.Open(outPath) + appendSignedLayer := func(layer, layerPath string) (err error) { + f, err := os.Open(layerPath) if err != nil { return err } @@ -195,7 +189,7 @@ func buildBootstrap(out io.Writer, ctx *renderContext) (err error) { reader := io.TeeReader(f, h) if err = arch.WriteHeader(&tar.Header{ - Name: layer + ".fs", + Name: layer, Size: stat.Size(), Mode: 0o600, }); err != nil { @@ -208,11 +202,261 @@ func buildBootstrap(out io.Writer, ctx *renderContext) (err error) { } digest := h.Sum(nil) - err = sign(layer+".fs.sig", digest) - if err != nil { - return err + err = sign(layer+".sig", digest) + + return + } + + allErofs := true + + for _, layer := range cfg.Layers { + if layer == "modules" { + continue // modules are in the initrd with boot v2 + } + + if !strings.HasSuffix(ctx.Host.Versions[layer], ".erofs") { + allErofs = false + break + } + } + + if allErofs { + layerPath, e := layersCombo(ctx, cfg, signer) + if e != nil { + err = e + return + } + + if err = appendSignedLayer("merged", layerPath); err != nil { + return + } + } else { + for _, layer := range cfg.Layers { + if layer == "modules" { + continue // modules are in the initrd with boot v2 + } + + layerPath, e := fetchHostLayer(ctx.Host, layer) + if e != nil { + err = e + return + } + + if err = appendSignedLayer(layer+".fs", layerPath); err != nil { + return + } } } return nil } + +func layersCombo(ctx *renderContext, cfg *config.Config, signer crypto.Signer) (path string, err error) { + key := layersComboKey(ctx.Host, cfg) + + return opMutex(key, func() (path string, err error) { + path = filepath.Join(*dataDir, "cache") + if err = os.MkdirAll(path, 0o700); err != nil { + return + } + + path = filepath.Join(path, key) + ".fs" + + if _, statErr := os.Stat(path); statErr == nil { + return // exists -> already done + } + + workdir, err := os.MkdirTemp("/tmp", "layers") + if err != nil { + return + } + + defer os.RemoveAll(workdir) + + tmpTar := filepath.Join(workdir, "output.tar") + + layers := append([]string{}, cfg.Layers...) + slices.Reverse(layers) + + cmdOut := new(bytes.Buffer) + + run := func(prog string, arg ...string) bool { + cmdOut.Reset() + + cmd := exec.Command(prog, arg...) + cmd.Stdout = cmdOut // os.Stdout + cmd.Stderr = os.Stderr + if e := cmd.Run(); e != nil { + err = fmt.Errorf("%s %q failed: %w", cmd.Path, cmd.Args, e) + return false + } + return true + } + + for i, layer := range layers { + if layer == "modules" { + continue // modules are in the initrd with boot v2 + } + + layerFile, e := fetchHostLayer(ctx.Host, layer) + if e != nil { + err = e + return + } + + mountPoint := filepath.Join(workdir, layer) + os.MkdirAll(mountPoint, 0700) + + if e := exec.Command("erofsfuse", layerFile, mountPoint).Run(); e != nil { + err = fmt.Errorf("erofsfuse %s %s failed: %w", layerFile, mountPoint, e) + return + } + + defer func() { + if err := exec.Command("umount", mountPoint).Run(); err != nil { + log.Printf("umount %s failed: %v", mountPoint, err) + } + }() + + mode := "--append" + if i == 0 { + mode = "--create" + } + + if !run("tar", mode, "-p", "-f", tmpTar, "-C", mountPoint, ".") { + return + } + + layers = append(layers, mountPoint) + } + + fsOut := filepath.Join(workdir, "output.fs") + if !run("mkfs.erofs", "-z", "lzma", "-C131072", "-Efragments,ztailpacking", + "-T0", "--all-time", "--ignore-mtime", "--tar=f", fsOut, tmpTar) { + return + } + + hashOut := filepath.Join(workdir, "output.hash") + + if !run("veritysetup", "format", fsOut, hashOut) { + return + } + + var rootHash []byte + for line := range strings.SplitSeq(cmdOut.String(), "\n") { + v, ok := strings.CutPrefix(line, "Root hash:") + if !ok { + continue + } + v = strings.TrimSpace(v) + + b, e := hex.DecodeString(v) + if e != nil { + err = fmt.Errorf("invalid root hash: %w", e) + return + } + + rootHash = b + break + } + + if len(rootHash) == 0 { + err = fmt.Errorf("root hash not found in output") + return + } + + sigBytes, err := signer.Sign(nil, rootHash, crypto.SHA256) + if err != nil { + err = fmt.Errorf("root hash signature failed: %w", err) + return + } + + outPath := path + ".tmp" + err = func() (err error) { + fsRd, e := os.Open(fsOut) + if e != nil { + return e + } + defer fsRd.Close() + hashRd, e := os.Open(hashOut) + if e != nil { + return e + } + defer hashRd.Close() + + fsStat, e := fsRd.Stat() + if e != nil { + return e + } + hashStat, e := hashRd.Stat() + if e != nil { + return e + } + + out, err := os.Create(outPath) + if err != nil { + return + } + defer out.Close() + + append := func(sz uint64, rd io.Reader) { + if err != nil { + return + } + szB := make([]byte, 8) + binary.BigEndian.PutUint64(szB, sz) + _, err = out.Write(szB) + if err != nil { + return + } + + _, err = io.Copy(out, rd) + } + + append(uint64(len(sigBytes)), bytes.NewBuffer(sigBytes)) + append(uint64(len(rootHash)), bytes.NewBuffer(rootHash)) + append(uint64(fsStat.Size()), fsRd) + append(uint64(hashStat.Size()), hashRd) + + if err != nil { + return + } + err = out.Close() + return + }() + if err != nil { + err = fmt.Errorf("assembly failed: %w", err) + return + } + + err = os.Rename(outPath, path) + return + }) +} + +func layersComboKey(host *localconfig.Host, cfg *config.Config) string { + key := new(strings.Builder) + key.WriteString("layers") + for _, layer := range cfg.Layers { + if layer == "modules" { + continue + } + key.WriteByte(':') + key.WriteString(layer) + if v, ok := host.Versions[layer]; ok { + key.WriteByte('@') + key.WriteString(v) + } + } + + return key.String() +} + +func fetchHostLayer(host *localconfig.Host, layer string) (path string, err error) { + layerVersion := host.Versions[layer] + if layerVersion == "" { + return "", fmt.Errorf("layer %q not mapped to a version", layer) + } + + return distFetch("layers", layer, layerVersion) +} diff --git a/cmd/dkl-local-server/mutex.go b/cmd/dkl-local-server/mutex.go new file mode 100644 index 0000000..cad3eab --- /dev/null +++ b/cmd/dkl-local-server/mutex.go @@ -0,0 +1,28 @@ +package main + +import "sync" + +var ( + opMutexes = map[string]*sync.Mutex{} + opsLock sync.Mutex +) + +func opMutex[T any](key string, op func() (T, error)) (T, error) { + lock := func() *sync.Mutex { + opsLock.Lock() + defer opsLock.Unlock() + + lock, ok := opMutexes[key] + if !ok { + lock = new(sync.Mutex) + opMutexes[key] = lock + } + + return lock + }() + + lock.Lock() + defer lock.Unlock() + + return op() +} diff --git a/cmd/dkl-local-server/ws-host.go b/cmd/dkl-local-server/ws-host.go index 0f28680..b344ac5 100644 --- a/cmd/dkl-local-server/ws-host.go +++ b/cmd/dkl-local-server/ws-host.go @@ -62,10 +62,10 @@ func (ws wsHost) register(rws *restful.WebService, alterRB func(*restful.RouteBu b("boot.qed"). Produces(mime.DISK + "+qed"). Doc("Get the " + ws.hostDoc + "'s boot disk image, QED (KVM)"), - b("boot.vmdk"). + b("boot.vdi"). Produces(mime.DISK + "+vdi"). Doc("Get the " + ws.hostDoc + "'s boot disk image, VDI (VirtualBox)"), - b("boot.qcow2"). + b("boot.vpc"). Produces(mime.DISK + "+vpc"). Doc("Get the " + ws.hostDoc + "'s boot disk image, VHD (Hyper-V)"), b("boot.vmdk").