dist & cache cleaning + preseeding

cmd/dkl-local-server/bootv2.go

@@ -132,15 +132,19 @@ func buildInitrd(out io.Writer, ctx *renderContext) (err error) {
 	return
 }
 
+func getSigner(clusterName string) (signer crypto.Signer, err error) {
+	ca, err := getUsableClusterCA(clusterName, "boot-signer")
+	if err != nil {
+		return
+	}
+	return ca.ParseKey()
+}
+
 func buildBootstrap(out io.Writer, ctx *renderContext) (err error) {
 	arch := tar.NewWriter(out)
 	defer arch.Close()
 
-	ca, err := getUsableClusterCA(ctx.Host.ClusterName, "boot-signer")
-	if err != nil {
-		return
-	}
-	signer, err := ca.ParseKey()
+	signer, err := getSigner(ctx.Host.ClusterName)
 	if err != nil {
 		return
 	}
@@ -245,7 +249,7 @@ func buildBootstrap(out io.Writer, ctx *renderContext) (err error) {
 	}
 
 	if allErofs {
-		layerPath, e := layersCombo(ctx, cfg, signer)
+		layerPath, e := layersCombo(ctx.Host, cfg, signer)
 		if e != nil {
 			err = e
 			return
@@ -275,184 +279,10 @@ func buildBootstrap(out io.Writer, ctx *renderContext) (err error) {
 	return nil
 }
 
-func layersCombo(ctx *renderContext, cfg *config.Config, signer crypto.Signer) (path string, err error) {
-	key := layersComboKey(ctx.Host, cfg)
-
+func layersCombo(host *localconfig.Host, cfg *config.Config, signer crypto.Signer) (path string, err error) {
+	key := layersComboKey(host, cfg)
 	return opMutex(key, func() (path string, err error) {
-		path = filepath.Join(*dataDir, "cache")
-		if err = os.MkdirAll(path, 0o700); err != nil {
-			return
-		}
-
-		path = filepath.Join(path, key) + ".fs"
-
-		if _, statErr := os.Stat(path); statErr == nil {
-			return // exists -> already done
-		}
-
-		workdir, err := os.MkdirTemp("/tmp", "layers")
-		if err != nil {
-			return
-		}
-
-		defer os.RemoveAll(workdir)
-
-		tmpTar := filepath.Join(workdir, "output.tar")
-
-		layers := slices.Clone(cfg.Layers)
-		slices.Reverse(layers)
-
-		cmdOut := new(bytes.Buffer)
-
-		run := func(prog string, arg ...string) bool {
-			cmdOut.Reset()
-
-			cmd := exec.Command(prog, arg...)
-			cmd.Stdout = cmdOut // os.Stdout
-			cmd.Stderr = os.Stderr
-			if e := cmd.Run(); e != nil {
-				err = fmt.Errorf("%s %q failed: %w", cmd.Path, cmd.Args, e)
-				return false
-			}
-			return true
-		}
-
-		for i, layer := range layers {
-			if layer == "modules" {
-				continue // modules are in the initrd with boot v2
-			}
-
-			layerFile, e := fetchHostLayer(ctx.Host, layer)
-			if e != nil {
-				err = e
-				return
-			}
-
-			mountPoint := filepath.Join(workdir, layer)
-			os.MkdirAll(mountPoint, 0700)
-
-			if e := exec.Command("erofsfuse", layerFile, mountPoint).Run(); e != nil {
-				err = fmt.Errorf("erofsfuse %s %s failed: %w", layerFile, mountPoint, e)
-				return
-			}
-
-			defer func() {
-				if err := exec.Command("umount", mountPoint).Run(); err != nil {
-					log.Printf("umount %s failed: %v", mountPoint, err)
-				}
-			}()
-
-			mode := "--append"
-			if i == 0 {
-				mode = "--create"
-			}
-
-			if !run("tar", mode, "-p", "-f", tmpTar, "-C", mountPoint, ".") {
-				return
-			}
-		}
-
-		fsOut := filepath.Join(workdir, "output.fs")
-		if !run("mkfs.erofs", "-z", "lzma", "-C131072", "-Efragments,ztailpacking",
-			"-T0", "--all-time", "--ignore-mtime", "--tar=f", fsOut, tmpTar) {
-			return
-		}
-
-		hashOut := filepath.Join(workdir, "output.hash")
-
-		if !run("veritysetup", "format", fsOut, hashOut) {
-			return
-		}
-
-		var rootHash []byte
-		for line := range strings.SplitSeq(cmdOut.String(), "\n") {
-			v, ok := strings.CutPrefix(line, "Root hash:")
-			if !ok {
-				continue
-			}
-			v = strings.TrimSpace(v)
-
-			b, e := hex.DecodeString(v)
-			if e != nil {
-				err = fmt.Errorf("invalid root hash: %w", e)
-				return
-			}
-
-			rootHash = b
-			break
-		}
-
-		if len(rootHash) == 0 {
-			err = fmt.Errorf("root hash not found in output")
-			return
-		}
-
-		sigBytes, err := signer.Sign(nil, rootHash, crypto.SHA256)
-		if err != nil {
-			err = fmt.Errorf("root hash signature failed: %w", err)
-			return
-		}
-
-		outPath := path + ".tmp"
-		err = func() (err error) {
-			fsRd, e := os.Open(fsOut)
-			if e != nil {
-				return e
-			}
-			defer fsRd.Close()
-			hashRd, e := os.Open(hashOut)
-			if e != nil {
-				return e
-			}
-			defer hashRd.Close()
-
-			fsStat, e := fsRd.Stat()
-			if e != nil {
-				return e
-			}
-			hashStat, e := hashRd.Stat()
-			if e != nil {
-				return e
-			}
-
-			out, err := os.Create(outPath)
-			if err != nil {
-				return
-			}
-			defer out.Close()
-
-			append := func(sz uint64, rd io.Reader) {
-				if err != nil {
-					return
-				}
-				szB := make([]byte, 8)
-				binary.BigEndian.PutUint64(szB, sz)
-				_, err = out.Write(szB)
-				if err != nil {
-					return
-				}
-
-				_, err = io.Copy(out, rd)
-			}
-
-			append(uint64(len(sigBytes)), bytes.NewBuffer(sigBytes))
-			append(uint64(len(rootHash)), bytes.NewBuffer(rootHash))
-			append(uint64(fsStat.Size()), fsRd)
-			append(uint64(hashStat.Size()), hashRd)
-
-			if err != nil {
-				return
-			}
-			err = out.Close()
-			return
-		}()
-		if err != nil {
-			err = fmt.Errorf("assembly failed: %w", err)
-			return
-		}
-
-		err = os.Rename(outPath, path)
-		return
+		return buildLayersCombo(host, cfg, signer)
 	})
 }
 
@@ -474,6 +304,184 @@ func layersComboKey(host *localconfig.Host, cfg *config.Config) string {
 	return key.String()
 }
 
+func buildLayersCombo(host *localconfig.Host, cfg *config.Config, signer crypto.Signer) (path string, err error) {
+	path = filepath.Join(*dataDir, "cache")
+	if err = os.MkdirAll(path, 0o700); err != nil {
+		return
+	}
+
+	key := layersComboKey(host, cfg)
+	path = filepath.Join(path, key) + ".fs"
+
+	if _, statErr := os.Stat(path); statErr == nil {
+		return // exists -> already done
+	}
+
+	workdir, err := os.MkdirTemp("/tmp", "layers")
+	if err != nil {
+		return
+	}
+
+	defer os.RemoveAll(workdir)
+
+	tmpTar := filepath.Join(workdir, "output.tar")
+
+	layers := slices.Clone(cfg.Layers)
+	slices.Reverse(layers)
+
+	cmdOut := new(bytes.Buffer)
+
+	run := func(prog string, arg ...string) bool {
+		cmdOut.Reset()
+
+		cmd := exec.Command(prog, arg...)
+		cmd.Stdout = cmdOut // os.Stdout
+		cmd.Stderr = os.Stderr
+		if e := cmd.Run(); e != nil {
+			err = fmt.Errorf("%s %q failed: %w", cmd.Path, cmd.Args, e)
+			return false
+		}
+		return true
+	}
+
+	for i, layer := range layers {
+		if layer == "modules" {
+			continue // modules are in the initrd with boot v2
+		}
+
+		layerFile, e := fetchHostLayer(host, layer)
+		if e != nil {
+			err = e
+			return
+		}
+
+		mountPoint := filepath.Join(workdir, layer)
+		os.MkdirAll(mountPoint, 0700)
+
+		if e := exec.Command("erofsfuse", layerFile, mountPoint).Run(); e != nil {
+			err = fmt.Errorf("erofsfuse %s %s failed: %w", layerFile, mountPoint, e)
+			return
+		}
+
+		defer func() {
+			if err := exec.Command("umount", mountPoint).Run(); err != nil {
+				log.Printf("umount %s failed: %v", mountPoint, err)
+			}
+		}()
+
+		mode := "--append"
+		if i == 0 {
+			mode = "--create"
+		}
+
+		if !run("tar", mode, "-p", "-f", tmpTar, "-C", mountPoint, ".") {
+			return
+		}
+	}
+
+	fsOut := filepath.Join(workdir, "output.fs")
+	if !run("mkfs.erofs", "-z", "lzma", "-C131072", "-Efragments,ztailpacking",
+		"-T0", "--all-time", "--ignore-mtime", "--tar=f", fsOut, tmpTar) {
+		return
+	}
+
+	hashOut := filepath.Join(workdir, "output.hash")
+
+	if !run("veritysetup", "format", fsOut, hashOut) {
+		return
+	}
+
+	var rootHash []byte
+	for line := range strings.SplitSeq(cmdOut.String(), "\n") {
+		v, ok := strings.CutPrefix(line, "Root hash:")
+		if !ok {
+			continue
+		}
+		v = strings.TrimSpace(v)
+
+		b, e := hex.DecodeString(v)
+		if e != nil {
+			err = fmt.Errorf("invalid root hash: %w", e)
+			return
+		}
+
+		rootHash = b
+		break
+	}
+
+	if len(rootHash) == 0 {
+		err = fmt.Errorf("root hash not found in output")
+		return
+	}
+
+	sigBytes, err := signer.Sign(nil, rootHash, crypto.SHA256)
+	if err != nil {
+		err = fmt.Errorf("root hash signature failed: %w", err)
+		return
+	}
+
+	outPath := path + ".tmp"
+	err = func() (err error) {
+		fsRd, e := os.Open(fsOut)
+		if e != nil {
+			return e
+		}
+		defer fsRd.Close()
+		hashRd, e := os.Open(hashOut)
+		if e != nil {
+			return e
+		}
+		defer hashRd.Close()
+
+		fsStat, e := fsRd.Stat()
+		if e != nil {
+			return e
+		}
+		hashStat, e := hashRd.Stat()
+		if e != nil {
+			return e
+		}
+
+		out, err := os.Create(outPath)
+		if err != nil {
+			return
+		}
+		defer out.Close()
+
+		append := func(sz uint64, rd io.Reader) {
+			if err != nil {
+				return
+			}
+			szB := make([]byte, 8)
+			binary.BigEndian.PutUint64(szB, sz)
+			_, err = out.Write(szB)
+			if err != nil {
+				return
+			}
+
+			_, err = io.Copy(out, rd)
+		}
+
+		append(uint64(len(sigBytes)), bytes.NewBuffer(sigBytes))
+		append(uint64(len(rootHash)), bytes.NewBuffer(rootHash))
+		append(uint64(fsStat.Size()), fsRd)
+		append(uint64(hashStat.Size()), hashRd)
+
+		if err != nil {
+			return
+		}
+		err = out.Close()
+		return
+	}()
+	if err != nil {
+		err = fmt.Errorf("assembly failed: %w", err)
+		return
+	}
+
+	err = os.Rename(outPath, path)
+	return
+}
+
 func fetchHostLayer(host *localconfig.Host, layer string) (path string, err error) {
 	layerVersion := host.Versions[layer]
 	if layerVersion == "" {

cmd/dkl-local-server/cache.go

@@ -0,0 +1,113 @@
+package main
+
+import (
+	"io/fs"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func updateCache() {
+	log.Print("updating cache")
+
+	cfg, err := readConfig()
+	if err != nil {
+		log.Printf("update cache failed: read config failed: %v", err)
+		return
+	}
+
+	errs := 0
+	inUse := map[string]bool{}
+
+	fetch := func(path ...string) {
+		assetPath, err := distFetch(path...)
+		if err != nil {
+			log.Printf("update cache: dist fetch failed: %v", err)
+			errs += 1
+		}
+		inUse[assetPath] = true
+	}
+
+	fetch("grub-support", *grubSupportVersion)
+
+	for _, host := range cfg.Hosts {
+		ctx, err := newRenderContext(host, cfg)
+		if err != nil {
+			log.Printf("update cache: %s: build context failed: %v", host.Name, err)
+			errs += 1
+			continue
+		}
+
+		_, cfg, err := ctx.Config()
+		if err != nil {
+			log.Printf("update cache: %s: render config failed: %v", host.Name, err)
+			errs += 1
+			continue
+		}
+
+		signer, err := getSigner(host.ClusterName)
+		if err != nil {
+			log.Printf("update cache: %s: get signer failed: %v", host.Name, err)
+			errs += 1
+			continue
+		}
+
+		fetch("kernels", host.Kernel)
+		fetch("layers", "modules", host.Kernel)
+		fetch("initrd", host.Initrd)
+
+		allErofs := true
+		for layer, version := range host.Versions {
+			fetch("layers", layer, version)
+
+			if layer != "modules" && !strings.HasSuffix(version, ".erofs") {
+				allErofs = false
+			}
+		}
+
+		if allErofs {
+			path, err := layersCombo(host, cfg, signer)
+			if err != nil {
+				log.Printf("update cache: layer combo failed: %v", err)
+				continue
+			}
+			inUse[path] = true
+		}
+	}
+
+	if errs != 0 {
+		log.Printf("update cache: %d errors, not cleaning", errs)
+		return
+	}
+
+	distDir := filepath.Join(*dataDir, "dist")
+	cacheDir := filepath.Join(*dataDir, "cache")
+
+	existing := []string{}
+
+	for _, dir := range []string{distDir, cacheDir} {
+		fs.WalkDir(os.DirFS(dir), ".", func(path string, d fs.DirEntry, err error) error {
+			if err != nil {
+				log.Printf("update cache: walking %s failed: %v", dir, err)
+				return nil
+			}
+			if !d.IsDir() {
+				existing = append(existing, filepath.Join(dir, path))
+			}
+			return nil
+		})
+	}
+
+	log.Printf("update cache: %d/%d assets in use", len(inUse), len(existing))
+
+	for _, path := range existing {
+		if inUse[path] {
+			continue
+		}
+		log.Print("update cache: removing ", path)
+		if err := os.Remove(path); err != nil {
+			log.Print("update cache: failed to remove: ", err)
+		}
+	}
+}

cmd/dkl-local-server/hash.go

@@ -6,7 +6,7 @@ import (
 	"encoding/json"
 )
 
-func hash(values ...interface{}) string {
+func hash(values ...any) string {
 	ba, err := json.Marshal(values)
 	if err != nil {
 		panic(err) // should not happen

cmd/dkl-local-server/secret-store.go

@@ -137,7 +137,9 @@ func unlockSecretStore(name string, passphrase []byte) (err httperr.Error) {
 	})
 
 	go updateState()
-	go migrateSecrets()
+
+	migrateSecrets() // we can probably remove it now
+	go updateCache()
 
 	return
 }

cmd/dkl-local-server/ws-configs.go

@@ -68,6 +68,8 @@ func writeNewConfig(cfgBytes []byte) (err error) {
 	err = os.Rename(out.Name(), cfgPath)
 
 	updateState()
+	go updateCache()
+
 	return
 }
 

cmd/dkl-local-server/ws.go

@@ -241,7 +241,7 @@ func wsError(resp *restful.Response, err error) {
 	}
 }
 
-func wsRender(resp *restful.Response, sslCfg *cfsslconfig.Config, tmplStr string, value interface{}) {
+func wsRender(resp *restful.Response, sslCfg *cfsslconfig.Config, tmplStr string, value any) {
 	tmpl, err := template.New("wsRender").Funcs(templateFuncs(sslCfg)).Parse(tmplStr)
 	if err != nil {
 		wsError(resp, err)