diff --git a/cmd/dkl-local-server/bootv2.go b/cmd/dkl-local-server/bootv2.go index 290f2cc..aaa1e04 100644 --- a/cmd/dkl-local-server/bootv2.go +++ b/cmd/dkl-local-server/bootv2.go @@ -132,15 +132,19 @@ func buildInitrd(out io.Writer, ctx *renderContext) (err error) { return } +func getSigner(clusterName string) (signer crypto.Signer, err error) { + ca, err := getUsableClusterCA(clusterName, "boot-signer") + if err != nil { + return + } + return ca.ParseKey() +} + func buildBootstrap(out io.Writer, ctx *renderContext) (err error) { arch := tar.NewWriter(out) defer arch.Close() - ca, err := getUsableClusterCA(ctx.Host.ClusterName, "boot-signer") - if err != nil { - return - } - signer, err := ca.ParseKey() + signer, err := getSigner(ctx.Host.ClusterName) if err != nil { return } @@ -245,7 +249,7 @@ func buildBootstrap(out io.Writer, ctx *renderContext) (err error) { } if allErofs { - layerPath, e := layersCombo(ctx, cfg, signer) + layerPath, e := layersCombo(ctx.Host, cfg, signer) if e != nil { err = e return @@ -275,184 +279,10 @@ func buildBootstrap(out io.Writer, ctx *renderContext) (err error) { return nil } -func layersCombo(ctx *renderContext, cfg *config.Config, signer crypto.Signer) (path string, err error) { - key := layersComboKey(ctx.Host, cfg) - +func layersCombo(host *localconfig.Host, cfg *config.Config, signer crypto.Signer) (path string, err error) { + key := layersComboKey(host, cfg) return opMutex(key, func() (path string, err error) { - path = filepath.Join(*dataDir, "cache") - if err = os.MkdirAll(path, 0o700); err != nil { - return - } - - path = filepath.Join(path, key) + ".fs" - - if _, statErr := os.Stat(path); statErr == nil { - return // exists -> already done - } - - workdir, err := os.MkdirTemp("/tmp", "layers") - if err != nil { - return - } - - defer os.RemoveAll(workdir) - - tmpTar := filepath.Join(workdir, "output.tar") - - layers := slices.Clone(cfg.Layers) - slices.Reverse(layers) - - cmdOut := new(bytes.Buffer) - - run := func(prog string, arg ...string) bool { - cmdOut.Reset() - - cmd := exec.Command(prog, arg...) - cmd.Stdout = cmdOut // os.Stdout - cmd.Stderr = os.Stderr - if e := cmd.Run(); e != nil { - err = fmt.Errorf("%s %q failed: %w", cmd.Path, cmd.Args, e) - return false - } - return true - } - - for i, layer := range layers { - if layer == "modules" { - continue // modules are in the initrd with boot v2 - } - - layerFile, e := fetchHostLayer(ctx.Host, layer) - if e != nil { - err = e - return - } - - mountPoint := filepath.Join(workdir, layer) - os.MkdirAll(mountPoint, 0700) - - if e := exec.Command("erofsfuse", layerFile, mountPoint).Run(); e != nil { - err = fmt.Errorf("erofsfuse %s %s failed: %w", layerFile, mountPoint, e) - return - } - - defer func() { - if err := exec.Command("umount", mountPoint).Run(); err != nil { - log.Printf("umount %s failed: %v", mountPoint, err) - } - }() - - mode := "--append" - if i == 0 { - mode = "--create" - } - - if !run("tar", mode, "-p", "-f", tmpTar, "-C", mountPoint, ".") { - return - } - } - - fsOut := filepath.Join(workdir, "output.fs") - if !run("mkfs.erofs", "-z", "lzma", "-C131072", "-Efragments,ztailpacking", - "-T0", "--all-time", "--ignore-mtime", "--tar=f", fsOut, tmpTar) { - return - } - - hashOut := filepath.Join(workdir, "output.hash") - - if !run("veritysetup", "format", fsOut, hashOut) { - return - } - - var rootHash []byte - for line := range strings.SplitSeq(cmdOut.String(), "\n") { - v, ok := strings.CutPrefix(line, "Root hash:") - if !ok { - continue - } - v = strings.TrimSpace(v) - - b, e := hex.DecodeString(v) - if e != nil { - err = fmt.Errorf("invalid root hash: %w", e) - return - } - - rootHash = b - break - } - - if len(rootHash) == 0 { - err = fmt.Errorf("root hash not found in output") - return - } - - sigBytes, err := signer.Sign(nil, rootHash, crypto.SHA256) - if err != nil { - err = fmt.Errorf("root hash signature failed: %w", err) - return - } - - outPath := path + ".tmp" - err = func() (err error) { - fsRd, e := os.Open(fsOut) - if e != nil { - return e - } - defer fsRd.Close() - hashRd, e := os.Open(hashOut) - if e != nil { - return e - } - defer hashRd.Close() - - fsStat, e := fsRd.Stat() - if e != nil { - return e - } - hashStat, e := hashRd.Stat() - if e != nil { - return e - } - - out, err := os.Create(outPath) - if err != nil { - return - } - defer out.Close() - - append := func(sz uint64, rd io.Reader) { - if err != nil { - return - } - szB := make([]byte, 8) - binary.BigEndian.PutUint64(szB, sz) - _, err = out.Write(szB) - if err != nil { - return - } - - _, err = io.Copy(out, rd) - } - - append(uint64(len(sigBytes)), bytes.NewBuffer(sigBytes)) - append(uint64(len(rootHash)), bytes.NewBuffer(rootHash)) - append(uint64(fsStat.Size()), fsRd) - append(uint64(hashStat.Size()), hashRd) - - if err != nil { - return - } - err = out.Close() - return - }() - if err != nil { - err = fmt.Errorf("assembly failed: %w", err) - return - } - - err = os.Rename(outPath, path) - return + return buildLayersCombo(host, cfg, signer) }) } @@ -474,6 +304,184 @@ func layersComboKey(host *localconfig.Host, cfg *config.Config) string { return key.String() } +func buildLayersCombo(host *localconfig.Host, cfg *config.Config, signer crypto.Signer) (path string, err error) { + path = filepath.Join(*dataDir, "cache") + if err = os.MkdirAll(path, 0o700); err != nil { + return + } + + key := layersComboKey(host, cfg) + path = filepath.Join(path, key) + ".fs" + + if _, statErr := os.Stat(path); statErr == nil { + return // exists -> already done + } + + workdir, err := os.MkdirTemp("/tmp", "layers") + if err != nil { + return + } + + defer os.RemoveAll(workdir) + + tmpTar := filepath.Join(workdir, "output.tar") + + layers := slices.Clone(cfg.Layers) + slices.Reverse(layers) + + cmdOut := new(bytes.Buffer) + + run := func(prog string, arg ...string) bool { + cmdOut.Reset() + + cmd := exec.Command(prog, arg...) + cmd.Stdout = cmdOut // os.Stdout + cmd.Stderr = os.Stderr + if e := cmd.Run(); e != nil { + err = fmt.Errorf("%s %q failed: %w", cmd.Path, cmd.Args, e) + return false + } + return true + } + + for i, layer := range layers { + if layer == "modules" { + continue // modules are in the initrd with boot v2 + } + + layerFile, e := fetchHostLayer(host, layer) + if e != nil { + err = e + return + } + + mountPoint := filepath.Join(workdir, layer) + os.MkdirAll(mountPoint, 0700) + + if e := exec.Command("erofsfuse", layerFile, mountPoint).Run(); e != nil { + err = fmt.Errorf("erofsfuse %s %s failed: %w", layerFile, mountPoint, e) + return + } + + defer func() { + if err := exec.Command("umount", mountPoint).Run(); err != nil { + log.Printf("umount %s failed: %v", mountPoint, err) + } + }() + + mode := "--append" + if i == 0 { + mode = "--create" + } + + if !run("tar", mode, "-p", "-f", tmpTar, "-C", mountPoint, ".") { + return + } + } + + fsOut := filepath.Join(workdir, "output.fs") + if !run("mkfs.erofs", "-z", "lzma", "-C131072", "-Efragments,ztailpacking", + "-T0", "--all-time", "--ignore-mtime", "--tar=f", fsOut, tmpTar) { + return + } + + hashOut := filepath.Join(workdir, "output.hash") + + if !run("veritysetup", "format", fsOut, hashOut) { + return + } + + var rootHash []byte + for line := range strings.SplitSeq(cmdOut.String(), "\n") { + v, ok := strings.CutPrefix(line, "Root hash:") + if !ok { + continue + } + v = strings.TrimSpace(v) + + b, e := hex.DecodeString(v) + if e != nil { + err = fmt.Errorf("invalid root hash: %w", e) + return + } + + rootHash = b + break + } + + if len(rootHash) == 0 { + err = fmt.Errorf("root hash not found in output") + return + } + + sigBytes, err := signer.Sign(nil, rootHash, crypto.SHA256) + if err != nil { + err = fmt.Errorf("root hash signature failed: %w", err) + return + } + + outPath := path + ".tmp" + err = func() (err error) { + fsRd, e := os.Open(fsOut) + if e != nil { + return e + } + defer fsRd.Close() + hashRd, e := os.Open(hashOut) + if e != nil { + return e + } + defer hashRd.Close() + + fsStat, e := fsRd.Stat() + if e != nil { + return e + } + hashStat, e := hashRd.Stat() + if e != nil { + return e + } + + out, err := os.Create(outPath) + if err != nil { + return + } + defer out.Close() + + append := func(sz uint64, rd io.Reader) { + if err != nil { + return + } + szB := make([]byte, 8) + binary.BigEndian.PutUint64(szB, sz) + _, err = out.Write(szB) + if err != nil { + return + } + + _, err = io.Copy(out, rd) + } + + append(uint64(len(sigBytes)), bytes.NewBuffer(sigBytes)) + append(uint64(len(rootHash)), bytes.NewBuffer(rootHash)) + append(uint64(fsStat.Size()), fsRd) + append(uint64(hashStat.Size()), hashRd) + + if err != nil { + return + } + err = out.Close() + return + }() + if err != nil { + err = fmt.Errorf("assembly failed: %w", err) + return + } + + err = os.Rename(outPath, path) + return +} + func fetchHostLayer(host *localconfig.Host, layer string) (path string, err error) { layerVersion := host.Versions[layer] if layerVersion == "" { diff --git a/cmd/dkl-local-server/cache.go b/cmd/dkl-local-server/cache.go new file mode 100644 index 0000000..112bdb8 --- /dev/null +++ b/cmd/dkl-local-server/cache.go @@ -0,0 +1,113 @@ +package main + +import ( + "io/fs" + "log" + "os" + "path/filepath" + "strings" +) + +func updateCache() { + log.Print("updating cache") + + cfg, err := readConfig() + if err != nil { + log.Printf("update cache failed: read config failed: %v", err) + return + } + + errs := 0 + inUse := map[string]bool{} + + fetch := func(path ...string) { + assetPath, err := distFetch(path...) + if err != nil { + log.Printf("update cache: dist fetch failed: %v", err) + errs += 1 + } + inUse[assetPath] = true + } + + fetch("grub-support", *grubSupportVersion) + + for _, host := range cfg.Hosts { + ctx, err := newRenderContext(host, cfg) + if err != nil { + log.Printf("update cache: %s: build context failed: %v", host.Name, err) + errs += 1 + continue + } + + _, cfg, err := ctx.Config() + if err != nil { + log.Printf("update cache: %s: render config failed: %v", host.Name, err) + errs += 1 + continue + } + + signer, err := getSigner(host.ClusterName) + if err != nil { + log.Printf("update cache: %s: get signer failed: %v", host.Name, err) + errs += 1 + continue + } + + fetch("kernels", host.Kernel) + fetch("layers", "modules", host.Kernel) + fetch("initrd", host.Initrd) + + allErofs := true + for layer, version := range host.Versions { + fetch("layers", layer, version) + + if layer != "modules" && !strings.HasSuffix(version, ".erofs") { + allErofs = false + } + } + + if allErofs { + path, err := layersCombo(host, cfg, signer) + if err != nil { + log.Printf("update cache: layer combo failed: %v", err) + continue + } + inUse[path] = true + } + } + + if errs != 0 { + log.Printf("update cache: %d errors, not cleaning", errs) + return + } + + distDir := filepath.Join(*dataDir, "dist") + cacheDir := filepath.Join(*dataDir, "cache") + + existing := []string{} + + for _, dir := range []string{distDir, cacheDir} { + fs.WalkDir(os.DirFS(dir), ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + log.Printf("update cache: walking %s failed: %v", dir, err) + return nil + } + if !d.IsDir() { + existing = append(existing, filepath.Join(dir, path)) + } + return nil + }) + } + + log.Printf("update cache: %d/%d assets in use", len(inUse), len(existing)) + + for _, path := range existing { + if inUse[path] { + continue + } + log.Print("update cache: removing ", path) + if err := os.Remove(path); err != nil { + log.Print("update cache: failed to remove: ", err) + } + } +} diff --git a/cmd/dkl-local-server/hash.go b/cmd/dkl-local-server/hash.go index 7b4cf31..3b663f1 100644 --- a/cmd/dkl-local-server/hash.go +++ b/cmd/dkl-local-server/hash.go @@ -6,7 +6,7 @@ import ( "encoding/json" ) -func hash(values ...interface{}) string { +func hash(values ...any) string { ba, err := json.Marshal(values) if err != nil { panic(err) // should not happen diff --git a/cmd/dkl-local-server/secret-store.go b/cmd/dkl-local-server/secret-store.go index de7e655..2e3a209 100644 --- a/cmd/dkl-local-server/secret-store.go +++ b/cmd/dkl-local-server/secret-store.go @@ -137,7 +137,9 @@ func unlockSecretStore(name string, passphrase []byte) (err httperr.Error) { }) go updateState() - go migrateSecrets() + + migrateSecrets() // we can probably remove it now + go updateCache() return } diff --git a/cmd/dkl-local-server/ws-configs.go b/cmd/dkl-local-server/ws-configs.go index c2015f6..c58243c 100644 --- a/cmd/dkl-local-server/ws-configs.go +++ b/cmd/dkl-local-server/ws-configs.go @@ -68,6 +68,8 @@ func writeNewConfig(cfgBytes []byte) (err error) { err = os.Rename(out.Name(), cfgPath) updateState() + go updateCache() + return } diff --git a/cmd/dkl-local-server/ws.go b/cmd/dkl-local-server/ws.go index 4f95ff3..62540a3 100644 --- a/cmd/dkl-local-server/ws.go +++ b/cmd/dkl-local-server/ws.go @@ -241,7 +241,7 @@ func wsError(resp *restful.Response, err error) { } } -func wsRender(resp *restful.Response, sslCfg *cfsslconfig.Config, tmplStr string, value interface{}) { +func wsRender(resp *restful.Response, sslCfg *cfsslconfig.Config, tmplStr string, value any) { tmpl, err := template.New("wsRender").Funcs(templateFuncs(sslCfg)).Parse(tmplStr) if err != nil { wsError(resp, err)