add all qemu-img convert disk formats

This allows the user to call it even after the store has been unlock in
order to get the admin token.

.dockerignore

@@ -0,0 +1,3 @@
+tmp
+dist
+test-run

.gitignore

@@ -1,5 +1,8 @@
 *.sw[po]
 modd-local.conf
 /tmp
-/go.work
+/test-dir2config
+/config.yaml
 /dist
+/go.work
+/go.work.sum

Dockerfile

@@ -1,23 +1,33 @@
 # ------------------------------------------------------------------------
-from mcluseau/golang-builder:1.19.4 as build
+from golang:1.24.3-bookworm as build
+
+run apt-get update && apt-get install -y git
+
+workdir /src
+
+copy go.mod go.sum ./
+run \
+  --mount=type=cache,id=gomod,target=/go/pkg/mod \
+  --mount=type=cache,id=gobuild,target=/root/.cache/go-build \
+  go mod download
+
+arg GIT_TAG
+
+copy . ./
+run \
+  --mount=type=cache,id=gomod,target=/go/pkg/mod \
+  --mount=type=cache,id=gobuild,target=/root/.cache/go-build \
+ go test ./... && \
+ hack/build ./...
 
 # ------------------------------------------------------------------------
-from debian:stretch
+from debian:bookworm
 entrypoint ["/bin/dkl-local-server"]
 
-env _uncache 1
+env _uncache=1
 run apt-get update \
- && apt-get install -y genisoimage gdisk dosfstools util-linux udev binutils systemd \
+ && yes |apt-get install -y genisoimage gdisk dosfstools util-linux udev binutils systemd \
+ grub2 grub-pc-bin grub-efi-amd64-bin ca-certificates curl openssh-client qemu-utils \
  && apt-get clean
 
-run yes |apt-get install -y grub2 grub-pc-bin grub-efi-amd64-bin \
+copy --from=build /src/dist/ /bin/
- && apt-get clean
-
-run apt-get install -y ca-certificates curl openssh-client \
- && apt-get clean
-
-run curl -L https://github.com/vmware/govmomi/releases/download/v0.21.0/govc_linux_amd64.gz | gunzip > /bin/govc && chmod +x /bin/govc
-
-copy upload-vmware.sh govc.env /var/lib/direktil/
-
-copy --from=build /go/bin/ /bin/

cmd/dkl-dir2config/assemble.go

@@ -0,0 +1,155 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+
+	"gopkg.in/yaml.v2"
+)
+
+func mergeIn(tgt, add map[any]any) {
+mergeLoop:
+	for k, v := range add {
+		switch v := v.(type) {
+		case map[any]any:
+			if tgtV, ok := tgt[k]; ok {
+				switch tgtV := tgtV.(type) {
+				case map[any]any:
+					mergeIn(tgtV, v)
+					continue mergeLoop
+				}
+			}
+		}
+
+		tgt[k] = v
+	}
+}
+
+func assemble(path string) (yamlBytes []byte, err error) {
+	obj := map[any]any{}
+
+	if Debug {
+		log.Printf("assemble %q", path)
+	}
+
+	err = eachFragment(path, searchList, func(r io.Reader) (err error) {
+		m := map[any]any{}
+		err = yaml.NewDecoder(r).Decode(&m)
+		if err != nil {
+			return
+		}
+
+		mergeIn(obj, m)
+		return
+	})
+	if err != nil {
+		err = fmt.Errorf("failed to assemble %q: %w", path, err)
+		return
+	}
+
+	yamlBytes, err = yaml.Marshal(obj)
+	if err != nil {
+		return
+	}
+
+	if Debug {
+		log.Printf("assemble %q result:\n%s", path, yamlBytes)
+	}
+
+	return
+}
+
+func eachFragment(path string, searchList []FS, walk func(io.Reader) error) (err error) {
+	var r io.ReadCloser
+
+	for len(searchList) != 0 {
+		fs := searchList[0]
+
+		r, err = fs.Open(path + ".yaml")
+		if os.IsNotExist(err) {
+			searchList = searchList[1:]
+			continue
+		}
+
+		if err != nil {
+			return
+		}
+
+		// found and open
+		break
+	}
+
+	if r == nil {
+		err = fmt.Errorf("%s: %w", path, os.ErrNotExist)
+		return
+	}
+
+	ba, err := io.ReadAll(r)
+	r.Close()
+
+	if err != nil {
+		return
+	}
+
+	if Debug {
+		log.Print("fragment:\n", string(ba))
+	}
+
+	in := bytes.NewBuffer(ba)
+
+	for {
+		var line string
+		line, err = in.ReadString('\n')
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return
+		}
+
+		line = strings.TrimSpace(line)
+		if len(line) == 0 {
+			continue
+		}
+
+		genCmd, found := strings.CutPrefix(line, "#!gen ")
+		if found {
+			cmdArgs := strings.Fields(genCmd)
+			if Debug {
+				log.Print("#!gen ", cmdArgs)
+			}
+
+			cmd := "gen/" + cmdArgs[0]
+			args := cmdArgs[1:]
+			genOutput, err := exec.Command(cmd, args...).Output()
+			if err != nil {
+				return fmt.Errorf("gen %v: %w", cmdArgs, err)
+			}
+			walk(bytes.NewBuffer(genOutput))
+			continue
+		}
+
+		includePath, found := strings.CutPrefix(line, "#!include ")
+		if !found {
+			continue
+		}
+
+		includePath = strings.TrimSpace(includePath)
+		if Debug {
+			log.Print("#!include ", includePath)
+		}
+		err = eachFragment(includePath, searchList, walk)
+		if err != nil {
+			return fmt.Errorf("include %q: %w", includePath, err)
+		}
+	}
+
+	in = bytes.NewBuffer(ba)
+	err = walk(in)
+	return
+}

cmd/dkl-dir2config/fs.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"io"
+	iofs "io/fs"
+)
+
+type FS interface {
+	Open(path string) (io.ReadCloser, error)
+	List(path string) ([]string, error)
+}
+
+type fsFS struct{ iofs.FS }
+
+func (fs fsFS) Open(path string) (io.ReadCloser, error) {
+	return fs.FS.Open(path)
+}
+
+func (fs fsFS) List(path string) (entries []string, err error) {
+	dirEnts, err := iofs.ReadDir(fs.FS, path)
+	if err != nil {
+		return
+	}
+
+	entries = make([]string, 0, len(dirEnts))
+	for _, ent := range dirEnts {
+		if ent.IsDir() {
+			continue
+		}
+		entries = append(entries, ent.Name())
+	}
+
+	return
+}

cmd/dkl-dir2config/git.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"io"
+	"sort"
+
+	"github.com/go-git/go-git/v5/plumbing/object"
+)
+
+type gitFS struct{ *object.Tree }
+
+func (fs gitFS) Open(path string) (r io.ReadCloser, err error) {
+	f, err := fs.Tree.File(path)
+	if err != nil {
+		return
+	}
+	return f.Reader()
+}
+
+func (fs gitFS) List(path string) (entries []string, err error) {
+	tree, err := fs.Tree.Tree(path)
+	if err != nil {
+		return
+	}
+
+	entries = make([]string, 0, len(tree.Entries))
+
+	for _, ent := range tree.Entries {
+		if !ent.Mode.IsFile() {
+			continue
+		}
+
+		entries = append(entries, ent.Name)
+	}
+
+	sort.Strings(entries)
+	return
+}

cmd/dkl-dir2config/main.go

@@ -2,9 +2,13 @@ package main
 
 import (
 	"flag"
+	"io/fs"
 	"log"
 	"os"
+	"path/filepath"
 
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
 	yaml "gopkg.in/yaml.v2"
 
 	"novit.tech/direktil/pkg/localconfig"
@@ -12,18 +16,27 @@ import (
 	"novit.tech/direktil/local-server/pkg/clustersconfig"
 )
 
+var Version = "dev"
+
 var (
+	Debug = false
+
 	dir     = flag.String("in", ".", "Source directory")
 	outPath = flag.String("out", "config.yaml", "Output file")
-	defaultsPath = flag.String("defaults", "defaults", "Path to the defaults")
+
+	base fs.FS
 
 	src *clustersconfig.Config
 	dst *localconfig.Config
 )
 
+func init() {
+	flag.BoolVar(&Debug, "debug", Debug, "debug")
+}
+
 func loadSrc() {
 	var err error
-	src, err = clustersconfig.FromDir(*dir, *defaultsPath)
+	src, err = clustersconfig.FromDir(read, assemble, listBase, listMerged)
 	if err != nil {
 		log.Fatal("failed to load config from dir: ", err)
 	}
@@ -34,6 +47,11 @@ func main() {
 
 	log.SetFlags(log.Ltime | log.Lmicroseconds | log.Lshortfile)
 
+	base = os.DirFS(*dir)
+	searchList = append(searchList, fsFS{base})
+
+	openIncludes()
+
 	loadSrc()
 
 	dst = &localconfig.Config{
@@ -50,8 +68,6 @@ func main() {
 
 	// ----------------------------------------------------------------------
 	for _, host := range src.Hosts {
-		loadSrc() // FIXME ugly fix of some template caching or something
-
 		log.Print("rendering host ", host.Name)
 		ctx, err := newRenderContext(host, src)
 
@@ -70,12 +86,12 @@ func main() {
 		}
 		ips = append(ips, host.IPs...)
 
-		if ctx.Group.Versions["modules"] == "" {
+		if ctx.Host.Versions["modules"] == "" {
 			// default modules' version to kernel's version
-			ctx.Group.Versions["modules"] = ctx.Group.Kernel
+			ctx.Host.Versions["modules"] = ctx.Host.Kernel
 		}
 
-		dst.Hosts = append(dst.Hosts, &localconfig.Host{
+		renderedHost := &localconfig.Host{
 			Name: host.Name,
 
 			ClusterName: ctx.Cluster.Name,
@@ -86,15 +102,21 @@ func main() {
 			MACs: macs,
 			IPs:  ips,
 
-			IPXE: ctx.Group.IPXE, // TODO render
+			IPXE: ctx.Host.IPXE, // TODO render
 
-			Kernel:   ctx.Group.Kernel,
+			Kernel:   ctx.Host.Kernel,
-			Initrd:   ctx.Group.Initrd,
+			Initrd:   ctx.Host.Initrd,
-			Versions: ctx.Group.Versions,
+			Versions: ctx.Host.Versions,
 
 			BootstrapConfig: ctx.BootstrapConfig(),
 			Config:          ctx.Config(),
-		})
+		}
+
+		if host.Template {
+			dst.HostTemplates = append(dst.HostTemplates, renderedHost)
+		} else {
+			dst.Hosts = append(dst.Hosts, renderedHost)
+		}
 	}
 
 	// ----------------------------------------------------------------------
@@ -105,8 +127,83 @@ func main() {
 
 	defer out.Close()
 
+	out.Write([]byte("# dkl-dir2config " + Version + "\n"))
+
 	if err = yaml.NewEncoder(out).Encode(dst); err != nil {
 		log.Fatal("failed to render output: ", err)
 	}
 
 }
+
+func cfgPath(subPath string) string { return filepath.Join(*dir, subPath) }
+
+func openIncludes() {
+	includesFile, err := base.Open("includes.yaml")
+
+	if os.IsNotExist(err) {
+		return
+	}
+	if err != nil {
+		log.Fatal("failed to open includes: ", err)
+	}
+
+	includes := make([]struct {
+		Path   string
+		Branch string
+		Tag    string
+	}, 0)
+
+	err = yaml.NewDecoder(includesFile).Decode(&includes)
+	if err != nil {
+		log.Fatal("failed to parse includes: ", err)
+	}
+
+	for _, include := range includes {
+		switch {
+		case include.Branch != "" || include.Tag != "":
+			p := cfgPath(include.Path) // FIXME parse git path to allow remote repos
+
+			var rev plumbing.Revision
+
+			switch {
+			case include.Branch != "":
+				log.Printf("opening include path %q as git, branch %q", p, include.Branch)
+				rev = plumbing.Revision(plumbing.NewBranchReferenceName(include.Branch))
+
+			case include.Tag != "":
+				log.Printf("opening include path %q as git, tag %q", p, include.Branch)
+				rev = plumbing.Revision(plumbing.NewTagReferenceName(include.Branch))
+			}
+
+			repo, err := git.PlainOpen(p)
+			if err != nil {
+				log.Fatal("failed to open: ", err)
+			}
+
+			revH, err := repo.ResolveRevision(rev)
+			if err != nil {
+				log.Fatalf("failed to resolve revision %s: %v", rev, err)
+			}
+
+			log.Print("  -> resolved to commit ", *revH)
+
+			commit, err := repo.CommitObject(*revH)
+			if err != nil {
+				log.Fatal("failed to get commit object: ", err)
+			}
+
+			tree, err := commit.Tree()
+			if err != nil {
+				log.Fatal("failed to open git tree: ", err)
+			}
+
+			searchList = append(searchList, gitFS{tree})
+
+		default:
+			p := cfgPath(include.Path)
+			log.Printf("opening include path %q as raw dir", p)
+
+			searchList = append(searchList, fsFS{os.DirFS(p)})
+		}
+	}
+}

cmd/dkl-dir2config/render-cluster.go

@@ -9,12 +9,12 @@ import (
 	"novit.tech/direktil/local-server/pkg/clustersconfig"
 )
 
-func clusterFuncs(clusterSpec *clustersconfig.Cluster) map[string]interface{} {
+func clusterFuncs(clusterSpec *clustersconfig.Cluster) map[string]any {
 	cluster := clusterSpec.Name
 
-	return map[string]interface{}{
+	return map[string]any{
-		"password": func(name string) (s string) {
+		"password": func(name, hash string) (s string) {
-			return fmt.Sprintf("{{ password %q %q }}", cluster, name)
+			return fmt.Sprintf("{{ password %q %q %q | quote }}", cluster, name, hash)
 		},
 
 		"token": func(name string) (s string) {
@@ -36,7 +36,7 @@ func clusterFuncs(clusterSpec *clustersconfig.Cluster) map[string]interface{} {
 			return fmt.Sprintf("{{ ca_dir %q %q }}", cluster, name), nil
 		},
 
-		"hosts_by_cluster": func(cluster string) (hosts []interface{}) {
+		"hosts_by_cluster": func(cluster string) (hosts []any) {
 			for _, host := range src.Hosts {
 				if host.Cluster == cluster {
 					hosts = append(hosts, asMap(host))
@@ -50,9 +50,9 @@ func clusterFuncs(clusterSpec *clustersconfig.Cluster) map[string]interface{} {
 			return
 		},
 
-		"hosts_by_group": func(group string) (hosts []interface{}) {
+		"hosts_by_group": func(group string) (hosts []any) {
 			for _, host := range src.Hosts {
-				if host.Group == group {
+				if host.Cluster == cluster && host.Group == group {
 					hosts = append(hosts, asMap(host))
 				}
 			}
@@ -63,6 +63,26 @@ func clusterFuncs(clusterSpec *clustersconfig.Cluster) map[string]interface{} {
 
 			return
 		},
+
+		"host_ip_from": func(hostName, net string) string {
+			host := src.Host(hostName)
+			if host == nil {
+				log.Printf("WARNING: no host named %q", hostName)
+				return "<no value>"
+			}
+
+			ipFrom := host.IPFrom
+			if ipFrom == nil {
+				ipFrom = map[string]string{}
+			}
+
+			ip, ok := ipFrom[net]
+			if !ok {
+				ip = host.IP
+			}
+
+			return ip
+		},
 	}
 }
 
@@ -101,12 +121,18 @@ func renderAddons(cluster *clustersconfig.Cluster) string {
 		return ""
 	}
 
-	addons := src.Addons[cluster.Addons]
+	buf := new(bytes.Buffer)
+
+	for _, addonSet := range cluster.Addons {
+		addons := src.Addons[addonSet]
 		if addons == nil {
-		log.Fatalf("cluster %q: no addons with name %q", cluster.Name, cluster.Addons)
+			log.Fatalf("cluster %q: no addons with name %q", cluster.Name, addonSet)
 		}
 
-	return string(renderClusterTemplates(cluster, "addons", addons))
+		buf.Write(renderClusterTemplates(cluster, "addons", addons))
+	}
+
+	return buf.String()
 }
 
 type namePod struct {

cmd/dkl-dir2config/render-context.go

@@ -2,15 +2,15 @@ package main
 
 import (
 	"bytes"
-	"crypto/sha1"
-	"encoding/hex"
 	"fmt"
 	"io"
 	"log"
+	"math/rand"
 	"path"
 	"reflect"
 	"strings"
 
+	"github.com/cespare/xxhash"
 	yaml "gopkg.in/yaml.v2"
 
 	"novit.tech/direktil/pkg/config"
@@ -23,9 +23,8 @@ type renderContext struct {
 	Annotations map[string]string
 
 	Host    *clustersconfig.Host
-	Group   *clustersconfig.Group
 	Cluster *clustersconfig.Cluster
-	Vars    map[string]interface{}
+	Vars    map[string]any
 
 	BootstrapConfigTemplate *clustersconfig.Template
 	ConfigTemplate          *clustersconfig.Template
@@ -41,34 +40,25 @@ func newRenderContext(host *clustersconfig.Host, cfg *clustersconfig.Config) (ct
 		return
 	}
 
-	group := cfg.Group(host.Group)
+	vars := make(map[string]any)
-	if group == nil {
-		err = fmt.Errorf("no group named %q", host.Group)
-		return
-	}
 
-	vars := make(map[string]interface{})
+	for _, oVars := range []map[string]any{
-
-	for _, oVars := range []map[string]interface{}{
 		cluster.Vars,
-		group.Vars,
 		host.Vars,
 	} {
 		mapMerge(vars, oVars)
 	}
 
 	return &renderContext{
-		Labels:      mergeLabels(cluster.Labels, group.Labels, host.Labels),
+		Labels:      mergeLabels(cluster.Labels, host.Labels),
-		Annotations: mergeLabels(cluster.Annotations, group.Annotations, host.Annotations),
+		Annotations: mergeLabels(cluster.Annotations, host.Annotations),
 
 		Host:    host,
-		Group:   group,
 		Cluster: cluster,
 		Vars:    vars,
 
-		BootstrapConfigTemplate: cfg.ConfigTemplate(group.BootstrapConfig),
+		BootstrapConfigTemplate: cfg.ConfigTemplate(host.BootstrapConfig),
-		ConfigTemplate:          cfg.ConfigTemplate(group.Config),
+		ConfigTemplate:          cfg.ConfigTemplate(host.Config),
-		StaticPodsTemplate:      cfg.StaticPodsTemplate(group.StaticPods),
 
 		clusterConfig: cfg,
 	}, nil
@@ -132,8 +122,6 @@ func (ctx *renderContext) Name() string {
 	switch {
 	case ctx.Host != nil:
 		return "host:" + ctx.Host.Name
-	case ctx.Group != nil:
-		return "group:" + ctx.Group.Name
 	case ctx.Cluster != nil:
 		return "cluster:" + ctx.Cluster.Name
 	default:
@@ -143,14 +131,14 @@ func (ctx *renderContext) Name() string {
 
 func (ctx *renderContext) BootstrapConfig() string {
 	if ctx.BootstrapConfigTemplate == nil {
-		log.Fatalf("no such (bootstrap) config: %q", ctx.Group.BootstrapConfig)
+		log.Fatalf("no such (bootstrap) config: %q", ctx.Host.BootstrapConfig)
 	}
 	return ctx.renderConfig(ctx.BootstrapConfigTemplate)
 }
 
 func (ctx *renderContext) Config() string {
 	if ctx.ConfigTemplate == nil {
-		log.Fatalf("no such config: %q", ctx.Group.Config)
+		log.Fatalf("no such config: %q", ctx.Host.Config)
 	}
 	return ctx.renderConfig(ctx.ConfigTemplate)
 }
@@ -166,38 +154,10 @@ func (ctx *renderContext) renderConfigTo(buf io.Writer, configTemplate *clusters
 
 	ctxMap := ctx.asMap()
 
-	templateFuncs := ctx.templateFuncs(ctxMap)
-
-	render := func(what string, t *clustersconfig.Template) (s string, err error) {
-		buf := &bytes.Buffer{}
-		err = t.Execute(ctxName, what, buf, ctxMap, templateFuncs)
-		if err != nil {
-			log.Printf("host %s: failed to render %s [%q]: %v", ctx.Host.Name, what, t.Name, err)
-			return
-		}
-
-		s = buf.String()
-		return
-	}
-
 	extraFuncs := ctx.templateFuncs(ctxMap)
 
-	extraFuncs["static_pods"] = func() (string, error) {
+	extraFuncs["static_pods_files"] = func(dir string) (string, error) {
-		name := ctx.Group.StaticPods
+		namePods := ctx.renderStaticPods()
-		if len(name) == 0 {
-			return "", fmt.Errorf("group %q has no static pods defined", ctx.Group.Name)
-		}
-
-		t := ctx.clusterConfig.StaticPodsTemplate(name)
-		if t == nil {
-			return "", fmt.Errorf("no static pods template named %q", name)
-		}
-
-		return render("static-pods", t)
-	}
-
-	extraFuncs["bootstrap_pods_files"] = func(dir string) (string, error) {
-		namePods := ctx.renderBootstrapPods()
 
 		defs := make([]config.FileDef, 0)
 
@@ -206,7 +166,7 @@ func (ctx *renderContext) renderConfigTo(buf io.Writer, configTemplate *clusters
 
 			ba, err := yaml.Marshal(namePod.Pod)
 			if err != nil {
-				return "", fmt.Errorf("bootstrap pod %s: failed to render: %v", name, err)
+				return "", fmt.Errorf("static pod %s: failed to render: %v", name, err)
 			}
 
 			defs = append(defs, config.FileDef{
@@ -220,33 +180,30 @@ func (ctx *renderContext) renderConfigTo(buf io.Writer, configTemplate *clusters
 		return string(ba), err
 	}
 
-	extraFuncs["machine_id"] = func() string {
+	extraFuncs["host_ip"] = func() string {
-		ba := sha1.Sum([]byte(ctx.Cluster.Name + "/" + ctx.Host.Name)) // TODO: check semantics of machine-id
+		if ctx.Host.Template {
-		return hex.EncodeToString(ba[:])
+			return "{{ host_ip }}"
 		}
+		return ctx.Host.IP
+	}
+	extraFuncs["host_name"] = func() string {
+		if ctx.Host.Template {
+			return "{{ host_name }}"
+		}
+		return ctx.Host.Name
+	}
+	extraFuncs["machine_id"] = func() string {
+		return "{{ machine_id }}"
+	}
+
+	extraFuncs["version"] = func() string { return Version }
 
 	if err := configTemplate.Execute(ctxName, "config", buf, ctxMap, extraFuncs); err != nil {
-		log.Fatalf("failed to render config %q for host %q: %v", ctx.Group.Config, ctx.Host.Name, err)
+		log.Fatalf("failed to render config %q for host %q: %v", ctx.Host.Config, ctx.Host.Name, err)
 	}
 }
 
-func (ctx *renderContext) StaticPods() (ba []byte, err error) {
+func (ctx *renderContext) templateFuncs(ctxMap map[string]any) map[string]interface{} {
-	if ctx.StaticPodsTemplate == nil {
-		log.Fatalf("no such static-pods: %q", ctx.Group.StaticPods)
-	}
-
-	ctxMap := ctx.asMap()
-
-	buf := bytes.NewBuffer(make([]byte, 0, 4096))
-	if err = ctx.StaticPodsTemplate.Execute(ctx.Name(), "static-pods", buf, ctxMap, ctx.templateFuncs(ctxMap)); err != nil {
-		return
-	}
-
-	ba = buf.Bytes()
-	return
-}
-
-func (ctx *renderContext) templateFuncs(ctxMap map[string]interface{}) map[string]interface{} {
 	cluster := ctx.Cluster.Name
 
 	getKeyCert := func(name, funcName string) (s string, err error) {
@@ -287,7 +244,25 @@ func (ctx *renderContext) templateFuncs(ctxMap map[string]interface{}) map[strin
 	}
 
 	funcs := clusterFuncs(ctx.Cluster)
-	for k, v := range map[string]interface{}{
+	for k, v := range map[string]any{
+		"default": func(value, defaultValue any) any {
+			switch v := value.(type) {
+			case string:
+				if v != "" {
+					return v
+				}
+			case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, float32, float64:
+				if v != 0 {
+					return v
+				}
+
+			default:
+				if v != nil {
+					return v
+				}
+			}
+			return defaultValue
+		},
 		"tls_key": func(name string) (string, error) {
 			return getKeyCert(name, "tls_key")
 		},
@@ -301,15 +276,18 @@ func (ctx *renderContext) templateFuncs(ctxMap map[string]interface{}) map[strin
 		},
 
 		"ssh_host_keys": func(dir string) (s string) {
-			return fmt.Sprintf("{{ ssh_host_keys %q %q %q}}",
+			return fmt.Sprintf("{{ ssh_host_keys %q %q \"\"}}",
-				dir, cluster, ctx.Host.Name)
+				dir, cluster)
+		},
+		"host_download_token": func() (s string) {
+			return "{{ host_download_token }}"
 		},
 
-		"hosts_of_group": func() (hosts []interface{}) {
+		"hosts_of_group": func() (hosts []any) {
-			hosts = make([]interface{}, 0)
+			hosts = make([]any, 0)
 
 			for _, host := range ctx.clusterConfig.Hosts {
-				if host.Group != ctx.Host.Group {
+				if host.Cluster == ctx.Cluster.Name && host.Group != ctx.Host.Group {
 					continue
 				}
 
@@ -321,12 +299,31 @@ func (ctx *renderContext) templateFuncs(ctxMap map[string]interface{}) map[strin
 
 		"hosts_of_group_count": func() (count int) {
 			for _, host := range ctx.clusterConfig.Hosts {
-				if host.Group == ctx.Host.Group {
+				if host.Cluster == ctx.Cluster.Name && host.Group == ctx.Host.Group {
 					count++
 				}
 			}
 			return
 		},
+
+		"shuffled_hosts_by_group": func(group string) (hosts []any) {
+			for _, host := range src.Hosts {
+				if host.Cluster == ctx.Cluster.Name && host.Group == group {
+					hosts = append(hosts, asMap(host))
+				}
+			}
+
+			if len(hosts) == 0 {
+				log.Printf("WARNING: no hosts in group %q", group)
+				return
+			}
+
+			seed := xxhash.Sum64String(ctx.Host.Name)
+			rng := rand.New(rand.NewSource(int64(seed)))
+			rng.Shuffle(len(hosts), func(i, j int) { hosts[i], hosts[j] = hosts[j], hosts[i] })
+
+			return
+		},
 	} {
 		funcs[k] = v
 	}

cmd/dkl-dir2config/render-host.go

@@ -10,18 +10,18 @@ import (
 	"novit.tech/direktil/local-server/pkg/clustersconfig"
 )
 
-func (ctx *renderContext) renderBootstrapPods() (pods []namePod) {
+func (ctx *renderContext) renderStaticPods() (pods []namePod) {
-	if ctx.Cluster.BootstrapPods == "" {
+	if ctx.Host.StaticPods == "" {
 		return
 	}
 
-	bootstrapPods, ok := src.BootstrapPods[ctx.Cluster.BootstrapPods]
+	staticPods, ok := src.StaticPods[ctx.Host.StaticPods]
 	if !ok {
-		log.Fatalf("no bootstrap pods template named %q", ctx.Cluster.BootstrapPods)
+		log.Fatalf("no static pods template named %q", ctx.Host.StaticPods)
 	}
 
-	// render bootstrap pods
+	// render static pods
-	parts := bytes.Split(ctx.renderHostTemplates("bootstrap-pods", bootstrapPods), []byte("\n---\n"))
+	parts := bytes.Split(ctx.renderHostTemplates("static-pods", staticPods), []byte("\n---\n"))
 	for _, part := range parts {
 		buf := bytes.NewBuffer(part)
 		dec := yaml.NewDecoder(buf)
@@ -35,7 +35,7 @@ func (ctx *renderContext) renderBootstrapPods() (pods []namePod) {
 			if err == io.EOF {
 				break
 			} else if err != nil {
-				log.Fatalf("bootstrap pod %d: failed to parse: %v\n%s", n, err, str)
+				log.Fatalf("static pod %d: failed to parse: %v\n%s", n, err, str)
 			}
 
 			if len(podMap) == 0 {
@@ -43,7 +43,7 @@ func (ctx *renderContext) renderBootstrapPods() (pods []namePod) {
 			}
 
 			if podMap["metadata"] == nil {
-				log.Fatalf("bootstrap pod %d: no metadata\n%s", n, buf.String())
+				log.Fatalf("static pod %d: no metadata\n%s", n, buf.String())
 			}
 
 			md := podMap["metadata"].(map[interface{}]interface{})
@@ -63,7 +63,7 @@ func (ctx *renderContext) renderHostTemplates(setName string,
 
 	log.Print("rendering host templates in ", setName)
 
-	buf := &bytes.Buffer{}
+	buf := bytes.NewBuffer(make([]byte, 0, 16<<10))
 
 	for _, t := range templates {
 		log.Print("- template: ", setName, ": ", t.Name)

cmd/dkl-dir2config/search-list.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"sort"
+)
+
+var searchList = make([]FS, 0)
+
+// read the first file matching path in the search list
+func read(path string) (ba []byte, err error) {
+	for _, fs := range searchList {
+		var r io.ReadCloser
+		r, err = fs.Open(path)
+		if os.IsNotExist(err) {
+			continue
+		}
+		if err != nil {
+			return
+		}
+
+		defer r.Close()
+		return io.ReadAll(r)
+	}
+
+	err = fmt.Errorf("%s: %w", path, os.ErrNotExist)
+	return
+}
+
+func listBase(path string) ([]string, error) {
+	return fsFS{base}.List(path)
+}
+
+func listMerged(path string) (entries []string, err error) {
+	seen := map[string]bool{}
+	for _, fs := range searchList {
+		var fsEnts []string
+		fsEnts, err = fs.List(path)
+		if os.IsNotExist(err) {
+			err = nil
+			continue
+		}
+		if err != nil {
+			return
+		}
+
+		for _, ent := range fsEnts {
+			if !seen[ent] {
+				entries = append(entries, ent)
+				seen[ent] = true
+			}
+		}
+	}
+
+	sort.Strings(entries)
+	return
+}

cmd/dkl-local-server/auth.go

@@ -1,28 +1,19 @@
 package main
 
 import (
-	"flag"
 	"log"
 	"net/http"
 )
 
-var (
+var adminToken string
-	hostsToken = flag.String("hosts-token", "", "Token to give to access /hosts (open is none)")
-	adminToken = flag.String("admin-token", "", "Token to give to access to admin actions (open is none)")
-)
-
-func authorizeHosts(r *http.Request) bool {
-	return authorizeToken(r, *hostsToken)
-}
 
 func authorizeAdmin(r *http.Request) bool {
-	return authorizeToken(r, *adminToken)
+	return authorizeToken(r, adminToken)
 }
 
 func authorizeToken(r *http.Request, token string) bool {
 	if token == "" {
-		// access is open
+		return false
-		return true
 	}
 
 	reqToken := r.Header.Get("Authorization")
@@ -34,13 +25,13 @@ func authorizeToken(r *http.Request, token string) bool {
 }
 
 func forbidden(w http.ResponseWriter, r *http.Request) {
-	log.Printf("denied access to %s from %s", r.RequestURI, r.RemoteAddr)
+	log.Printf("denied access to %s from %s", r.URL.Path, r.RemoteAddr)
 	http.Error(w, "Forbidden", http.StatusForbidden)
 }
 
-func requireToken(token string, handler http.Handler) http.Handler {
+func requireToken(token *string, handler http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		if !authorizeToken(req, token) {
+		if !authorizeToken(req, *token) {
 			forbidden(w, req)
 			return
 		}
@@ -49,9 +40,5 @@ func requireToken(token string, handler http.Handler) http.Handler {
 }
 
 func requireAdmin(handler http.Handler) http.Handler {
-	return requireToken(*adminToken, handler)
+	return requireToken(&adminToken, handler)
-}
-
-func requireHosts(handler http.Handler) http.Handler {
-	return requireToken(*hostsToken, handler)
 }

cmd/dkl-local-server/boot-img.go

@@ -6,7 +6,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -18,7 +17,7 @@ import (
 )
 
 func buildBootImg(out io.Writer, ctx *renderContext) (err error) {
-	bootImg, err := ioutil.TempFile(os.TempDir(), "boot.img-")
+	bootImg, err := os.CreateTemp(os.TempDir(), "boot.img-")
 	if err != nil {
 		return
 	}
@@ -30,7 +29,7 @@ func buildBootImg(out io.Writer, ctx *renderContext) (err error) {
 	}
 
 	// send the result
-	bootImg.Seek(0, os.SEEK_SET)
+	bootImg.Seek(0, io.SeekStart)
 	io.Copy(out, bootImg)
 	return
 }
@@ -57,7 +56,55 @@ func buildBootImgGZ(out io.Writer, ctx *renderContext) (err error) {
 	return
 }
 
-var grubSupportVersion = flag.String("grub-support", "1.0.1", "GRUB support version")
+func buildBootImgQemuConvert(out io.Writer, ctx *renderContext, format string) (err error) {
+	imgPath, err := func() (imgPath string, err error) {
+		bootImg, err := os.CreateTemp(os.TempDir(), "boot.img-")
+		if err != nil {
+			return
+		}
+		defer rmTempFile(bootImg)
+
+		err = setupBootImage(bootImg, ctx)
+		if err != nil {
+			return
+		}
+
+		err = bootImg.Sync()
+		if err != nil {
+			return
+		}
+
+		imgPath = bootImg.Name() + "." + format
+
+		err = run("qemu-img", "convert", "-f", "raw", "-O", format, bootImg.Name(), imgPath)
+		if err != nil {
+			return
+		}
+		return
+	}()
+
+	defer os.Remove(imgPath)
+
+	if err != nil {
+		return
+	}
+
+	// send the result
+	img, err := os.Open(imgPath)
+	if err != nil {
+		return
+	}
+
+	io.Copy(out, img)
+	return
+}
+func qemuImgBootImg(format string) func(out io.Writer, ctx *renderContext) (err error) {
+	return func(out io.Writer, ctx *renderContext) (err error) {
+		return buildBootImgQemuConvert(out, ctx, format)
+	}
+}
+
+var grubSupportVersion = flag.String("grub-support", "1.1.0", "GRUB support version")
 
 func setupBootImage(bootImg *os.File, ctx *renderContext) (err error) {
 	path, err := ctx.distFetch("grub-support", *grubSupportVersion)

cmd/dkl-local-server/boot-iso.go

@@ -3,7 +3,6 @@ package main
 import (
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"os"
 	"os/exec"
@@ -13,189 +12,8 @@ import (
 	"github.com/cespare/xxhash"
 )
 
-// deprecated
+func buildBootISO(out io.Writer, ctx *renderContext) (err error) {
-func buildBootISO(out io.Writer, ctx *renderContext) error {
+	tempDir, err := os.MkdirTemp("/tmp", "iso-v2-")
-	tempDir, err := ioutil.TempDir("/tmp", "iso-")
-	if err != nil {
-		return err
-	}
-
-	defer os.RemoveAll(tempDir)
-
-	cp := func(src, dst string) error {
-		log.Printf("iso: adding %s as %s", src, dst)
-		in, err := os.Open(src)
-		if err != nil {
-			return err
-		}
-
-		defer in.Close()
-
-		outPath := filepath.Join(tempDir, dst)
-
-		if err := os.MkdirAll(filepath.Dir(outPath), 0755); err != nil {
-			return err
-		}
-
-		out, err := os.Create(outPath)
-		if err != nil {
-			return err
-		}
-
-		defer out.Close()
-
-		_, err = io.Copy(out, in)
-		return err
-	}
-
-	err = func() error {
-		// grub
-
-		if err := os.MkdirAll(filepath.Join(tempDir, "grub"), 0755); err != nil {
-			return err
-		}
-		err = ioutil.WriteFile(filepath.Join(tempDir, "grub", "grub.cfg"), []byte(`
-search --set=root --file /config.yaml
-
-insmod all_video
-set timeout=3
-
-menuentry "Direktil" {
-    linux  /vmlinuz direktil.boot=DEVNAME=sr0 direktil.boot.fs=iso9660 `+ctx.CmdLine+`
-    initrd /initrd
-}
-`), 0644)
-		if err != nil {
-			return err
-		}
-
-		coreImgPath := filepath.Join(tempDir, "grub", "core.img")
-		grubCfgPath := filepath.Join(tempDir, "grub", "grub.cfg")
-
-		cmd := exec.Command("grub-mkstandalone",
-			"--format=i386-pc",
-			"--output="+coreImgPath,
-			"--install-modules=linux normal iso9660 biosdisk memdisk search tar ls",
-			"--modules=linux normal iso9660 biosdisk search",
-			"--locales=",
-			"--fonts=",
-			"boot/grub/grub.cfg="+grubCfgPath,
-		)
-		cmd.Stdout = os.Stdout
-		cmd.Stderr = os.Stderr
-		if err := cmd.Run(); err != nil {
-			return err
-		}
-
-		defer os.Remove(coreImgPath)
-		defer os.Remove(grubCfgPath)
-
-		out, err := os.Create(filepath.Join(tempDir, "grub", "bios.img"))
-		if err != nil {
-			return err
-		}
-
-		defer out.Close()
-
-		b, err := ioutil.ReadFile("/usr/lib/grub/i386-pc/cdboot.img")
-		if err != nil {
-			return err
-		}
-
-		if _, err := out.Write(b); err != nil {
-			return err
-		}
-
-		b, err = ioutil.ReadFile(coreImgPath)
-		if err != nil {
-			return err
-		}
-
-		if _, err := out.Write(b); err != nil {
-			return err
-		}
-
-		return nil
-	}()
-	if err != nil {
-		return err
-	}
-
-	// config
-	cfgBytes, cfg, err := ctx.Config()
-	if err != nil {
-		return err
-	}
-
-	ioutil.WriteFile(filepath.Join(tempDir, "config.yaml"), cfgBytes, 0600)
-
-	// kernel and initrd
-	type distCopy struct {
-		Src []string
-		Dst string
-	}
-
-	copies := []distCopy{
-		{Src: []string{"kernels", ctx.Host.Kernel}, Dst: "vmlinuz"},
-		{Src: []string{"initrd", ctx.Host.Initrd}, Dst: "initrd"},
-	}
-
-	// layers
-	for _, layer := range cfg.Layers {
-		layerVersion := ctx.Host.Versions[layer]
-		if layerVersion == "" {
-			return fmt.Errorf("layer %q not mapped to a version", layer)
-		}
-
-		copies = append(copies,
-			distCopy{
-				Src: []string{"layers", layer, layerVersion},
-				Dst: filepath.Join("current", "layers", layer+".fs"),
-			})
-	}
-
-	for _, copy := range copies {
-		outPath, err := ctx.distFetch(copy.Src...)
-		if err != nil {
-			return err
-		}
-
-		err = cp(outPath, copy.Dst)
-		if err != nil {
-			return err
-		}
-	}
-
-	// build the ISO
-	mkisofs, err := exec.LookPath("genisoimage")
-	if err != nil {
-		mkisofs, err = exec.LookPath("mkisofs")
-	}
-	if err != nil {
-		return err
-	}
-
-	cmd := exec.Command(mkisofs,
-		"-quiet",
-		"-joliet",
-		"-joliet-long",
-		"-rock",
-		"-translation-table",
-		"-no-emul-boot",
-		"-boot-load-size", "4",
-		"-boot-info-table",
-		"-eltorito-boot", "grub/bios.img",
-		"-eltorito-catalog", "grub/boot.cat",
-		tempDir,
-	)
-	cmd.Stdout = out
-	cmd.Stderr = os.Stderr
-
-	return cmd.Run()
-}
-
-func buildBootISOv2(out io.Writer, ctx *renderContext) (err error) {
-	tempDir, err := ioutil.TempDir("/tmp", "iso-v2-")
 	if err != nil {
 		return
 	}
@@ -251,7 +69,7 @@ func buildBootISOv2(out io.Writer, ctx *renderContext) (err error) {
 		f.Write([]byte("direktil marker file\n"))
 		f.Close()
 
-		err = ioutil.WriteFile(filepath.Join(tempDir, "grub", "grub.cfg"), []byte(`
+		err = os.WriteFile(filepath.Join(tempDir, "grub", "grub.cfg"), []byte(`
 search --set=root --file /`+tag+`
 
 insmod all_video
@@ -294,7 +112,7 @@ menuentry "Direktil" {
 
 		defer out.Close()
 
-		b, err := ioutil.ReadFile("/usr/lib/grub/i386-pc/cdboot.img")
+		b, err := os.ReadFile("/usr/lib/grub/i386-pc/cdboot.img")
 		if err != nil {
 			return err
 		}
@@ -303,7 +121,7 @@ menuentry "Direktil" {
 			return err
 		}
 
-		b, err = ioutil.ReadFile(coreImgPath)
+		b, err = os.ReadFile(coreImgPath)
 		if err != nil {
 			return err
 		}
@@ -320,7 +138,7 @@ menuentry "Direktil" {
 
 	// kernel and initrd
 	buildRes(fetchKernel, "vmlinuz")
-	buildRes(buildInitrdV2, "initrd")
+	buildRes(buildInitrd, "initrd")
 
 	// build the ISO
 	mkisofs, err := exec.LookPath("genisoimage")

cmd/dkl-local-server/boot-tar.go

@@ -4,7 +4,6 @@ import (
 	"archive/tar"
 	"bytes"
 	"io"
-	"io/ioutil"
 	"log"
 	"os"
 
@@ -37,7 +36,7 @@ func buildBootTar(out io.Writer, ctx *renderContext) (err error) {
 		return
 	}
 
-	kernelBytes, err := ioutil.ReadFile(kernelPath)
+	kernelBytes, err := os.ReadFile(kernelPath)
 	if err != nil {
 		return
 	}
@@ -49,7 +48,7 @@ func buildBootTar(out io.Writer, ctx *renderContext) (err error) {
 
 	// initrd
 	initrd := new(bytes.Buffer)
-	err = buildInitrdV2(initrd, ctx)
+	err = buildInitrd(initrd, ctx)
 	if err != nil {
 		return
 	}
@@ -98,7 +97,7 @@ func buildBootEFITar(out io.Writer, ctx *renderContext) (err error) {
 		return
 	}
 
-	kernelBytes, err := ioutil.ReadFile(kernelPath)
+	kernelBytes, err := os.ReadFile(kernelPath)
 	if err != nil {
 		return
 	}
@@ -110,7 +109,7 @@ func buildBootEFITar(out io.Writer, ctx *renderContext) (err error) {
 
 	// initrd
 	initrd := new(bytes.Buffer)
-	err = buildInitrdV2(initrd, ctx)
+	err = buildInitrd(initrd, ctx)
 	if err != nil {
 		return
 	}

cmd/dkl-local-server/bootv2.go

@@ -3,7 +3,6 @@ package main
 import (
 	"archive/tar"
 	"encoding/json"
-	"flag"
 	"fmt"
 	"io"
 	"log"
@@ -15,8 +14,6 @@ import (
 	"novit.tech/direktil/pkg/cpiocat"
 )
 
-var initrdV2 = flag.String("initrd-v2", "2.1.0", "initrd V2 version (temporary flag)") // FIXME
-
 func renderBootstrapConfig(w http.ResponseWriter, r *http.Request, ctx *renderContext, asJson bool) (err error) {
 	log.Printf("sending bootstrap config for %q", ctx.Host.Name)
 
@@ -34,7 +31,7 @@ func renderBootstrapConfig(w http.ResponseWriter, r *http.Request, ctx *renderCo
 	return nil
 }
 
-func buildInitrdV2(out io.Writer, ctx *renderContext) (err error) {
+func buildInitrd(out io.Writer, ctx *renderContext) (err error) {
 	_, cfg, err := ctx.Config()
 	if err != nil {
 		return
@@ -43,7 +40,7 @@ func buildInitrdV2(out io.Writer, ctx *renderContext) (err error) {
 	cat := cpiocat.New(out)
 
 	// initrd
-	initrdPath, err := ctx.distFetch("initrd", *initrdV2)
+	initrdPath, err := ctx.distFetch("initrd", ctx.Host.Initrd)
 	if err != nil {
 		return
 	}
@@ -73,7 +70,9 @@ func buildInitrdV2(out io.Writer, ctx *renderContext) (err error) {
 
 	// ssh keys
 	// FIXME we want a bootstrap-stage key instead of the real host key
-	cat.AppendBytes(cfg.FileContent("/etc/ssh/ssh_host_rsa_key"), "id_rsa", 0600)
+	for _, format := range []string{"rsa", "dsa", "ecdsa", "ed25519"} {
+		cat.AppendBytes(cfg.FileContent("/etc/ssh/ssh_host_"+format+"_key"), "id_"+format, 0600)
+	}
 
 	return cat.Close()
 }

cmd/dkl-local-server/cas-cleaner.go

@@ -12,13 +12,15 @@ var (
 )
 
 func casCleaner() {
-	for {
+	for range time.Tick(*cacheCleanDelay) {
+		if !wPublicState.Get().Store.Open {
+			continue
+		}
+
 		err := cleanCAS()
 		if err != nil {
 			log.Print("warn: couldn't clean cache: ", err)
 		}
-
-		time.Sleep(*cacheCleanDelay)
 	}
 }
 

cmd/dkl-local-server/cluster-render-context.go

@@ -1,32 +1,88 @@
 package main
 
 import (
+	"crypto/rand"
+	"encoding/base32"
 	"encoding/json"
 	"fmt"
 	"log"
 	"path"
+	"strconv"
+	"strings"
 
+	cfsslconfig "github.com/cloudflare/cfssl/config"
 	"github.com/cloudflare/cfssl/csr"
 	yaml "gopkg.in/yaml.v2"
 
+	"novit.tech/direktil/pkg/bootstrapconfig"
 	"novit.tech/direktil/pkg/config"
 )
 
-var templateFuncs = map[string]interface{}{
+func templateFuncs(sslCfg *cfsslconfig.Config) map[string]any {
-	"password": func(cluster, name string) (password string, err error) {
+	getKeyCert := func(cluster, caName, name, profile, label, reqJson string) (kc KeyCert, err error) {
-		password = secretData.Password(cluster, name)
+		certReq := &csr.CertificateRequest{
-		if len(password) == 0 {
+			KeyRequest: csr.NewKeyRequest(),
-			err = fmt.Errorf("password %q not defined for cluster %q", name, cluster)
 		}
+
+		err = json.Unmarshal([]byte(reqJson), certReq)
+		if err != nil {
+			log.Print("CSR unmarshal failed on: ", reqJson)
 			return
+		}
+
+		return getUsableKeyCert(cluster, caName, name, profile, label, certReq, sslCfg)
+	}
+
+	hash := func(plain, seed []byte, hashAlg string) (hashed string, err error) {
+		switch hashAlg {
+		case "sha512crypt":
+			return sha512crypt(plain, seed)
+
+		case "bootstrap":
+			return bootstrapconfig.JoinSeedAndHash(seed, bootstrapconfig.PasswordHashFromSeed(seed, plain)), nil
+
+		default:
+			return "", fmt.Errorf("unknown hash alg: %q", hashAlg)
+		}
+	}
+
+	return map[string]any{
+		"quote": strconv.Quote,
+
+		"password": func(cluster, name, hashAlg string) (password string, err error) {
+			key := cluster + "/" + name
+
+			seed, err := seeds.GetOrCreate(key, func() (seed []byte, err error) {
+				seed = make([]byte, 16)
+				_, err = rand.Read(seed)
+				return
+			})
+
+			if err != nil {
+				return "", fmt.Errorf("failed to get seed: %w", err)
+			}
+
+			password, err = clusterPasswords.GetOrCreate(key, func() (password string, err error) {
+				raw := make([]byte, 10)
+				_, err = rand.Read(raw)
+				if err != nil {
+					return "", fmt.Errorf("failed to generate password: %w", err)
+				}
+
+				password = strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(raw))
+				return
+			})
+			if err != nil {
+				return
+			}
+
+			return hash([]byte(password), seed, hashAlg)
 		},
 
-	"token": func(cluster, name string) (s string, err error) {
+		"token": getOrCreateClusterToken,
-		return secretData.Token(cluster, name)
-	},
 
 		"ca_key": func(cluster, name string) (s string, err error) {
-		ca, err := secretData.CA(cluster, name)
+			ca, err := getUsableClusterCA(cluster, name)
 			if err != nil {
 				return
 			}
@@ -36,7 +92,7 @@ var templateFuncs = map[string]interface{}{
 		},
 
 		"ca_crt": func(cluster, name string) (s string, err error) {
-		ca, err := secretData.CA(cluster, name)
+			ca, err := getUsableClusterCA(cluster, name)
 			if err != nil {
 				return
 			}
@@ -46,7 +102,7 @@ var templateFuncs = map[string]interface{}{
 		},
 
 		"ca_dir": func(cluster, name string) (s string, err error) {
-		ca, err := secretData.CA(cluster, name)
+			ca, err := getUsableClusterCA(cluster, name)
 			if err != nil {
 				return
 			}
@@ -88,7 +144,7 @@ var templateFuncs = map[string]interface{}{
 		},
 
 		"tls_dir": func(dir, cluster, caName, name, profile, label, reqJson string) (s string, err error) {
-		ca, err := secretData.CA(cluster, caName)
+			ca, err := getUsableClusterCA(cluster, caName)
 			if err != nil {
 				return
 			}
@@ -116,47 +172,7 @@ var templateFuncs = map[string]interface{}{
 				},
 			})
 		},
-
-	"ssh_host_keys": func(dir, cluster, host string) (s string, err error) {
-		pairs, err := secretData.SSHKeyPairs(cluster, host)
-		if err != nil {
-			return
 	}
-
-		files := make([]config.FileDef, 0, len(pairs)*2)
-
-		for _, pair := range pairs {
-			basePath := path.Join(dir, "ssh_host_"+pair.Type+"_key")
-			files = append(files, []config.FileDef{
-				{
-					Path:    basePath,
-					Mode:    0600,
-					Content: pair.Private,
-				},
-				{
-					Path:    basePath + ".pub",
-					Mode:    0644,
-					Content: pair.Public,
-				},
-			}...)
-		}
-
-		return asYaml(files)
-	},
-}
-
-func getKeyCert(cluster, caName, name, profile, label, reqJson string) (kc *KeyCert, err error) {
-	certReq := &csr.CertificateRequest{
-		KeyRequest: csr.NewKeyRequest(),
-	}
-
-	err = json.Unmarshal([]byte(reqJson), certReq)
-	if err != nil {
-		log.Print("CSR unmarshal failed on: ", reqJson)
-		return
-	}
-
-	return secretData.KeyCert(cluster, caName, name, profile, label, certReq)
 }
 
 func asYaml(v interface{}) (string, error) {

cmd/dkl-local-server/host-download-tokens.go

@@ -0,0 +1,3 @@
+package main
+
+var hostDownloadTokens = KVSecrets[string]{"hosts/download-tokens"}

cmd/dkl-local-server/httperr.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"net/http"
+
+	"m.cluseau.fr/go/httperr"
+)
+
+var (
+	ErrNotFound     = httperr.NotFound
+	ErrUnauthorized = httperr.StdStatus(http.StatusUnauthorized)
+	ErrForbidden    = httperr.StdStatus(http.StatusForbidden)
+	ErrInternal     = httperr.StdStatus(http.StatusInternalServerError)
+	ErrInvalidToken = httperr.NewStd(1000, http.StatusForbidden, "invalid token")
+	ErrStoreLocked  = httperr.NewStd(1001, http.StatusServiceUnavailable, "store is locked")
+)

cmd/dkl-local-server/initrd.go

@@ -2,13 +2,9 @@ package main
 
 import (
 	"encoding/json"
-	"fmt"
-	"io"
 	"log"
 	"net/http"
-	"os"
 
-	cpio "github.com/cavaliergopher/cpio"
 	yaml "gopkg.in/yaml.v2"
 )
 
@@ -28,84 +24,3 @@ func renderConfig(w http.ResponseWriter, r *http.Request, ctx *renderContext, as
 
 	return nil
 }
-
-func buildInitrd(out io.Writer, ctx *renderContext) error {
-	_, cfg, err := ctx.Config()
-
-	if err != nil {
-		return err
-	}
-
-	// send initrd basis
-	initrdPath, err := ctx.distFetch("initrd", ctx.Host.Initrd)
-	if err != nil {
-		return err
-	}
-
-	err = writeFile(out, initrdPath)
-	if err != nil {
-		return err
-	}
-
-	// and our extra archive
-	archive := cpio.NewWriter(out)
-
-	// - required dirs
-	for _, dir := range []string{
-		"boot",
-		"boot/current",
-		"boot/current/layers",
-	} {
-		archive.WriteHeader(&cpio.Header{
-			Name: dir,
-			Mode: cpio.FileMode(0600 | os.ModeDir),
-		})
-	}
-
-	// - the layers
-	for _, layer := range cfg.Layers {
-		layerVersion := ctx.Host.Versions[layer]
-		if layerVersion == "" {
-			return fmt.Errorf("layer %q not mapped to a version", layer)
-		}
-
-		path, err := ctx.distFetch("layers", layer, layerVersion)
-		if err != nil {
-			return err
-		}
-
-		stat, err := os.Stat(path)
-		if err != nil {
-			return err
-		}
-
-		archive.WriteHeader(&cpio.Header{
-			Name: "boot/current/layers/" + layer + ".fs",
-			Mode: 0600,
-			Size: stat.Size(),
-		})
-
-		if err = writeFile(archive, path); err != nil {
-			return err
-		}
-	}
-
-	// - the configuration
-	ba, err := yaml.Marshal(cfg)
-	if err != nil {
-		return err
-	}
-
-	archive.WriteHeader(&cpio.Header{
-		Name: "boot/config.yaml",
-		Mode: 0600,
-		Size: int64(len(ba)),
-	})
-
-	archive.Write(ba)
-
-	// finalize the archive
-	archive.Flush()
-	archive.Close()
-	return nil
-}

cmd/dkl-local-server/main.go

@@ -21,13 +21,15 @@ const (
 	etcDir = "/etc/direktil"
 )
 
+var Version = "dev"
+
 var (
 	address    = flag.String("address", ":7606", "HTTP listen address")
 	tlsAddress = flag.String("tls-address", "", "HTTPS listen address")
 	certFile   = flag.String("tls-cert", etcDir+"/server.crt", "Server TLS certificate")
 	keyFile    = flag.String("tls-key", etcDir+"/server.key", "Server TLS key")
 
-	autoUnlock = flag.String("auto-unlock", "", "Auto-unlock store (testing only!)")
+	autoUnlock = flag.String("auto-unlock", "", "Auto-unlock store (testing only!) env: DLS_AUTO_UNLOCK")
 
 	casStore cas.Store
 )
@@ -41,6 +43,9 @@ func main() {
 		log.Fatal("no listen address given")
 	}
 
+	log.Print("Direktil local-server version ", Version)
+	wPublicState.Change(func(s *PublicState) { s.ServerVersion = Version })
+
 	computeUIHash()
 
 	openSecretStore()
@@ -52,12 +57,12 @@ func main() {
 		}
 		if autoUnlock != "" {
 			log.Printf("auto-unlocking the store")
-			err := unlockSecretStore([]byte(autoUnlock))
+			err := unlockSecretStore("test", []byte(autoUnlock))
-			if err != nil {
+			if err.Any() {
 				log.Fatal(err)
 			}
 
-			log.Print("store auto-unlocked, admin token is ", *adminToken)
+			log.Print("store auto-unlocked")
 		}
 
 		os.Setenv("DLS_AUTO_UNLOCK", "")

cmd/dkl-local-server/render-context.go

@@ -2,12 +2,15 @@ package main
 
 import (
 	"bytes"
+	"crypto/sha1"
 	"crypto/sha256"
 	"encoding/hex"
+	"fmt"
 	"io"
 	"log"
 	"net/http"
 	"net/url"
+	"path"
 	"path/filepath"
 	"text/template"
 
@@ -25,7 +28,7 @@ var cmdlineParam = restful.QueryParameter("cmdline", "Linux kernel cmdline addit
 
 type renderContext struct {
 	Host      *localconfig.Host
-	SSLConfig string
+	SSLConfig *cfsslconfig.Config
 
 	// Linux kernel extra cmdline
 	CmdLine string `yaml:"-"`
@@ -61,12 +64,7 @@ func renderCtx(w http.ResponseWriter, r *http.Request, ctx *renderContext, what
 	return nil
 }
 
-var prevSSLConfig = "-"
+func sslConfigFromLocalConfig(cfg *localconfig.Config) (sslCfg *cfsslconfig.Config, err error) {
-
-func newRenderContext(host *localconfig.Host, cfg *localconfig.Config) (ctx *renderContext, err error) {
-	if prevSSLConfig != cfg.SSLConfig {
-		var sslCfg *cfsslconfig.Config
-
 	if len(cfg.SSLConfig) == 0 {
 		sslCfg = &cfsslconfig.Config{}
 	} else {
@@ -75,18 +73,18 @@ func newRenderContext(host *localconfig.Host, cfg *localconfig.Config) (ctx *ren
 			return
 		}
 	}
+	return
+}
 
-		err = loadSecretData(sslCfg)
+func newRenderContext(host *localconfig.Host, cfg *localconfig.Config) (ctx *renderContext, err error) {
+	sslCfg, err := sslConfigFromLocalConfig(cfg)
 	if err != nil {
 		return
 	}
 
-		prevSSLConfig = cfg.SSLConfig
-	}
-
 	return &renderContext{
-		SSLConfig: cfg.SSLConfig,
 		Host:      host,
+		SSLConfig: sslCfg,
 	}, nil
 }
 
@@ -120,7 +118,7 @@ func (ctx *renderContext) BootstrapConfig() (ba []byte, cfg *bsconfig.Config, er
 
 func (ctx *renderContext) render(templateText string) (ba []byte, err error) {
 	tmpl, err := template.New(ctx.Host.Name + "/config").
-		Funcs(templateFuncs).
+		Funcs(ctx.TemplateFuncs()).
 		Parse(templateText)
 
 	if err != nil {
@@ -132,13 +130,6 @@ func (ctx *renderContext) render(templateText string) (ba []byte, err error) {
 		return
 	}
 
-	if secretData.Changed() {
-		err = secretData.Save()
-		if err != nil {
-			return
-		}
-	}
-
 	ba = buf.Bytes()
 	return
 }
@@ -180,3 +171,80 @@ func asMap(v interface{}) map[string]interface{} {
 
 	return result
 }
+
+func (ctx *renderContext) TemplateFuncs() map[string]any {
+	funcs := templateFuncs(ctx.SSLConfig)
+
+	for name, method := range map[string]any{
+		"host_ip": func() (s string) {
+			return ctx.Host.IPs[0]
+		},
+		"host_name": func() (s string) {
+			return ctx.Host.Name
+		},
+		"machine_id": func() (s string) {
+			ba := sha1.Sum([]byte(ctx.Host.ClusterName + "/" + ctx.Host.Name))
+			return hex.EncodeToString(ba[:])
+		},
+
+		"ssh_host_keys": func(dir, cluster, host string) (s string, err error) {
+			if host == "" {
+				host = ctx.Host.Name
+			}
+			if host != ctx.Host.Name {
+				err = fmt.Errorf("wrong host name")
+				return
+			}
+
+			pairs, err := getSSHKeyPairs(host)
+			if err != nil {
+				return
+			}
+
+			files := make([]config.FileDef, 0, len(pairs)*2)
+
+			for _, pair := range pairs {
+				basePath := path.Join(dir, "ssh_host_"+pair.Type+"_key")
+				files = append(files, []config.FileDef{
+					{
+						Path:    basePath,
+						Mode:    0600,
+						Content: pair.Private,
+					},
+					{
+						Path:    basePath + ".pub",
+						Mode:    0644,
+						Content: pair.Public,
+					},
+				}...)
+			}
+
+			return asYaml(files)
+		},
+		"host_download_token": func() (token string, err error) {
+			key := ctx.Host.Name
+			token, found, err := hostDownloadTokens.Get(key)
+			if err != nil {
+				return
+			}
+
+			if !found {
+				token, err = newToken(32)
+				if err != nil {
+					return
+				}
+
+				err = hostDownloadTokens.Put(key, token)
+				if err != nil {
+					return
+				}
+			}
+
+			return
+		},
+	} {
+		funcs[name] = method
+	}
+
+	return funcs
+}

cmd/dkl-local-server/secret-store.go

@@ -1,22 +1,25 @@
 package main
 
 import (
-	"crypto/rand"
-	"encoding/base32"
 	"encoding/json"
 	"log"
 	"net/http"
 	"os"
 	"path/filepath"
+	"sort"
+	"strings"
 	"sync"
 
+	restful "github.com/emicklei/go-restful"
 	"m.cluseau.fr/go/httperr"
+
 	"novit.tech/direktil/local-server/secretstore"
 )
 
 var secStore *secretstore.Store
 
-func secStorePath(name string) string { return filepath.Join(*dataDir, "secrets", name) }
+func secStoreRoot() string            { return filepath.Join(*dataDir, "secrets") }
+func secStorePath(name string) string { return filepath.Join(secStoreRoot(), name) }
 func secKeysStorePath() string        { return secStorePath(".keys") }
 
 func openSecretStore() {
@@ -56,11 +59,11 @@ func openSecretStore() {
 var (
 	unlockMutex = sync.Mutex{}
 
-	ErrStoreAlreadyUnlocked = httperr.NewStd(http.StatusConflict, 1, "store already unlocked")
+	ErrStoreAlreadyUnlocked = httperr.NewStd(1, http.StatusConflict, "store already unlocked")
-	ErrInvalidPassphrase    = httperr.NewStd(http.StatusBadRequest, 2, "invalid passphrase")
+	ErrInvalidPassphrase    = httperr.NewStd(2, http.StatusBadRequest, "invalid passphrase")
 )
 
-func unlockSecretStore(passphrase []byte) *httperr.Error {
+func unlockSecretStore(name string, passphrase []byte) (err httperr.Error) {
 	unlockMutex.Lock()
 	defer unlockMutex.Unlock()
 
@@ -69,9 +72,9 @@ func unlockSecretStore(passphrase []byte) *httperr.Error {
 	}
 
 	if secStore.IsNew() {
-		err := secStore.Init(passphrase)
+		err := secStore.Init(name, passphrase)
 		if err != nil {
-			return httperr.New(http.StatusInternalServerError, err)
+			return httperr.Internal(err)
 		}
 
 		err = secStore.SaveTo(secKeysStorePath())
@@ -79,7 +82,7 @@ func unlockSecretStore(passphrase []byte) *httperr.Error {
 			log.Print("secret store save error: ", err)
 			secStore.Close()
 
-			return httperr.New(http.StatusInternalServerError, err)
+			return httperr.Internal(err)
 		}
 
 	} else {
@@ -94,31 +97,39 @@ func unlockSecretStore(passphrase []byte) *httperr.Error {
 			log.Print("failed to read admin token: ", err)
 			secStore.Close()
 
-			return httperr.New(http.StatusInternalServerError, err)
+			return httperr.Internal(err)
 		}
 
-		randBytes := make([]byte, 32)
+		token, err = newToken(32)
-		_, err := rand.Read(randBytes)
 		if err != nil {
-			log.Print("rand read error: ", err)
 			secStore.Close()
-
+			return httperr.Internal(err)
-			return httperr.New(http.StatusInternalServerError, err)
 		}
 
-		token = base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(randBytes)
 		err = writeSecret("admin-token", token)
 		if err != nil {
 			log.Print("write error: ", err)
 			secStore.Close()
 
-			return httperr.New(http.StatusInternalServerError, err)
+			return httperr.Internal(err)
 		}
 
 		log.Print("wrote new admin token")
 	}
 
-	*adminToken = token
+	adminToken = token
+
+	{
+		token, err := newToken(16)
+		if err != nil {
+			secStore.Close()
+			return httperr.Internal(err)
+		}
+
+		wState.Change(func(v *State) {
+			v.Store.DownloadToken = token
+		})
+	}
 
 	wPublicState.Change(func(v *PublicState) {
 		v.Store.New = false
@@ -126,8 +137,9 @@ func unlockSecretStore(passphrase []byte) *httperr.Error {
 	})
 
 	go updateState()
+	go migrateSecrets()
 
-	return nil
+	return
 }
 
 func readSecret(name string, value any) (err error) {
@@ -147,7 +159,13 @@ func readSecret(name string, value any) (err error) {
 }
 
 func writeSecret(name string, value any) (err error) {
-	f, err := os.Create(secStorePath(name + ".data.new"))
+	path := secStorePath(name + ".data.new")
+
+	if err = os.MkdirAll(filepath.Dir(path), 0700); err != nil {
+		return
+	}
+
+	f, err := os.Create(path)
 	if err != nil {
 		return
 	}
@@ -167,5 +185,213 @@ func writeSecret(name string, value any) (err error) {
 		return
 	}
 
-	return os.Rename(f.Name(), secStorePath(name+".data"))
+	err = os.Rename(f.Name(), secStorePath(name+".data"))
+	if err != nil {
+		return
+	}
+
+	go updateState()
+	return
+}
+
+var secL sync.Mutex
+
+func updateSecret[T any](name string, update func(*T)) (err error) {
+	secL.Lock()
+	defer secL.Unlock()
+
+	v := new(T)
+	err = readSecret(name, v)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return
+		}
+		err = nil
+	}
+
+	update(v)
+
+	return writeSecret(name, *v)
+}
+
+func updateSecretWithKey[T any](name, key string, update func(v *T)) (err error) {
+	secL.Lock()
+	defer secL.Unlock()
+
+	kvs := map[string]*T{}
+
+	err = readSecret(name, &kvs)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return
+		}
+		err = nil
+	}
+
+	update(kvs[key])
+
+	return writeSecret(name, kvs)
+}
+
+type KVSecrets[T any] struct{ Name string }
+
+func (s KVSecrets[T]) Data() (kvs map[string]T, err error) {
+	kvs = make(map[string]T)
+	err = readSecret(s.Name, &kvs)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return
+		}
+		err = nil
+	}
+	return
+}
+
+type KV[T any] struct {
+	K string
+	V T
+}
+
+func (s KVSecrets[T]) List(prefix string) (list []KV[T], err error) {
+	data, err := s.Data()
+	if err != nil {
+		return
+	}
+
+	list = make([]KV[T], 0, len(data))
+
+	for k, v := range data {
+		if !strings.HasPrefix(k, prefix) {
+			continue
+		}
+		list = append(list, KV[T]{k, v})
+	}
+
+	sort.Slice(list, func(i, j int) bool { return list[i].K < list[j].K })
+
+	return
+}
+
+func (s KVSecrets[T]) Keys(prefix string) (keys []string, err error) {
+	kvs, err := s.Data()
+	if err != nil {
+		return
+	}
+
+	keys = make([]string, 0, len(kvs))
+
+	for k := range kvs {
+		if !strings.HasPrefix(k, prefix) {
+			continue
+		}
+		keys = append(keys, k[len(prefix):])
+	}
+
+	sort.Strings(keys)
+
+	return
+}
+
+func (s KVSecrets[T]) Get(key string) (v T, found bool, err error) {
+	kvs, err := s.Data()
+	if err != nil {
+		return
+	}
+
+	v, found = kvs[key]
+	return
+}
+
+func (s KVSecrets[T]) Put(key string, v T) (err error) {
+	secL.Lock()
+	defer secL.Unlock()
+
+	kvs, err := s.Data()
+	if err != nil {
+		return
+	}
+
+	kvs[key] = v
+	err = writeSecret(s.Name, kvs)
+	return
+}
+
+func (s KVSecrets[T]) Del(key string) (err error) {
+	secL.Lock()
+	defer secL.Unlock()
+
+	kvs, err := s.Data()
+	if err != nil {
+		return
+	}
+
+	delete(kvs, key)
+	err = writeSecret(s.Name, kvs)
+	return
+}
+
+func (s KVSecrets[T]) GetOrCreate(key string, create func() (T, error)) (v T, err error) {
+	v, found, err := s.Get(key)
+	if err != nil {
+		return
+	}
+
+	if !found {
+		v, err = create()
+		if err != nil {
+			return
+		}
+
+		err = s.Put(key, v)
+	}
+
+	return
+}
+
+func (s KVSecrets[T]) WsList(resp *restful.Response, prefix string) {
+	keys, err := s.Keys(prefix)
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+
+	resp.WriteEntity(keys)
+}
+
+func (s KVSecrets[T]) WsGet(resp *restful.Response, key string) {
+	keys, found, err := s.Get(key)
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+
+	if !found {
+		wsNotFound(resp)
+		return
+	}
+
+	resp.WriteEntity(keys)
+}
+
+func (s KVSecrets[T]) WsPut(req *restful.Request, resp *restful.Response, key string) {
+	v := new(T)
+	err := req.ReadEntity(v)
+	if err != nil {
+		wsBadRequest(resp, err.Error())
+		return
+	}
+
+	err = s.Put(key, *v)
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+}
+
+func (s KVSecrets[T]) WsDel(req *restful.Request, resp *restful.Response, key string) {
+	err := s.Del(key)
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
 }

cmd/dkl-local-server/secrets-migrate.go

@@ -0,0 +1,88 @@
+package main
+
+import (
+	"log"
+	"os"
+
+	cfsslconfig "github.com/cloudflare/cfssl/config"
+)
+
+func migrateSecrets() {
+	if _, err := os.Stat(secretDataPath()); err != nil {
+		if os.IsNotExist(err) {
+			return
+		}
+
+		log.Print("not migrating old secrets: ", err)
+
+		return
+	}
+
+	log.Print("migrating old secrets")
+
+	log := log.New(log.Default().Writer(), "secrets migration: ", log.Flags()|log.Lmsgprefix)
+
+	// load secrets
+	cfg, err := readConfig()
+	if err != nil {
+		log.Fatal(err)
+		return
+	}
+
+	var sslCfg *cfsslconfig.Config
+
+	if len(cfg.SSLConfig) == 0 {
+		sslCfg = &cfsslconfig.Config{}
+	} else {
+		sslCfg, err = cfsslconfig.LoadConfig([]byte(cfg.SSLConfig))
+		if err != nil {
+			return
+		}
+	}
+
+	secretData, err := loadSecretData(sslCfg)
+	if err != nil {
+		log.Fatal(err)
+		return
+	}
+
+	for clusterName, cluster := range secretData.clusters {
+		for k, v := range cluster.Tokens {
+			err = clusterTokens.Put(clusterName+"/"+k, v)
+			if err != nil {
+				log.Fatal(err)
+				return
+			}
+		}
+
+		for k, v := range cluster.Passwords {
+			err = clusterPasswords.Put(clusterName+"/"+k, v)
+			if err != nil {
+				log.Fatal(err)
+				return
+			}
+		}
+
+		for caName, ca := range cluster.CAs {
+			clusterCAs.Put(clusterName+"/"+caName, CA{Key: ca.Key, Cert: ca.Cert})
+
+			for signedName, signed := range ca.Signed {
+				err = clusterCASignedKeys.Put(clusterName+"/"+caName+"/"+signedName, *signed)
+				if err != nil {
+					log.Fatal(err)
+				}
+			}
+		}
+
+		for hostName, pairs := range cluster.SSHKeyPairs {
+			err = sshHostKeys.Put(hostName, pairs)
+			if err != nil {
+				log.Fatal(err)
+			}
+		}
+	}
+
+	if err := os.Rename(secretDataPath(), secretDataPath()+".migrated"); err != nil {
+		log.Fatal("failed to rename migrated secrets: ", err)
+	}
+}

cmd/dkl-local-server/secrets.go

@@ -1,45 +1,19 @@
 package main
 
 import (
-	"crypto"
-	"crypto/rand"
-	"encoding/base32"
 	"encoding/json"
 	"errors"
-	"fmt"
-	"io/ioutil"
-	"net"
 	"os"
 	"path/filepath"
-	"sort"
-	"sync"
 	"time"
 
-	"github.com/cespare/xxhash"
 	"github.com/cloudflare/cfssl/certinfo"
 	"github.com/cloudflare/cfssl/config"
-	"github.com/cloudflare/cfssl/csr"
-	"github.com/cloudflare/cfssl/helpers"
-	"github.com/cloudflare/cfssl/initca"
 	"github.com/cloudflare/cfssl/log"
-	"github.com/cloudflare/cfssl/signer"
-	"github.com/cloudflare/cfssl/signer/local"
-	"k8s.io/apimachinery/pkg/util/validation"
-	"k8s.io/apimachinery/pkg/util/validation/field"
-)
-
-var (
-	secretData *SecretData
-	DontSave   = false
 )
 
 type SecretData struct {
-	l sync.Mutex
-
-	prevHash uint64
-
 	clusters map[string]*ClusterSecrets
-	changed  bool
 	config   *config.Config
 }
 
@@ -50,13 +24,6 @@ type ClusterSecrets struct {
 	SSHKeyPairs map[string][]SSHKeyPair
 }
 
-type CA struct {
-	Key  []byte
-	Cert []byte
-
-	Signed map[string]*KeyCert
-}
-
 type KeyCert struct {
 	Key     []byte
 	Cert    []byte
@@ -67,21 +34,18 @@ func secretDataPath() string {
 	return filepath.Join(*dataDir, "secret-data.json")
 }
 
-func loadSecretData(config *config.Config) (err error) {
+func loadSecretData(config *config.Config) (sd *SecretData, err error) {
 	log.Info("Loading secret data")
 
-	sd := &SecretData{
+	sd = &SecretData{
 		clusters: make(map[string]*ClusterSecrets),
-		changed:  false,
 		config:   config,
 	}
 
-	ba, err := ioutil.ReadFile(secretDataPath())
+	ba, err := os.ReadFile(secretDataPath())
 	if err != nil {
 		if os.IsNotExist(err) {
-			sd.changed = true
 			err = nil
-			secretData = sd
 			return
 		}
 		return
@@ -91,218 +55,6 @@ func loadSecretData(config *config.Config) (err error) {
 		return
 	}
 
-	sd.prevHash = xxhash.Sum64(ba)
-
-	secretData = sd
-	return
-}
-
-func (sd *SecretData) Changed() bool {
-	return sd.changed
-}
-
-func (sd *SecretData) Save() (err error) {
-	if DontSave {
-		return
-	}
-
-	sd.l.Lock()
-	defer sd.l.Unlock()
-
-	ba, err := json.Marshal(sd.clusters)
-	if err != nil {
-		return
-	}
-
-	h := xxhash.Sum64(ba)
-	if h == sd.prevHash {
-		return
-	}
-
-	log.Info("Saving secret data")
-	err = ioutil.WriteFile(secretDataPath(), ba, 0600)
-
-	if err == nil {
-		sd.prevHash = h
-	}
-
-	return
-}
-
-func newClusterSecrets() *ClusterSecrets {
-	return &ClusterSecrets{
-		CAs:       make(map[string]*CA),
-		Tokens:    make(map[string]string),
-		Passwords: make(map[string]string),
-	}
-}
-
-func (sd *SecretData) cluster(name string) (cs *ClusterSecrets) {
-	cs, ok := sd.clusters[name]
-	if ok {
-		return
-	}
-
-	sd.l.Lock()
-	defer sd.l.Unlock()
-
-	log.Info("secret-data: new cluster: ", name)
-
-	cs = newClusterSecrets()
-	sd.clusters[name] = cs
-	sd.changed = true
-	return
-}
-
-func (sd *SecretData) Passwords(cluster string) (passwords []string) {
-	cs := sd.cluster(cluster)
-
-	passwords = make([]string, 0, len(cs.Passwords))
-	for name := range cs.Passwords {
-		passwords = append(passwords, name)
-	}
-
-	sort.Strings(passwords)
-
-	return
-}
-
-func (sd *SecretData) Password(cluster, name string) (password string) {
-	cs := sd.cluster(cluster)
-
-	if cs.Passwords == nil {
-		cs.Passwords = make(map[string]string)
-	}
-
-	password = cs.Passwords[name]
-	return
-}
-
-func (sd *SecretData) SetPassword(cluster, name, password string) {
-	cs := sd.cluster(cluster)
-
-	if cs.Passwords == nil {
-		cs.Passwords = make(map[string]string)
-	}
-
-	cs.Passwords[name] = password
-	sd.changed = true
-}
-
-func (sd *SecretData) Token(cluster, name string) (token string, err error) {
-	cs := sd.cluster(cluster)
-
-	token = cs.Tokens[name]
-	if token != "" {
-		return
-	}
-
-	sd.l.Lock()
-	defer sd.l.Unlock()
-
-	log.Info("secret-data: new token in cluster ", cluster, ": ", name)
-
-	b := make([]byte, 16)
-	_, err = rand.Read(b)
-	if err != nil {
-		return
-	}
-
-	token = base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(b)
-
-	cs.Tokens[name] = token
-	sd.changed = true
-	return
-}
-
-func (sd *SecretData) RenewCACert(cluster, name string) (err error) {
-	cs := sd.cluster(cluster)
-
-	ca := cs.CAs[name]
-
-	var signer crypto.Signer
-	signer, err = helpers.ParsePrivateKeyPEM(ca.Key)
-	if err != nil {
-		return
-	}
-
-	newCert, _, err := initca.NewFromSigner(newCACertReq(), signer)
-	if err != nil {
-		return
-	}
-
-	sd.l.Lock()
-	defer sd.l.Unlock()
-
-	cs.CAs[name].Cert = newCert
-	sd.changed = true
-
-	return
-}
-
-func newCACertReq() *csr.CertificateRequest {
-	return &csr.CertificateRequest{
-		CN: "Direktil Local Server",
-		KeyRequest: &csr.KeyRequest{
-			A: "ecdsa",
-			S: 521, // 256, 384, 521
-		},
-		Names: []csr.Name{
-			{
-				C: "NC",
-				O: "novit.nc",
-			},
-		},
-	}
-}
-
-func (sd *SecretData) CA(cluster, name string) (ca *CA, err error) {
-
-	defer func() {
-		if err != nil {
-			err = fmt.Errorf("cluster %s CA %s: %w", cluster, name, err)
-		}
-	}()
-
-	cs := sd.cluster(cluster)
-
-	ca, ok := cs.CAs[name]
-	if ok {
-		checkErr := checkCertUsable(ca.Cert)
-		if checkErr != nil {
-			log.Infof("secret-data cluster %s: CA %s: regenerating certificate: %v", cluster, name, checkErr)
-
-			err = sd.RenewCACert(cluster, name)
-			if err != nil {
-				err = fmt.Errorf("renew: %w", err)
-			}
-		}
-
-		return
-	}
-
-	sd.l.Lock()
-	defer sd.l.Unlock()
-
-	log.Info("secret-data: new CA in cluster ", cluster, ": ", name)
-
-	req := newCACertReq()
-
-	cert, _, key, err := initca.New(req)
-	if err != nil {
-		err = fmt.Errorf("initca: %w", err)
-		return
-	}
-
-	ca = &CA{
-		Key:    key,
-		Cert:   cert,
-		Signed: make(map[string]*KeyCert),
-	}
-
-	cs.CAs[name] = ca
-	sd.changed = true
-
 	return
 }
 
@@ -321,104 +73,3 @@ func checkCertUsable(certPEM []byte) error {
 
 	return nil
 }
-
-func (sd *SecretData) KeyCert(cluster, caName, name, profile, label string, req *csr.CertificateRequest) (kc *KeyCert, err error) {
-	for idx, host := range req.Hosts {
-		if ip := net.ParseIP(host); ip != nil {
-			// valid IP (v4 or v6)
-			continue
-		}
-
-		if host == "*" {
-			continue
-		}
-
-		if errs := validation.IsDNS1123Subdomain(host); len(errs) == 0 {
-			continue
-		}
-		if errs := validation.IsWildcardDNS1123Subdomain(host); len(errs) == 0 {
-			continue
-		}
-
-		path := field.NewPath(cluster, name, "hosts").Index(idx)
-		return nil, fmt.Errorf("%v: %q is not an IP or FQDN", path, host)
-	}
-
-	if req.CA != nil {
-		err = errors.New("no CA section allowed here")
-		return
-	}
-
-	ca, err := sd.CA(cluster, caName)
-	if err != nil {
-		return
-	}
-
-	logPrefix := fmt.Sprintf("secret-data: cluster %s: CA %s:", cluster, caName)
-
-	rh := hash(req)
-	kc, ok := ca.Signed[name]
-	if ok && rh == kc.ReqHash {
-		err = checkCertUsable(kc.Cert)
-		if err == nil {
-			return
-		}
-		log.Infof("%s regenerating certificate: ", err)
-
-	} else if ok {
-		log.Infof("%s CSR changed for %s: hash=%q previous=%q", name, rh, kc.ReqHash)
-	} else {
-		log.Infof("%s new CSR for %s", logPrefix, name)
-	}
-
-	sd.l.Lock()
-	defer sd.l.Unlock()
-
-	sgr, err := ca.Signer(sd.config.Signing)
-	if err != nil {
-		return
-	}
-
-	generator := &csr.Generator{Validator: func(_ *csr.CertificateRequest) error { return nil }}
-
-	csr, key, err := generator.ProcessRequest(req)
-	if err != nil {
-		return
-	}
-
-	signReq := signer.SignRequest{
-		Request: string(csr),
-		Profile: profile,
-		Label:   label,
-	}
-
-	cert, err := sgr.Sign(signReq)
-	if err != nil {
-		return
-	}
-
-	kc = &KeyCert{
-		Key:     key,
-		Cert:    cert,
-		ReqHash: rh,
-	}
-
-	ca.Signed[name] = kc
-	sd.changed = true
-
-	return
-}
-
-func (ca *CA) Signer(policy *config.Signing) (result *local.Signer, err error) {
-	caCert, err := helpers.ParseCertificatePEM(ca.Cert)
-	if err != nil {
-		return
-	}
-
-	caKey, err := helpers.ParsePrivateKeyPEM(ca.Key)
-	if err != nil {
-		return
-	}
-
-	return local.NewSigner(caKey, caCert, signer.DefaultSigAlgo(caKey), policy)
-}

cmd/dkl-local-server/sha512crypt.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"encoding/base64"
+
+	crypthash "github.com/sergeymakinen/go-crypt/hash"
+	"github.com/sergeymakinen/go-crypt/sha512"
+)
+
+// for some reason, no implementation of crypt's sha512 is clean enough :(
+
+func sha512crypt(password, seed []byte) (string, error) {
+	// loose salt entropy because of character restriction in the salt
+	salt := []byte(base64.RawStdEncoding.EncodeToString(seed))[:sha512.MaxSaltLength]
+	// - base64 allows '+' where the salt accepts '.'
+	for i, c := range salt {
+		if c == '+' {
+			salt[i] = '.'
+		}
+	}
+
+	scheme := struct {
+		HashPrefix string
+		Rounds     uint32 `hash:"param:rounds,omitempty"`
+		Salt       []byte
+		Sum        [86]byte
+	}{
+		HashPrefix: sha512.Prefix,
+		Rounds:     sha512.DefaultRounds,
+		Salt:       salt,
+	}
+
+	key, err := sha512.Key([]byte(password), scheme.Salt, scheme.Rounds)
+	if err != nil {
+		return "", err
+	}
+	crypthash.LittleEndianEncoding.Encode(scheme.Sum[:], key)
+	return crypthash.Marshal(scheme)
+}

cmd/dkl-local-server/ssh-secrets.go

@@ -10,39 +10,20 @@ import (
 	"crypto/x509"
 	"encoding/asn1"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"os/exec"
 )
 
+var sshHostKeys = KVSecrets[[]SSHKeyPair]{"hosts/ssh-host-keys"}
+
 type SSHKeyPair struct {
 	Type    string
 	Public  string
 	Private string
 }
 
-func (sd *SecretData) SSHKeyPairs(cluster, host string) (pairs []SSHKeyPair, err error) {
+func getSSHKeyPairs(host string) (pairs []SSHKeyPair, err error) {
-	cs := sd.cluster(cluster)
+	pairs, _, err = sshHostKeys.Get(host)
-
-	if cs.SSHKeyPairs == nil {
-		cs.SSHKeyPairs = map[string][]SSHKeyPair{}
-	}
-
-	outFile, err := ioutil.TempFile("/tmp", "dls-key.")
-	if err != nil {
-		return
-	}
-
-	outPath := outFile.Name()
-
-	removeTemp := func() {
-		os.Remove(outPath)
-		os.Remove(outPath + ".pub")
-	}
-
-	defer removeTemp()
-
-	pairs = cs.SSHKeyPairs[host]
 
 	didGenerate := false
 
@@ -59,46 +40,65 @@ genLoop:
 			}
 		}
 
-		didGenerate = true
+		err = func() (err error) {
+			outFile, err := os.CreateTemp("/tmp", "dls-key.")
+			if err != nil {
+				return
+			}
+
+			outPath := outFile.Name()
+
+			removeTemp := func() {
+				os.Remove(outPath)
+				os.Remove(outPath + ".pub")
+			}
 
 			removeTemp()
+			defer removeTemp()
 
 			var out, privKey, pubKey []byte
 
-		out, err = exec.Command("ssh-keygen",
+			cmd := exec.Command("ssh-keygen",
 				"-N", "",
 				"-C", "root@"+host,
 				"-f", outPath,
-			"-t", keyType).CombinedOutput()
+				"-t", keyType)
+			out, err = cmd.CombinedOutput()
 			if err != nil {
 				err = fmt.Errorf("ssh-keygen failed: %v: %s", err, string(out))
 				return
 			}
 
-		privKey, err = ioutil.ReadFile(outPath)
+			privKey, err = os.ReadFile(outPath)
 			if err != nil {
 				return
 			}
 
-		os.Remove(outPath)
+			pubKey, err = os.ReadFile(outPath + ".pub")
-
-		pubKey, err = ioutil.ReadFile(outPath + ".pub")
 			if err != nil {
 				return
 			}
 
-		os.Remove(outPath + ".pub")
-
 			pairs = append(pairs, SSHKeyPair{
 				Type:    keyType,
 				Public:  string(pubKey),
 				Private: string(privKey),
 			})
+			didGenerate = true
+
+			return
+		}()
+
+		if err != nil {
+			return
+		}
 	}
 
 	if didGenerate {
-		cs.SSHKeyPairs[host] = pairs
+		err = sshHostKeys.Put(host, pairs)
-		err = sd.Save()
+		if err != nil {
+			return
+		}
 	}
 
 	return

cmd/dkl-local-server/ssh-secrets_test.go

@@ -2,16 +2,9 @@ package main
 
 import "testing"
 
-func init() {
-	DontSave = true
-}
-
 func TestSSHKeyGet(t *testing.T) {
-	sd := &SecretData{
+	// TODO needs fake secret store
-		clusters: make(map[string]*ClusterSecrets),
+	// if _, err := getSSHKeyPairs("host"); err != nil {
-	}
+	// 	t.Error(err)
-
+	// }
-	if _, err := sd.SSHKeyPairs("test", "host"); err != nil {
-		t.Error(err)
-	}
 }

cmd/dkl-local-server/state.go

@@ -8,6 +8,7 @@ import (
 )
 
 type PublicState struct {
+	ServerVersion string
 	UIHash        string
 	Store         struct {
 		New  bool
@@ -20,25 +21,39 @@ var wPublicState = watchable.New[PublicState]()
 type State struct {
 	HasConfig bool
 
+	Store struct {
+		DownloadToken string
+		KeyNames      []string
+	}
+
 	Clusters []ClusterState
 	Hosts    []HostState
 	Config   *localconfig.Config
 
 	Downloads map[string]DownloadSpec
+
+	HostTemplates []string
 }
 
 type ClusterState struct {
 	Name      string
 	Addons    bool
-	// TODO CAs
+	Passwords []string
-	// TODO passwords
+	Tokens    []string
-	// TODO tokens
+	CAs       []CAState
 }
 
 type HostState struct {
 	Name    string
 	Cluster string
 	IPs     []string
+
+	Template string `json:",omitempty"`
+}
+
+type CAState struct {
+	Name   string
+	Signed []string
 }
 
 var wState = watchable.New[State]()
@@ -50,14 +65,21 @@ func init() {
 func updateState() {
 	log.Print("updating state")
 
+	// store key names
+	keyNames := make([]string, 0, len(secStore.Keys))
+	for _, key := range secStore.Keys {
+		keyNames = append(keyNames, key.Name)
+	}
+
+	// config
 	cfg, err := readConfig()
 	if err != nil {
-		wState.Change(func(v *State) { v.HasConfig = false; v.Config = nil })
+		wState.Change(func(v *State) { v.HasConfig = false; v.Config = nil; v.Store.KeyNames = keyNames })
 		return
 	}
 
 	if secStore.IsNew() || !secStore.Unlocked() {
-		wState.Change(func(v *State) { v.HasConfig = false; v.Config = nil })
+		wState.Change(func(v *State) { v.HasConfig = false; v.Config = nil; v.Store.KeyNames = keyNames })
 		return
 	}
 
@@ -68,25 +90,75 @@ func updateState() {
 			Name:   cluster.Name,
 			Addons: len(cluster.Addons) != 0,
 		}
+
+		c.Passwords, err = clusterPasswords.Keys(c.Name + "/")
+		if err != nil {
+			log.Print("failed to read cluster passwords: ", err)
+		}
+		c.Tokens, err = clusterTokens.Keys(c.Name + "/")
+		if err != nil {
+			log.Print("failed to read cluster tokens: ", err)
+		}
+
+		caNames, err := clusterCAs.Keys(c.Name + "/")
+		if err != nil {
+			log.Print("failed to read cluster CAs: ", err)
+		}
+		for _, caName := range caNames {
+			ca := CAState{Name: caName}
+
+			signedNames, err := clusterCASignedKeys.Keys(c.Name + "/" + caName + "/")
+			if err != nil {
+				log.Print("failed to read cluster CA signed keys: ", err)
+			}
+			for _, signedName := range signedNames {
+				ca.Signed = append(ca.Signed, signedName)
+			}
+
+			c.CAs = append(c.CAs, ca)
+		}
+
 		clusters = append(clusters, c)
 	}
 
-	hosts := make([]HostState, 0, len(cfg.Hosts))
+	hfts, err := hostsFromTemplate.List("")
+	if err != nil {
+		log.Print("failed to read hosts from template: ", err)
+	}
+
+	hosts := make([]HostState, 0, len(cfg.Hosts)+len(hfts))
 	for _, host := range cfg.Hosts {
 		h := HostState{
 			Name:    host.Name,
 			Cluster: host.ClusterName,
 			IPs:     host.IPs,
 		}
-
 		hosts = append(hosts, h)
 	}
 
+	for _, kv := range hfts {
+		name, hft := kv.K, kv.V
+		h := HostState{
+			Name:    name,
+			Cluster: hft.ClusterName(cfg),
+			IPs:     []string{hft.IP},
+
+			Template: hft.Template,
+		}
+		hosts = append(hosts, h)
+	}
+
+	hostTemplates := make([]string, len(cfg.HostTemplates))
+	for i, ht := range cfg.HostTemplates {
+		hostTemplates[i] = ht.Name
+	}
+
 	// done
 	wState.Change(func(v *State) {
 		v.HasConfig = true
-		//v.Config = cfg
+		v.Store.KeyNames = keyNames
 		v.Clusters = clusters
 		v.Hosts = hosts
+		v.HostTemplates = hostTemplates
 	})
 }

cmd/dkl-local-server/tls-ca.go

@@ -0,0 +1,199 @@
+package main
+
+import (
+	"crypto"
+	"crypto/x509"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"log"
+	"net"
+
+	"github.com/cloudflare/cfssl/config"
+	"github.com/cloudflare/cfssl/csr"
+	"github.com/cloudflare/cfssl/helpers"
+	"github.com/cloudflare/cfssl/initca"
+	"github.com/cloudflare/cfssl/signer"
+	"github.com/cloudflare/cfssl/signer/local"
+	"k8s.io/apimachinery/pkg/util/validation"
+)
+
+type CA struct {
+	Key  []byte
+	Cert []byte
+
+	Signed map[string]*KeyCert
+}
+
+func (ca *CA) Init() (err error) {
+	req := ca.newReq()
+
+	cert, _, key, err := initca.New(req)
+	if err != nil {
+		err = fmt.Errorf("initca: %w", err)
+		return
+	}
+
+	ca.Key = key
+	ca.Cert = cert
+
+	return
+}
+
+func (ca *CA) RenewCert() (err error) {
+	var signer crypto.Signer
+	signer, err = helpers.ParsePrivateKeyPEM(ca.Key)
+	if err != nil {
+		return
+	}
+
+	newCert, _, err := initca.NewFromSigner(ca.newReq(), signer)
+	if err != nil {
+		return
+	}
+
+	ca.Cert = newCert
+
+	return
+}
+
+func (_ CA) newReq() *csr.CertificateRequest {
+	return &csr.CertificateRequest{
+		CN: "Direktil Local Server",
+		KeyRequest: &csr.KeyRequest{
+			A: "ecdsa",
+			S: 521, // 256, 384, 521
+		},
+		Names: []csr.Name{
+			{
+				O: "novit.io",
+			},
+		},
+	}
+}
+
+func (ca CA) Signer(policy *config.Signing) (result *local.Signer, err error) {
+	caCert, err := helpers.ParseCertificatePEM(ca.Cert)
+	if err != nil {
+		return
+	}
+
+	caKey, err := helpers.ParsePrivateKeyPEM(ca.Key)
+	if err != nil {
+		return
+	}
+
+	return local.NewSigner(caKey, caCert, signer.DefaultSigAlgo(caKey), policy)
+}
+
+func getUsableKeyCert(cluster, caName, name, profile, label string, req *csr.CertificateRequest, cfg *config.Config) (kc KeyCert, err error) {
+	log := log.New(log.Default().Writer(), cluster+": CA "+caName+": ", log.Flags()|log.Lmsgprefix)
+
+	ca, err := getUsableClusterCA(cluster, caName)
+	if err != nil {
+		return
+	}
+
+	for _, host := range req.Hosts {
+		if ip := net.ParseIP(host); ip != nil {
+			// valid IP (v4 or v6)
+			continue
+		}
+
+		if host == "*" {
+			continue
+		}
+
+		if errs := validation.IsDNS1123Subdomain(host); len(errs) == 0 {
+			continue
+		}
+		if errs := validation.IsWildcardDNS1123Subdomain(host); len(errs) == 0 {
+			continue
+		}
+
+		err = fmt.Errorf("%q is not an IP or FQDN", host)
+		return
+	}
+
+	if req.CA != nil {
+		err = errors.New("no CA section allowed here")
+		return
+	}
+
+	rh := hash(req)
+
+	key := cluster + "/" + caName + "/" + name
+
+	kc, found, err := clusterCASignedKeys.Get(key)
+	if err != nil {
+		return
+	}
+
+	if found {
+		if rh == kc.ReqHash {
+			err = func() (err error) {
+				err = checkCertUsable(kc.Cert)
+				if err != nil {
+					return
+				}
+
+				pool := x509.NewCertPool()
+				if !pool.AppendCertsFromPEM(ca.Cert) {
+					panic("unexpected invalid CA certificate at this point")
+				}
+
+				certBlock, _ := pem.Decode(kc.Cert)
+				cert, err := x509.ParseCertificate(certBlock.Bytes)
+				if err != nil {
+					return
+				}
+
+				_, err = cert.Verify(x509.VerifyOptions{Roots: pool})
+				return
+			}()
+			if err == nil {
+				return // all good, no need to create or renew
+			}
+
+			log.Print("regenerating certificate: ", err)
+
+		} else {
+			log.Printf("CSR changed for %s: hash=%q previous=%q", name, rh, kc.ReqHash)
+		}
+	} else {
+		log.Print("new CSR for ", name)
+	}
+
+	sgr, err := ca.Signer(cfg.Signing)
+	if err != nil {
+		return
+	}
+
+	generator := &csr.Generator{Validator: func(_ *csr.CertificateRequest) error { return nil }}
+
+	csr, tlsKey, err := generator.ProcessRequest(req)
+	if err != nil {
+		return
+	}
+
+	signReq := signer.SignRequest{
+		Request: string(csr),
+		Profile: profile,
+		Label:   label,
+	}
+
+	cert, err := sgr.Sign(signReq)
+	if err != nil {
+		return
+	}
+
+	kc = KeyCert{
+		Key:     tlsKey,
+		Cert:    cert,
+		ReqHash: rh,
+	}
+
+	err = clusterCASignedKeys.Put(key, kc)
+
+	return
+}

cmd/dkl-local-server/token.go

@@ -0,0 +1,24 @@
+package main
+
+import (
+	"crypto/rand"
+	"encoding/base32"
+	"log"
+	"net/http"
+
+	"m.cluseau.fr/go/httperr"
+)
+
+func newToken(sizeInBytes int) (token string, err error) {
+	randBytes := make([]byte, sizeInBytes)
+
+	_, err = rand.Read(randBytes)
+	if err != nil {
+		log.Print("rand read error: ", err)
+		err = httperr.New(http.StatusInternalServerError, err)
+		return
+	}
+
+	token = base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(randBytes)
+	return
+}

cmd/dkl-local-server/upstream.go

@@ -19,7 +19,7 @@ import (
 )
 
 var (
-	upstreamURL = flag.String("upstream", "https://direktil.novit.nc/dist", "Upstream server for dist elements")
+	upstreamURL = flag.String("upstream", "https://dkl.novit.io/dist", "Upstream server for dist elements")
 )
 
 func (ctx *renderContext) distFetch(path ...string) (outPath string, err error) {

cmd/dkl-local-server/ws-auth.go

@@ -7,24 +7,20 @@ import (
 )
 
 func adminAuth(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) {
-	tokenAuth(req, resp, chain, *adminToken)
+	tokenAuth(req, resp, chain, adminToken)
-}
-
-func hostsAuth(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) {
-	tokenAuth(req, resp, chain, *hostsToken, *adminToken)
 }
 
 func tokenAuth(req *restful.Request, resp *restful.Response, chain *restful.FilterChain, allowedTokens ...string) {
 	token := getToken(req)
 
 	for _, allowedToken := range allowedTokens {
-		if allowedToken == "" || token == allowedToken {
+		if allowedToken != "" && token == allowedToken {
 			chain.ProcessFilter(req, resp)
 			return
 		}
 	}
 
-	resp.WriteErrorString(401, "401: Not Authorized")
+	wsError(resp, ErrUnauthorized)
 	return
 }
 
@@ -38,7 +34,7 @@ func getToken(req *restful.Request) string {
 	}
 
 	if !strings.HasPrefix(token, bearerPrefix) {
-		return ""
+		return token
 	}
 
 	return token[len(bearerPrefix):]

cmd/dkl-local-server/ws-cluster-cas.go

@@ -0,0 +1,96 @@
+package main
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/cloudflare/cfssl/helpers"
+	"github.com/cloudflare/cfssl/log"
+	restful "github.com/emicklei/go-restful"
+)
+
+var clusterCAs = newClusterSecretKV[CA]("CAs")
+
+func wsClusterCAs(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	clusterCAs.WsList(resp, clusterName+"/")
+}
+
+func wsClusterCA(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	name := req.PathParameter("ca-name")
+
+	clusterCAs.WsGet(resp, clusterName+"/"+name)
+}
+
+func getUsableClusterCA(cluster, name string) (ca CA, err error) {
+	defer func() {
+		if err != nil {
+			err = fmt.Errorf("cluster %s CA %s: %w", cluster, name, err)
+		}
+	}()
+
+	key := cluster + "/" + name
+
+	ca, found, err := clusterCAs.Get(key)
+	if err != nil {
+		return
+	}
+
+	if !found {
+		log.Info("new CA in cluster ", cluster, ": ", name)
+
+		err = ca.Init()
+		if err != nil {
+			return
+		}
+
+		err = clusterCAs.Put(key, ca)
+		if err != nil {
+			return
+		}
+
+		return
+	}
+
+	checkErr := checkCertUsable(ca.Cert)
+	if checkErr != nil {
+		log.Infof("cluster %s: CA %s: regenerating certificate: %v", cluster, name, checkErr)
+
+		prevCerts, _ := helpers.ParseCertificatesPEM(ca.Cert)
+
+		err = ca.RenewCert()
+		if err != nil {
+			err = fmt.Errorf("renew: %w", err)
+		}
+
+		now := time.Now()
+		for _, cert := range prevCerts {
+			if cert.NotAfter.After(now) {
+				continue
+			}
+			certPEM := helpers.EncodeCertificatePEM(cert)
+			ca.Cert = append(ca.Cert, certPEM...)
+		}
+
+		err = clusterCAs.Put(key, ca)
+	}
+
+	return
+}
+
+var clusterCASignedKeys = newClusterSecretKV[KeyCert]("CA-signed-keys")
+
+func wsClusterCASignedKeys(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	caName := req.PathParameter("ca-name")
+	clusterCASignedKeys.WsList(resp, clusterName+"/"+caName+"/")
+}
+
+func wsClusterCASignedKey(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	caName := req.PathParameter("ca-name")
+	name := req.PathParameter("signed-name")
+
+	clusterCASignedKeys.WsGet(resp, clusterName+"/"+caName+"/"+name)
+}

cmd/dkl-local-server/ws-cluster-passwords.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	restful "github.com/emicklei/go-restful"
+)
+
+var seeds = newClusterSecretKV[[]byte]("seeds")
+
+var clusterPasswords = newClusterSecretKV[string]("passwords")
+
+func wsClusterPasswords(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	clusterPasswords.WsList(resp, clusterName+"/")
+}
+
+func wsClusterPassword(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	name := req.PathParameter("password-name")
+
+	clusterPasswords.WsGet(resp, clusterName+"/"+name)
+}
+
+func wsClusterSetPassword(req *restful.Request, resp *restful.Response) {
+	cluster := wsReadCluster(req, resp)
+	if cluster == nil {
+		return
+	}
+
+	name := req.PathParameter("password-name")
+
+	clusterPasswords.WsPut(req, resp, cluster.Name+"/"+name)
+}

cmd/dkl-local-server/ws-cluster-tokens.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"crypto/rand"
+	"encoding/base32"
+
+	restful "github.com/emicklei/go-restful"
+)
+
+var clusterTokens = newClusterSecretKV[string]("tokens")
+
+func getOrCreateClusterToken(cluster, name string) (token string, err error) {
+	key := cluster + "/" + name
+
+	token, found, err := clusterTokens.Get(key)
+
+	if err != nil || found {
+		return
+	}
+
+	b := make([]byte, 16)
+	_, err = rand.Read(b)
+	if err != nil {
+		return
+	}
+
+	token = base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(b)
+
+	err = clusterTokens.Put(key, token)
+	return
+}
+
+func wsClusterTokens(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	clusterTokens.WsList(resp, clusterName+"/")
+}
+
+func wsClusterToken(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	name := req.PathParameter("token-name")
+
+	clusterTokens.WsGet(resp, clusterName+"/"+name)
+}

cmd/dkl-local-server/ws-clusters.go

@@ -2,13 +2,22 @@ package main
 
 import (
 	"log"
-	"sort"
+	"net/url"
+	"strconv"
 
 	restful "github.com/emicklei/go-restful"
 
+	"novit.tech/direktil/local-server/pkg/mime"
 	"novit.tech/direktil/pkg/localconfig"
 )
 
+var clusterSecretKVs = []string{}
+
+func newClusterSecretKV[T any](name string) KVSecrets[T] {
+	clusterSecretKVs = append(clusterSecretKVs, name)
+	return KVSecrets[T]{"clusters/" + name}
+}
+
 func wsListClusters(req *restful.Request, resp *restful.Response) {
 	cfg := wsReadConfig(resp)
 	if cfg == nil {
@@ -33,7 +42,7 @@ func wsReadCluster(req *restful.Request, resp *restful.Response) (cluster *local
 
 	cluster = cfg.Cluster(clusterName)
 	if cluster == nil {
-		wsNotFound(req, resp)
+		wsNotFound(resp)
 		return
 	}
 
@@ -57,152 +66,58 @@ func wsClusterAddons(req *restful.Request, resp *restful.Response) {
 
 	if len(cluster.Addons) == 0 {
 		log.Printf("cluster %q has no addons defined", cluster.Name)
-		wsNotFound(req, resp)
+		wsNotFound(resp)
 		return
 	}
 
-	wsRender(resp, cluster.Addons, cluster)
+	cfg := wsReadConfig(resp)
-}
+	if cfg == nil {
-
-func wsClusterPasswords(req *restful.Request, resp *restful.Response) {
-	cluster := wsReadCluster(req, resp)
-	if cluster == nil {
 		return
 	}
 
-	resp.WriteEntity(secretData.Passwords(cluster.Name))
+	sslCfg, err := sslConfigFromLocalConfig(cfg)
-}
-
-func wsClusterPassword(req *restful.Request, resp *restful.Response) {
-	cluster := wsReadCluster(req, resp)
-	if cluster == nil {
-		return
-	}
-
-	name := req.PathParameter("password-name")
-
-	resp.WriteEntity(secretData.Password(cluster.Name, name))
-}
-
-func wsClusterSetPassword(req *restful.Request, resp *restful.Response) {
-	cluster := wsReadCluster(req, resp)
-	if cluster == nil {
-		return
-	}
-
-	name := req.PathParameter("password-name")
-
-	var password string
-	if err := req.ReadEntity(&password); err != nil {
-		wsError(resp, err) // FIXME this is a BadRequest
-		return
-	}
-
-	secretData.SetPassword(cluster.Name, name, password)
-
-	if err := secretData.Save(); err != nil {
-		wsError(resp, err)
-		return
-	}
-}
-
-func wsClusterToken(req *restful.Request, resp *restful.Response) {
-	cluster := wsReadCluster(req, resp)
-	if cluster == nil {
-		return
-	}
-
-	name := req.PathParameter("token-name")
-
-	token, err := secretData.Token(cluster.Name, name)
 	if err != nil {
 		wsError(resp, err)
 		return
 	}
 
-	resp.WriteEntity(token)
+	wsRender(resp, sslCfg, cluster.Addons, cluster)
-}
-
-func wsClusterBootstrapPods(req *restful.Request, resp *restful.Response) {
-	cluster := wsReadCluster(req, resp)
-	if cluster == nil {
-		return
-	}
-
-	if len(cluster.BootstrapPods) == 0 {
-		log.Printf("cluster %q has no bootstrap pods defined", cluster.Name)
-		wsNotFound(req, resp)
-		return
-	}
-
-	wsRender(resp, cluster.BootstrapPods, cluster)
-}
-
-func wsClusterCAs(req *restful.Request, resp *restful.Response) {
-	cs := secretData.clusters[req.PathParameter("cluster-name")]
-	if cs == nil {
-		wsNotFound(req, resp)
-		return
-	}
-
-	keys := make([]string, 0, len(cs.CAs))
-	for k := range cs.CAs {
-		keys = append(keys, k)
-	}
-
-	sort.Strings(keys)
-
-	resp.WriteJson(keys, restful.MIME_JSON)
 }
 
 func wsClusterCACert(req *restful.Request, resp *restful.Response) {
-	cs := secretData.clusters[req.PathParameter("cluster-name")]
+	clusterName := req.PathParameter("cluster-name")
-	if cs == nil {
+	caName := req.PathParameter("ca-name")
-		wsNotFound(req, resp)
+
-		return
+	ca, found, err := clusterCAs.Get(clusterName + "/" + caName)
-	}
+	if err != nil {
-
+		wsError(resp, err)
-	ca := cs.CAs[req.PathParameter("ca-name")]
+		return
-	if ca == nil {
+	}
-		wsNotFound(req, resp)
+	if !found {
+		wsNotFound(resp)
 		return
 	}
 
+	resp.Header().Set("Content-Type", mime.CERT)
 	resp.Write(ca.Cert)
 }
 
 func wsClusterSignedCert(req *restful.Request, resp *restful.Response) {
-	cs := secretData.clusters[req.PathParameter("cluster-name")]
+	clusterName := req.PathParameter("cluster-name")
-	if cs == nil {
+	caName := req.PathParameter("ca-name")
-		wsNotFound(req, resp)
-		return
-	}
-
-	ca := cs.CAs[req.PathParameter("ca-name")]
-	if ca == nil {
-		wsNotFound(req, resp)
-		return
-	}
-
 	name := req.QueryParameter("name")
 
-	if name == "" {
+	kc, found, err := clusterCASignedKeys.Get(clusterName + "/" + caName + "/" + name)
-		keys := make([]string, 0, len(ca.Signed))
+	if err != nil {
-		for k := range ca.Signed {
+		wsError(resp, err)
-			keys = append(keys, k)
+		return
 	}
-
+	if !found {
-		sort.Strings(keys)
+		wsNotFound(resp)
-
-		resp.WriteJson(keys, restful.MIME_JSON)
-		return
-	}
-
-	kc := ca.Signed[name]
-	if kc == nil {
-		wsNotFound(req, resp)
 		return
 	}
 
+	resp.AddHeader("Content-Type", mime.CERT)
+	resp.AddHeader("Content-Disposition", "attachment; filename="+strconv.Quote(clusterName+"_"+caName+"_"+url.PathEscape(name)+".crt"))
 	resp.Write(kc.Cert)
 }

cmd/dkl-local-server/ws-configs.go

@@ -3,7 +3,6 @@ package main
 import (
 	"compress/gzip"
 	"io"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 
@@ -18,11 +17,14 @@ func wsUploadConfig(req *restful.Request, resp *restful.Response) {
 
 	if err != nil {
 		wsError(resp, err)
+		return
 	}
+
+	resp.WriteEntity(true)
 }
 
 func writeNewConfig(reader io.Reader) (err error) {
-	out, err := ioutil.TempFile(*dataDir, ".config-upload")
+	out, err := os.CreateTemp(*dataDir, ".config-upload")
 	if err != nil {
 		return
 	}
@@ -38,11 +40,18 @@ func writeNewConfig(reader io.Reader) (err error) {
 	cfgPath := configFilePath()
 	in, err := os.Open(cfgPath)
 
-	if err == nil {
+	if err != nil {
+		if os.IsNotExist(err) {
+			// nothing to backup
+		} else {
+			return // real error
+		}
+	} else {
 		err = backupCurrentConfig(in)
-	} else if !os.IsNotExist(err) {
+		if err != nil {
 			return
 		}
+	}
 
 	err = os.Rename(out.Name(), cfgPath)
 

cmd/dkl-local-server/ws-downloads.go

@@ -58,7 +58,7 @@ func wsDownload(req *restful.Request, resp *restful.Response) {
 	asset := req.PathParameter("asset")
 
 	if token == "" || asset == "" {
-		wsNotFound(req, resp)
+		wsNotFound(resp)
 		return
 	}
 
@@ -81,6 +81,7 @@ func wsDownload(req *restful.Request, resp *restful.Response) {
 		}
 
 		if !found {
+			wsNotFound(resp)
 			return
 		}
 
@@ -95,11 +96,11 @@ func wsDownload(req *restful.Request, resp *restful.Response) {
 	})
 
 	if !found {
-		wsNotFound(req, resp)
+		wsNotFound(resp)
 		return
 	}
 
-	log.Printf("download via token %q", token)
+	log.Printf("download via token: %s %q asset %q", spec.Kind, spec.Name, asset)
 
 	cfg, err := readConfig()
 	if err != nil {
@@ -115,7 +116,7 @@ func wsDownload(req *restful.Request, resp *restful.Response) {
 	case "cluster":
 		cluster := cfg.ClusterByName(spec.Name)
 		if cluster == nil {
-			wsNotFound(req, resp)
+			wsNotFound(resp)
 			return
 		}
 
@@ -125,13 +126,13 @@ func wsDownload(req *restful.Request, resp *restful.Response) {
 			resp.Write([]byte(cluster.Addons))
 
 		default:
-			wsNotFound(req, resp)
+			wsNotFound(resp)
 		}
 
 	case "host":
-		host := cfg.Host(spec.Name)
+		host := hostOrTemplate(cfg, spec.Name)
 		if host == nil {
-			wsNotFound(req, resp)
+			wsNotFound(resp)
 			return
 		}
 
@@ -145,6 +146,6 @@ func wsDownload(req *restful.Request, resp *restful.Response) {
 		renderHost(resp.ResponseWriter, req.Request, asset, host, cfg)
 
 	default:
-		wsNotFound(req, resp)
+		wsNotFound(resp)
 	}
 }

cmd/dkl-local-server/ws-host.go

@@ -13,21 +13,23 @@ import (
 	"novit.tech/direktil/local-server/pkg/mime"
 )
 
-var trustXFF = flag.Bool("trust-xff", true, "Trust the X-Forwarded-For header")
+var (
+	allowDetectedHost = flag.Bool("allow-detected-host", false, "Allow access to host assets from its IP (insecure but enables unattended netboot)")
+	trustXFF          = flag.Bool("trust-xff", false, "Trust the X-Forwarded-For header")
+)
 
 type wsHost struct {
-	prefix  string
 	hostDoc string
-	getHost func(req *restful.Request) string
+	getHost func(req *restful.Request) (hostName string, err error)
 }
 
-func (ws *wsHost) register(rws *restful.WebService, alterRB func(*restful.RouteBuilder)) {
+func (ws wsHost) register(rws *restful.WebService, alterRB func(*restful.RouteBuilder)) {
 	b := func(what string) *restful.RouteBuilder {
-		return rws.GET(ws.prefix + "/" + what).To(ws.render)
+		return rws.GET("/" + what).To(ws.render)
 	}
 
 	for _, rb := range []*restful.RouteBuilder{
-		rws.GET(ws.prefix).To(ws.get).
+		rws.GET("").To(ws.get).
 			Doc("Get the "+ws.hostDoc+"'s details").
 			Returns(200, "OK", localconfig.Host{}),
 
@@ -44,13 +46,30 @@ func (ws *wsHost) register(rws *restful.WebService, alterRB func(*restful.RouteB
 			Produces(mime.DISK).
 			Doc("Get the " + ws.hostDoc + "'s boot disk image"),
 
+		// - raw + compressed
 		b("boot.img.gz").
 			Produces(mime.DISK + "+gzip").
-			Doc("Get the " + ws.hostDoc + "'s boot disk image (gzip compressed)"),
+			Doc("Get the " + ws.hostDoc + "'s boot disk image, gzip compressed"),
-
 		b("boot.img.lz4").
 			Produces(mime.DISK + "+lz4").
-			Doc("Get the " + ws.hostDoc + "'s boot disk image (lz4 compressed)"),
+			Doc("Get the " + ws.hostDoc + "'s boot disk image, lz4 compressed"),
+
+		// - other formats
+		b("boot.qcow2").
+			Produces(mime.DISK + "+qcow2").
+			Doc("Get the " + ws.hostDoc + "'s boot disk image, QCOW2 (KVM, Xen)"),
+		b("boot.qed").
+			Produces(mime.DISK + "+qed").
+			Doc("Get the " + ws.hostDoc + "'s boot disk image, QED (KVM)"),
+		b("boot.vmdk").
+			Produces(mime.DISK + "+vdi").
+			Doc("Get the " + ws.hostDoc + "'s boot disk image, VDI (VirtualBox)"),
+		b("boot.qcow2").
+			Produces(mime.DISK + "+vpc").
+			Doc("Get the " + ws.hostDoc + "'s boot disk image, VHD (Hyper-V)"),
+		b("boot.vmdk").
+			Produces(mime.DISK + "+vmdk").
+			Doc("Get the " + ws.hostDoc + "'s boot disk image, VMDK (VMware)"),
 
 		// metal/local HDD upgrades
 		b("boot.tar").
@@ -71,62 +90,57 @@ func (ws *wsHost) register(rws *restful.WebService, alterRB func(*restful.RouteB
 			Produces(mime.IPXE).
 			Doc("Get the " + ws.hostDoc + "'s IPXE code (for netboot)"),
 
+		// boot support
 		b("kernel").
 			Produces(mime.OCTET).
 			Doc("Get the " + ws.hostDoc + "'s kernel (ie: for netboot)"),
-
 		b("initrd").
 			Produces(mime.OCTET).
 			Doc("Get the " + ws.hostDoc + "'s initial RAM disk (ie: for netboot)"),
 
-		// boot v2
 		// - bootstrap config
 		b("bootstrap-config").
 			Produces(mime.YAML).
 			Doc("Get the " + ws.hostDoc + "'s bootstrap configuration"),
 		b("bootstrap-config.json").
 			Doc("Get the " + ws.hostDoc + "'s bootstrap configuration (as JSON)"),
-		// - initrd
-		b("initrd-v2").
-			Produces(mime.OCTET).
-			Doc("Get the " + ws.hostDoc + "'s initial RAM disk (v2)"),
 		// - bootstrap
 		b("bootstrap.tar").
 			Produces(mime.TAR).
 			Doc("Get the " + ws.hostDoc + "'s bootstrap seed archive"),
-		b("boot-v2.iso").
-			Produces(mime.ISO).
-			Param(cmdlineParam).
-			Doc("Get the " + ws.hostDoc + "'s boot CD-ROM image (v2)"),
 	} {
 		alterRB(rb)
 		rws.Route(rb)
 	}
 }
 
-func (ws *wsHost) host(req *restful.Request, resp *restful.Response) (host *localconfig.Host, cfg *localconfig.Config) {
+func (ws wsHost) host(req *restful.Request, resp *restful.Response) (host *localconfig.Host, cfg *localconfig.Config) {
-	hostname := ws.getHost(req)
+	hostname, err := ws.getHost(req)
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
 	if hostname == "" {
-		wsNotFound(req, resp)
+		wsNotFound(resp)
 		return
 	}
 
-	cfg, err := readConfig()
+	cfg, err = readConfig()
 	if err != nil {
 		wsError(resp, err)
 		return
 	}
 
-	host = cfg.Host(hostname)
+	host = hostOrTemplate(cfg, hostname)
 	if host == nil {
-		log.Print("no host named ", hostname)
+		wsNotFound(resp)
-		wsNotFound(req, resp)
 		return
 	}
+
 	return
 }
 
-func (ws *wsHost) get(req *restful.Request, resp *restful.Response) {
+func (ws wsHost) get(req *restful.Request, resp *restful.Response) {
 	host, _ := ws.host(req, resp)
 	if host == nil {
 		return
@@ -135,7 +149,7 @@ func (ws *wsHost) get(req *restful.Request, resp *restful.Response) {
 	resp.WriteEntity(host)
 }
 
-func (ws *wsHost) render(req *restful.Request, resp *restful.Response) {
+func (ws wsHost) render(req *restful.Request, resp *restful.Response) {
 	host, cfg := ws.host(req, resp)
 	if host == nil {
 		return
@@ -167,12 +181,17 @@ func renderHost(w http.ResponseWriter, r *http.Request, what string, host *local
 	case "kernel":
 		err = renderKernel(w, r, ctx)
 
+	// boot v2
+	case "bootstrap-config":
+		err = renderBootstrapConfig(w, r, ctx, false)
+	case "bootstrap-config.json":
+		err = renderBootstrapConfig(w, r, ctx, true)
 	case "initrd":
 		err = renderCtx(w, r, ctx, what, buildInitrd)
-
+	case "bootstrap.tar":
+		err = renderCtx(w, r, ctx, what, buildBootstrap)
 	case "boot.iso":
 		err = renderCtx(w, r, ctx, what, buildBootISO)
-
 	case "boot.tar":
 		err = renderCtx(w, r, ctx, what, buildBootTar)
 	case "boot-efi.tar":
@@ -180,24 +199,20 @@ func renderHost(w http.ResponseWriter, r *http.Request, what string, host *local
 
 	case "boot.img":
 		err = renderCtx(w, r, ctx, what, buildBootImg)
-
 	case "boot.img.gz":
 		err = renderCtx(w, r, ctx, what, buildBootImgGZ)
-
 	case "boot.img.lz4":
 		err = renderCtx(w, r, ctx, what, buildBootImgLZ4)
-
+	case "boot.qcow2":
-	// boot v2
+		err = renderCtx(w, r, ctx, what, qemuImgBootImg("qcow2"))
-	case "bootstrap-config":
+	case "boot.qed":
-		err = renderBootstrapConfig(w, r, ctx, false)
+		err = renderCtx(w, r, ctx, what, qemuImgBootImg("qed"))
-	case "bootstrap-config.json":
+	case "boot.vdi":
-		err = renderBootstrapConfig(w, r, ctx, true)
+		err = renderCtx(w, r, ctx, what, qemuImgBootImg("vdi"))
-	case "initrd-v2":
+	case "boot.vmdk":
-		err = renderCtx(w, r, ctx, what, buildInitrdV2)
+		err = renderCtx(w, r, ctx, what, qemuImgBootImg("vmdk"))
-	case "bootstrap.tar":
+	case "boot.vpc":
-		err = renderCtx(w, r, ctx, what, buildBootstrap)
+		err = renderCtx(w, r, ctx, what, qemuImgBootImg("vpc"))
-	case "boot-v2.iso":
-		err = renderCtx(w, r, ctx, what, buildBootISOv2)
 
 	default:
 		http.NotFound(w, r)

cmd/dkl-local-server/ws-hosts-from-templates.go

@@ -0,0 +1,117 @@
+package main
+
+import (
+	"log"
+	"net/netip"
+
+	"github.com/emicklei/go-restful"
+
+	"novit.tech/direktil/pkg/localconfig"
+)
+
+var hostsFromTemplate = KVSecrets[HostFromTemplate]{"hosts-from-template"}
+
+type HostFromTemplate struct {
+	Template string
+	IP       string
+}
+
+func (hft HostFromTemplate) ClusterName(cfg *localconfig.Config) string {
+	for _, ht := range cfg.HostTemplates {
+		if ht.Name == hft.Template {
+			return ht.ClusterName
+		}
+	}
+	return ""
+}
+
+func hostOrTemplate(cfg *localconfig.Config, name string) (host *localconfig.Host) {
+	host = cfg.Host(name)
+	if host != nil {
+		log.Print("no host named ", name)
+		return
+	}
+
+	hft, found, err := hostsFromTemplate.Get(name)
+	if err != nil {
+		log.Print("failed to read store: ", err)
+		return
+	}
+
+	if !found {
+		log.Print("no host from template named ", name)
+		return
+	}
+
+	ht := cfg.HostTemplate(hft.Template)
+	if ht == nil {
+		log.Print("no host template named ", name)
+		return
+	}
+
+	host = &localconfig.Host{}
+	*host = *ht
+	host.Name = name
+	host.IPs = []string{hft.IP}
+	return
+}
+
+func wsHostsFromTemplateList(req *restful.Request, resp *restful.Response) {
+	hostsFromTemplate.WsList(resp, "")
+}
+
+func wsHostsFromTemplateSet(req *restful.Request, resp *restful.Response) {
+	name := req.PathParameter("name")
+
+	cfg, err := readConfig()
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+
+	v := HostFromTemplate{}
+	if err := req.ReadEntity(&v); err != nil {
+		wsBadRequest(resp, err.Error())
+		return
+	}
+
+	if v.Template == "" {
+		wsBadRequest(resp, "template is required")
+		return
+	}
+	if v.IP == "" {
+		wsBadRequest(resp, "ip is required")
+		return
+	}
+	if _, err := netip.ParseAddr(v.IP); err != nil {
+		wsBadRequest(resp, "bad IP: "+err.Error())
+		return
+	}
+
+	found := false
+	for _, ht := range cfg.HostTemplates {
+		if ht.Name != v.Template {
+			continue
+		}
+		found = true
+		break
+	}
+
+	if !found {
+		wsBadRequest(resp, "no host template with this name")
+		return
+	}
+
+	if err := hostsFromTemplate.Put(name, v); err != nil {
+		wsError(resp, err)
+		return
+	}
+
+	updateState()
+}
+
+func wsHostsFromTemplateDelete(req *restful.Request, resp *restful.Response) {
+	name := req.PathParameter("name")
+	hostsFromTemplate.WsDel(req, resp, name)
+	updateState()
+}

cmd/dkl-local-server/ws-public.go

@@ -1,23 +1,195 @@
 package main
 
 import (
+	"archive/tar"
+	"bytes"
+	"io"
+	"io/fs"
+	"log"
 	"net/http"
+	"os"
+	"path/filepath"
 
 	restful "github.com/emicklei/go-restful"
+	"m.cluseau.fr/go/httperr"
+	"novit.tech/direktil/local-server/secretstore"
 )
 
+type NamedPassphrase struct {
+	Name       string
+	Passphrase []byte
+}
+
 func wsUnlockStore(req *restful.Request, resp *restful.Response) {
-	var passphrase string
+	np := NamedPassphrase{}
-	err := req.ReadEntity(&passphrase)
+	err := req.ReadEntity(&np)
 	if err != nil {
 		resp.WriteError(http.StatusBadRequest, err)
 		return
 	}
 
-	if err := unlockSecretStore([]byte(passphrase)); err != nil {
+	defer secretstore.Memzero(np.Passphrase)
+
+	if secStore.IsNew() {
+		if len(np.Name) == 0 {
+			wsBadRequest(resp, "no name given")
+			return
+		}
+	}
+
+	if len(np.Passphrase) == 0 {
+		wsBadRequest(resp, "no passphrase given")
+		return
+	}
+
+	if secStore.Unlocked() {
+		if secStore.HasKey(np.Passphrase) {
+			resp.WriteEntity(adminToken)
+		} else {
+			wsError(resp, ErrUnauthorized)
+		}
+		return
+	}
+
+	if err := unlockSecretStore(np.Name, np.Passphrase); err.Any() {
 		err.WriteJSON(resp.ResponseWriter)
 		return
 	}
 
-	resp.WriteEntity(*adminToken)
+	resp.WriteEntity(adminToken)
+}
+
+func wsStoreDownload(req *restful.Request, resp *restful.Response) {
+	token := req.QueryParameter("token")
+	if token != wState.Get().Store.DownloadToken {
+		wsError(resp, ErrInvalidToken)
+		return
+	}
+
+	buf := new(bytes.Buffer)
+	arch := tar.NewWriter(buf)
+
+	root := os.DirFS(secStoreRoot())
+
+	err := fs.WalkDir(root, ".", func(path string, d fs.DirEntry, readErr error) (err error) {
+		if readErr != nil {
+			err = readErr
+			return
+		}
+
+		if path == "." {
+			return
+		}
+
+		fi, err := d.Info()
+		if err != nil {
+			return
+		}
+
+		hdr, err := tar.FileInfoHeader(fi, "")
+		if err != nil {
+			return
+		}
+
+		hdr.Name = path
+		hdr.Uid = 0
+		hdr.Gid = 0
+
+		err = arch.WriteHeader(hdr)
+		if err != nil {
+			return
+		}
+
+		if fi.IsDir() {
+			return
+		}
+
+		f, err := root.Open(path)
+		if err != nil {
+			return
+		}
+		defer f.Close()
+
+		io.Copy(arch, f)
+
+		return
+	})
+
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+
+	err = arch.Close()
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+
+	buf.WriteTo(resp)
+}
+
+func wsStoreUpload(req *restful.Request, resp *restful.Response) {
+	if !secStore.IsNew() {
+		wsError(resp, httperr.BadRequest("store is not new"))
+		return
+	}
+
+	buf := new(bytes.Buffer)
+
+	_, err := io.Copy(buf, req.Request.Body)
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+
+	arch := tar.NewReader(buf)
+
+	root := secStoreRoot()
+
+	for {
+		hdr, err := arch.Next()
+		if err == io.EOF {
+			err = nil
+			break
+		} else if err != nil {
+			wsError(resp, err)
+			return
+		}
+
+		log.Print(hdr.Name)
+
+		fullPath := filepath.Join(root, hdr.Name)
+
+		switch {
+		case hdr.FileInfo().IsDir():
+			err = os.MkdirAll(fullPath, 0700)
+			if err != nil {
+				wsError(resp, err)
+				return
+			}
+
+		default:
+			content, err := io.ReadAll(io.LimitReader(arch, hdr.Size))
+			if err != nil {
+				wsError(resp, err)
+				return
+			}
+
+			err = os.WriteFile(fullPath, content, 0600)
+			if err != nil {
+				wsError(resp, err)
+				return
+			}
+		}
+	}
+
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+
+	openSecretStore()
+
+	resp.WriteEntity(map[string]any{"ok": true})
 }

cmd/dkl-local-server/ws-store.go

@@ -0,0 +1,80 @@
+package main
+
+import (
+	"strconv"
+	"strings"
+
+	restful "github.com/emicklei/go-restful"
+	"novit.tech/direktil/local-server/secretstore"
+)
+
+func wsStoreAddKey(req *restful.Request, resp *restful.Response) {
+	np := NamedPassphrase{}
+
+	err := req.ReadEntity(&np)
+	if err != nil {
+		wsBadRequest(resp, err.Error())
+		return
+	}
+
+	np.Name = strings.TrimSpace(np.Name)
+
+	if len(np.Name) == 0 {
+		wsBadRequest(resp, "no name given")
+		return
+	}
+
+	if len(np.Passphrase) == 0 {
+		wsBadRequest(resp, "no passphrase given")
+		return
+	}
+
+	for _, k := range secStore.Keys {
+		if k.Name == np.Name {
+			wsBadRequest(resp, "there's already a passphrase named "+strconv.Quote(np.Name))
+			return
+		}
+	}
+
+	secStore.AddKey(np.Name, np.Passphrase)
+	defer updateState()
+
+	err = secStore.SaveTo(secKeysStorePath())
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+}
+
+func wsStoreDelKey(req *restful.Request, resp *restful.Response) {
+	name := ""
+
+	err := req.ReadEntity(&name)
+	if err != nil {
+		wsBadRequest(resp, err.Error())
+		return
+	}
+
+	newKeys := make([]secretstore.KeyEntry, 0, len(secStore.Keys))
+	for _, k := range secStore.Keys {
+		if k.Name == name {
+			continue
+		}
+
+		newKeys = append(newKeys, k)
+	}
+
+	if len(newKeys) == 0 {
+		wsBadRequest(resp, "can't remove the last key from the store")
+		return
+	}
+
+	secStore.Keys = newKeys
+	defer updateState()
+
+	err = secStore.SaveTo(secKeysStorePath())
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+}

cmd/dkl-local-server/ws.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"log"
 	"net"
@@ -8,7 +9,9 @@ import (
 	"strings"
 	"text/template"
 
+	cfsslconfig "github.com/cloudflare/cfssl/config"
 	"github.com/emicklei/go-restful"
+	"m.cluseau.fr/go/httperr"
 
 	"novit.tech/direktil/pkg/localconfig"
 
@@ -21,12 +24,19 @@ func registerWS(rest *restful.Container) {
 		ws := &restful.WebService{}
 		ws.
 			Path("/public").
-			Produces("application/json").
+			Produces(mime.JSON).
-			Consumes("application/json").
+			Consumes(mime.JSON).
 			Route(ws.POST("/unlock-store").To(wsUnlockStore).
-				Reads("").
+				Reads(NamedPassphrase{}).
 				Writes("").
 				Doc("Try to unlock the store")).
+			Route(ws.GET("/store.tar").To(wsStoreDownload).
+				Produces(mime.TAR).
+				Param(ws.QueryParameter("token", "the download token")).
+				Doc("Fetch the encrypted store")).
+			Route(ws.POST("/store.tar").To(wsStoreUpload).
+				Consumes(mime.TAR).
+				Doc("Upload an existing store")).
 			Route(ws.GET("/downloads/{token}/{asset}").To(wsDownload).
 				Param(ws.PathParameter("token", "the download token")).
 				Param(ws.PathParameter("asset", "the requested asset")).
@@ -36,70 +46,93 @@ func registerWS(rest *restful.Container) {
 	}
 
 	// Admin-level APIs
-	ws := &restful.WebService{}
+	ws := (&restful.WebService{}).
-	ws.
+		Filter(requireSecStore).
 		Filter(adminAuth).
-		HeaderParameter("Authorization", "Admin bearer token")
+		Param(restful.HeaderParameter("Authorization", "Admin bearer token").Required(true)).
+		Produces(mime.JSON)
+
+	// - store management
+	ws.Route(ws.POST("/store/add-key").To(wsStoreAddKey).
+		Consumes(mime.JSON).Reads(NamedPassphrase{}).
+		Doc("Add an unlock key to the store"))
+	ws.Route(ws.POST("/store/delete-key").To(wsStoreDelKey).
+		Consumes(mime.JSON).Reads("").
+		Doc("Remove an unlock key to the store (by its name)"))
 
 	// - downloads
 	ws.Route(ws.POST("/authorize-download").To(wsAuthorizeDownload).
+		Consumes(mime.JSON).Reads(DownloadSpec{}).
+		Produces(mime.JSON).
 		Doc("Create a download token for the given download"))
 
 	// - configs API
 	ws.Route(ws.POST("/configs").To(wsUploadConfig).
+		Consumes(mime.YAML).Param(ws.BodyParameter("config", "The new full configuration")).
+		Produces(mime.JSON).Writes(true).
 		Doc("Upload a new current configuration, archiving the previous one"))
 
 	// - clusters API
 	ws.Route(ws.GET("/clusters").To(wsListClusters).
 		Doc("List clusters"))
 
-	ws.Route(ws.GET("/clusters/{cluster-name}").To(wsCluster).
+	ws.Route(ws.GET("/hosts-from-template").To(wsHostsFromTemplateList).
-		Doc("Get cluster details"))
+		Doc("List host template instances"))
+	ws.Route(ws.POST("/hosts-from-template/{name}").To(wsHostsFromTemplateSet).
+		Reads(HostFromTemplate{}).
+		Doc("Create or update a host template instance"))
+	ws.Route(ws.DELETE("/hosts-from-template/{name}").To(wsHostsFromTemplateDelete).
+		Reads(HostFromTemplate{}).
+		Doc("Delete a host template instance"))
 
-	ws.Route(ws.GET("/clusters/{cluster-name}/addons").To(wsClusterAddons).
+	const (
+		GET = http.MethodGet
+		PUT = http.MethodPut
+	)
+
+	cluster := func(method, subPath string) *restful.RouteBuilder {
+		return ws.Method(method).Path("/clusters/{cluster-name}" + subPath).
+			Param(ws.PathParameter("cluster-name", "name of the cluster"))
+	}
+
+	for _, builder := range []*restful.RouteBuilder{
+		cluster(GET, "").To(wsCluster).
+			Doc("Get cluster details"),
+
+		cluster(GET, "/addons").To(wsClusterAddons).
 			Produces(mime.YAML).
 			Doc("Get cluster addons").
 			Returns(http.StatusOK, "OK", nil).
-		Returns(http.StatusNotFound, "The cluster does not exists or does not have addons defined", nil))
+			Returns(http.StatusNotFound, "The cluster does not exists or does not have addons defined", nil),
 
-	ws.Route(ws.GET("/clusters/{cluster-name}/bootstrap-pods").To(wsClusterBootstrapPods).
+		cluster(GET, "/tokens").To(wsClusterTokens).
-		Produces(mime.YAML).
+			Doc("List cluster's tokens"),
-		Doc("Get cluster bootstrap pods YAML definitions").
+		cluster(GET, "/tokens/{token-name}").To(wsClusterToken).
-		Returns(http.StatusOK, "OK", nil).
+			Doc("Get cluster's token"),
-		Returns(http.StatusNotFound, "The cluster does not exists or does not have bootstrap pods defined", nil))
 
-	ws.Route(ws.GET("/clusters/{cluster-name}/passwords").To(wsClusterPasswords).
+		cluster(GET, "/passwords").To(wsClusterPasswords).
-		Doc("List cluster's passwords"))
+			Doc("List cluster's passwords"),
-	ws.Route(ws.GET("/clusters/{cluster-name}/passwords/{password-name}").To(wsClusterPassword).
+		cluster(GET, "/passwords/{password-name}").To(wsClusterPassword).
-		Doc("Get cluster's password"))
+			Doc("Get cluster's password"),
-	ws.Route(ws.PUT("/clusters/{cluster-name}/passwords/{password-name}").To(wsClusterSetPassword).
+		cluster(PUT, "/passwords/{password-name}").To(wsClusterSetPassword).
-		Doc("Set cluster's password"))
+			Doc("Set cluster's password"),
 
-	ws.Route(ws.GET("/clusters/{cluster-name}/ca").To(wsClusterCAs).
+		cluster(GET, "/CAs").To(wsClusterCAs).
-		Doc("Get cluster CAs"))
+			Doc("Get cluster CAs"),
-	ws.Route(ws.GET("/clusters/{cluster-name}/ca/{ca-name}/certificate").To(wsClusterCACert).
+		cluster(GET, "/CAs/{ca-name}/certificate").To(wsClusterCACert).
 			Produces(mime.CACERT).
-		Doc("Get cluster CA's certificate"))
+			Doc("Get cluster CA's certificate"),
-	ws.Route(ws.GET("/clusters/{cluster-name}/ca/{ca-name}/signed").To(wsClusterSignedCert).
+		cluster(GET, "/CAs/{ca-name}/signed").To(wsClusterSignedCert).
 			Produces(mime.CERT).
 			Param(ws.QueryParameter("name", "signed reference name").Required(true)).
-		Doc("Get cluster's certificate signed by the CA"))
+			Doc("Get cluster's certificate signed by the CA"),
-
+	} {
-	ws.Route(ws.GET("/clusters/{cluster-name}/tokens/{token-name}").To(wsClusterToken).
+		ws.Route(builder)
-		Doc("Get cluster's token"))
+	}
 
 	ws.Route(ws.GET("/hosts").To(wsListHosts).
 		Doc("List hosts"))
 
-	(&wsHost{
-		prefix:  "/hosts/{host-name}",
-		hostDoc: "given host",
-		getHost: func(req *restful.Request) string {
-			return req.PathParameter("host-name")
-		},
-	}).register(ws, func(rb *restful.RouteBuilder) {
-	})
-
 	ws.Route(ws.GET("/ssh-acls").To(wsSSH_ACL_List))
 	ws.Route(ws.GET("/ssh-acls/{acl-name}").To(wsSSH_ACL_Get))
 	ws.Route(ws.PUT("/ssh-acls/{acl-name}").To(wsSSH_ACL_Set))
@@ -107,11 +140,28 @@ func registerWS(rest *restful.Container) {
 	rest.Add(ws)
 
 	// Hosts API
-	ws = &restful.WebService{}
+	ws = (&restful.WebService{}).
-	ws.Produces("application/json")
+		Filter(requireSecStore).
-	ws.Path("/me")
+		Filter(adminAuth).
-	ws.Filter(hostsAuth).
+		Path("/hosts/{host-name}").
-		HeaderParameter("Authorization", "Host or admin bearer token")
+		Param(ws.HeaderParameter("Authorization", "Host or admin bearer token"))
+
+	(&wsHost{
+		hostDoc: "given host",
+		getHost: func(req *restful.Request) (string, error) {
+			return req.PathParameter("host-name"), nil
+		},
+	}).register(ws, func(rb *restful.RouteBuilder) {
+		rb.Param(ws.PathParameter("host-name", "host's name"))
+	})
+
+	rest.Add(ws)
+
+	// Detected host API
+	ws = (&restful.WebService{}).
+		Filter(requireSecStore).
+		Path("/me").
+		Param(ws.HeaderParameter("Authorization", "Host or admin bearer token"))
 
 	(&wsHost{
 		hostDoc: "detected host",
@@ -120,10 +170,51 @@ func registerWS(rest *restful.Container) {
 		rb.Notes("In this case, the host is detected from the remote IP")
 	})
 
+	// Hosts by token API
+	ws = (&restful.WebService{}).
+		Filter(requireSecStore).
+		Path("/hosts-by-token/{host-token}").
+		Param(ws.PathParameter("host-token", "host's download token"))
+
+	(&wsHost{
+		hostDoc: "token's host",
+		getHost: func(req *restful.Request) (host string, err error) {
+			reqToken := req.PathParameter("host-token")
+
+			data, err := hostDownloadTokens.Data()
+			if err != nil {
+				return
+			}
+
+			for h, token := range data {
+				if token == reqToken {
+					host = h
+					return
+				}
+			}
+
+			return
+		},
+	}).register(ws, func(rb *restful.RouteBuilder) {
+		rb.Notes("In this case, the host is detected from the token")
+	})
+
 	rest.Add(ws)
 }
 
-func detectHost(req *restful.Request) string {
+func requireSecStore(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) {
+	if !secStore.Unlocked() {
+		wsError(resp, ErrStoreLocked)
+		return
+	}
+	chain.ProcessFilter(req, resp)
+}
+
+func detectHost(req *restful.Request) (hostName string, err error) {
+	if !*allowDetectedHost {
+		return
+	}
+
 	r := req.Request
 	remoteAddr := r.RemoteAddr
 
@@ -141,17 +232,17 @@ func detectHost(req *restful.Request) string {
 
 	cfg, err := readConfig()
 	if err != nil {
-		return ""
+		return
 	}
 
 	host := cfg.HostByIP(hostIP)
 
 	if host == nil {
 		log.Print("no host found for IP ", hostIP)
-		return ""
+		return
 	}
 
-	return host.Name
+	return host.Name, nil
 }
 
 func wsReadConfig(resp *restful.Response) *localconfig.Config {
@@ -165,19 +256,28 @@ func wsReadConfig(resp *restful.Response) *localconfig.Config {
 	return cfg
 }
 
-func wsNotFound(req *restful.Request, resp *restful.Response) {
+func wsNotFound(resp *restful.Response) {
-	http.NotFound(resp.ResponseWriter, req.Request)
+	wsError(resp, ErrNotFound)
+}
+
+func wsBadRequest(resp *restful.Response, err string) {
+	httperr.New(http.StatusBadRequest, errors.New(err)).WriteJSON(resp.ResponseWriter)
 }
 
 func wsError(resp *restful.Response, err error) {
 	log.Output(2, fmt.Sprint("request failed: ", err))
-	resp.WriteErrorString(
+
-		http.StatusInternalServerError,
+	switch err := err.(type) {
-		http.StatusText(http.StatusInternalServerError))
+	case httperr.Error:
+		err.WriteJSON(resp.ResponseWriter)
+
+	default:
+		httperr.Internal(err).WriteJSON(resp.ResponseWriter)
+	}
 }
 
-func wsRender(resp *restful.Response, tmplStr string, value interface{}) {
+func wsRender(resp *restful.Response, sslCfg *cfsslconfig.Config, tmplStr string, value interface{}) {
-	tmpl, err := template.New("wsRender").Funcs(templateFuncs).Parse(tmplStr)
+	tmpl, err := template.New("wsRender").Funcs(templateFuncs(sslCfg)).Parse(tmplStr)
 	if err != nil {
 		wsError(resp, err)
 		return

go.mod

@@ -1,43 +1,51 @@
 module novit.tech/direktil/local-server
 
-go 1.19
+go 1.21
 
 require (
-	github.com/cavaliergopher/cpio v1.0.1
 	github.com/cespare/xxhash v1.1.0
-	github.com/cloudflare/cfssl v1.6.3
+	github.com/cloudflare/cfssl v1.6.5
 	github.com/dustin/go-humanize v1.0.1
 	github.com/emicklei/go-restful v2.16.0+incompatible
 	github.com/emicklei/go-restful-openapi v1.4.1
+	github.com/go-git/go-git/v5 v5.12.0
 	github.com/mcluseau/go-swagger-ui v0.0.0-20191019002626-fd9128c24a34
 	github.com/miolini/datacounter v1.0.3
 	github.com/oklog/ulid v1.3.1
 	github.com/pierrec/lz4 v2.6.1+incompatible
-	golang.org/x/crypto v0.5.0
+	github.com/sergeymakinen/go-crypt v1.0.0
+	golang.org/x/crypto v0.22.0
 	gopkg.in/src-d/go-billy.v4 v4.3.2
 	gopkg.in/src-d/go-git.v4 v4.13.1
 	gopkg.in/yaml.v2 v2.4.0
-	k8s.io/apimachinery v0.26.1
+	k8s.io/apimachinery v0.29.3
-	m.cluseau.fr/go v0.0.0-20230206224905-5322a9bff2ec
+	m.cluseau.fr/go v0.0.0-20230809064045-12c5a121c766
-	novit.tech/direktil/pkg v0.0.0-20230201224712-5e39572dc50e
+	novit.tech/direktil/pkg v0.0.0-20240415130406-0d2e181a4ed6
 )
 
 replace github.com/zmap/zlint/v3 => github.com/zmap/zlint/v3 v3.3.1
 
 require (
-	github.com/Microsoft/go-winio v0.6.0 // indirect
+	dario.cat/mergo v1.0.0 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/ProtonMail/go-crypto v1.0.0 // indirect
+	github.com/cavaliergopher/cpio v1.0.1 // indirect
+	github.com/cloudflare/circl v1.3.7 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/frankban/quicktest v1.5.0 // indirect
-	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
-	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-git/go-billy/v5 v5.5.0 // indirect
-	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
-	github.com/go-openapi/spec v0.20.8 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
-	github.com/go-openapi/swag v0.22.3 // indirect
+	github.com/go-openapi/jsonreference v0.21.0 // indirect
-	github.com/go-sql-driver/mysql v1.7.0 // indirect
+	github.com/go-openapi/spec v0.21.0 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/gobuffalo/envy v1.10.2 // indirect
 	github.com/gobuffalo/packd v1.0.2 // indirect
 	github.com/gobuffalo/packr v1.30.1 // indirect
-	github.com/google/certificate-transparency-go v1.1.4 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/google/certificate-transparency-go v1.1.8 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jmoiron/sqlx v1.3.5 // indirect
 	github.com/joho/godotenv v1.5.1 // indirect
@@ -45,27 +53,29 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
 	github.com/kisielk/sqlstruct v0.0.0-20210630145711-dae28ed37023 // indirect
-	github.com/lib/pq v1.10.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/rogpeppe/go-internal v1.9.0 // indirect
+	github.com/pjbgf/sha1cd v0.3.0 // indirect
-	github.com/sergi/go-diff v1.3.1 // indirect
+	github.com/rogpeppe/go-internal v1.12.0 // indirect
+	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
+	github.com/skeema/knownhosts v1.2.2 // indirect
 	github.com/src-d/gcfg v1.4.0 // indirect
-	github.com/weppos/publicsuffix-go v0.20.0 // indirect
+	github.com/weppos/publicsuffix-go v0.30.2 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
-	github.com/zmap/zcrypto v0.0.0-20230205235340-d51ce4775101 // indirect
+	github.com/zmap/zcrypto v0.0.0-20231219022726-a1f61fb1661c // indirect
-	github.com/zmap/zlint/v3 v3.1.0 // indirect
+	github.com/zmap/zlint/v3 v3.5.0 // indirect
-	golang.org/x/mod v0.7.0 // indirect
+	golang.org/x/mod v0.17.0 // indirect
-	golang.org/x/net v0.5.0 // indirect
+	golang.org/x/net v0.24.0 // indirect
-	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/text v0.6.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
-	golang.org/x/tools v0.5.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
-	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
+	golang.org/x/tools v0.20.0 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/klog/v2 v2.90.0 // indirect
+	k8s.io/klog/v2 v2.120.1 // indirect
-	k8s.io/utils v0.0.0-20230202215443-34013725500c // indirect
+	k8s.io/utils v0.0.0-20240310230437-4693a0247e57 // indirect
 )

govc.env

@@ -1,8 +0,0 @@
-export GOVC_DATACENTER=
-export GOVC_PASSWORD=
-export GOVC_URL=
-export GOVC_USERNAME=
-export GOVC_INSECURE=1
-export GOVC_DATASTORE=
-export NOVIT_VM_FOLDER=
-export NOVIT_ISO_FOLDER=

hack/build

@@ -0,0 +1,3 @@
+#! /bin/sh
+set -ex
+go build -o dist/ -trimpath -ldflags "-X main.Version=${GIT_TAG:-$(git describe --always --dirty)}" $*

hack/docker-build

@@ -0,0 +1,8 @@
+#! /bin/bash
+set -ex
+case "$1" in
+commit) tag=$(git describe --always --dirty) ;;
+"")     tag=latest ;;
+*)      tag=$1 ;;
+esac
+docker build -t novit.tech/direktil/local-server:$tag .

hack/install

@@ -0,0 +1,4 @@
+#! /bin/sh
+set -ex
+go install -trimpath -ldflags "-X main.Version=$(git describe --always --dirty)" \
+    ./cmd/dkl-dir2config

html/ui/app.css

@@ -17,3 +17,7 @@
     max-height: 100pt;
     overflow: auto;
 }
+
+.cluster {
+    max-width: 50%;
+}

html/ui/index.html

@@ -22,10 +22,10 @@
             <button class="link" @click="copyText(session.token)">&#x1F5D0;</button>
         </span>
 
-        <span id="uiHash">ui <code>{{ uiHash || '-----' }}</code></span>
+        <span>server <code>{{ serverVersion || '-----' }}</code></span>
+        <span>ui     <code>{{ uiHash || '-----' }}</code></span>
 
-        <span class="green" v-if="publicState">&#x1F5F2;</span>
+        <span :class="publicState ? 'green' : 'red'">&#x1F5F2;</span>
-        <span class="red"   v-else            >&#x1F5F2;</span>
     </div>
 </header>
 
@@ -40,16 +40,23 @@
 </template>
 <template v-else-if="publicState.Store.New">
     <p>Store is new.</p>
-    <form @submit="unlockStore" action="/public/unlock-store">
+    <p>Option 1: initialize a new store</p>
-        <input type="password" v-model="forms.store.pass1" name="passphrase" />
+    <form @submit="unlockStore">
-        <input type="password" v-model="forms.store.pass2" />
+        <input type="text"     v-model="forms.store.name"  name="name" placeholder="Name" /><br/>
+        <input type="password" v-model="forms.store.pass1" name="passphrase" required placeholder="Passphrase" />
+        <input type="password" v-model="forms.store.pass2" required placeholder="Passphrase confirmation" />
         <input type="submit" value="initialize" :disabled="!forms.store.pass1 || forms.store.pass1 != forms.store.pass2" />
     </form>
+    <p>Option 2: upload a previously downloaded store</p>
+    <form @submit="uploadStore">
+        <input type="file" ref="storeUpload" />
+        <input type="submit" value="upload" />
+    </form>
 </template>
 <template v-else-if="!publicState.Store.Open">
     <p>Store is not open.</p>
-    <form @submit="unlockStore" action="/public/unlock-store">
+    <form @submit="unlockStore">
-        <input type="password" name="passphrase" v-model="forms.store.pass1" />
+        <input type="password" name="passphrase" v-model="forms.store.pass1" required placeholder="Passphrase" />
         <input type="submit" value="unlock" :disabled="!forms.store.pass1" />
     </form>
 </template>
@@ -57,9 +64,9 @@
     <p v-if="!session.token">Not logged in.</p>
     <p v-else>Invalid token</p>
 
-    <form @submit="setToken">
+    <form @submit="unlockStore">
-        <input type="password" v-model="forms.setToken" />
+        <input type="password" v-model="forms.store.pass1" required placeholder="Passphrase" />
-        <input type="submit" value="set token"/>
+        <input type="submit" value="log in"/>
     </form>
 </template>
 <template v-else>
@@ -79,7 +86,52 @@
     </div>
   </div>
 
-  <pre v-if="false">{{ state }}</pre>
+  <h2>Admin actions</h2>
+
+  <h3>Config</h3>
+  <form @submit="uploadConfig">
+      <input type="file" ref="configUpload" required />
+      <input type="submit" value="upload config" />
+  </form>
+
+  <h3>Store</h3>
+  <p><a :href="'/public/store.tar?token='+state.Store.DownloadToken" target="_blank">Download</a></p>
+  <form @submit="storeAddKey" action="/store/add-key">
+      <p>Add an unlock phrase:</p>
+      <input type="text"     v-model="forms.store.name"  name="name" required placeholder="Name" /><br/>
+      <input type="password" v-model="forms.store.pass1" name="passphrase" autocomplete="new-password" required placeholder="Phrase" />
+      <input type="password" v-model="forms.store.pass2" autocomplete="new-password" required placeholder="Phrase confirmation" />
+      <input type="submit" value="add unlock phrase" :disabled="!forms.store.pass1 || forms.store.pass1 != forms.store.pass2" />
+  </form>
+  <form @submit="storeDelKey" action="/store/delete-key">
+      <p>Remove an unlock phrase:</p>
+      <input type="text" v-model="forms.delKey.name"  name="name" required placeholder="Name" />
+      <input type="submit" value="remove unlock phrase" />
+
+      <p v-if="state.Store.KeyNames">Available names:
+        <template v-for="k,i in state.Store.KeyNames">{{i?", ":""}}<code @click="forms.delKey.name=k">{{k}}</code></template>.</p>
+  </form>
+
+<template v-if="state.HostTemplates && state.HostTemplates.length">
+  <h3>Hosts from template</h3>
+  <form @submit="hostFromTemplateAdd" action="">
+      <p>Add a host from template instance:</p>
+      <input type="text" v-model="forms.hostFromTemplate.name" required placeholder="Name" />
+      <select v-model="forms.hostFromTemplate.Template" required>
+        <option v-for="name in state.HostTemplates" :value="name">{{name}}</option>
+      </select>
+      <input type="text" v-model="forms.hostFromTemplate.IP" required placeholder="IP" />
+      <input type="submit" value="add instance" />
+  </form>
+  <form @submit="hostFromTemplateDel" action="">
+      <p>Remove a host from template instance:</p>
+      <select v-model="forms.hostFromTemplateDel" required>
+        <option v-for="h in hostsFromTemplate" :value="h.Name">{{h.Name}}</option>
+      </select>
+      <input type="submit" value="delete instance" :disabled="!forms.hostFromTemplateDel" />
+  </form>
+</template>
+
 </template>
 </div>
 

html/ui/js/Cluster.js

@@ -1,16 +1,35 @@
 
 import Downloads from './Downloads.js';
+import GetCopy from './GetCopy.js';
 
 export default {
-    components: { Downloads },
+    components: { Downloads, GetCopy },
     props: [ 'cluster', 'token', 'state' ],
     template: `
 <div class="cluster">
   <div class="title">Cluster {{ cluster.Name }}</div>
+  <div class="section">Tokens</div>
+  <section class="links">
+    <GetCopy v-for="n in cluster.Tokens" :token="token" :name="n" :href="'/clusters/'+cluster.Name+'/tokens/'+n" />
+  </section>
+  <div class="section">Passwords</div>
+  <section class="links">
+    <GetCopy v-for="n in cluster.Passwords" :token="token" :name="n" :href="'/clusters/'+cluster.Name+'/passwords/'+n" />
+  </section>
   <div class="section">Downloads</div>
   <section class="downloads">
     <Downloads :token="token" :state="state" kind="cluster" :name="cluster.Name" />
   </section>
+  <div class="section">CAs</div>
+  <table><tr><th>Name</th><th>Certificate</th><th>Signed certificates</th></tr>
+  <tr v-for="ca in cluster.CAs">
+    <td>{{ ca.Name }}</td>
+    <td><GetCopy :token="token" name="cert" :href="'/clusters/'+cluster.Name+'/CAs/'+ca.Name+'/certificate'" /></td>
+    <td><template v-for="signed in ca.Signed">
+      {{" "}}
+      <GetCopy :token="token" :name="signed" :href="'/clusters/'+cluster.Name+'/CAs/'+ca.Name+'/signed?name='+signed" />
+    </template></td>
+  </tr></table>
 </div>
 `
 }

html/ui/js/Downloads.js

@@ -10,18 +10,18 @@ export default {
                 cluster: ['addons'],
                 host: [
                     "kernel",
-                    "initrd-v2",
+                    "initrd",
                     "bootstrap.tar",
-                    "boot-v2.iso",
+                    "boot.img.lz4",
-                    "config",
                     "boot.iso",
+                    "config",
+                    "bootstrap-config",
                     "boot.tar",
                     "boot-efi.tar",
-                    "boot.img",
                     "boot.img.gz",
-                    "boot.img.lz4",
+                    "boot.img",
-                    "bootstrap-config",
+                    "boot.qcow2",
-                    "initrd",
+                    "boot.vmdk",
                     "ipxe",
                 ],
             }[this.kind]
@@ -50,7 +50,7 @@ export default {
             fetch('/authorize-download', {
                 method: 'POST',
                 body: JSON.stringify({Kind: this.kind, Name: this.name, Assets: this.assets}),
-                headers: { 'Authorization': 'Bearer ' + this.token },
+                headers: { 'Authorization': 'Bearer ' + this.token, 'Content-Type': 'application/json' },
             }).then((resp) => resp.json())
               .then((token) => { this.selectedAssets = {}; this.createDisabled = false })
               .catch((e) => { alert('failed to create link'); this.createDisabled = false })

html/ui/js/GetCopy.js

@@ -0,0 +1,32 @@
+export default {
+    props: [ 'name', 'href', 'token' ],
+    data() { return {showCopied: false} },
+    template: `<span class="notif"><div v-if="showCopied">copied!</div><a :href="href" @click="fetchAndSave()">{{name}}</a>&nbsp;<a href="#" class="copy" @click="fetchAndCopy()">&#x1F5D0;</a></span>`,
+    methods: {
+        fetch() {
+            event.preventDefault()
+            return fetch(this.href, {
+                method: 'GET',
+                headers: { 'Authorization': 'Bearer ' + this.token },
+            })
+        },
+        handleFetchError(e) {
+            console.log("failed to get value:", e)
+            alert('failed to get value')
+        },
+        fetchAndSave() {
+            this.fetch().then(resp => resp.blob()).then((value) => {
+                window.open(URL.createObjectURL(value), "_blank")
+            }).catch(this.handleFetchError)
+        },
+        fetchAndCopy() {
+            this.fetch()
+            .then((resp) => resp.headers.get("content-type") == "application/json" ? resp.json() : resp.text())
+            .then((value) => {
+                window.navigator.clipboard.writeText(value)
+                this.showCopied = true
+                setTimeout(() => { this.showCopied = false }, 1000)
+            }).catch(this.handleFetchError)
+        },
+    },
+}

html/ui/js/Host.js

@@ -8,8 +8,9 @@ export default {
 <div class="host">
   <div class="title">Host {{ host.Name }}</div>
   <section>
+    <div><small>Cluster: {{ host.Cluster }}<template v-if="host.Template"> ({{ host.Template }})</template></small></div>
     <template v-for="ip in host.IPs">
-        {{ ip }}
+      <code>{{ ip }}</code>{{" "}}
     </template>
   </section>
   <div class="section">Downloads</div>

html/ui/js/app.js

@@ -9,11 +9,16 @@ createApp({
     data() {
         return {
             forms: {
-                store: {  },
+                store: {},
+                storeUpload: {},
+                delKey: {},
+                hostFromTemplate: {},
+                hostFromTemplateDel: "",
             },
             session: {},
             error: null,
             publicState: null,
+            serverVersion: null,
             uiHash: null,
             watchingState: false,
             state: null,
@@ -39,6 +44,7 @@ createApp({
             deep: true,
             handler(v) {
                 if (v) {
+                    this.serverVersion = v.ServerVersion
                     if (this.uiHash && v.UIHash != this.uiHash) {
                         console.log("reloading")
                         location.reload()
@@ -50,6 +56,12 @@ createApp({
         }
     },
 
+    computed: {
+        hostsFromTemplate() {
+            return (this.state.Hosts||[]).filter((h) => h.Template)
+        },
+    },
+
     methods: {
         copyText(text) {
             event.preventDefault()
@@ -60,8 +72,34 @@ createApp({
             this.session.token = this.forms.setToken
             this.forms.setToken = null
         },
+        uploadStore() {
+            event.preventDefault()
+            this.apiPost('/public/store.tar', this.$refs.storeUpload.files[0], (v) => {
+                this.forms.store = {}
+            }, "application/tar")
+        },
+        namedPassphrase(name, passphrase) {
+            return {Name: this.forms.store.name, Passphrase: btoa(this.forms.store.pass1)}
+        },
+        storeAddKey() {
+            this.apiPost('/store/add-key', this.namedPassphrase(), (v) => {
+                this.forms.store = {}
+            })
+        },
+        storeDelKey() {
+            event.preventDefault()
+
+            let name = this.forms.delKey.name
+
+            if (!confirm("Remove key named "+JSON.stringify(name)+"?")) {
+                return
+            }
+            this.apiPost('/store/delete-key', name , (v) => {
+                this.forms.delKey = {}
+            })
+        },
         unlockStore() {
-            this.apiPost('/public/unlock-store', this.forms.store.pass1, (v) => {
+            this.apiPost('/public/unlock-store', this.namedPassphrase(), (v) => {
                 this.forms.store = {}
 
                 if (v) {
@@ -73,7 +111,23 @@ createApp({
                 }
             })
         },
-        apiPost(action, data, onload) {
+        uploadConfig() {
+            this.apiPost('/configs', this.$refs.configUpload.files[0], (v) => {}, "text/vnd.yaml")
+        },
+        hostFromTemplateAdd() {
+            let v = this.forms.hostFromTemplate;
+            this.apiPost('/hosts-from-template/'+v.name, v, (v) => { this.forms.hostFromTemplate = {} });
+        },
+        hostFromTemplateDel() {
+            event.preventDefault()
+
+            let v = this.forms.hostFromTemplateDel;
+            if (!confirm("delete host template instance "+v+"?")) {
+                return
+            }
+            this.apiDelete('/hosts-from-template/'+v, (v) => { this.forms.hostFromTemplateDel = "" });
+        },
+        apiPost(action, data, onload, contentType = 'application/json') {
             event.preventDefault()
 
             if (data === undefined) {
@@ -92,7 +146,7 @@ createApp({
             var xhr = new XMLHttpRequest()
 
             xhr.responseType = 'json'
-            // TODO spinner, pending aciton notification, or something
+            // TODO spinner, pending action notification, or something
             xhr.onerror = () => {
                 // this.actionResults.splice(idx, 1, {...item, done: true, failed: true })
             }
@@ -110,11 +164,37 @@ createApp({
 
             xhr.open("POST", action)
             xhr.setRequestHeader('Accept', 'application/json')
-            xhr.setRequestHeader('Content-Type', 'application/json')
+            xhr.setRequestHeader('Content-Type', contentType)
             if (this.session.token) {
                 xhr.setRequestHeader('Authorization', 'Bearer '+this.session.token)
             }
+
+            if (contentType == "application/json") {
                 xhr.send(JSON.stringify(data))
+            } else {
+                xhr.send(data)
+            }
+        },
+        apiDelete(action, data, onload) {
+            event.preventDefault()
+
+            var xhr = new XMLHttpRequest()
+            xhr.onload = (r) => {
+                if (xhr.status != 200) {
+                    this.error = xhr.response
+                    return
+                }
+                this.error = null
+                if (onload) {
+                    onload(xhr.response)
+                }
+            }
+            xhr.open("DELETE", action)
+            xhr.setRequestHeader('Accept', 'application/json')
+            if (this.session.token) {
+                xhr.setRequestHeader('Authorization', 'Bearer '+this.session.token)
+            }
+            xhr.send()
         },
         download(url) {
             event.target.target = '_blank'

html/ui/style.css

@@ -45,7 +45,7 @@ th, tr:last-child > td {
     background: #333;
     color: #eee;
   }
-  a[href], button.link {
+  a[href], a[href]:visited, button.link {
     border: none;
     color: #31b0fa;
   }
@@ -124,4 +124,29 @@ header .utils > * {
 .sheets section {
     margin: 2pt 6pt 6pt 6pt;
 }
+.sheets > *:last-child > table:last-child > tr:last-child > td {
+    border-bottom: none;
+}
 
+.notif {
+    display: inline-block;
+    position: relative;
+}
+.notif > div:first-child {
+    position: absolute;
+    min-width: 100%; height: 100%;
+    background: white;
+    opacity: 75%;
+    text-align: center;
+}
+
+.links > * { margin-left: 1ex; }
+.links > *:first-child { margin-left: 0; }
+
+@media (prefers-color-scheme: dark) {
+  .notif > div:first-child {
+    background: black;
+  }
+}
+
+.copy { font-size: small; }

modd.conf

@@ -3,20 +3,24 @@ modd.conf {}
 **/*.go go.mod go.sum {
   prep: go test ./...
   prep: mkdir -p dist
-  prep: go build -o dist/ -trimpath ./...
+  prep: hack/build ./...
   #prep: docker build --build-arg GOPROXY=$GOPROXY -t dls .
-  #daemon +sigterm: /var/lib/direktil/test-run
+  #daemon +sigterm: bash test-run
 }
 
 html/**/* {
-  prep: go build -o dist/ -trimpath ./cmd/dkl-local-server
+  prep: hack/build ./cmd/dkl-local-server
 }
 
-dist/dkl-local-server {
+#dist/dkl-local-server {
-  prep: mkdir -p tmp
+#  prep: mkdir -p tmp
-  daemon +sigterm: dist/dkl-local-server -data tmp -auto-unlock test
+#  daemon +sigterm: dist/dkl-local-server -data tmp -auto-unlock test
+#}
+
+dist/dkl-dir2config {
+#  prep: dist/dkl-dir2config --debug --in test-dir2config
 }
 
-**/*.proto !dist/**/* {
+#**/*.proto !dist/**/* {
-  prep: for mod in @mods; do protoc -I ./ --go_out=plugins=grpc,paths=source_relative:. $mod; done
+#  prep: for mod in @mods; do protoc -I ./ --go_out=plugins=grpc,paths=source_relative:. $mod; done
-}
+#}

pkg/clustersconfig/clustersconfig.go

@@ -4,7 +4,6 @@ import (
 	"flag"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"log"
 	"net"
 	"os"
@@ -25,11 +24,9 @@ var (
 
 type Config struct {
 	Hosts        []*Host
-	Groups        []*Group
 	Clusters     []*Cluster
 	Configs      []*Template
-	StaticPods    []*Template            `yaml:"static_pods"`
+	StaticPods   map[string][]*Template `yaml:"static_pods"`
-	BootstrapPods map[string][]*Template `yaml:"bootstrap_pods"`
 	Addons       map[string][]*Template
 	SSLConfig    string         `yaml:"ssl_config"`
 	CertRequests []*CertRequest `yaml:"cert_requests"`
@@ -44,7 +41,7 @@ func FromBytes(data []byte) (*Config, error) {
 }
 
 func FromFile(path string) (*Config, error) {
-	ba, err := ioutil.ReadFile(path)
+	ba, err := os.ReadFile(path)
 	if err != nil {
 		return nil, err
 	}
@@ -89,15 +86,6 @@ func (c *Config) HostByMAC(mac string) *Host {
 	return nil
 }
 
-func (c *Config) Group(name string) *Group {
-	for _, group := range c.Groups {
-		if group.Name == name {
-			return group
-		}
-	}
-	return nil
-}
-
 func (c *Config) Cluster(name string) *Cluster {
 	for _, cluster := range c.Clusters {
 		if cluster.Name == name {
@@ -116,15 +104,6 @@ func (c *Config) ConfigTemplate(name string) *Template {
 	return nil
 }
 
-func (c *Config) StaticPodsTemplate(name string) *Template {
-	for _, s := range c.StaticPods {
-		if s.Name == name {
-			return s
-		}
-	}
-	return nil
-}
-
 func (c *Config) CSR(name string) *CertRequest {
 	for _, s := range c.CertRequests {
 		if s.Name == name {
@@ -140,23 +119,25 @@ func (c *Config) SaveTo(path string) error {
 		return err
 	}
 
-	return ioutil.WriteFile(path, ba, 0600)
+	return os.WriteFile(path, ba, 0600)
 }
 
 type Template struct {
 	Name     string
 	Template string
-
-	parsedTemplate *template.Template
 }
 
 func (t *Template) Execute(contextName, elementName string, wr io.Writer, data interface{}, extraFuncs map[string]interface{}) error {
-	if t.parsedTemplate == nil {
 	var templateFuncs = map[string]interface{}{
 		"indent": func(indent, s string) (indented string) {
 			indented = indent + strings.Replace(s, "\n", "\n"+indent, -1)
 			return
 		},
+		"yaml": func(v any) (s string, err error) {
+			ba, err := yaml.Marshal(v)
+			s = string(ba)
+			return
+		},
 	}
 
 	for name, f := range extraFuncs {
@@ -169,8 +150,6 @@ func (t *Template) Execute(contextName, elementName string, wr io.Writer, data i
 	if err != nil {
 		return err
 	}
-		t.parsedTemplate = tmpl
-	}
 
 	if *templateDetailsDir != "" {
 		templateID++
@@ -181,7 +160,7 @@ func (t *Template) Execute(contextName, elementName string, wr io.Writer, data i
 		base += string(filepath.Separator)
 		log.Print("writing template details: ", base, "{in,data,out}")
 
-		if err := ioutil.WriteFile(base+"in", []byte(t.Template), 0600); err != nil {
+		if err := os.WriteFile(base+"in", []byte(t.Template), 0600); err != nil {
 			return err
 		}
 
@@ -190,7 +169,7 @@ func (t *Template) Execute(contextName, elementName string, wr io.Writer, data i
 			return err
 		}
 
-		if err := ioutil.WriteFile(base+"data", yamlBytes, 0600); err != nil {
+		if err := os.WriteFile(base+"data", yamlBytes, 0600); err != nil {
 			return err
 		}
 
@@ -204,41 +183,37 @@ func (t *Template) Execute(contextName, elementName string, wr io.Writer, data i
 		wr = io.MultiWriter(wr, out)
 	}
 
-	return t.parsedTemplate.Execute(wr, data)
+	return tmpl.Execute(wr, data)
 }
 
 // Host represents a host served by this server.
 type Host struct {
 	WithRev
 
-	Name        string
+	Template bool `json:",omitempty"`
-	Labels      map[string]string
-	Annotations map[string]string
 
-	MAC     string
+	Name        string
+	Labels      map[string]string `json:",omitempty"`
+	Annotations map[string]string `json:",omitempty"`
+
+	MAC     string `json:",omitempty"`
 	IP      string
-	IPs     []string
+	IPs     []string `json:",omitempty"`
 	Cluster string
 	Group   string
-	Vars    Vars
-}
 
-// Group represents a group of hosts and provides their configuration.
+	Net    string
-type Group struct {
+	IPFrom map[string]string `json:",omitempty" yaml:"ip_from"`
-	WithRev
 
-	Name        string
+	IPXE            string `json:",omitempty"`
-	Labels      map[string]string
-	Annotations map[string]string
-
-	Master          bool
-	IPXE            string
 	Kernel          string
 	Initrd          string
 	BootstrapConfig string `yaml:"bootstrap_config"`
 	Config          string
-	StaticPods      string `yaml:"static_pods"`
 	Versions        map[string]string
+
+	StaticPods string `yaml:"static_pods"`
+
 	Vars Vars
 }
 
@@ -254,12 +229,12 @@ type Cluster struct {
 	Annotations map[string]string
 
 	Domain  string
-	Addons        string
+	Addons  []string
-	BootstrapPods string `yaml:"bootstrap_pods"`
 	Subnets struct {
 		Services string
 		Pods     string
 	}
+
 	Vars Vars
 }
 
@@ -274,7 +249,7 @@ func (c *Cluster) DNSSvcIP() net.IP {
 func (c *Cluster) NthSvcIP(n byte) net.IP {
 	_, cidr, err := net.ParseCIDR(c.Subnets.Services)
 	if err != nil {
-		panic(fmt.Errorf("Invalid services CIDR: %v", err))
+		panic(fmt.Errorf("invalid services CIDR: %v", err))
 	}
 
 	ip := cidr.IP

pkg/clustersconfig/dir.go

@@ -3,7 +3,6 @@ package clustersconfig
 import (
 	"fmt"
 	"io/ioutil"
-	"log"
 	"os"
 	"path"
 	"path/filepath"
@@ -15,40 +14,38 @@ import (
 // Debug enables debug logs from this package.
 var Debug = false
 
-func FromDir(dirPath, defaultsPath string) (*Config, error) {
+func FromDir(
-	if Debug {
+	read func(path string) ([]byte, error),
-		log.Printf("loading config from dir %s (defaults from %s)", dirPath, defaultsPath)
+	assemble func(path string) ([]byte, error),
-	}
+	listBase func(path string) ([]string, error),
+	listMerged func(path string) ([]string, error),
+) (*Config, error) {
 
-	defaults, err := NewDefaults(defaultsPath)
+	load := func(dir, name string, out any) (err error) {
+		ba, err := assemble(filepath.Join(dir, name))
 		if err != nil {
-		return nil, fmt.Errorf("failed to load defaults: %v", err)
+			return
 		}
-
+		err = yaml.UnmarshalStrict(ba, out)
-	store := &dirStore{dirPath}
-	load := func(dir, name string, out Rev) error {
-		ba, err := store.Get(path.Join(dir, name))
 		if err != nil {
-			return fmt.Errorf("failed to load %s/%s from dir: %v", dir, name, err)
+			return
-		}
-		if err = defaults.Load(dir, ".yaml", out, ba); err != nil {
-			return fmt.Errorf("failed to enrich %s/%s from defaults: %v", dir, name, err)
 		}
 		return nil
 	}
 
 	config := &Config{
 		Addons:     make(map[string][]*Template),
-		BootstrapPods: make(map[string][]*Template),
+		StaticPods: make(map[string][]*Template),
 	}
 
 	// load clusters
-	names, err := store.List("clusters")
+	names, err := listBase("clusters")
 	if err != nil {
 		return nil, fmt.Errorf("failed to list clusters: %v", err)
 	}
 
 	for _, name := range names {
+		name, _ = strings.CutSuffix(name, ".yaml")
 		cluster := &Cluster{Name: name}
 		if err := load("clusters", name, cluster); err != nil {
 			return nil, err
@@ -57,103 +54,14 @@ func FromDir(dirPath, defaultsPath string) (*Config, error) {
 		config.Clusters = append(config.Clusters, cluster)
 	}
 
-	// load groups
-	names, err = store.List("groups")
-	if err != nil {
-		return nil, fmt.Errorf("failed to list groups: %v", err)
-	}
-
-	read := func(rev, filePath string) (data []byte, fromDefaults bool, err error) {
-		data, err = store.Get(filePath)
-		if err != nil {
-			err = fmt.Errorf("faild to read %s: %v", filePath, err)
-			return
-		}
-
-		if data != nil {
-			return // ok
-		}
-
-		if len(rev) == 0 {
-			err = fmt.Errorf("entry not found: %s", filePath)
-			return
-		}
-
-		data, err = defaults.ReadAll(rev, filePath+".yaml")
-		if err != nil {
-			err = fmt.Errorf("failed to read %s:%s: %v", rev, filePath, err)
-			return
-		}
-
-		fromDefaults = true
-		return
-	}
-
-	template := func(rev, dir, name string, templates *[]*Template) (ref string, err error) {
-		ref = name
-		if len(name) == 0 {
-			return
-		}
-
-		ba, fromDefaults, err := read(rev, path.Join(dir, name))
-		if err != nil {
-			return
-		}
-
-		if fromDefaults {
-			ref = rev + ":" + name
-		}
-
-		if !hasTemplate(ref, *templates) {
-			if Debug {
-				log.Printf("new template in %s: %s", dir, ref)
-			}
-
-			*templates = append(*templates, &Template{
-				Name:     ref,
-				Template: string(ba),
-			})
-		}
-
-		return
-	}
-
-	for _, name := range names {
-		group := &Group{Name: name}
-		if err := load("groups", name, group); err != nil {
-			return nil, err
-		}
-
-		group.BootstrapConfig, err = template(group.Rev(), "configs", group.BootstrapConfig, &config.Configs)
-		if err != nil {
-			return nil, fmt.Errorf("failed to load config for group %q: %v", name, err)
-		}
-
-		group.Config, err = template(group.Rev(), "configs", group.Config, &config.Configs)
-		if err != nil {
-			return nil, fmt.Errorf("failed to load config for group %q: %v", name, err)
-		}
-
-		if Debug {
-			log.Printf("group %q: config=%q static_pods=%q",
-				group.Name, group.Config, group.StaticPods)
-		}
-
-		group.StaticPods, err = template(group.Rev(), "static-pods", group.StaticPods, &config.StaticPods)
-		if err != nil {
-			return nil, fmt.Errorf("failed to load static pods for group %q: %v", name, err)
-		}
-
-		config.Groups = append(config.Groups, group)
-	}
-
 	// load hosts
-	names, err = store.List("hosts")
+	names, err = listBase("hosts")
 	if err != nil {
 		return nil, fmt.Errorf("failed to list hosts: %v", err)
 	}
 
 	for _, name := range names {
+		name, _ = strings.CutSuffix(name, ".yaml")
 		o := &Host{Name: name}
 		if err := load("hosts", name, o); err != nil {
 			return nil, err
@@ -163,28 +71,20 @@ func FromDir(dirPath, defaultsPath string) (*Config, error) {
 	}
 
 	// load config templates
-	loadTemplates := func(rev, dir string, templates *[]*Template) error {
+	loadTemplates := func(dir string, templates *[]*Template) error {
-		names, err := store.List(dir)
+		names, err := listMerged(dir)
 		if err != nil {
 			return fmt.Errorf("failed to list %s: %v", dir, err)
 		}
 
-		if len(rev) != 0 {
+		for _, fullName := range names {
-			var defaultsNames []string
+			name, _ := strings.CutSuffix(fullName, ".yaml")
-			defaultsNames, err = defaults.List(rev, dir)
-			if err != nil {
-				return fmt.Errorf("failed to list %s:%s: %v", rev, dir, err)
-			}
 
-			names = append(names, defaultsNames...)
-		}
-
-		for _, name := range names {
 			if hasTemplate(name, *templates) {
 				continue
 			}
 
-			ba, _, err := read(rev, path.Join(dir, name))
+			ba, err := read(path.Join(dir, fullName))
 			if err != nil {
 				return err
 			}
@@ -198,53 +98,57 @@ func FromDir(dirPath, defaultsPath string) (*Config, error) {
 		return nil
 	}
 
+	loadTemplates("configs", &config.Configs)
+
 	// cluster addons
 	for _, cluster := range config.Clusters {
-		addonSet := cluster.Addons
+		addonSets := cluster.Addons
-		if len(addonSet) == 0 {
+		if len(addonSets) == 0 {
 			continue
 		}
 
+		for _, addonSet := range addonSets {
 			if _, ok := config.Addons[addonSet]; ok {
 				continue
 			}
 
 			templates := make([]*Template, 0)
-		if err = loadTemplates(cluster.Rev(), path.Join("addons", addonSet), &templates); err != nil {
+			if err = loadTemplates(path.Join("addons", addonSet), &templates); err != nil {
 				return nil, err
 			}
 
 			config.Addons[addonSet] = templates
 		}
+	}
 
-	// cluster bootstrap pods
+	// cluster static pods
-	for _, cluster := range config.Clusters {
+	for _, host := range config.Hosts {
-		bpSet := cluster.BootstrapPods
+		bpSet := host.StaticPods
 		if bpSet == "" {
 			continue
 		}
 
-		if _, ok := config.BootstrapPods[bpSet]; ok {
+		if _, ok := config.StaticPods[bpSet]; ok {
 			continue
 		}
 
 		templates := make([]*Template, 0)
-		if err = loadTemplates(cluster.Rev(), path.Join("bootstrap-pods", bpSet), &templates); err != nil {
+		if err = loadTemplates(path.Join("static-pods", bpSet), &templates); err != nil {
 			return nil, err
 		}
 
-		config.BootstrapPods[bpSet] = templates
+		config.StaticPods[bpSet] = templates
 	}
 
 	// load SSL configuration
-	if ba, err := ioutil.ReadFile(filepath.Join(dirPath, "ssl-config.json")); err == nil {
+	if ba, err := read("ssl-config.json"); err == nil {
 		config.SSLConfig = string(ba)
 
 	} else if !os.IsNotExist(err) {
 		return nil, err
 	}
 
-	if ba, err := ioutil.ReadFile(filepath.Join(dirPath, "cert-requests.yaml")); err == nil {
+	if ba, err := read("cert-requests.yaml"); err == nil {
 		reqs := make([]*CertRequest, 0)
 		if err = yaml.Unmarshal(ba, &reqs); err != nil {
 			return nil, err

pkg/mime/mime.go

@@ -1,6 +1,7 @@
 package mime
 
 const (
+	JSON   = "application/json"
 	YAML   = "text/vnd.yaml"
 	TAR    = "application/tar"
 	DISK   = "application/x-diskimage"

secretstore/mem.go

@@ -1,5 +1,7 @@
 package secretstore
 
+func Memzero(ba []byte) { memzero(ba) }
+
 func memzero(ba []byte) {
 	for i := range ba {
 		ba[i] = 0

secretstore/secret-store.go

@@ -2,29 +2,34 @@ package secretstore
 
 import (
 	"bufio"
+	"bytes"
 	"crypto/aes"
 	"crypto/cipher"
 	"crypto/sha512"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"log"
 	"os"
+	"strconv"
 	"syscall"
 
 	"golang.org/x/crypto/argon2"
 )
 
 type Store struct {
+	Salt [aes.BlockSize]byte
+	Keys []KeyEntry
+
 	unlocked bool
 	key      [32]byte
-	salt     [aes.BlockSize]byte
-	keys     []keyEntry
 }
 
-type keyEntry struct {
+type KeyEntry struct {
-	hash   [64]byte
+	Name   string
-	encKey [32]byte
+	Hash   [64]byte
+	EncKey [32]byte
 }
 
 func New() (s *Store) {
@@ -77,30 +82,32 @@ func (s *Store) Close() {
 }
 
 func (s *Store) IsNew() bool {
-	return len(s.keys) == 0
+	return len(s.Keys) == 0
 }
 
 func (s *Store) Unlocked() bool {
 	return s.unlocked
 }
 
-func (s *Store) Init(passphrase []byte) (err error) {
+func (s *Store) Init(name string, passphrase []byte) (err error) {
 	err = randRead(s.key[:])
 	if err != nil {
 		return
 	}
-	err = randRead(s.salt[:])
+	err = randRead(s.Salt[:])
 	if err != nil {
 		return
 	}
 
-	s.AddKey(passphrase)
+	s.AddKey(name, passphrase)
 
 	s.unlocked = true
 
 	return
 }
 
+var jsonFormatHdr = []byte("{json}")
+
 func (s *Store) ReadFrom(in io.Reader) (n int64, err error) {
 	memzero(s.key[:])
 	s.unlocked = false
@@ -117,69 +124,77 @@ func (s *Store) ReadFrom(in io.Reader) (n int64, err error) {
 		n += int64(nr)
 	}
 
-	// read the salt
+	// read the file's start (json header or start of salt)
-	readFull(s.salt[:])
+
+	readFull(s.Salt[:len(jsonFormatHdr)])
+	if err != nil {
+		return
+	}
+
+	if !bytes.Equal(s.Salt[:len(jsonFormatHdr)], jsonFormatHdr) {
+		// old key file
+
+		// finish reading the salt
+		readFull(s.Salt[len(jsonFormatHdr):])
 		if err != nil {
 			return
 		}
 
 		// read the (encrypted) keys
-	s.keys = make([]keyEntry, 0)
+		s.Keys = make([]KeyEntry, 0)
 		for {
-		k := keyEntry{}
+			k := KeyEntry{Name: "key-" + strconv.Itoa(len(s.Keys))}
-		readFull(k.hash[:])
+			readFull(k.Hash[:])
 			if err != nil {
 				if err == io.EOF {
 					err = nil
 				}
 				return
 			}
-		readFull(k.encKey[:])
+			readFull(k.EncKey[:])
 			if err != nil {
 				return
 			}
 
-		s.keys = append(s.keys, k)
+			s.Keys = append(s.Keys, k)
 		}
+	}
+
+	err = json.NewDecoder(in).Decode(s)
+	return
 }
 
 func (s *Store) WriteTo(out io.Writer) (n int64, err error) {
-	write := func(ba []byte) {
+	_, err = out.Write(jsonFormatHdr)
-		var nr int
-		nr, err = out.Write(ba)
-		n += int64(nr)
-	}
-
-	write(s.salt[:])
 	if err != nil {
 		return
 	}
-
+	err = json.NewEncoder(out).Encode(s)
-	for _, k := range s.keys {
-		write(k.hash[:])
-		if err != nil {
-			return
-		}
-
-		write(k.encKey[:])
-		if err != nil {
-			return
-		}
-	}
-
 	return
 }
 
 var ErrNoSuchKey = errors.New("no such key")
 
+func (s *Store) HasKey(passphrase []byte) bool {
+	key, hash := s.keyPairFromPassword(passphrase)
+	defer memzero(key[:])
+
+	for _, k := range s.Keys {
+		if k.Hash == hash {
+			return true
+		}
+	}
+
+	return false
+}
+
 func (s *Store) Unlock(passphrase []byte) (ok bool) {
 	key, hash := s.keyPairFromPassword(passphrase)
-	memzero(passphrase)
 	defer memzero(key[:])
 
 	var idx = -1
-	for i := range s.keys {
+	for i := range s.Keys {
-		if hash == s.keys[i].hash {
+		if hash == s.Keys[i].Hash {
 			idx = i
 			break
 		}
@@ -189,28 +204,28 @@ func (s *Store) Unlock(passphrase []byte) (ok bool) {
 		return
 	}
 
-	s.decryptTo(s.key[:], s.keys[idx].encKey[:], &key)
+	s.decryptTo(s.key[:], s.Keys[idx].EncKey[:], &key)
 
 	s.unlocked = true
 	return true
 }
 
-func (s *Store) AddKey(passphrase []byte) {
+func (s *Store) AddKey(name string, passphrase []byte) {
 	key, hash := s.keyPairFromPassword(passphrase)
 	memzero(passphrase)
 
 	defer memzero(key[:])
 
-	k := keyEntry{hash: hash}
+	k := KeyEntry{Name: name, Hash: hash}
 
 	encKey := s.encrypt(s.key[:], &key)
-	copy(k.encKey[:], encKey)
+	copy(k.EncKey[:], encKey)
 
-	s.keys = append(s.keys, k)
+	s.Keys = append(s.Keys, k)
 }
 
 func (s *Store) keyPairFromPassword(password []byte) (key [32]byte, hash [64]byte) {
-	keySlice := argon2.IDKey(password, s.salt[:], 1, 64*1024, 4, 32)
+	keySlice := argon2.IDKey(password, s.Salt[:], 1, 64*1024, 4, 32)
 
 	copy(key[:], keySlice)
 	memzero(keySlice)
@@ -236,12 +251,12 @@ func (s *Store) NewDecrypter(iv [aes.BlockSize]byte) cipher.Stream {
 
 func (s *Store) encrypt(src []byte, key *[32]byte) (dst []byte) {
 	dst = make([]byte, len(src))
-	newEncrypter(s.salt, key).XORKeyStream(dst, src)
+	newEncrypter(s.Salt, key).XORKeyStream(dst, src)
 	return
 }
 
 func (s *Store) decryptTo(dst []byte, src []byte, key *[32]byte) {
-	newDecrypter(s.salt, key).XORKeyStream(dst, src)
+	newDecrypter(s.Salt, key).XORKeyStream(dst, src)
 }
 
 func newEncrypter(iv [aes.BlockSize]byte, key *[32]byte) cipher.Stream {

upload-vmware.sh

@@ -1,27 +0,0 @@
-set -e
-
-dir=/var/lib/direktil/
-PATH=$PATH:$dir
-cd $dir
-
-if [ ! -f govc.env ]; then
-    echo ERROR: govc.env file not found in dir $dir ; exit 1
-fi
-source govc.env
-
-if [ $# != 2 ]; then
-    echo "Usage: $0 <VM_NAME> <NOVIT_HOST>" ; exit 1
-fi
-
-if [[ -z $NOVIT_VM_FOLDER || -z $NOVIT_ISO_FOLDER ]]; then
-    echo "ERROR: All GOVC env vars (including NOVIT_VM_FOLDER and NOVIT_ISO_FOLDER) must be provided" ; exit 1
-fi
-
-VM=$1
-HOST=$2
-
-govc vm.power -off "$NOVIT_VM_FOLDER/$VM" || true
-sleep 5
-curl localhost:7606/hosts/$HOST/boot.iso | govc datastore.upload - "$NOVIT_ISO_FOLDER/$VM.iso"
-sleep 5
-govc vm.power -on "$NOVIT_VM_FOLDER/$VM"