use GIT_TAG to override Version

This allows the user to call it even after the store has been unlock in
order to get the admin token.

.dockerignore

@@ -0,0 +1,2 @@
+tmp
+dist

.gitignore

@@ -1,2 +1,8 @@
 *.sw[po]
 modd-local.conf
+/tmp
+/test-dir2config
+/config.yaml
+/dist
+/go.work
+/go.work.sum

Dockerfile

@@ -1,23 +1,33 @@
 # ------------------------------------------------------------------------
-from golang:1.11.5 as build
+from golang:1.21.5-bullseye as build
 
-env pkg novit.nc/direktil/local-server
-copy vendor /go/src/${pkg}/vendor
-copy pkg    /go/src/${pkg}/pkg
-copy cmd    /go/src/${pkg}/cmd
-workdir /go/src/${pkg}
-run go test ./... \
- && go install ./cmd/...
+run apt-get update && apt-get install -y git
+
+workdir /src
+
+copy go.mod go.sum ./
+run \
+  --mount=type=cache,id=gomod,target=/go/pkg/mod \
+  --mount=type=cache,id=gobuild,target=/root/.cache/go-build \
+  go mod download
+
+arg GIT_TAG
+
+copy . ./
+run \
+  --mount=type=cache,id=gomod,target=/go/pkg/mod \
+  --mount=type=cache,id=gobuild,target=/root/.cache/go-build \
+ go test ./... && \
+ hack/build ./...
 
 # ------------------------------------------------------------------------
-from debian:stretch
+from debian:bullseye
 entrypoint ["/bin/dkl-local-server"]
 
+env _uncache 1
 run apt-get update \
- && apt-get install -y genisoimage gdisk dosfstools util-linux udev \
+ && yes |apt-get install -y genisoimage gdisk dosfstools util-linux udev binutils systemd \
+ grub2 grub-pc-bin grub-efi-amd64-bin ca-certificates curl openssh-client \
  && apt-get clean
 
-run yes |apt-get install -y grub2 grub-pc-bin grub-efi-amd64-bin \
- && apt-get clean
-
-copy --from=build /go/bin/ /bin/
+copy --from=build /src/dist/ /bin/

cmd/dkl-dir2config/assemble.go

@@ -0,0 +1,138 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"strings"
+
+	"gopkg.in/yaml.v2"
+)
+
+func mergeIn(tgt, add map[any]any) {
+mergeLoop:
+	for k, v := range add {
+		switch v := v.(type) {
+		case map[any]any:
+			if tgtV, ok := tgt[k]; ok {
+				switch tgtV := tgtV.(type) {
+				case map[any]any:
+					mergeIn(tgtV, v)
+					continue mergeLoop
+				}
+			}
+		}
+
+		tgt[k] = v
+	}
+}
+
+func assemble(path string) (yamlBytes []byte, err error) {
+	obj := map[any]any{}
+
+	if Debug {
+		log.Printf("assemble %q", path)
+	}
+
+	err = eachFragment(path, searchList, func(r io.Reader) (err error) {
+		m := map[any]any{}
+		err = yaml.NewDecoder(r).Decode(&m)
+		if err != nil {
+			return
+		}
+
+		mergeIn(obj, m)
+		return
+	})
+	if err != nil {
+		err = fmt.Errorf("failed to assemble %q: %w", path, err)
+		return
+	}
+
+	yamlBytes, err = yaml.Marshal(obj)
+	if err != nil {
+		return
+	}
+
+	if Debug {
+		log.Printf("assemble %q result:\n%s", path, yamlBytes)
+	}
+
+	return
+}
+
+func eachFragment(path string, searchList []FS, walk func(io.Reader) error) (err error) {
+	var r io.ReadCloser
+
+	for len(searchList) != 0 {
+		fs := searchList[0]
+
+		r, err = fs.Open(path + ".yaml")
+		if os.IsNotExist(err) {
+			searchList = searchList[1:]
+			continue
+		}
+
+		if err != nil {
+			return
+		}
+
+		// found and open
+		break
+	}
+
+	if r == nil {
+		err = fmt.Errorf("%s: %w", path, os.ErrNotExist)
+		return
+	}
+
+	ba, err := io.ReadAll(r)
+	r.Close()
+
+	if err != nil {
+		return
+	}
+
+	if Debug {
+		log.Print("fragment:\n", string(ba))
+	}
+
+	in := bytes.NewBuffer(ba)
+
+	for {
+		var line string
+		line, err = in.ReadString('\n')
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return
+		}
+
+		line = strings.TrimSpace(line)
+		if len(line) == 0 {
+			continue
+		}
+
+		includePath, found := strings.CutPrefix(line, "#!include ")
+		if !found {
+			continue // or break?
+		}
+
+		includePath = strings.TrimSpace(includePath)
+		if Debug {
+			log.Print("#!include ", includePath)
+		}
+		err = eachFragment(includePath, searchList, walk)
+		if err != nil {
+			err = fmt.Errorf("include %q: %w", includePath, err)
+			return
+		}
+	}
+
+	in = bytes.NewBuffer(ba)
+	err = walk(in)
+	return
+}

cmd/dkl-dir2config/fs.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"io"
+	iofs "io/fs"
+)
+
+type FS interface {
+	Open(path string) (io.ReadCloser, error)
+	List(path string) ([]string, error)
+}
+
+type fsFS struct{ iofs.FS }
+
+func (fs fsFS) Open(path string) (io.ReadCloser, error) {
+	return fs.FS.Open(path)
+}
+
+func (fs fsFS) List(path string) (entries []string, err error) {
+	dirEnts, err := iofs.ReadDir(fs.FS, path)
+	if err != nil {
+		return
+	}
+
+	entries = make([]string, 0, len(dirEnts))
+	for _, ent := range dirEnts {
+		if ent.IsDir() {
+			continue
+		}
+		entries = append(entries, ent.Name())
+	}
+
+	return
+}

cmd/dkl-dir2config/git.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"io"
+	"sort"
+
+	"github.com/go-git/go-git/v5/plumbing/object"
+)
+
+type gitFS struct{ *object.Tree }
+
+func (fs gitFS) Open(path string) (r io.ReadCloser, err error) {
+	f, err := fs.Tree.File(path)
+	if err != nil {
+		return
+	}
+	return f.Reader()
+}
+
+func (fs gitFS) List(path string) (entries []string, err error) {
+	tree, err := fs.Tree.Tree(path)
+	if err != nil {
+		return
+	}
+
+	entries = make([]string, 0, len(tree.Entries))
+
+	for _, ent := range tree.Entries {
+		if !ent.Mode.IsFile() {
+			continue
+		}
+
+		entries = append(entries, ent.Name)
+	}
+
+	sort.Strings(entries)
+	return
+}

cmd/dkl-dir2config/main.go

@@ -1,30 +1,42 @@
 package main
 
 import (
-	"bytes"
 	"flag"
-	"fmt"
+	"io/fs"
 	"log"
 	"os"
+	"path/filepath"
 
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
 	yaml "gopkg.in/yaml.v2"
-	"novit.nc/direktil/pkg/localconfig"
 
-	"novit.nc/direktil/local-server/pkg/clustersconfig"
+	"novit.tech/direktil/pkg/localconfig"
+
+	"novit.tech/direktil/local-server/pkg/clustersconfig"
 )
 
+var Version = "dev"
+
 var (
+	Debug = false
+
 	dir     = flag.String("in", ".", "Source directory")
 	outPath = flag.String("out", "config.yaml", "Output file")
-	defaultsPath = flag.String("defaults", "defaults", "Path to the defaults")
+
+	base fs.FS
 
 	src *clustersconfig.Config
 	dst *localconfig.Config
 )
 
+func init() {
+	flag.BoolVar(&Debug, "debug", Debug, "debug")
+}
+
 func loadSrc() {
 	var err error
-	src, err = clustersconfig.FromDir(*dir, *defaultsPath)
+	src, err = clustersconfig.FromDir(read, assemble, listBase, listMerged)
 	if err != nil {
 		log.Fatal("failed to load config from dir: ", err)
 	}
@@ -35,6 +47,16 @@ func main() {
 
 	log.SetFlags(log.Ltime | log.Lmicroseconds | log.Lshortfile)
 
+	base = os.DirFS(*dir)
+	searchList = append(searchList, fsFS{base})
+
+	openIncludes()
+
+	if false {
+		assemble("hosts/m1")
+		log.Fatal("--- debug: end ---")
+	}
+
 	loadSrc()
 
 	dst = &localconfig.Config{
@@ -51,8 +73,6 @@ func main() {
 
 	// ----------------------------------------------------------------------
 	for _, host := range src.Hosts {
-		loadSrc() // FIXME ugly fix of some template caching or something
-
 		log.Print("rendering host ", host.Name)
 		ctx, err := newRenderContext(host, src)
 
@@ -71,22 +91,29 @@ func main() {
 		}
 		ips = append(ips, host.IPs...)
 
-		if ctx.Group.Versions["modules"] == "" {
+		if ctx.Host.Versions["modules"] == "" {
 			// default modules' version to kernel's version
-			ctx.Group.Versions["modules"] = ctx.Group.Kernel
+			ctx.Host.Versions["modules"] = ctx.Host.Kernel
 		}
 
 		dst.Hosts = append(dst.Hosts, &localconfig.Host{
 			Name: host.Name,
+
+			ClusterName: ctx.Cluster.Name,
+
+			Labels:      ctx.Labels,
+			Annotations: ctx.Annotations,
+
 			MACs: macs,
 			IPs:  ips,
 
-			IPXE: ctx.Group.IPXE, // TODO render
+			IPXE: ctx.Host.IPXE, // TODO render
 
-			Kernel:   ctx.Group.Kernel,
-			Initrd:   ctx.Group.Initrd,
-			Versions: ctx.Group.Versions,
+			Kernel:   ctx.Host.Kernel,
+			Initrd:   ctx.Host.Initrd,
+			Versions: ctx.Host.Versions,
 
+			BootstrapConfig: ctx.BootstrapConfig(),
 			Config:          ctx.Config(),
 		})
 	}
@@ -99,39 +126,83 @@ func main() {
 
 	defer out.Close()
 
+	out.Write([]byte("# dkl-dir2config " + Version + "\n"))
+
 	if err = yaml.NewEncoder(out).Encode(dst); err != nil {
 		log.Fatal("failed to render output: ", err)
 	}
 
 }
 
-func renderAddons(cluster *clustersconfig.Cluster) string {
-	if len(cluster.Addons) == 0 {
-		return ""
+func cfgPath(subPath string) string { return filepath.Join(*dir, subPath) }
+
+func openIncludes() {
+	includesFile, err := base.Open("includes.yaml")
+
+	if os.IsNotExist(err) {
+		return
 	}
-
-	addons := src.Addons[cluster.Addons]
-	if addons == nil {
-		log.Fatalf("cluster %q: no addons with name %q", cluster.Name, cluster.Addons)
-	}
-
-	clusterAsMap := asMap(cluster)
-	clusterAsMap["kubernetes_svc_ip"] = cluster.KubernetesSvcIP().String()
-	clusterAsMap["dns_svc_ip"] = cluster.DNSSvcIP().String()
-
-	buf := &bytes.Buffer{}
-
-	for _, addon := range addons {
-		fmt.Fprintf(buf, "---\n# addon: %s\n", addon.Name)
-		err := addon.Execute(buf, clusterAsMap, nil)
-
 	if err != nil {
-			log.Fatalf("cluster %q: addons %q: failed to render %q: %v",
-				cluster.Name, cluster.Addons, addon.Name, err)
+		log.Fatal("failed to open includes: ", err)
 	}
 
-		fmt.Fprintln(buf)
+	includes := make([]struct {
+		Path   string
+		Branch string
+		Tag    string
+	}, 0)
+
+	err = yaml.NewDecoder(includesFile).Decode(&includes)
+	if err != nil {
+		log.Fatal("failed to parse includes: ", err)
 	}
 
-	return buf.String()
+	for _, include := range includes {
+		switch {
+		case include.Branch != "" || include.Tag != "":
+			p := cfgPath(include.Path) // FIXME parse git path to allow remote repos
+
+			var rev plumbing.Revision
+
+			switch {
+			case include.Branch != "":
+				log.Printf("opening include path %q as git, branch %q", p, include.Branch)
+				rev = plumbing.Revision(plumbing.NewBranchReferenceName(include.Branch))
+
+			case include.Tag != "":
+				log.Printf("opening include path %q as git, tag %q", p, include.Branch)
+				rev = plumbing.Revision(plumbing.NewTagReferenceName(include.Branch))
+			}
+
+			repo, err := git.PlainOpen(p)
+			if err != nil {
+				log.Fatal("failed to open: ", err)
+			}
+
+			revH, err := repo.ResolveRevision(rev)
+			if err != nil {
+				log.Fatalf("failed to resolve revision %s: %v", rev, err)
+			}
+
+			log.Print("  -> resolved to commit ", *revH)
+
+			commit, err := repo.CommitObject(*revH)
+			if err != nil {
+				log.Fatal("failed to get commit object: ", err)
+			}
+
+			tree, err := commit.Tree()
+			if err != nil {
+				log.Fatal("failed to open git tree: ", err)
+			}
+
+			searchList = append(searchList, gitFS{tree})
+
+		default:
+			p := cfgPath(include.Path)
+			log.Printf("opening include path %q as raw dir", p)
+
+			searchList = append(searchList, fsFS{os.DirFS(p)})
+		}
+	}
 }

cmd/dkl-dir2config/render-cluster.go

@@ -0,0 +1,122 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"path"
+
+	"novit.tech/direktil/local-server/pkg/clustersconfig"
+)
+
+func clusterFuncs(clusterSpec *clustersconfig.Cluster) map[string]interface{} {
+	cluster := clusterSpec.Name
+
+	return map[string]interface{}{
+		"password": func(name, hash string) (s string) {
+			return fmt.Sprintf("{{ password %q %q %q | quote }}", cluster, name, hash)
+		},
+
+		"token": func(name string) (s string) {
+			return fmt.Sprintf("{{ token %q %q }}", cluster, name)
+		},
+
+		"ca_key": func(name string) (s string, err error) {
+			// TODO check CA exists
+			// ?ctx.clusterConfig.CA(name)
+			return fmt.Sprintf("{{ ca_key %q %q }}", cluster, name), nil
+		},
+
+		"ca_crt": func(name string) (s string, err error) {
+			// TODO check CA exists
+			return fmt.Sprintf("{{ ca_crt %q %q }}", cluster, name), nil
+		},
+
+		"ca_dir": func(name string) (s string, err error) {
+			return fmt.Sprintf("{{ ca_dir %q %q }}", cluster, name), nil
+		},
+
+		"hosts_by_cluster": func(cluster string) (hosts []interface{}) {
+			for _, host := range src.Hosts {
+				if host.Cluster == cluster {
+					hosts = append(hosts, asMap(host))
+				}
+			}
+
+			if len(hosts) == 0 {
+				log.Printf("WARNING: no hosts in cluster %q", cluster)
+			}
+
+			return
+		},
+
+		"hosts_by_group": func(group string) (hosts []interface{}) {
+			for _, host := range src.Hosts {
+				if host.Cluster == cluster && host.Group == group {
+					hosts = append(hosts, asMap(host))
+				}
+			}
+
+			if len(hosts) == 0 {
+				log.Printf("WARNING: no hosts in group %q", group)
+			}
+
+			return
+		},
+	}
+}
+
+func renderClusterTemplates(cluster *clustersconfig.Cluster, setName string,
+	templates []*clustersconfig.Template) []byte {
+	clusterAsMap := asMap(cluster)
+	clusterAsMap["kubernetes_svc_ip"] = cluster.KubernetesSvcIP().String()
+	clusterAsMap["dns_svc_ip"] = cluster.DNSSvcIP().String()
+
+	funcs := clusterFuncs(cluster)
+
+	log.Print("rendering cluster templates in ", setName)
+
+	buf := &bytes.Buffer{}
+
+	contextName := "cluster:" + cluster.Name
+
+	for _, t := range templates {
+		log.Print("- template: ", setName, ": ", t.Name)
+		fmt.Fprintf(buf, "---\n# %s: %s\n", setName, t.Name)
+		err := t.Execute(contextName, path.Join(setName, t.Name), buf, clusterAsMap, funcs)
+
+		if err != nil {
+			log.Fatalf("cluster %q: %s: failed to render %q: %v",
+				cluster.Name, setName, t.Name, err)
+		}
+
+		fmt.Fprintln(buf)
+	}
+
+	return buf.Bytes()
+}
+
+func renderAddons(cluster *clustersconfig.Cluster) string {
+	if len(cluster.Addons) == 0 {
+		return ""
+	}
+
+	buf := new(bytes.Buffer)
+
+	for _, addonSet := range cluster.Addons {
+		addons := src.Addons[addonSet]
+		if addons == nil {
+			log.Fatalf("cluster %q: no addons with name %q", cluster.Name, addonSet)
+		}
+
+		buf.Write(renderClusterTemplates(cluster, "addons", addons))
+	}
+
+	return buf.String()
+}
+
+type namePod struct {
+	Namespace string
+	Name      string
+	Pod       map[string]interface{}
+}

cmd/dkl-dir2config/render-context.go

@@ -2,19 +2,31 @@ package main
 
 import (
 	"bytes"
+	"crypto/sha1"
+	"encoding/hex"
 	"fmt"
+	"io"
 	"log"
+	"path"
+	"reflect"
+	"strings"
 
 	yaml "gopkg.in/yaml.v2"
 
-	"novit.nc/direktil/local-server/pkg/clustersconfig"
+	"novit.tech/direktil/pkg/config"
+
+	"novit.tech/direktil/local-server/pkg/clustersconfig"
 )
 
 type renderContext struct {
+	Labels      map[string]string
+	Annotations map[string]string
+
 	Host    *clustersconfig.Host
-	Group              *clustersconfig.Group
 	Cluster *clustersconfig.Cluster
-	Vars               map[string]interface{}
+	Vars    map[string]any
+
+	BootstrapConfigTemplate *clustersconfig.Template
 	ConfigTemplate          *clustersconfig.Template
 	StaticPodsTemplate      *clustersconfig.Template
 
@@ -28,95 +40,156 @@ func newRenderContext(host *clustersconfig.Host, cfg *clustersconfig.Config) (ct
 		return
 	}
 
-	group := cfg.Group(host.Group)
-	if group == nil {
-		err = fmt.Errorf("no group named %q", host.Group)
-		return
-	}
+	vars := make(map[string]any)
 
-	vars := make(map[string]interface{})
-
-	for _, oVars := range []map[string]interface{}{
+	for _, oVars := range []map[string]any{
 		cluster.Vars,
-		group.Vars,
 		host.Vars,
 	} {
-		for k, v := range oVars {
-			vars[k] = v
-		}
+		mapMerge(vars, oVars)
 	}
 
 	return &renderContext{
+		Labels:      mergeLabels(cluster.Labels, host.Labels),
+		Annotations: mergeLabels(cluster.Annotations, host.Annotations),
+
 		Host:    host,
-		Group:              group,
 		Cluster: cluster,
 		Vars:    vars,
-		ConfigTemplate:     cfg.ConfigTemplate(group.Config),
-		StaticPodsTemplate: cfg.StaticPodsTemplate(group.StaticPods),
+
+		BootstrapConfigTemplate: cfg.ConfigTemplate(host.BootstrapConfig),
+		ConfigTemplate:          cfg.ConfigTemplate(host.Config),
 
 		clusterConfig: cfg,
 	}, nil
 }
 
+func mergeLabels(sources ...map[string]string) map[string]string {
+	ret := map[string]string{}
+
+	for _, src := range sources {
+		for k, v := range src {
+			ret[k] = v
+		}
+	}
+
+	return ret
+}
+
+func mapMerge(target, source map[string]interface{}) {
+	for k, v := range source {
+		target[k] = genericMerge(target[k], v)
+	}
+}
+
+func genericMerge(target, source interface{}) (result interface{}) {
+	srcV := reflect.ValueOf(source)
+	tgtV := reflect.ValueOf(target)
+
+	if srcV.Kind() == reflect.Map && tgtV.Kind() == reflect.Map {
+		// XXX maybe more specific later
+		result = map[interface{}]interface{}{}
+		resultV := reflect.ValueOf(result)
+
+		tgtIt := tgtV.MapRange()
+		for tgtIt.Next() {
+			sv := srcV.MapIndex(tgtIt.Key())
+			if sv.Kind() == 0 {
+				resultV.SetMapIndex(tgtIt.Key(), tgtIt.Value())
+				continue
+			}
+
+			merged := genericMerge(tgtIt.Value().Interface(), sv.Interface())
+			resultV.SetMapIndex(tgtIt.Key(), reflect.ValueOf(merged))
+		}
+
+		srcIt := srcV.MapRange()
+		for srcIt.Next() {
+			if resultV.MapIndex(srcIt.Key()).Kind() != 0 {
+				continue // already done
+			}
+
+			resultV.SetMapIndex(srcIt.Key(), srcIt.Value())
+		}
+
+		return
+	}
+
+	return source
+}
+
+func (ctx *renderContext) Name() string {
+	switch {
+	case ctx.Host != nil:
+		return "host:" + ctx.Host.Name
+	case ctx.Cluster != nil:
+		return "cluster:" + ctx.Cluster.Name
+	default:
+		return "unknown"
+	}
+}
+
+func (ctx *renderContext) BootstrapConfig() string {
+	if ctx.BootstrapConfigTemplate == nil {
+		log.Fatalf("no such (bootstrap) config: %q", ctx.Host.BootstrapConfig)
+	}
+	return ctx.renderConfig(ctx.BootstrapConfigTemplate)
+}
+
 func (ctx *renderContext) Config() string {
 	if ctx.ConfigTemplate == nil {
-		log.Fatalf("no such config: %q", ctx.Group.Config)
-	}
-
-	ctxMap := ctx.asMap()
-
-	templateFuncs := ctx.templateFuncs(ctxMap)
-
-	render := func(what string, t *clustersconfig.Template) (s string, err error) {
-		buf := &bytes.Buffer{}
-		err = t.Execute(buf, ctxMap, templateFuncs)
-		if err != nil {
-			log.Printf("host %s: failed to render %s [%q]: %v", ctx.Host.Name, what, t.Name, err)
-			return
-		}
-
-		s = buf.String()
-		return
-	}
-
-	extraFuncs := ctx.templateFuncs(ctxMap)
-
-	extraFuncs["static_pods"] = func() (string, error) {
-		name := ctx.Group.StaticPods
-		if len(name) == 0 {
-			return "", fmt.Errorf("group %q has no static pods defined", ctx.Group.Name)
-		}
-
-		t := ctx.clusterConfig.StaticPodsTemplate(name)
-		if t == nil {
-			return "", fmt.Errorf("no static pods template named %q", name)
-		}
-
-		return render("static pods", t)
-	}
-
-	buf := bytes.NewBuffer(make([]byte, 0, 4096))
-	if err := ctx.ConfigTemplate.Execute(buf, ctxMap, extraFuncs); err != nil {
-		log.Fatalf("failed to render config %q for host %q: %v", ctx.Group.Config, ctx.Host.Name, err)
+		log.Fatalf("no such config: %q", ctx.Host.Config)
+	}
+	return ctx.renderConfig(ctx.ConfigTemplate)
 }
 
+func (ctx *renderContext) renderConfig(configTemplate *clustersconfig.Template) string {
+	buf := new(strings.Builder)
+	ctx.renderConfigTo(buf, configTemplate)
 	return buf.String()
 }
 
-func (ctx *renderContext) StaticPods() (ba []byte, err error) {
-	if ctx.StaticPodsTemplate == nil {
-		log.Fatalf("no such static-pods: %q", ctx.Group.StaticPods)
-	}
+func (ctx *renderContext) renderConfigTo(buf io.Writer, configTemplate *clustersconfig.Template) {
+	ctxName := ctx.Name()
 
 	ctxMap := ctx.asMap()
 
-	buf := bytes.NewBuffer(make([]byte, 0, 4096))
-	if err = ctx.StaticPodsTemplate.Execute(buf, ctxMap, ctx.templateFuncs(ctxMap)); err != nil {
-		return
+	extraFuncs := ctx.templateFuncs(ctxMap)
+
+	extraFuncs["static_pods_files"] = func(dir string) (string, error) {
+		namePods := ctx.renderStaticPods()
+
+		defs := make([]config.FileDef, 0)
+
+		for _, namePod := range namePods {
+			name := namePod.Namespace + "_" + namePod.Name
+
+			ba, err := yaml.Marshal(namePod.Pod)
+			if err != nil {
+				return "", fmt.Errorf("static pod %s: failed to render: %v", name, err)
 			}
 
-	ba = buf.Bytes()
-	return
+			defs = append(defs, config.FileDef{
+				Path:    path.Join(dir, name+".yaml"),
+				Mode:    0640,
+				Content: string(ba),
+			})
+		}
+
+		ba, err := yaml.Marshal(defs)
+		return string(ba), err
+	}
+
+	extraFuncs["machine_id"] = func() string {
+		ba := sha1.Sum([]byte(ctx.Cluster.Name + "/" + ctx.Host.Name)) // TODO: check semantics of machine-id
+		return hex.EncodeToString(ba[:])
+	}
+
+	extraFuncs["version"] = func() string { return Version }
+
+	if err := configTemplate.Execute(ctxName, "config", buf, ctxMap, extraFuncs); err != nil {
+		log.Fatalf("failed to render config %q for host %q: %v", ctx.Host.Config, ctx.Host.Name, err)
+	}
 }
 
 func (ctx *renderContext) templateFuncs(ctxMap map[string]interface{}) map[string]interface{} {
@@ -135,7 +208,7 @@ func (ctx *renderContext) templateFuncs(ctxMap map[string]interface{}) map[strin
 		}
 
 		buf := &bytes.Buffer{}
-		err = req.Execute(buf, ctxMap, nil)
+		err = req.Execute(ctx.Name(), "req:"+name, buf, ctxMap, nil)
 		if err != nil {
 			return
 		}
@@ -159,26 +232,26 @@ func (ctx *renderContext) templateFuncs(ctxMap map[string]interface{}) map[strin
 		return
 	}
 
-	return map[string]interface{}{
-		"token": func(name string) (s string) {
-			return fmt.Sprintf("{{ token %q %q }}", cluster, name)
-		},
+	funcs := clusterFuncs(ctx.Cluster)
+	for k, v := range map[string]interface{}{
+		"default": func(value, defaultValue any) any {
+			switch v := value.(type) {
+			case string:
+				if v != "" {
+					return v
+				}
+			case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, float32, float64:
+				if v != 0 {
+					return v
+				}
 
-		"ca_key": func(name string) (s string, err error) {
-			// TODO check CA exists
-			// ?ctx.clusterConfig.CA(name)
-			return fmt.Sprintf("{{ ca_key %q %q }}", cluster, name), nil
+			default:
+				if v != nil {
+					return v
+				}
+			}
+			return defaultValue
 		},
-
-		"ca_crt": func(name string) (s string, err error) {
-			// TODO check CA exists
-			return fmt.Sprintf("{{ ca_crt %q %q }}", cluster, name), nil
-		},
-
-		"ca_dir": func(name string) (s string, err error) {
-			return fmt.Sprintf("{{ ca_dir %q %q }}", cluster, name), nil
-		},
-
 		"tls_key": func(name string) (string, error) {
 			return getKeyCert(name, "tls_key")
 		},
@@ -191,6 +264,14 @@ func (ctx *renderContext) templateFuncs(ctxMap map[string]interface{}) map[strin
 			return getKeyCert(name, "tls_dir")
 		},
 
+		"ssh_host_keys": func(dir string) (s string) {
+			return fmt.Sprintf("{{ ssh_host_keys %q %q \"\"}}",
+				dir, cluster)
+		},
+		"host_download_token": func() (s string) {
+			return "{{ host_download_token }}"
+		},
+
 		"hosts_of_group": func() (hosts []interface{}) {
 			hosts = make([]interface{}, 0)
 
@@ -213,7 +294,10 @@ func (ctx *renderContext) templateFuncs(ctxMap map[string]interface{}) map[strin
 			}
 			return
 		},
+	} {
+		funcs[k] = v
 	}
+	return funcs
 }
 
 func (ctx *renderContext) asMap() map[string]interface{} {

cmd/dkl-dir2config/render-context_test.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"testing"
+
+	yaml "gopkg.in/yaml.v2"
+)
+
+func TestMerge(t *testing.T) {
+	if v := genericMerge("a", "b"); v != "b" {
+		t.Errorf("got %q", v)
+	}
+
+    if v := unparse(genericMerge(parse(`
+a: t
+b: t
+m:
+  a1: t
+  b1: t
+`), parse(`
+a: s
+c: s
+m:
+  a1: s
+  c1: s
+`))); "\n"+v != `
+a: s
+b: t
+c: s
+m:
+  a1: s
+  b1: t
+  c1: s
+` {
+    t.Errorf("got\n%s", v)
+}
+}
+
+func parse(s string) (r interface{}) {
+	r = map[string]interface{}{}
+	err := yaml.Unmarshal([]byte(s), r)
+	if err != nil {
+		panic(err)
+	}
+	return
+}
+
+func unparse(v interface{}) (s string) {
+	ba, err := yaml.Marshal(v)
+	if err != nil {
+		panic(err)
+	}
+	return string(ba)
+}

cmd/dkl-dir2config/render-host.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"log"
+
+	yaml "gopkg.in/yaml.v2"
+	"novit.tech/direktil/local-server/pkg/clustersconfig"
+)
+
+func (ctx *renderContext) renderStaticPods() (pods []namePod) {
+	if ctx.Host.StaticPods == "" {
+		return
+	}
+
+	staticPods, ok := src.StaticPods[ctx.Host.StaticPods]
+	if !ok {
+		log.Fatalf("no static pods template named %q", ctx.Host.StaticPods)
+	}
+
+	// render static pods
+	parts := bytes.Split(ctx.renderHostTemplates("static-pods", staticPods), []byte("\n---\n"))
+	for _, part := range parts {
+		buf := bytes.NewBuffer(part)
+		dec := yaml.NewDecoder(buf)
+
+		for n := 0; ; n++ {
+			str := buf.String()
+
+			podMap := map[string]interface{}{}
+			err := dec.Decode(podMap)
+
+			if err == io.EOF {
+				break
+			} else if err != nil {
+				log.Fatalf("static pod %d: failed to parse: %v\n%s", n, err, str)
+			}
+
+			if len(podMap) == 0 {
+				continue
+			}
+
+			if podMap["metadata"] == nil {
+				log.Fatalf("static pod %d: no metadata\n%s", n, buf.String())
+			}
+
+			md := podMap["metadata"].(map[interface{}]interface{})
+
+			namespace := md["namespace"].(string)
+			name := md["name"].(string)
+
+			pods = append(pods, namePod{namespace, name, podMap})
+		}
+	}
+
+	return
+}
+
+func (ctx *renderContext) renderHostTemplates(setName string,
+	templates []*clustersconfig.Template) []byte {
+
+	log.Print("rendering host templates in ", setName)
+
+	buf := &bytes.Buffer{}
+
+	for _, t := range templates {
+		log.Print("- template: ", setName, ": ", t.Name)
+		fmt.Fprintf(buf, "---\n# %s: %s\n", setName, t.Name)
+
+		ctx.renderConfigTo(buf, t)
+		fmt.Fprintln(buf)
+	}
+
+	return buf.Bytes()
+}

cmd/dkl-dir2config/search-list.go

@@ -0,0 +1,60 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"sort"
+)
+
+var searchList = make([]FS, 0)
+
+// read the first file matching path in the search list
+func read(path string) (ba []byte, err error) {
+	for _, fs := range searchList {
+		var r io.ReadCloser
+		r, err = fs.Open(path)
+		if os.IsNotExist(err) {
+			continue
+		}
+		if err != nil {
+			return
+		}
+
+		defer r.Close()
+		return ioutil.ReadAll(r)
+	}
+
+	err = fmt.Errorf("%s: %w", path, os.ErrNotExist)
+	return
+}
+
+func listBase(path string) ([]string, error) {
+	return fsFS{base}.List(path)
+}
+
+func listMerged(path string) (entries []string, err error) {
+	seen := map[string]bool{}
+	for _, fs := range searchList {
+		var fsEnts []string
+		fsEnts, err = fs.List(path)
+		if os.IsNotExist(err) {
+			err = nil
+			continue
+		}
+		if err != nil {
+			return
+		}
+
+		for _, ent := range fsEnts {
+			if !seen[ent] {
+				entries = append(entries, ent)
+				seen[ent] = true
+			}
+		}
+	}
+
+	sort.Strings(entries)
+	return
+}

cmd/dkl-local-server/auth.go

@@ -1,22 +1,14 @@
 package main
 
 import (
-	"flag"
 	"log"
 	"net/http"
 )
 
-var (
-	hostsToken = flag.String("hosts-token", "", "Token to give to access /hosts (open is none)")
-	adminToken = flag.String("admin-token", "", "Token to give to access to admin actions (open is none)")
-)
-
-func authorizeHosts(r *http.Request) bool {
-	return authorizeToken(r, *hostsToken)
-}
+var adminToken string
 
 func authorizeAdmin(r *http.Request) bool {
-	return authorizeToken(r, *adminToken)
+	return authorizeToken(r, adminToken)
 }
 
 func authorizeToken(r *http.Request, token string) bool {
@@ -26,11 +18,28 @@ func authorizeToken(r *http.Request, token string) bool {
 	}
 
 	reqToken := r.Header.Get("Authorization")
-
+	if reqToken != "" {
 		return reqToken == "Bearer "+token
 	}
 
+	return r.URL.Query().Get("token") == token
+}
+
 func forbidden(w http.ResponseWriter, r *http.Request) {
-	log.Printf("denied access to %s from %s", r.RequestURI, r.RemoteAddr)
+	log.Printf("denied access to %s from %s", r.URL.Path, r.RemoteAddr)
 	http.Error(w, "Forbidden", http.StatusForbidden)
 }
+
+func requireToken(token string, handler http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		if !authorizeToken(req, token) {
+			forbidden(w, req)
+			return
+		}
+		handler.ServeHTTP(w, req)
+	})
+}
+
+func requireAdmin(handler http.Handler) http.Handler {
+	return requireToken(adminToken, handler)
+}

cmd/dkl-local-server/boot-img.go

@@ -3,6 +3,7 @@ package main
 import (
 	"archive/tar"
 	"compress/gzip"
+	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -56,8 +57,10 @@ func buildBootImgGZ(out io.Writer, ctx *renderContext) (err error) {
 	return
 }
 
+var grubSupportVersion = flag.String("grub-support", "1.0.1", "GRUB support version")
+
 func setupBootImage(bootImg *os.File, ctx *renderContext) (err error) {
-	path, err := ctx.distFetch("grub-support", "1.0.0")
+	path, err := ctx.distFetch("grub-support", *grubSupportVersion)
 	if err != nil {
 		return
 	}
@@ -81,7 +84,10 @@ func setupBootImage(bootImg *os.File, ctx *renderContext) (err error) {
 		return
 	}
 
-	devb, err := exec.Command("losetup", "--find", "--show", "--partscan", bootImg.Name()).CombinedOutput()
+	log.Print("running losetup...")
+	cmd := exec.Command("losetup", "--find", "--show", "--partscan", bootImg.Name())
+	cmd.Stderr = os.Stderr
+	devb, err := cmd.Output()
 	if err != nil {
 		return
 	}

cmd/dkl-local-server/boot-iso.go

@@ -8,28 +8,25 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strconv"
+
+	"github.com/cespare/xxhash"
 )
 
-func buildBootISO(out io.Writer, ctx *renderContext) error {
-	tempDir, err := ioutil.TempDir("/tmp", "iso-")
+func buildBootISO(out io.Writer, ctx *renderContext) (err error) {
+	tempDir, err := ioutil.TempDir("/tmp", "iso-v2-")
 	if err != nil {
-		return err
+		return
 	}
 
 	defer os.RemoveAll(tempDir)
 
-	cp := func(src, dst string) error {
-		log.Printf("iso: adding %s as %s", src, dst)
-		in, err := os.Open(src)
-		if err != nil {
-			return err
-		}
-
-		defer in.Close()
+	buildRes := func(build func(out io.Writer, ctx *renderContext) error, dst string) (err error) {
+		log.Printf("iso-v2: building %s", dst)
 
 		outPath := filepath.Join(tempDir, dst)
 
-		if err := os.MkdirAll(filepath.Dir(outPath), 0755); err != nil {
+		if err = os.MkdirAll(filepath.Dir(outPath), 0755); err != nil {
 			return err
 		}
 
@@ -37,32 +34,55 @@ func buildBootISO(out io.Writer, ctx *renderContext) error {
 		if err != nil {
 			return err
 		}
-
 		defer out.Close()
 
-		_, err = io.Copy(out, in)
+		err = build(out, ctx)
+		if err != nil {
+			return
+		}
+
 		return err
 	}
 
-	err = func() error {
+	err = func() (err error) {
 		// grub
 
-		if err := os.MkdirAll(filepath.Join(tempDir, "grub"), 0755); err != nil {
-			return err
+		if err = os.MkdirAll(filepath.Join(tempDir, "grub"), 0755); err != nil {
+			return
 		}
+
+		// create a tag file
+		bootstrapBytes, _, err := ctx.BootstrapConfig()
+		if err != nil {
+			return
+		}
+
+		h := xxhash.New()
+		fmt.Fprintln(h, ctx.Host.Kernel)
+		h.Write(bootstrapBytes)
+
+		tag := "dkl-" + strconv.FormatUint(h.Sum64(), 32) + ".tag"
+
+		f, err := os.Create(filepath.Join(tempDir, tag))
+		if err != nil {
+			return
+		}
+		f.Write([]byte("direktil marker file\n"))
+		f.Close()
+
 		err = ioutil.WriteFile(filepath.Join(tempDir, "grub", "grub.cfg"), []byte(`
-search --set=root --file /config.yaml
+search --set=root --file /`+tag+`
 
 insmod all_video
 set timeout=3
 
 menuentry "Direktil" {
-    linux  /vmlinuz direktil.boot=DEVNAME=sr0 direktil.boot.fs=iso9660
+    linux  /vmlinuz `+ctx.CmdLine+`
     initrd /initrd
 }
 `), 0644)
 		if err != nil {
-			return err
+			return
 		}
 
 		coreImgPath := filepath.Join(tempDir, "grub", "core.img")
@@ -117,50 +137,9 @@ menuentry "Direktil" {
 		return err
 	}
 
-	// config
-	cfgBytes, cfg, err := ctx.Config()
-	if err != nil {
-		return err
-	}
-
-	ioutil.WriteFile(filepath.Join(tempDir, "config.yaml"), cfgBytes, 0600)
-
 	// kernel and initrd
-	type distCopy struct {
-		Src []string
-		Dst string
-	}
-
-	copies := []distCopy{
-		{Src: []string{"kernels", ctx.Host.Kernel}, Dst: "vmlinuz"},
-		{Src: []string{"initrd", ctx.Host.Initrd}, Dst: "initrd"},
-	}
-
-	// layers
-	for _, layer := range cfg.Layers {
-		layerVersion := ctx.Host.Versions[layer]
-		if layerVersion == "" {
-			return fmt.Errorf("layer %q not mapped to a version", layer)
-		}
-
-		copies = append(copies,
-			distCopy{
-				Src: []string{"layers", layer, layerVersion},
-				Dst: filepath.Join("current", "layers", layer+".fs"),
-			})
-	}
-
-	for _, copy := range copies {
-		outPath, err := ctx.distFetch(copy.Src...)
-		if err != nil {
-			return err
-		}
-
-		err = cp(outPath, copy.Dst)
-		if err != nil {
-			return err
-		}
-	}
+	buildRes(fetchKernel, "vmlinuz")
+	buildRes(buildInitrd, "initrd")
 
 	// build the ISO
 	mkisofs, err := exec.LookPath("genisoimage")

cmd/dkl-local-server/boot-tar.go

@@ -2,11 +2,13 @@ package main
 
 import (
 	"archive/tar"
-	"fmt"
+	"bytes"
 	"io"
+	"io/ioutil"
 	"log"
 	"os"
-	"path/filepath"
+
+	"novit.tech/direktil/local-server/pkg/utf16"
 )
 
 func rmTempFile(f *os.File) {
@@ -21,7 +23,7 @@ func buildBootTar(out io.Writer, ctx *renderContext) (err error) {
 	defer arch.Close()
 
 	archAdd := func(path string, ba []byte) (err error) {
-		err = arch.WriteHeader(&tar.Header{Name: path, Size: int64(len(ba))})
+		err = arch.WriteHeader(&tar.Header{Name: path, Mode: 0640, Size: int64(len(ba))})
 		if err != nil {
 			return
 		}
@@ -29,70 +31,95 @@ func buildBootTar(out io.Writer, ctx *renderContext) (err error) {
 		return
 	}
 
-	// config
-	cfgBytes, cfg, err := ctx.Config()
+	// kernel
+	kernelPath, err := ctx.distFetch("kernels", ctx.Host.Kernel)
 	if err != nil {
-		return err
+		return
 	}
 
-	archAdd("config.yaml", cfgBytes)
-
-	// add "current" elements
-	type distCopy struct {
-		Src []string
-		Dst string
-	}
-
-	// kernel and initrd
-	copies := []distCopy{
-		{Src: []string{"kernels", ctx.Host.Kernel}, Dst: "current/vmlinuz"},
-		{Src: []string{"initrd", ctx.Host.Initrd}, Dst: "current/initrd"},
-	}
-
-	// layers
-	for _, layer := range cfg.Layers {
-		layerVersion := ctx.Host.Versions[layer]
-		if layerVersion == "" {
-			return fmt.Errorf("layer %q not mapped to a version", layer)
-		}
-
-		copies = append(copies,
-			distCopy{
-				Src: []string{"layers", layer, layerVersion},
-				Dst: filepath.Join("current", "layers", layer+".fs"),
-			})
-	}
-
-	for _, copy := range copies {
-		outPath, err := ctx.distFetch(copy.Src...)
+	kernelBytes, err := ioutil.ReadFile(kernelPath)
 	if err != nil {
-			return err
+		return
 	}
 
-		f, err := os.Open(outPath)
+	err = archAdd("current/vmlinuz", kernelBytes)
 	if err != nil {
-			return err
+		return
 	}
 
-		defer f.Close()
-
-		stat, err := f.Stat()
+	// initrd
+	initrd := new(bytes.Buffer)
+	err = buildInitrd(initrd, ctx)
 	if err != nil {
-			return err
+		return
 	}
 
-		if err = arch.WriteHeader(&tar.Header{
-			Name: copy.Dst,
-			Size: stat.Size(),
-		}); err != nil {
-			return err
-		}
-
-		_, err = io.Copy(arch, f)
+	err = archAdd("current/initrd", initrd.Bytes())
 	if err != nil {
-			return err
-		}
+		return
 	}
 
+	// done
+	return nil
+}
+
+func buildBootEFITar(out io.Writer, ctx *renderContext) (err error) {
+	arch := tar.NewWriter(out)
+	defer arch.Close()
+
+	archAdd := func(path string, ba []byte) (err error) {
+		err = arch.WriteHeader(&tar.Header{Name: path, Mode: 0640, Size: int64(len(ba))})
+		if err != nil {
+			return
+		}
+		_, err = arch.Write(ba)
+		return
+	}
+
+	const (
+		prefix    = "EFI/dkl/"
+		efiPrefix = "\\EFI\\dkl\\"
+	)
+
+	// boot.csv
+	// -> annoyingly it's UTF-16...
+	bootCsvBytes := utf16.FromUTF8([]byte("" +
+		"current_kernel.efi,dkl current,initrd=" + efiPrefix + "current_initrd.img,Direktil current\n" +
+		"previous_kernel.efi,dkl previous,initrd=" + efiPrefix + "previous_initrd.img,Direktil previous\n"))
+
+	err = archAdd(prefix+"BOOT.CSV", []byte(bootCsvBytes))
+	if err != nil {
+		return
+	}
+
+	// kernel
+	kernelPath, err := ctx.distFetch("kernels", ctx.Host.Kernel)
+	if err != nil {
+		return
+	}
+
+	kernelBytes, err := ioutil.ReadFile(kernelPath)
+	if err != nil {
+		return
+	}
+
+	err = archAdd(prefix+"current_kernel.efi", kernelBytes)
+	if err != nil {
+		return
+	}
+
+	// initrd
+	initrd := new(bytes.Buffer)
+	err = buildInitrd(initrd, ctx)
+	if err != nil {
+		return
+	}
+
+	err = archAdd(prefix+"current_initrd.img", initrd.Bytes())
+	if err != nil {
+		return
+	}
+
+	// done
 	return nil
 }

cmd/dkl-local-server/bootv2.go

@@ -0,0 +1,142 @@
+package main
+
+import (
+	"archive/tar"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+
+	yaml "gopkg.in/yaml.v2"
+
+	"novit.tech/direktil/pkg/cpiocat"
+)
+
+func renderBootstrapConfig(w http.ResponseWriter, r *http.Request, ctx *renderContext, asJson bool) (err error) {
+	log.Printf("sending bootstrap config for %q", ctx.Host.Name)
+
+	_, cfg, err := ctx.BootstrapConfig()
+	if err != nil {
+		return err
+	}
+
+	if asJson {
+		err = json.NewEncoder(w).Encode(cfg)
+	} else {
+		err = yaml.NewEncoder(w).Encode(cfg)
+	}
+
+	return nil
+}
+
+func buildInitrd(out io.Writer, ctx *renderContext) (err error) {
+	_, cfg, err := ctx.Config()
+	if err != nil {
+		return
+	}
+
+	cat := cpiocat.New(out)
+
+	// initrd
+	initrdPath, err := ctx.distFetch("initrd", ctx.Host.Initrd)
+	if err != nil {
+		return
+	}
+	cat.AppendArchFile(initrdPath)
+
+	// embedded layers (modules)
+	for _, layer := range cfg.Layers {
+		switch layer {
+		case "modules":
+
+			layerVersion := ctx.Host.Versions[layer]
+			modulesPath, err := ctx.distFetch("layers", layer, layerVersion)
+			if err != nil {
+				return err
+			}
+
+			cat.AppendFile(modulesPath, "modules.sqfs")
+		}
+	}
+
+	// config
+	cfgBytes, _, err := ctx.BootstrapConfig()
+	if err != nil {
+		return
+	}
+	cat.AppendBytes(cfgBytes, "config.yaml", 0600)
+
+	// ssh keys
+	// FIXME we want a bootstrap-stage key instead of the real host key
+	for _, format := range []string{"rsa", "dsa", "ecdsa", "ed25519"} {
+		cat.AppendBytes(cfg.FileContent("/etc/ssh/ssh_host_"+format+"_key"), "id_"+format, 0600)
+	}
+
+	return cat.Close()
+}
+
+func buildBootstrap(out io.Writer, ctx *renderContext) (err error) {
+	arch := tar.NewWriter(out)
+	defer arch.Close()
+
+	// config
+	cfgBytes, cfg, err := ctx.Config()
+	if err != nil {
+		return err
+	}
+
+	err = arch.WriteHeader(&tar.Header{Name: "config.yaml", Size: int64(len(cfgBytes))})
+	if err != nil {
+		return
+	}
+
+	_, err = arch.Write(cfgBytes)
+	if err != nil {
+		return
+	}
+
+	// layers
+	for _, layer := range cfg.Layers {
+		if layer == "modules" {
+			continue // modules are with the kernel in boot v2
+		}
+
+		layerVersion := ctx.Host.Versions[layer]
+		if layerVersion == "" {
+			return fmt.Errorf("layer %q not mapped to a version", layer)
+		}
+
+		outPath, err := ctx.distFetch("layers", layer, layerVersion)
+		if err != nil {
+			return err
+		}
+
+		f, err := os.Open(outPath)
+		if err != nil {
+			return err
+		}
+
+		defer f.Close()
+
+		stat, err := f.Stat()
+		if err != nil {
+			return err
+		}
+
+		if err = arch.WriteHeader(&tar.Header{
+			Name: layer + ".fs",
+			Size: stat.Size(),
+		}); err != nil {
+			return err
+		}
+
+		_, err = io.Copy(arch, f)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

cmd/dkl-local-server/cas-cleaner.go

@@ -12,20 +12,22 @@ var (
 )
 
 func casCleaner() {
-	for {
+	for range time.Tick(*cacheCleanDelay) {
+		if !wPublicState.Get().Store.Open {
+			continue
+		}
+
 		err := cleanCAS()
 		if err != nil {
 			log.Print("warn: couldn't clean cache: ", err)
 		}
-
-		time.Sleep(*cacheCleanDelay)
 	}
 }
 
 func cleanCAS() error {
 	cfg, err := readConfig()
 	if err != nil {
-		return err
+		return nil
 	}
 
 	activeTags := make([]string, len(cfg.Hosts))

cmd/dkl-local-server/cluster-render-context.go

@@ -0,0 +1,185 @@
+package main
+
+import (
+	"crypto/rand"
+	"encoding/base32"
+	"encoding/json"
+	"fmt"
+	"log"
+	"path"
+	"strconv"
+	"strings"
+
+	cfsslconfig "github.com/cloudflare/cfssl/config"
+	"github.com/cloudflare/cfssl/csr"
+	yaml "gopkg.in/yaml.v2"
+
+	"novit.tech/direktil/pkg/bootstrapconfig"
+	"novit.tech/direktil/pkg/config"
+)
+
+func templateFuncs(sslCfg *cfsslconfig.Config) map[string]any {
+	getKeyCert := func(cluster, caName, name, profile, label, reqJson string) (kc KeyCert, err error) {
+		certReq := &csr.CertificateRequest{
+			KeyRequest: csr.NewKeyRequest(),
+		}
+
+		err = json.Unmarshal([]byte(reqJson), certReq)
+		if err != nil {
+			log.Print("CSR unmarshal failed on: ", reqJson)
+			return
+		}
+
+		return getUsableKeyCert(cluster, caName, name, profile, label, certReq, sslCfg)
+	}
+
+	hash := func(plain, seed []byte, hashAlg string) (hashed string, err error) {
+		switch hashAlg {
+		case "sha512crypt":
+			return sha512crypt(plain, seed)
+
+		case "bootstrap":
+			return bootstrapconfig.JoinSeedAndHash(seed, bootstrapconfig.PasswordHashFromSeed(seed, plain)), nil
+
+		default:
+			return "", fmt.Errorf("unknown hash alg: %q", hashAlg)
+		}
+	}
+
+	return map[string]any{
+		"quote": strconv.Quote,
+
+		"password": func(cluster, name, hashAlg string) (password string, err error) {
+			key := cluster + "/" + name
+
+			seed, err := seeds.GetOrCreate(key, func() (seed []byte, err error) {
+				seed = make([]byte, 16)
+				_, err = rand.Read(seed)
+				return
+			})
+
+			if err != nil {
+				return "", fmt.Errorf("failed to get seed: %w", err)
+			}
+
+			password, err = clusterPasswords.GetOrCreate(key, func() (password string, err error) {
+				raw := make([]byte, 10)
+				_, err = rand.Read(raw)
+				if err != nil {
+					return "", fmt.Errorf("failed to generate password: %w", err)
+				}
+
+				password = strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(raw))
+				return
+			})
+			if err != nil {
+				return
+			}
+
+			return hash([]byte(password), seed, hashAlg)
+		},
+
+		"token": getOrCreateClusterToken,
+
+		"ca_key": func(cluster, name string) (s string, err error) {
+			ca, err := getUsableClusterCA(cluster, name)
+			if err != nil {
+				return
+			}
+
+			s = string(ca.Key)
+			return
+		},
+
+		"ca_crt": func(cluster, name string) (s string, err error) {
+			ca, err := getUsableClusterCA(cluster, name)
+			if err != nil {
+				return
+			}
+
+			s = string(ca.Cert)
+			return
+		},
+
+		"ca_dir": func(cluster, name string) (s string, err error) {
+			ca, err := getUsableClusterCA(cluster, name)
+			if err != nil {
+				return
+			}
+
+			dir := "/etc/tls-ca/" + name
+
+			return asYaml([]config.FileDef{
+				{
+					Path:    path.Join(dir, "ca.crt"),
+					Mode:    0644,
+					Content: string(ca.Cert),
+				},
+				{
+					Path:    path.Join(dir, "ca.key"),
+					Mode:    0600,
+					Content: string(ca.Key),
+				},
+			})
+		},
+
+		"tls_key": func(cluster, caName, name, profile, label, reqJson string) (s string, err error) {
+			kc, err := getKeyCert(cluster, caName, name, profile, label, reqJson)
+			if err != nil {
+				return
+			}
+
+			s = string(kc.Key)
+			return
+		},
+
+		"tls_crt": func(cluster, caName, name, profile, label, reqJson string) (s string, err error) {
+			kc, err := getKeyCert(cluster, caName, name, profile, label, reqJson)
+			if err != nil {
+				return
+			}
+
+			s = string(kc.Cert)
+			return
+		},
+
+		"tls_dir": func(dir, cluster, caName, name, profile, label, reqJson string) (s string, err error) {
+			ca, err := getUsableClusterCA(cluster, caName)
+			if err != nil {
+				return
+			}
+
+			kc, err := getKeyCert(cluster, caName, name, profile, label, reqJson)
+			if err != nil {
+				return
+			}
+
+			return asYaml([]config.FileDef{
+				{
+					Path:    path.Join(dir, "ca.crt"),
+					Mode:    0644,
+					Content: string(ca.Cert),
+				},
+				{
+					Path:    path.Join(dir, "tls.crt"),
+					Mode:    0644,
+					Content: string(kc.Cert),
+				},
+				{
+					Path:    path.Join(dir, "tls.key"),
+					Mode:    0600,
+					Content: string(kc.Key),
+				},
+			})
+		},
+	}
+}
+
+func asYaml(v interface{}) (string, error) {
+	ba, err := yaml.Marshal(v)
+	if err != nil {
+		return "", err
+	}
+
+	return string(ba), nil
+}

cmd/dkl-local-server/data.go

@@ -4,7 +4,7 @@ import (
 	"flag"
 	"path/filepath"
 
-	"novit.nc/direktil/pkg/localconfig"
+	"novit.tech/direktil/pkg/localconfig"
 )
 
 var (

cmd/dkl-local-server/host-download-tokens.go

@@ -0,0 +1,3 @@
+package main
+
+var hostDownloadTokens = KVSecrets[string]{"hosts/download-tokens"}

cmd/dkl-local-server/httperr.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"net/http"
+
+	"m.cluseau.fr/go/httperr"
+)
+
+var (
+	ErrNotFound     = httperr.NotFound
+	ErrUnauthorized = httperr.StdStatus(http.StatusUnauthorized)
+	ErrForbidden    = httperr.StdStatus(http.StatusForbidden)
+	ErrInternal     = httperr.StdStatus(http.StatusInternalServerError)
+	ErrInvalidToken = httperr.NewStd(1000, http.StatusForbidden, "invalid token")
+	ErrStoreLocked  = httperr.NewStd(1001, http.StatusServiceUnavailable, "store is locked")
+)

cmd/dkl-local-server/initrd.go

@@ -2,13 +2,9 @@ package main
 
 import (
 	"encoding/json"
-	"fmt"
-	"io"
 	"log"
 	"net/http"
-	"os"
 
-	cpio "github.com/cavaliercoder/go-cpio"
 	yaml "gopkg.in/yaml.v2"
 )
 
@@ -28,84 +24,3 @@ func renderConfig(w http.ResponseWriter, r *http.Request, ctx *renderContext, as
 
 	return nil
 }
-
-func buildInitrd(out io.Writer, ctx *renderContext) error {
-	_, cfg, err := ctx.Config()
-
-	if err != nil {
-		return err
-	}
-
-	// send initrd basis
-	initrdPath, err := ctx.distFetch("initrd", ctx.Host.Initrd)
-	if err != nil {
-		return err
-	}
-
-	err = writeFile(out, initrdPath)
-	if err != nil {
-		return err
-	}
-
-	// and our extra archive
-	archive := cpio.NewWriter(out)
-
-	// - required dirs
-	for _, dir := range []string{
-		"boot",
-		"boot/current",
-		"boot/current/layers",
-	} {
-		archive.WriteHeader(&cpio.Header{
-			Name: dir,
-			Mode: 0600 | cpio.ModeDir,
-		})
-	}
-
-	// - the layers
-	for _, layer := range cfg.Layers {
-		layerVersion := ctx.Host.Versions[layer]
-		if layerVersion == "" {
-			return fmt.Errorf("layer %q not mapped to a version", layer)
-		}
-
-		path, err := ctx.distFetch("layers", layer, layerVersion)
-		if err != nil {
-			return err
-		}
-
-		stat, err := os.Stat(path)
-		if err != nil {
-			return err
-		}
-
-		archive.WriteHeader(&cpio.Header{
-			Name: "boot/current/layers/" + layer + ".fs",
-			Mode: 0600,
-			Size: stat.Size(),
-		})
-
-		if err = writeFile(archive, path); err != nil {
-			return err
-		}
-	}
-
-	// - the configuration
-	ba, err := yaml.Marshal(cfg)
-	if err != nil {
-		return err
-	}
-
-	archive.WriteHeader(&cpio.Header{
-		Name: "boot/config.yaml",
-		Mode: 0600,
-		Size: int64(len(ba)),
-	})
-
-	archive.Write(ba)
-
-	// finalize the archive
-	archive.Flush()
-	archive.Close()
-	return nil
-}

cmd/dkl-local-server/kernel.go

@@ -1,8 +1,10 @@
 package main
 
 import (
+	"io"
 	"log"
 	"net/http"
+	"os"
 )
 
 func renderKernel(w http.ResponseWriter, r *http.Request, ctx *renderContext) error {
@@ -15,3 +17,19 @@ func renderKernel(w http.ResponseWriter, r *http.Request, ctx *renderContext) er
 	http.ServeFile(w, r, path)
 	return nil
 }
+
+func fetchKernel(out io.Writer, ctx *renderContext) (err error) {
+	path, err := ctx.distFetch("kernels", ctx.Host.Kernel)
+	if err != nil {
+		return err
+	}
+
+	in, err := os.Open(path)
+	if err != nil {
+		return
+	}
+	defer in.Close()
+
+	_, err = io.Copy(out, in)
+	return
+}

cmd/dkl-local-server/main.go

@@ -4,44 +4,86 @@ import (
 	"flag"
 	"log"
 	"net/http"
+	"os"
 	"path/filepath"
 
 	restful "github.com/emicklei/go-restful"
-	"github.com/mcluseau/go-swagger-ui"
-	"novit.nc/direktil/pkg/cas"
+	swaggerui "github.com/mcluseau/go-swagger-ui"
+	"m.cluseau.fr/go/watchable/streamsse"
 
-	"novit.nc/direktil/local-server/pkg/apiutils"
+	"novit.tech/direktil/pkg/cas"
+
+	dlshtml "novit.tech/direktil/local-server/html"
+	"novit.tech/direktil/local-server/pkg/apiutils"
 )
 
 const (
 	etcDir = "/etc/direktil"
 )
 
+var Version = "dev"
+
 var (
 	address    = flag.String("address", ":7606", "HTTP listen address")
 	tlsAddress = flag.String("tls-address", "", "HTTPS listen address")
 	certFile   = flag.String("tls-cert", etcDir+"/server.crt", "Server TLS certificate")
 	keyFile    = flag.String("tls-key", etcDir+"/server.key", "Server TLS key")
 
+	autoUnlock = flag.String("auto-unlock", "", "Auto-unlock store (testing only!) env: DLS_AUTO_UNLOCK")
+
 	casStore cas.Store
 )
 
 func main() {
+	log.SetFlags(log.LstdFlags | log.Lshortfile)
+
 	flag.Parse()
 
 	if *address == "" && *tlsAddress == "" {
 		log.Fatal("no listen address given")
 	}
 
+	log.Print("Direktil local-server version ", Version)
+	wPublicState.Change(func(s *PublicState) { s.ServerVersion = Version })
+
+	computeUIHash()
+
+	openSecretStore()
+
+	{
+		autoUnlock := *autoUnlock
+		if autoUnlock == "" {
+			autoUnlock = os.Getenv("DLS_AUTO_UNLOCK")
+		}
+		if autoUnlock != "" {
+			log.Printf("auto-unlocking the store")
+			err := unlockSecretStore("test", []byte(autoUnlock))
+			if err.Any() {
+				log.Fatal(err)
+			}
+
+			log.Print("store auto-unlocked, token is ", adminToken)
+		}
+
+		os.Setenv("DLS_AUTO_UNLOCK", "")
+	}
+
 	casStore = cas.NewDir(filepath.Join(*dataDir, "cache"))
 	go casCleaner()
 
 	apiutils.Setup(func() {
-		restful.Add(buildWS())
+		registerWS(restful.DefaultContainer)
 	})
 
 	swaggerui.HandleAt("/swagger-ui/")
 
+	staticHandler := http.FileServer(http.FS(dlshtml.FS))
+	http.Handle("/favicon.ico", staticHandler)
+	http.Handle("/ui/", staticHandler)
+
+	http.Handle("/public-state", streamsse.StreamHandler(wPublicState))
+	http.Handle("/state", requireAdmin(streamsse.StreamHandler(wState)))
+
 	if *address != "" {
 		log.Print("HTTP listening on ", *address)
 		go log.Fatal(http.ListenAndServe(*address, nil))

cmd/dkl-local-server/render-context.go

@@ -4,36 +4,49 @@ import (
 	"bytes"
 	"crypto/sha256"
 	"encoding/hex"
-	"encoding/json"
+	"fmt"
 	"io"
 	"log"
 	"net/http"
+	"net/url"
 	"path"
 	"path/filepath"
 	"text/template"
 
 	cfsslconfig "github.com/cloudflare/cfssl/config"
-	"github.com/cloudflare/cfssl/csr"
+	restful "github.com/emicklei/go-restful"
 	yaml "gopkg.in/yaml.v2"
 
-	"novit.nc/direktil/pkg/config"
-	"novit.nc/direktil/pkg/localconfig"
+	"novit.tech/direktil/pkg/config"
+	"novit.tech/direktil/pkg/localconfig"
+
+	bsconfig "novit.tech/direktil/pkg/bootstrapconfig"
 )
 
+var cmdlineParam = restful.QueryParameter("cmdline", "Linux kernel cmdline addition")
+
 type renderContext struct {
 	Host      *localconfig.Host
-	SSLConfig string
+	SSLConfig *cfsslconfig.Config
+
+	// Linux kernel extra cmdline
+	CmdLine string `yaml:"-"`
 }
 
 func renderCtx(w http.ResponseWriter, r *http.Request, ctx *renderContext, what string,
 	create func(out io.Writer, ctx *renderContext) error) error {
-	log.Printf("sending %s for %q", what, ctx.Host.Name)
 
 	tag, err := ctx.Tag()
 	if err != nil {
 		return err
 	}
 
+	ctx.CmdLine = r.URL.Query().Get(cmdlineParam.Data().Name)
+
+	if ctx.CmdLine != "" {
+		what = what + "?cmdline=" + url.QueryEscape(ctx.CmdLine)
+	}
+
 	// get it or create it
 	content, meta, err := casStore.GetOrCreate(tag, what, func(out io.Writer) error {
 		log.Printf("building %s for %q", what, ctx.Host.Name)
@@ -45,16 +58,12 @@ func renderCtx(w http.ResponseWriter, r *http.Request, ctx *renderContext, what
 	}
 
 	// serve it
+	log.Printf("sending %s for %q", what, ctx.Host.Name)
 	http.ServeContent(w, r, what, meta.ModTime(), content)
 	return nil
 }
 
-var prevSSLConfig = "-"
-
-func newRenderContext(host *localconfig.Host, cfg *localconfig.Config) (ctx *renderContext, err error) {
-	if prevSSLConfig != cfg.SSLConfig {
-		var sslCfg *cfsslconfig.Config
-
+func sslConfigFromLocalConfig(cfg *localconfig.Config) (sslCfg *cfsslconfig.Config, err error) {
 	if len(cfg.SSLConfig) == 0 {
 		sslCfg = &cfsslconfig.Config{}
 	} else {
@@ -63,25 +72,53 @@ func newRenderContext(host *localconfig.Host, cfg *localconfig.Config) (ctx *ren
 			return
 		}
 	}
+	return
+}
 
-		err = loadSecretData(sslCfg)
+func newRenderContext(host *localconfig.Host, cfg *localconfig.Config) (ctx *renderContext, err error) {
+	sslCfg, err := sslConfigFromLocalConfig(cfg)
 	if err != nil {
 		return
 	}
 
-		prevSSLConfig = cfg.SSLConfig
-	}
-
 	return &renderContext{
-		SSLConfig: cfg.SSLConfig,
 		Host:      host,
+		SSLConfig: sslCfg,
 	}, nil
 }
 
 func (ctx *renderContext) Config() (ba []byte, cfg *config.Config, err error) {
+	ba, err = ctx.render(ctx.Host.Config)
+	if err != nil {
+		return
+	}
+
+	cfg = &config.Config{}
+	if err = yaml.Unmarshal(ba, cfg); err != nil {
+		return
+	}
+
+	return
+}
+
+func (ctx *renderContext) BootstrapConfig() (ba []byte, cfg *bsconfig.Config, err error) {
+	ba, err = ctx.render(ctx.Host.BootstrapConfig)
+	if err != nil {
+		return
+	}
+
+	cfg = &bsconfig.Config{}
+	if err = yaml.Unmarshal(ba, cfg); err != nil {
+		return
+	}
+
+	return
+}
+
+func (ctx *renderContext) render(templateText string) (ba []byte, err error) {
 	tmpl, err := template.New(ctx.Host.Name + "/config").
-		Funcs(ctx.templateFuncs()).
-		Parse(ctx.Host.Config)
+		Funcs(ctx.TemplateFuncs()).
+		Parse(templateText)
 
 	if err != nil {
 		return
@@ -92,147 +129,10 @@ func (ctx *renderContext) Config() (ba []byte, cfg *config.Config, err error) {
 		return
 	}
 
-	if secretData.Changed() {
-		err = secretData.Save()
-		if err != nil {
-			return
-		}
-	}
-
 	ba = buf.Bytes()
-
-	cfg = &config.Config{}
-
-	if err = yaml.Unmarshal(buf.Bytes(), cfg); err != nil {
 	return
 }
 
-	return
-}
-
-func (ctx *renderContext) templateFuncs() map[string]interface{} {
-	getKeyCert := func(cluster, caName, name, profile, label, reqJson string) (kc *KeyCert, err error) {
-		certReq := &csr.CertificateRequest{
-			KeyRequest: csr.NewBasicKeyRequest(),
-		}
-
-		err = json.Unmarshal([]byte(reqJson), certReq)
-		if err != nil {
-			log.Print("CSR unmarshal failed on: ", reqJson)
-			return
-		}
-
-		return secretData.KeyCert(cluster, caName, name, profile, label, certReq)
-	}
-
-	asYaml := func(v interface{}) (string, error) {
-		ba, err := yaml.Marshal(v)
-		if err != nil {
-			return "", err
-		}
-
-		return string(ba), nil
-	}
-
-	return map[string]interface{}{
-		"token": func(cluster, name string) (s string, err error) {
-			return secretData.Token(cluster, name)
-		},
-
-		"ca_key": func(cluster, name string) (s string, err error) {
-			ca, err := secretData.CA(cluster, name)
-			if err != nil {
-				return
-			}
-
-			s = string(ca.Key)
-			return
-		},
-
-		"ca_crt": func(cluster, name string) (s string, err error) {
-			ca, err := secretData.CA(cluster, name)
-			if err != nil {
-				return
-			}
-
-			s = string(ca.Cert)
-			return
-		},
-
-		"ca_dir": func(cluster, name string) (s string, err error) {
-			ca, err := secretData.CA(cluster, name)
-			if err != nil {
-				return
-			}
-
-			dir := "/etc/tls-ca/" + name
-
-			return asYaml([]config.FileDef{
-				{
-					Path:    path.Join(dir, "ca.crt"),
-					Mode:    0644,
-					Content: string(ca.Cert),
-				},
-				{
-					Path:    path.Join(dir, "ca.key"),
-					Mode:    0600,
-					Content: string(ca.Key),
-				},
-			})
-		},
-
-		"tls_key": func(cluster, caName, name, profile, label, reqJson string) (s string, err error) {
-			kc, err := getKeyCert(cluster, caName, name, profile, label, reqJson)
-			if err != nil {
-				return
-			}
-
-			s = string(kc.Key)
-			return
-		},
-
-		"tls_crt": func(cluster, caName, name, profile, label, reqJson string) (s string, err error) {
-			kc, err := getKeyCert(cluster, caName, name, profile, label, reqJson)
-			if err != nil {
-				return
-			}
-
-			s = string(kc.Cert)
-			return
-		},
-
-		"tls_dir": func(dir, cluster, caName, name, profile, label, reqJson string) (s string, err error) {
-			ca, err := secretData.CA(cluster, caName)
-			if err != nil {
-				return
-			}
-
-			kc, err := getKeyCert(cluster, caName, name, profile, label, reqJson)
-			if err != nil {
-				return
-			}
-
-			return asYaml([]config.FileDef{
-				{
-					Path:    path.Join(dir, "ca.crt"),
-					Mode:    0644,
-					Content: string(ca.Cert),
-				},
-				{
-					Path:    path.Join(dir, "tls.crt"),
-					Mode:    0644,
-					Content: string(kc.Cert),
-				},
-				{
-					Path:    path.Join(dir, "tls.key"),
-					Mode:    0600,
-					Content: string(kc.Key),
-				},
-			})
-		},
-	}
-}
-
 func (ctx *renderContext) distFilePath(path ...string) string {
 	return filepath.Join(append([]string{*dataDir, "dist"}, path...)...)
 }
@@ -270,3 +170,69 @@ func asMap(v interface{}) map[string]interface{} {
 
 	return result
 }
+
+func (ctx *renderContext) TemplateFuncs() map[string]any {
+	funcs := templateFuncs(ctx.SSLConfig)
+
+	for name, method := range map[string]any{
+		"ssh_host_keys": func(dir, cluster, host string) (s string, err error) {
+			if host == "" {
+				host = ctx.Host.Name
+			}
+			if host != ctx.Host.Name {
+				err = fmt.Errorf("wrong host name")
+				return
+			}
+
+			pairs, err := getSSHKeyPairs(host)
+			if err != nil {
+				return
+			}
+
+			files := make([]config.FileDef, 0, len(pairs)*2)
+
+			for _, pair := range pairs {
+				basePath := path.Join(dir, "ssh_host_"+pair.Type+"_key")
+				files = append(files, []config.FileDef{
+					{
+						Path:    basePath,
+						Mode:    0600,
+						Content: pair.Private,
+					},
+					{
+						Path:    basePath + ".pub",
+						Mode:    0644,
+						Content: pair.Public,
+					},
+				}...)
+			}
+
+			return asYaml(files)
+		},
+		"host_download_token": func() (token string, err error) {
+			key := ctx.Host.Name
+			token, found, err := hostDownloadTokens.Get(key)
+			if err != nil {
+				return
+			}
+
+			if !found {
+				token, err = newToken(32)
+				if err != nil {
+					return
+				}
+
+				err = hostDownloadTokens.Put(key, token)
+				if err != nil {
+					return
+				}
+			}
+
+			return
+		},
+	} {
+		funcs[name] = method
+	}
+
+	return funcs
+}

cmd/dkl-local-server/secret-store.go

@@ -0,0 +1,350 @@
+package main
+
+import (
+	"encoding/json"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+
+	restful "github.com/emicklei/go-restful"
+	"m.cluseau.fr/go/httperr"
+
+	"novit.tech/direktil/local-server/secretstore"
+)
+
+var secStore *secretstore.Store
+
+func secStoreRoot() string            { return filepath.Join(*dataDir, "secrets") }
+func secStorePath(name string) string { return filepath.Join(secStoreRoot(), name) }
+func secKeysStorePath() string        { return secStorePath(".keys") }
+
+func openSecretStore() {
+	var err error
+
+	keysPath := secKeysStorePath()
+
+	if err := os.MkdirAll(filepath.Dir(filepath.Dir(keysPath)), 0755); err != nil {
+		log.Fatal("failed to create dirs: ", err)
+	}
+
+	if err := os.MkdirAll(filepath.Dir(keysPath), 0700); err != nil {
+		log.Fatal("failed to secret store dir: ", err)
+	}
+
+	secStore, err = secretstore.Open(keysPath)
+
+	switch {
+	case err == nil:
+		wPublicState.Change(func(v *PublicState) {
+			v.Store.New = false
+			v.Store.Open = false
+		})
+
+	case os.IsNotExist(err):
+		secStore = secretstore.New()
+		wPublicState.Change(func(v *PublicState) {
+			v.Store.New = true
+			v.Store.Open = false
+		})
+
+	default:
+		log.Fatal("failed to open keys store: ", err)
+	}
+}
+
+var (
+	unlockMutex = sync.Mutex{}
+
+	ErrStoreAlreadyUnlocked = httperr.NewStd(1, http.StatusConflict, "store already unlocked")
+	ErrInvalidPassphrase    = httperr.NewStd(2, http.StatusBadRequest, "invalid passphrase")
+)
+
+func unlockSecretStore(name string, passphrase []byte) (err httperr.Error) {
+	unlockMutex.Lock()
+	defer unlockMutex.Unlock()
+
+	if secStore.Unlocked() {
+		return ErrStoreAlreadyUnlocked
+	}
+
+	if secStore.IsNew() {
+		err := secStore.Init(name, passphrase)
+		if err != nil {
+			return httperr.Internal(err)
+		}
+
+		err = secStore.SaveTo(secKeysStorePath())
+		if err != nil {
+			log.Print("secret store save error: ", err)
+			secStore.Close()
+
+			return httperr.Internal(err)
+		}
+
+	} else {
+		if !secStore.Unlock([]byte(passphrase)) {
+			return ErrInvalidPassphrase
+		}
+	}
+
+	token := ""
+	if err := readSecret("admin-token", &token); err != nil {
+		if !os.IsNotExist(err) {
+			log.Print("failed to read admin token: ", err)
+			secStore.Close()
+
+			return httperr.Internal(err)
+		}
+
+		token, err = newToken(32)
+		if err != nil {
+			secStore.Close()
+			return httperr.Internal(err)
+		}
+
+		err = writeSecret("admin-token", token)
+		if err != nil {
+			log.Print("write error: ", err)
+			secStore.Close()
+
+			return httperr.Internal(err)
+		}
+
+		log.Print("wrote new admin token")
+	}
+
+	adminToken = token
+
+	{
+		token, err := newToken(16)
+		if err != nil {
+			secStore.Close()
+			return httperr.Internal(err)
+		}
+
+		wState.Change(func(v *State) {
+			v.Store.DownloadToken = token
+		})
+	}
+
+	wPublicState.Change(func(v *PublicState) {
+		v.Store.New = false
+		v.Store.Open = true
+	})
+
+	go updateState()
+	go migrateSecrets()
+
+	return
+}
+
+func readSecret(name string, value any) (err error) {
+	f, err := os.Open(secStorePath(name + ".data"))
+	if err != nil {
+		return
+	}
+
+	defer f.Close()
+
+	in, err := secStore.NewReader(f)
+	if err != nil {
+		return
+	}
+
+	return json.NewDecoder(in).Decode(value)
+}
+
+func writeSecret(name string, value any) (err error) {
+	path := secStorePath(name + ".data.new")
+
+	if err = os.MkdirAll(filepath.Dir(path), 0700); err != nil {
+		return
+	}
+
+	f, err := os.Create(path)
+	if err != nil {
+		return
+	}
+
+	err = func() (err error) {
+		defer f.Close()
+
+		out, err := secStore.NewWriter(f)
+		if err != nil {
+			return
+		}
+
+		return json.NewEncoder(out).Encode(value)
+	}()
+
+	if err != nil {
+		return
+	}
+
+	err = os.Rename(f.Name(), secStorePath(name+".data"))
+	if err != nil {
+		return
+	}
+
+	go updateState()
+	return
+}
+
+var secL sync.Mutex
+
+func updateSecret[T any](name string, update func(*T)) (err error) {
+	secL.Lock()
+	defer secL.Unlock()
+
+	v := new(T)
+	err = readSecret(name, v)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return
+		}
+		err = nil
+	}
+
+	update(v)
+
+	return writeSecret(name, *v)
+}
+
+func updateSecretWithKey[T any](name, key string, update func(v *T)) (err error) {
+	secL.Lock()
+	defer secL.Unlock()
+
+	kvs := map[string]*T{}
+
+	err = readSecret(name, &kvs)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return
+		}
+		err = nil
+	}
+
+	update(kvs[key])
+
+	return writeSecret(name, kvs)
+}
+
+type KVSecrets[T any] struct{ Name string }
+
+func (s KVSecrets[T]) Data() (kvs map[string]T, err error) {
+	kvs = make(map[string]T)
+	err = readSecret(s.Name, &kvs)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return
+		}
+		err = nil
+	}
+	return
+}
+
+func (s KVSecrets[T]) Keys(prefix string) (keys []string, err error) {
+	kvs, err := s.Data()
+	if err != nil {
+		return
+	}
+
+	keys = make([]string, 0, len(kvs))
+
+	for k := range kvs {
+		if !strings.HasPrefix(k, prefix) {
+			continue
+		}
+		keys = append(keys, k[len(prefix):])
+	}
+
+	sort.Strings(keys)
+
+	return
+}
+
+func (s KVSecrets[T]) Get(key string) (v T, found bool, err error) {
+	kvs, err := s.Data()
+	if err != nil {
+		return
+	}
+
+	v, found = kvs[key]
+	return
+}
+
+func (s KVSecrets[T]) Put(key string, v T) (err error) {
+	secL.Lock()
+	defer secL.Unlock()
+
+	kvs, err := s.Data()
+	if err != nil {
+		return
+	}
+
+	kvs[key] = v
+	err = writeSecret(s.Name, kvs)
+	return
+}
+
+func (s KVSecrets[T]) GetOrCreate(key string, create func() (T, error)) (v T, err error) {
+	v, found, err := s.Get(key)
+	if err != nil {
+		return
+	}
+
+	if !found {
+		v, err = create()
+		if err != nil {
+			return
+		}
+
+		err = s.Put(key, v)
+	}
+
+	return
+}
+
+func (s KVSecrets[T]) WsList(resp *restful.Response, prefix string) {
+	keys, err := s.Keys(prefix)
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+
+	resp.WriteEntity(keys)
+}
+
+func (s KVSecrets[T]) WsGet(resp *restful.Response, key string) {
+	keys, found, err := s.Get(key)
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+
+	if !found {
+		wsNotFound(resp)
+		return
+	}
+
+	resp.WriteEntity(keys)
+}
+
+func (s KVSecrets[T]) WsPut(req *restful.Request, resp *restful.Response, key string) {
+	v := new(T)
+	err := req.ReadEntity(v)
+	if err != nil {
+		wsBadRequest(resp, err.Error())
+		return
+	}
+
+	err = s.Put(key, *v)
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+}

cmd/dkl-local-server/secrets-migrate.go

@@ -0,0 +1,88 @@
+package main
+
+import (
+	"log"
+	"os"
+
+	cfsslconfig "github.com/cloudflare/cfssl/config"
+)
+
+func migrateSecrets() {
+	if _, err := os.Stat(secretDataPath()); err != nil {
+		if os.IsNotExist(err) {
+			return
+		}
+
+		log.Print("not migrating old secrets: ", err)
+
+		return
+	}
+
+	log.Print("migrating old secrets")
+
+	log := log.New(log.Default().Writer(), "secrets migration: ", log.Flags()|log.Lmsgprefix)
+
+	// load secrets
+	cfg, err := readConfig()
+	if err != nil {
+		log.Fatal(err)
+		return
+	}
+
+	var sslCfg *cfsslconfig.Config
+
+	if len(cfg.SSLConfig) == 0 {
+		sslCfg = &cfsslconfig.Config{}
+	} else {
+		sslCfg, err = cfsslconfig.LoadConfig([]byte(cfg.SSLConfig))
+		if err != nil {
+			return
+		}
+	}
+
+	secretData, err := loadSecretData(sslCfg)
+	if err != nil {
+		log.Fatal(err)
+		return
+	}
+
+	for clusterName, cluster := range secretData.clusters {
+		for k, v := range cluster.Tokens {
+			err = clusterTokens.Put(clusterName+"/"+k, v)
+			if err != nil {
+				log.Fatal(err)
+				return
+			}
+		}
+
+		for k, v := range cluster.Passwords {
+			err = clusterPasswords.Put(clusterName+"/"+k, v)
+			if err != nil {
+				log.Fatal(err)
+				return
+			}
+		}
+
+		for caName, ca := range cluster.CAs {
+			clusterCAs.Put(clusterName+"/"+caName, CA{Key: ca.Key, Cert: ca.Cert})
+
+			for signedName, signed := range ca.Signed {
+				err = clusterCASignedKeys.Put(clusterName+"/"+caName+"/"+signedName, *signed)
+				if err != nil {
+					log.Fatal(err)
+				}
+			}
+		}
+
+		for hostName, pairs := range cluster.SSHKeyPairs {
+			err = sshHostKeys.Put(hostName, pairs)
+			if err != nil {
+				log.Fatal(err)
+			}
+		}
+	}
+
+	if err := os.Rename(secretDataPath(), secretDataPath()+".migrated"); err != nil {
+		log.Fatal("failed to rename migrated secrets: ", err)
+	}
+}

cmd/dkl-local-server/secrets.go

@@ -1,50 +1,28 @@
 package main
 
 import (
-	"crypto/rand"
-	"encoding/base32"
 	"encoding/json"
 	"errors"
-	"fmt"
 	"io/ioutil"
-	"net"
 	"os"
 	"path/filepath"
-	"sync"
+	"time"
 
+	"github.com/cloudflare/cfssl/certinfo"
 	"github.com/cloudflare/cfssl/config"
-	"github.com/cloudflare/cfssl/csr"
-	"github.com/cloudflare/cfssl/helpers"
-	"github.com/cloudflare/cfssl/initca"
 	"github.com/cloudflare/cfssl/log"
-	"github.com/cloudflare/cfssl/signer"
-	"github.com/cloudflare/cfssl/signer/local"
-	"k8s.io/apimachinery/pkg/util/validation"
-	"k8s.io/apimachinery/pkg/util/validation/field"
-)
-
-var (
-	secretData *SecretData
 )
 
 type SecretData struct {
-	l sync.Mutex
-
 	clusters map[string]*ClusterSecrets
-	changed  bool
 	config   *config.Config
 }
 
 type ClusterSecrets struct {
 	CAs         map[string]*CA
 	Tokens      map[string]string
-}
-
-type CA struct {
-	Key  []byte
-	Cert []byte
-
-	Signed map[string]*KeyCert
+	Passwords   map[string]string
+	SSHKeyPairs map[string][]SSHKeyPair
 }
 
 type KeyCert struct {
@@ -57,21 +35,18 @@ func secretDataPath() string {
 	return filepath.Join(*dataDir, "secret-data.json")
 }
 
-func loadSecretData(config *config.Config) (err error) {
+func loadSecretData(config *config.Config) (sd *SecretData, err error) {
 	log.Info("Loading secret data")
 
-	sd := &SecretData{
+	sd = &SecretData{
 		clusters: make(map[string]*ClusterSecrets),
-		changed:  false,
 		config:   config,
 	}
 
 	ba, err := ioutil.ReadFile(secretDataPath())
 	if err != nil {
 		if os.IsNotExist(err) {
-			sd.changed = true
 			err = nil
-			secretData = sd
 			return
 		}
 		return
@@ -81,207 +56,21 @@ func loadSecretData(config *config.Config) (err error) {
 		return
 	}
 
-	secretData = sd
 	return
 }
 
-func (sd *SecretData) Changed() bool {
-	return sd.changed
-}
-
-func (sd *SecretData) Save() error {
-	sd.l.Lock()
-	defer sd.l.Unlock()
-
-	log.Info("Saving secret data")
-	ba, err := json.Marshal(sd.clusters)
+func checkCertUsable(certPEM []byte) error {
+	cert, err := certinfo.ParseCertificatePEM(certPEM)
 	if err != nil {
 		return err
 	}
-	return ioutil.WriteFile(secretDataPath(), ba, 0600)
+
+	certDuration := cert.NotAfter.Sub(cert.NotBefore)
+	delayBeforeRegen := certDuration / 3 // TODO allow configuration
+
+	if cert.NotAfter.Sub(time.Now()) < delayBeforeRegen {
+		return errors.New("too old")
 	}
 
-func (sd *SecretData) cluster(name string) (cs *ClusterSecrets) {
-	cs, ok := sd.clusters[name]
-	if ok {
-		return
-	}
-
-	sd.l.Lock()
-	defer sd.l.Unlock()
-
-	log.Info("secret-data: new cluster: ", name)
-
-	cs = &ClusterSecrets{
-		CAs:    make(map[string]*CA),
-		Tokens: make(map[string]string),
-	}
-	sd.clusters[name] = cs
-	sd.changed = true
-	return
-}
-
-func (sd *SecretData) Token(cluster, name string) (token string, err error) {
-	cs := sd.cluster(cluster)
-
-	token = cs.Tokens[name]
-	if token != "" {
-		return
-	}
-
-	sd.l.Lock()
-	defer sd.l.Unlock()
-
-	log.Info("secret-data: new token in cluster ", cluster, ": ", name)
-
-	b := make([]byte, 16)
-	_, err = rand.Read(b)
-	if err != nil {
-		return
-	}
-
-	token = base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(b)
-
-	cs.Tokens[name] = token
-	sd.changed = true
-	return
-}
-
-func (sd *SecretData) CA(cluster, name string) (ca *CA, err error) {
-	cs := sd.cluster(cluster)
-
-	ca, ok := cs.CAs[name]
-	if ok {
-		return
-	}
-
-	sd.l.Lock()
-	defer sd.l.Unlock()
-
-	log.Info("secret-data: new CA in cluster ", cluster, ": ", name)
-
-	req := &csr.CertificateRequest{
-		CN: "Direktil Local Server",
-		KeyRequest: &csr.BasicKeyRequest{
-			A: "ecdsa",
-			S: 521, // 256, 384, 521
-		},
-		Names: []csr.Name{
-			{
-				C: "NC",
-				O: "novit.nc",
-			},
-		},
-	}
-
-	cert, _, key, err := initca.New(req)
-	if err != nil {
-		return
-	}
-
-	ca = &CA{
-		Key:    key,
-		Cert:   cert,
-		Signed: make(map[string]*KeyCert),
-	}
-
-	cs.CAs[name] = ca
-	sd.changed = true
-
-	return
-}
-
-func (sd *SecretData) KeyCert(cluster, caName, name, profile, label string, req *csr.CertificateRequest) (kc *KeyCert, err error) {
-	for idx, host := range req.Hosts {
-		if ip := net.ParseIP(host); ip != nil {
-			// valid IP (v4 or v6)
-			continue
-		}
-
-		if host == "*" {
-			continue
-		}
-
-		if errs := validation.IsDNS1123Subdomain(host); len(errs) == 0 {
-			continue
-		}
-		if errs := validation.IsWildcardDNS1123Subdomain(host); len(errs) == 0 {
-			continue
-		}
-
-		path := field.NewPath(cluster, name, "hosts").Index(idx)
-		return nil, fmt.Errorf("%v: %q is not an IP or FQDN", path, host)
-	}
-
-	if req.CA != nil {
-		err = errors.New("no CA section allowed here")
-		return
-	}
-
-	ca, err := sd.CA(cluster, caName)
-	if err != nil {
-		return
-	}
-
-	rh := hash(req)
-	kc, ok := ca.Signed[name]
-	if ok && rh == kc.ReqHash {
-		return
-	} else if ok {
-		log.Infof("secret-data: cluster %s: CA %s: CSR changed for %s: hash=%q previous=%q",
-			cluster, caName, name, rh, kc.ReqHash)
-	} else {
-		log.Infof("secret-data: cluster %s: CA %s: new CSR for %s", cluster, caName, name)
-	}
-
-	sd.l.Lock()
-	defer sd.l.Unlock()
-
-	sgr, err := ca.Signer(sd.config.Signing)
-	if err != nil {
-		return
-	}
-
-	generator := &csr.Generator{Validator: func(_ *csr.CertificateRequest) error { return nil }}
-
-	csr, key, err := generator.ProcessRequest(req)
-	if err != nil {
-		return
-	}
-
-	signReq := signer.SignRequest{
-		Request: string(csr),
-		Profile: profile,
-		Label:   label,
-	}
-
-	cert, err := sgr.Sign(signReq)
-	if err != nil {
-		return
-	}
-
-	kc = &KeyCert{
-		Key:     key,
-		Cert:    cert,
-		ReqHash: rh,
-	}
-
-	ca.Signed[name] = kc
-	sd.changed = true
-
-	return
-}
-
-func (ca *CA) Signer(policy *config.Signing) (result *local.Signer, err error) {
-	caCert, err := helpers.ParseCertificatePEM(ca.Cert)
-	if err != nil {
-		return
-	}
-
-	caKey, err := helpers.ParsePrivateKeyPEM(ca.Key)
-	if err != nil {
-		return
-	}
-
-	return local.NewSigner(caKey, caCert, signer.DefaultSigAlgo(caKey), policy)
+	return nil
 }

cmd/dkl-local-server/sha512crypt.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"encoding/base64"
+
+	crypthash "github.com/sergeymakinen/go-crypt/hash"
+	"github.com/sergeymakinen/go-crypt/sha512"
+)
+
+// for some reason, no implementation of crypt's sha512 is clean enough :(
+
+func sha512crypt(password, seed []byte) (string, error) {
+	// loose salt entropy because of character restriction in the salt
+	salt := []byte(base64.RawStdEncoding.EncodeToString(seed))[:sha512.MaxSaltLength]
+	// - base64 allows '+' where the salt accepts '.'
+	for i, c := range salt {
+		if c == '+' {
+			salt[i] = '.'
+		}
+	}
+
+	scheme := struct {
+		HashPrefix string
+		Rounds     uint32 `hash:"param:rounds,omitempty"`
+		Salt       []byte
+		Sum        [86]byte
+	}{
+		HashPrefix: sha512.Prefix,
+		Rounds:     sha512.DefaultRounds,
+		Salt:       salt,
+	}
+
+	key, err := sha512.Key([]byte(password), scheme.Salt, scheme.Rounds)
+	if err != nil {
+		return "", err
+	}
+	crypthash.LittleEndianEncoding.Encode(scheme.Sum[:], key)
+	return crypthash.Marshal(scheme)
+}

cmd/dkl-local-server/ssh-secrets.go

@@ -0,0 +1,168 @@
+package main
+
+import (
+	"crypto/dsa"
+	"crypto/ecdsa"
+	"crypto/ed25519"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/asn1"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+)
+
+var sshHostKeys = KVSecrets[[]SSHKeyPair]{"hosts/ssh-host-keys"}
+
+type SSHKeyPair struct {
+	Type    string
+	Public  string
+	Private string
+}
+
+func getSSHKeyPairs(host string) (pairs []SSHKeyPair, err error) {
+	pairs, _, err = sshHostKeys.Get(host)
+
+	didGenerate := false
+
+genLoop:
+	for _, keyType := range []string{
+		"rsa",
+		"dsa",
+		"ecdsa",
+		"ed25519",
+	} {
+		for _, pair := range pairs {
+			if pair.Type == keyType {
+				continue genLoop
+			}
+		}
+
+		err = func() (err error) {
+			outFile, err := ioutil.TempFile("/tmp", "dls-key.")
+			if err != nil {
+				return
+			}
+
+			outPath := outFile.Name()
+
+			removeTemp := func() {
+				os.Remove(outPath)
+				os.Remove(outPath + ".pub")
+			}
+
+			removeTemp()
+			defer removeTemp()
+
+			var out, privKey, pubKey []byte
+
+			cmd := exec.Command("ssh-keygen",
+				"-N", "",
+				"-C", "root@"+host,
+				"-f", outPath,
+				"-t", keyType)
+			out, err = cmd.CombinedOutput()
+			if err != nil {
+				err = fmt.Errorf("ssh-keygen failed: %v: %s", err, string(out))
+				return
+			}
+
+			privKey, err = ioutil.ReadFile(outPath)
+			if err != nil {
+				return
+			}
+
+			pubKey, err = ioutil.ReadFile(outPath + ".pub")
+			if err != nil {
+				return
+			}
+
+			pairs = append(pairs, SSHKeyPair{
+				Type:    keyType,
+				Public:  string(pubKey),
+				Private: string(privKey),
+			})
+			didGenerate = true
+
+			return
+		}()
+
+		if err != nil {
+			return
+		}
+	}
+
+	if didGenerate {
+		err = sshHostKeys.Put(host, pairs)
+		if err != nil {
+			return
+		}
+	}
+
+	return
+}
+
+func sshKeyGenDSA() (data []byte, pubKey interface{}, err error) {
+	privKey := &dsa.PrivateKey{}
+
+	err = dsa.GenerateParameters(&privKey.Parameters, rand.Reader, dsa.L1024N160)
+	if err != nil {
+		return
+	}
+
+	err = dsa.GenerateKey(privKey, rand.Reader)
+	if err != nil {
+		return
+	}
+
+	data, err = asn1.Marshal(*privKey)
+	//data, err = x509.MarshalPKCS8PrivateKey(privKey)
+	if err != nil {
+		return
+	}
+
+	pubKey = privKey.PublicKey
+	return
+}
+
+func sshKeyGenRSA() (data []byte, pubKey interface{}, err error) {
+	privKey, err := rsa.GenerateKey(rand.Reader, 2048)
+	if err != nil {
+		return
+	}
+
+	data = x509.MarshalPKCS1PrivateKey(privKey)
+	pubKey = privKey.Public()
+
+	return
+}
+
+func sshKeyGenECDSA() (data []byte, pubKey interface{}, err error) {
+	privKey, err := ecdsa.GenerateKey(elliptic.P521(), rand.Reader)
+	if err != nil {
+		return
+	}
+
+	data, err = x509.MarshalPKCS8PrivateKey(privKey)
+	if err != nil {
+		return
+	}
+
+	pubKey = privKey.Public()
+
+	return
+}
+
+func sshKeyGenED25519() (data []byte, pubKey interface{}, err error) {
+	pubKey, privKey, err := ed25519.GenerateKey(rand.Reader)
+
+	data, err = x509.MarshalPKCS8PrivateKey(privKey)
+	if err != nil {
+		return
+	}
+
+	return
+}

cmd/dkl-local-server/ssh-secrets_test.go

@@ -0,0 +1,10 @@
+package main
+
+import "testing"
+
+func TestSSHKeyGet(t *testing.T) {
+	// TODO needs fake secret store
+	// if _, err := getSSHKeyPairs("host"); err != nil {
+	// 	t.Error(err)
+	// }
+}

cmd/dkl-local-server/state.go

@@ -0,0 +1,138 @@
+package main
+
+import (
+	"log"
+
+	"m.cluseau.fr/go/watchable"
+	"novit.tech/direktil/pkg/localconfig"
+)
+
+type PublicState struct {
+	ServerVersion string
+	UIHash        string
+	Store         struct {
+		New  bool
+		Open bool
+	}
+}
+
+var wPublicState = watchable.New[PublicState]()
+
+type State struct {
+	HasConfig bool
+
+	Store struct {
+		DownloadToken string
+		KeyNames      []string
+	}
+
+	Clusters []ClusterState
+	Hosts    []HostState
+	Config   *localconfig.Config
+
+	Downloads map[string]DownloadSpec
+}
+
+type ClusterState struct {
+	Name      string
+	Addons    bool
+	Passwords []string
+	Tokens    []string
+	CAs       []CAState
+}
+
+type HostState struct {
+	Name    string
+	Cluster string
+	IPs     []string
+}
+
+type CAState struct {
+	Name   string
+	Signed []string
+}
+
+var wState = watchable.New[State]()
+
+func init() {
+	wState.Set(State{Downloads: map[string]DownloadSpec{}})
+}
+
+func updateState() {
+	log.Print("updating state")
+
+	// store key names
+	keyNames := make([]string, 0, len(secStore.Keys))
+	for _, key := range secStore.Keys {
+		keyNames = append(keyNames, key.Name)
+	}
+
+	// config
+	cfg, err := readConfig()
+	if err != nil {
+		wState.Change(func(v *State) { v.HasConfig = false; v.Config = nil; v.Store.KeyNames = keyNames })
+		return
+	}
+
+	if secStore.IsNew() || !secStore.Unlocked() {
+		wState.Change(func(v *State) { v.HasConfig = false; v.Config = nil; v.Store.KeyNames = keyNames })
+		return
+	}
+
+	// remove heavy data
+	clusters := make([]ClusterState, 0, len(cfg.Clusters))
+	for _, cluster := range cfg.Clusters {
+		c := ClusterState{
+			Name:   cluster.Name,
+			Addons: len(cluster.Addons) != 0,
+		}
+
+		c.Passwords, err = clusterPasswords.Keys(c.Name + "/")
+		if err != nil {
+			log.Print("failed to read cluster passwords: ", err)
+		}
+		c.Tokens, err = clusterTokens.Keys(c.Name + "/")
+		if err != nil {
+			log.Print("failed to read cluster tokens: ", err)
+		}
+
+		caNames, err := clusterCAs.Keys(c.Name + "/")
+		if err != nil {
+			log.Print("failed to read cluster CAs: ", err)
+		}
+		for _, caName := range caNames {
+			ca := CAState{Name: caName}
+
+			signedNames, err := clusterCASignedKeys.Keys(c.Name + "/" + caName + "/")
+			if err != nil {
+				log.Print("failed to read cluster CA signed keys: ", err)
+			}
+			for _, signedName := range signedNames {
+				ca.Signed = append(ca.Signed, signedName)
+			}
+
+			c.CAs = append(c.CAs, ca)
+		}
+
+		clusters = append(clusters, c)
+	}
+
+	hosts := make([]HostState, 0, len(cfg.Hosts))
+	for _, host := range cfg.Hosts {
+		h := HostState{
+			Name:    host.Name,
+			Cluster: host.ClusterName,
+			IPs:     host.IPs,
+		}
+
+		hosts = append(hosts, h)
+	}
+
+	// done
+	wState.Change(func(v *State) {
+		v.HasConfig = true
+		v.Store.KeyNames = keyNames
+		v.Clusters = clusters
+		v.Hosts = hosts
+	})
+}

cmd/dkl-local-server/tls-ca.go

@@ -0,0 +1,178 @@
+package main
+
+import (
+	"crypto"
+	"errors"
+	"fmt"
+	"log"
+	"net"
+
+	"github.com/cloudflare/cfssl/config"
+	"github.com/cloudflare/cfssl/csr"
+	"github.com/cloudflare/cfssl/helpers"
+	"github.com/cloudflare/cfssl/initca"
+	"github.com/cloudflare/cfssl/signer"
+	"github.com/cloudflare/cfssl/signer/local"
+	"k8s.io/apimachinery/pkg/util/validation"
+)
+
+type CA struct {
+	Key  []byte
+	Cert []byte
+
+	Signed map[string]*KeyCert
+}
+
+func (ca *CA) Init() (err error) {
+	req := ca.newReq()
+
+	cert, _, key, err := initca.New(req)
+	if err != nil {
+		err = fmt.Errorf("initca: %w", err)
+		return
+	}
+
+	ca.Key = key
+	ca.Cert = cert
+
+	return
+}
+
+func (ca *CA) RenewCert() (err error) {
+	var signer crypto.Signer
+	signer, err = helpers.ParsePrivateKeyPEM(ca.Key)
+	if err != nil {
+		return
+	}
+
+	newCert, _, err := initca.NewFromSigner(ca.newReq(), signer)
+	if err != nil {
+		return
+	}
+
+	ca.Cert = newCert
+
+	return
+}
+
+func (_ CA) newReq() *csr.CertificateRequest {
+	return &csr.CertificateRequest{
+		CN: "Direktil Local Server",
+		KeyRequest: &csr.KeyRequest{
+			A: "ecdsa",
+			S: 521, // 256, 384, 521
+		},
+		Names: []csr.Name{
+			{
+				O: "novit.io",
+			},
+		},
+	}
+}
+
+func (ca CA) Signer(policy *config.Signing) (result *local.Signer, err error) {
+	caCert, err := helpers.ParseCertificatePEM(ca.Cert)
+	if err != nil {
+		return
+	}
+
+	caKey, err := helpers.ParsePrivateKeyPEM(ca.Key)
+	if err != nil {
+		return
+	}
+
+	return local.NewSigner(caKey, caCert, signer.DefaultSigAlgo(caKey), policy)
+}
+
+func getUsableKeyCert(cluster, caName, name, profile, label string, req *csr.CertificateRequest, cfg *config.Config) (kc KeyCert, err error) {
+	log := log.New(log.Default().Writer(), cluster+": CA "+caName+": ", log.Flags()|log.Lmsgprefix)
+
+	ca, err := getUsableClusterCA(cluster, caName)
+	if err != nil {
+		return
+	}
+
+	for _, host := range req.Hosts {
+		if ip := net.ParseIP(host); ip != nil {
+			// valid IP (v4 or v6)
+			continue
+		}
+
+		if host == "*" {
+			continue
+		}
+
+		if errs := validation.IsDNS1123Subdomain(host); len(errs) == 0 {
+			continue
+		}
+		if errs := validation.IsWildcardDNS1123Subdomain(host); len(errs) == 0 {
+			continue
+		}
+
+		err = fmt.Errorf("%q is not an IP or FQDN", host)
+		return
+	}
+
+	if req.CA != nil {
+		err = errors.New("no CA section allowed here")
+		return
+	}
+
+	rh := hash(req)
+
+	key := cluster + "/" + caName + "/" + name
+
+	kc, found, err := clusterCASignedKeys.Get(key)
+	if err != nil {
+		return
+	}
+
+	if found {
+		if rh == kc.ReqHash {
+			err = checkCertUsable(kc.Cert)
+			if err == nil {
+				return // all good, no need to create or renew
+			}
+
+			log.Print("regenerating certificate: ", err)
+
+		} else {
+			log.Printf("CSR changed for %s: hash=%q previous=%q", name, rh, kc.ReqHash)
+		}
+	} else {
+		log.Print("new CSR for ", name)
+	}
+
+	sgr, err := ca.Signer(cfg.Signing)
+	if err != nil {
+		return
+	}
+
+	generator := &csr.Generator{Validator: func(_ *csr.CertificateRequest) error { return nil }}
+
+	csr, tlsKey, err := generator.ProcessRequest(req)
+	if err != nil {
+		return
+	}
+
+	signReq := signer.SignRequest{
+		Request: string(csr),
+		Profile: profile,
+		Label:   label,
+	}
+
+	cert, err := sgr.Sign(signReq)
+	if err != nil {
+		return
+	}
+
+	kc = KeyCert{
+		Key:     tlsKey,
+		Cert:    cert,
+		ReqHash: rh,
+	}
+
+	err = clusterCASignedKeys.Put(key, kc)
+
+	return
+}

cmd/dkl-local-server/token.go

@@ -0,0 +1,24 @@
+package main
+
+import (
+	"crypto/rand"
+	"encoding/base32"
+	"log"
+	"net/http"
+
+	"m.cluseau.fr/go/httperr"
+)
+
+func newToken(sizeInBytes int) (token string, err error) {
+	randBytes := make([]byte, sizeInBytes)
+
+	_, err = rand.Read(randBytes)
+	if err != nil {
+		log.Print("rand read error: ", err)
+		err = httperr.New(http.StatusInternalServerError, err)
+		return
+	}
+
+	token = base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(randBytes)
+	return
+}

cmd/dkl-local-server/ui.go

@@ -0,0 +1,45 @@
+package main
+
+import (
+	"encoding/base32"
+	"io"
+	"io/fs"
+	"log"
+	"strings"
+
+	"github.com/cespare/xxhash"
+	dlshtml "novit.tech/direktil/local-server/html"
+)
+
+func computeUIHash() {
+	xxh := xxhash.New()
+
+	err := fs.WalkDir(dlshtml.FS, "ui", func(path string, entry fs.DirEntry, walkErr error) (err error) {
+		if walkErr != nil {
+			err = walkErr
+			return
+		}
+
+		if entry.IsDir() {
+			return
+		}
+
+		f, err := dlshtml.FS.Open(path)
+		if err != nil {
+			return
+		}
+		defer f.Close()
+
+		io.Copy(xxh, f)
+
+		return nil
+	})
+
+	if err != nil {
+		log.Fatal("failed to hash UI: ", err)
+	}
+
+	h := strings.ToLower(base32.HexEncoding.WithPadding(base32.NoPadding).EncodeToString(xxh.Sum(nil)))[:5]
+	log.Printf("UI hash: %s", h)
+	wPublicState.Change(func(v *PublicState) { v.UIHash = h })
+}

cmd/dkl-local-server/upstream.go

@@ -1,18 +1,25 @@
 package main
 
 import (
+	"crypto/sha1"
+	"encoding/hex"
 	"flag"
+	"fmt"
 	"io"
 	"log"
 	"net/http"
 	"os"
 	gopath "path"
 	"path/filepath"
+	"strconv"
 	"time"
+
+	"github.com/dustin/go-humanize"
+	"github.com/miolini/datacounter"
 )
 
 var (
-	upstreamURL = flag.String("upstream", "https://direktil.novit.nc/dist", "Upstream server for dist elements")
+	upstreamURL = flag.String("upstream", "https://dkl.novit.io/dist", "Upstream server for dist elements")
 )
 
 func (ctx *renderContext) distFetch(path ...string) (outPath string, err error) {
@@ -39,42 +46,82 @@ func (ctx *renderContext) distFetch(path ...string) (outPath string, err error)
 		return
 	}
 
-	tempOutPath := filepath.Join(filepath.Dir(outPath), "._part_"+filepath.Base(outPath))
-
-	done := make(chan error, 1)
-	go func() {
 	defer resp.Body.Close()
-		defer close(done)
 
-		out, err := os.Create(tempOutPath)
-		if err != nil {
-			done <- err
+	if resp.StatusCode != 200 {
+		err = fmt.Errorf("wrong status: %s", resp.Status)
 		return
 	}
 
-		defer out.Close()
+	length, _ := strconv.Atoi(resp.Header.Get("Content-Length"))
+
+	fOut, err := os.Create(filepath.Join(filepath.Dir(outPath), "._part_"+filepath.Base(outPath)))
+	if err != nil {
+		return
+	}
+
+	hash := sha1.New()
+
+	body := datacounter.NewReaderCounter(resp.Body)
+	out := io.MultiWriter(fOut, hash)
+
+	done := make(chan error, 1)
+	go func() {
+		_, err = io.Copy(out, body)
+		fOut.Close()
+
+		if err != nil {
+			os.Remove(fOut.Name())
+		}
 
-		_, err = io.Copy(out, resp.Body)
 		done <- err
+		close(done)
 	}()
 
+	start := time.Now()
+
 wait:
 	select {
 	case <-time.After(10 * time.Second):
-		log.Print("still fetching ", subPath, "...")
+		status := ""
+		if length != 0 {
+			count := body.Count()
+			elapsedDuration := time.Since(start)
+
+			progress := float64(count) / float64(length)
+
+			elapsed := float64(elapsedDuration)
+			remaining := time.Duration(elapsed/progress - elapsed)
+
+			status = fmt.Sprintf(" (%.2f%%, ETA %v, %s/s)",
+				progress*100,
+				remaining.Truncate(time.Second),
+				humanize.Bytes(uint64(float64(count)/elapsedDuration.Seconds())))
+		}
+		log.Printf("still fetching %s%s...", subPath, status)
 		goto wait
 
 	case err = <-done:
 		if err != nil {
 			log.Print("fetch of ", subPath, " failed: ", err)
-			os.Remove(tempOutPath)
 			return
 		}
-
-		log.Print("fetch of ", subPath, " finished")
 	}
 
-	err = os.Rename(tempOutPath, outPath)
+	hexSum := hex.EncodeToString(hash.Sum(nil))
+	log.Printf("fetch of %s finished (SHA1 checksum: %s)", subPath, hexSum)
+
+	if remoteSum := resp.Header.Get("X-Content-SHA1"); remoteSum != "" {
+		log.Printf("fetch of %s: remote SHA1 checksum: %s", subPath, remoteSum)
+		if remoteSum != hexSum {
+			err = fmt.Errorf("wrong SHA1 checksum: server=%s local=%s", remoteSum, hexSum)
+			log.Print("fetch of ", subPath, ": ", err)
+			os.Remove(fOut.Name())
+			return
+		}
+	}
+
+	err = os.Rename(fOut.Name(), outPath)
 
 	return
 }

cmd/dkl-local-server/ws-auth.go

@@ -7,24 +7,20 @@ import (
 )
 
 func adminAuth(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) {
-	tokenAuth(req, resp, chain, *adminToken)
-}
-
-func hostsAuth(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) {
-	tokenAuth(req, resp, chain, *hostsToken, *adminToken)
+	tokenAuth(req, resp, chain, adminToken)
 }
 
 func tokenAuth(req *restful.Request, resp *restful.Response, chain *restful.FilterChain, allowedTokens ...string) {
 	token := getToken(req)
 
 	for _, allowedToken := range allowedTokens {
-		if allowedToken == "" || token == allowedToken {
+		if allowedToken != "" && token == allowedToken {
 			chain.ProcessFilter(req, resp)
 			return
 		}
 	}
 
-	resp.WriteErrorString(401, "401: Not Authorized")
+	wsError(resp, ErrUnauthorized)
 	return
 }
 
@@ -33,8 +29,12 @@ func getToken(req *restful.Request) string {
 
 	token := req.HeaderParameter("Authorization")
 
+	if token == "" {
+		return req.QueryParameter("token")
+	}
+
 	if !strings.HasPrefix(token, bearerPrefix) {
-		return ""
+		return token
 	}
 
 	return token[len(bearerPrefix):]

cmd/dkl-local-server/ws-cluster-cas.go

@@ -0,0 +1,83 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/cloudflare/cfssl/log"
+	restful "github.com/emicklei/go-restful"
+)
+
+var clusterCAs = newClusterSecretKV[CA]("CAs")
+
+func wsClusterCAs(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	clusterCAs.WsList(resp, clusterName+"/")
+}
+
+func wsClusterCA(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	name := req.PathParameter("ca-name")
+
+	clusterCAs.WsGet(resp, clusterName+"/"+name)
+}
+
+func getUsableClusterCA(cluster, name string) (ca CA, err error) {
+	defer func() {
+		if err != nil {
+			err = fmt.Errorf("cluster %s CA %s: %w", cluster, name, err)
+		}
+	}()
+
+	key := cluster + "/" + name
+
+	ca, found, err := clusterCAs.Get(key)
+	if err != nil {
+		return
+	}
+
+	if !found {
+		log.Info("new CA in cluster ", cluster, ": ", name)
+
+		err = ca.Init()
+		if err != nil {
+			return
+		}
+
+		err = clusterCAs.Put(key, ca)
+		if err != nil {
+			return
+		}
+
+		return
+	}
+
+	checkErr := checkCertUsable(ca.Cert)
+	if checkErr != nil {
+		log.Infof("cluster %s: CA %s: regenerating certificate: %v", cluster, name, checkErr)
+
+		err = ca.RenewCert()
+		if err != nil {
+			err = fmt.Errorf("renew: %w", err)
+		}
+
+		err = clusterCAs.Put(key, ca)
+	}
+
+	return
+}
+
+var clusterCASignedKeys = newClusterSecretKV[KeyCert]("CA-signed-keys")
+
+func wsClusterCASignedKeys(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	caName := req.PathParameter("ca-name")
+	clusterCASignedKeys.WsList(resp, clusterName+"/"+caName+"/")
+}
+
+func wsClusterCASignedKey(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	caName := req.PathParameter("ca-name")
+	name := req.PathParameter("signed-name")
+
+	clusterCASignedKeys.WsGet(resp, clusterName+"/"+caName+"/"+name)
+}

cmd/dkl-local-server/ws-cluster-passwords.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	restful "github.com/emicklei/go-restful"
+)
+
+var seeds = newClusterSecretKV[[]byte]("seeds")
+
+var clusterPasswords = newClusterSecretKV[string]("passwords")
+
+func wsClusterPasswords(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	clusterPasswords.WsList(resp, clusterName+"/")
+}
+
+func wsClusterPassword(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	name := req.PathParameter("password-name")
+
+	clusterPasswords.WsGet(resp, clusterName+"/"+name)
+}
+
+func wsClusterSetPassword(req *restful.Request, resp *restful.Response) {
+	cluster := wsReadCluster(req, resp)
+	if cluster == nil {
+		return
+	}
+
+	name := req.PathParameter("password-name")
+
+	clusterPasswords.WsPut(req, resp, cluster.Name+"/"+name)
+}

cmd/dkl-local-server/ws-cluster-tokens.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"crypto/rand"
+	"encoding/base32"
+
+	restful "github.com/emicklei/go-restful"
+)
+
+var clusterTokens = newClusterSecretKV[string]("tokens")
+
+func getOrCreateClusterToken(cluster, name string) (token string, err error) {
+	key := cluster + "/" + name
+
+	token, found, err := clusterTokens.Get(key)
+
+	if err != nil || found {
+		return
+	}
+
+	b := make([]byte, 16)
+	_, err = rand.Read(b)
+	if err != nil {
+		return
+	}
+
+	token = base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(b)
+
+	err = clusterTokens.Put(key, token)
+	return
+}
+
+func wsClusterTokens(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	clusterTokens.WsList(resp, clusterName+"/")
+}
+
+func wsClusterToken(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	name := req.PathParameter("token-name")
+
+	clusterTokens.WsGet(resp, clusterName+"/"+name)
+}

cmd/dkl-local-server/ws-clusters.go

@@ -2,11 +2,22 @@ package main
 
 import (
 	"log"
+	"net/url"
+	"strconv"
 
 	restful "github.com/emicklei/go-restful"
-	"novit.nc/direktil/pkg/localconfig"
+
+	"novit.tech/direktil/local-server/pkg/mime"
+	"novit.tech/direktil/pkg/localconfig"
 )
 
+var clusterSecretKVs = []string{}
+
+func newClusterSecretKV[T any](name string) KVSecrets[T] {
+	clusterSecretKVs = append(clusterSecretKVs, name)
+	return KVSecrets[T]{"clusters/" + name}
+}
+
 func wsListClusters(req *restful.Request, resp *restful.Response) {
 	cfg := wsReadConfig(resp)
 	if cfg == nil {
@@ -31,7 +42,7 @@ func wsReadCluster(req *restful.Request, resp *restful.Response) (cluster *local
 
 	cluster = cfg.Cluster(clusterName)
 	if cluster == nil {
-		wsNotFound(req, resp)
+		wsNotFound(resp)
 		return
 	}
 
@@ -55,9 +66,58 @@ func wsClusterAddons(req *restful.Request, resp *restful.Response) {
 
 	if len(cluster.Addons) == 0 {
 		log.Printf("cluster %q has no addons defined", cluster.Name)
-		wsNotFound(req, resp)
+		wsNotFound(resp)
 		return
 	}
 
-	resp.Write([]byte(cluster.Addons))
+	cfg := wsReadConfig(resp)
+	if cfg == nil {
+		return
+	}
+
+	sslCfg, err := sslConfigFromLocalConfig(cfg)
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+
+	wsRender(resp, sslCfg, cluster.Addons, cluster)
+}
+
+func wsClusterCACert(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	caName := req.PathParameter("ca-name")
+
+	ca, found, err := clusterCAs.Get(clusterName + "/" + caName)
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+	if !found {
+		wsNotFound(resp)
+		return
+	}
+
+	resp.Header().Set("Content-Type", mime.CERT)
+	resp.Write(ca.Cert)
+}
+
+func wsClusterSignedCert(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	caName := req.PathParameter("ca-name")
+	name := req.QueryParameter("name")
+
+	kc, found, err := clusterCASignedKeys.Get(clusterName + "/" + caName + "/" + name)
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+	if !found {
+		wsNotFound(resp)
+		return
+	}
+
+	resp.AddHeader("Content-Type", mime.CERT)
+	resp.AddHeader("Content-Disposition", "attachment; filename="+strconv.Quote(clusterName+"_"+caName+"_"+url.PathEscape(name)+".crt"))
+	resp.Write(kc.Cert)
 }

cmd/dkl-local-server/ws-configs.go

@@ -18,7 +18,10 @@ func wsUploadConfig(req *restful.Request, resp *restful.Response) {
 
 	if err != nil {
 		wsError(resp, err)
+		return
 	}
+
+	resp.WriteEntity(true)
 }
 
 func writeNewConfig(reader io.Reader) (err error) {
@@ -35,8 +38,30 @@ func writeNewConfig(reader io.Reader) (err error) {
 		return
 	}
 
-	archivesPath := filepath.Join(*dataDir, "archives")
 	cfgPath := configFilePath()
+	in, err := os.Open(cfgPath)
+
+	if err != nil {
+		if os.IsNotExist(err) {
+			// nothing to backup
+		} else {
+			return // real error
+		}
+	} else {
+		err = backupCurrentConfig(in)
+		if err != nil {
+			return
+		}
+	}
+
+	err = os.Rename(out.Name(), cfgPath)
+
+	updateState()
+	return
+}
+
+func backupCurrentConfig(in io.ReadCloser) (err error) {
+	archivesPath := filepath.Join(*dataDir, "archives")
 
 	err = os.MkdirAll(archivesPath, 0700)
 	if err != nil {
@@ -52,11 +77,6 @@ func writeNewConfig(reader io.Reader) (err error) {
 
 	defer bck.Close()
 
-	in, err := os.Open(cfgPath)
-	if err != nil {
-		return
-	}
-
 	gz, err := gzip.NewWriterLevel(bck, 2)
 	if err != nil {
 		return
@@ -66,10 +86,5 @@ func writeNewConfig(reader io.Reader) (err error) {
 	gz.Close()
 	in.Close()
 
-	if err != nil {
-		return
-	}
-
-	err = os.Rename(out.Name(), cfgPath)
 	return
 }

cmd/dkl-local-server/ws-downloads.go

@@ -0,0 +1,151 @@
+package main
+
+import (
+	"crypto/rand"
+	"encoding/base32"
+	"log"
+	"net/http"
+	"strconv"
+	"time"
+
+	restful "github.com/emicklei/go-restful"
+	"m.cluseau.fr/go/cow"
+)
+
+type DownloadSpec struct {
+	Kind   string
+	Name   string
+	Assets []string
+
+	createdAt time.Time
+}
+
+func wsAuthorizeDownload(req *restful.Request, resp *restful.Response) {
+	var spec DownloadSpec
+
+	if err := req.ReadEntity(&spec); err != nil {
+		wsError(resp, err)
+		return
+	}
+
+	if spec.Kind == "" || spec.Name == "" || len(spec.Assets) == 0 {
+		resp.WriteErrorString(http.StatusBadRequest, "missing data")
+		return
+	}
+
+	randBytes := make([]byte, 32)
+	_, err := rand.Read(randBytes)
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+
+	token := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(randBytes)
+
+	spec.createdAt = time.Now()
+
+	wState.Change(func(v *State) {
+		cow.MapSet(&v.Downloads, token, spec)
+	})
+
+	log.Printf("download token created for %s %q, assets %q", spec.Kind, spec.Name, spec.Assets)
+
+	resp.WriteAsJson(token)
+}
+
+func wsDownload(req *restful.Request, resp *restful.Response) {
+	token := req.PathParameter("token")
+	asset := req.PathParameter("asset")
+
+	if token == "" || asset == "" {
+		wsNotFound(resp)
+		return
+	}
+
+	var spec DownloadSpec
+	found := false
+	wState.Change(func(v *State) {
+		var ok bool
+		spec, ok = v.Downloads[token]
+		if !ok {
+			return
+		}
+
+		newAssets := make([]string, 0, len(spec.Assets))
+		for _, a := range spec.Assets {
+			if a == asset {
+				found = true
+			} else {
+				newAssets = append(newAssets, a)
+			}
+		}
+
+		if !found {
+			wsNotFound(resp)
+			return
+		}
+
+		cow.Map(&v.Downloads)
+
+		if len(newAssets) == 0 {
+			delete(v.Downloads, token)
+		} else {
+			spec.Assets = newAssets
+			v.Downloads[token] = spec
+		}
+	})
+
+	if !found {
+		wsNotFound(resp)
+		return
+	}
+
+	log.Printf("download via token: %s %q asset %q", spec.Kind, spec.Name, asset)
+
+	cfg, err := readConfig()
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+
+	setHeader := func(ext string) {
+		resp.AddHeader("Content-Disposition", "attachment; filename="+strconv.Quote(spec.Kind+"_"+spec.Name+"_"+asset+ext))
+	}
+
+	switch spec.Kind {
+	case "cluster":
+		cluster := cfg.ClusterByName(spec.Name)
+		if cluster == nil {
+			wsNotFound(resp)
+			return
+		}
+
+		switch asset {
+		case "addons":
+			setHeader(".yaml")
+			resp.Write([]byte(cluster.Addons))
+
+		default:
+			wsNotFound(resp)
+		}
+
+	case "host":
+		host := cfg.Host(spec.Name)
+		if host == nil {
+			wsNotFound(resp)
+			return
+		}
+
+		switch asset {
+		case "config", "bootstrap-config":
+			setHeader(".yaml")
+		default:
+			setHeader("")
+		}
+
+		renderHost(resp.ResponseWriter, req.Request, asset, host, cfg)
+
+	default:
+		wsNotFound(resp)
+	}
+}

cmd/dkl-local-server/ws-host.go

@@ -8,26 +8,30 @@ import (
 
 	restful "github.com/emicklei/go-restful"
 
-	"novit.nc/direktil/local-server/pkg/mime"
-	"novit.nc/direktil/pkg/localconfig"
+	"novit.tech/direktil/pkg/localconfig"
+
+	"novit.tech/direktil/local-server/pkg/mime"
 )
 
-var trustXFF = flag.Bool("trust-xff", true, "Trust the X-Forwarded-For header")
+var (
+	allowDetectedHost = flag.Bool("allow-detected-host", false, "Allow access to host assets from its IP (insecure but enables unattended netboot)")
+	trustXFF          = flag.Bool("trust-xff", false, "Trust the X-Forwarded-For header")
+)
 
 type wsHost struct {
-	prefix  string
 	hostDoc string
-	getHost func(req *restful.Request) string
+	getHost func(req *restful.Request) (hostName string, err error)
 }
 
-func (ws *wsHost) register(rws *restful.WebService, alterRB func(*restful.RouteBuilder)) {
+func (ws wsHost) register(rws *restful.WebService, alterRB func(*restful.RouteBuilder)) {
 	b := func(what string) *restful.RouteBuilder {
-		return rws.GET(ws.prefix + "/" + what).To(ws.render)
+		return rws.GET("/" + what).To(ws.render)
 	}
 
 	for _, rb := range []*restful.RouteBuilder{
-		rws.GET(ws.prefix).To(ws.get).
-			Doc("Get the " + ws.hostDoc + "'s details"),
+		rws.GET("").To(ws.get).
+			Doc("Get the "+ws.hostDoc+"'s details").
+			Returns(200, "OK", localconfig.Host{}),
 
 		// raw configuration
 		b("config").
@@ -54,10 +58,14 @@ func (ws *wsHost) register(rws *restful.WebService, alterRB func(*restful.RouteB
 		b("boot.tar").
 			Produces(mime.TAR).
 			Doc("Get the " + ws.hostDoc + "'s /boot archive (ie: for metal upgrades)"),
+		b("boot-efi.tar").
+			Produces(mime.TAR).
+			Doc("Get the " + ws.hostDoc + "'s /boot archive (ie: for metal upgrades)"),
 
 		// read-only ISO support
 		b("boot.iso").
 			Produces(mime.ISO).
+			Param(cmdlineParam).
 			Doc("Get the " + ws.hostDoc + "'s boot CD-ROM image"),
 
 		// netboot support
@@ -65,27 +73,42 @@ func (ws *wsHost) register(rws *restful.WebService, alterRB func(*restful.RouteB
 			Produces(mime.IPXE).
 			Doc("Get the " + ws.hostDoc + "'s IPXE code (for netboot)"),
 
+		// boot support
 		b("kernel").
 			Produces(mime.OCTET).
 			Doc("Get the " + ws.hostDoc + "'s kernel (ie: for netboot)"),
-
 		b("initrd").
 			Produces(mime.OCTET).
 			Doc("Get the " + ws.hostDoc + "'s initial RAM disk (ie: for netboot)"),
+
+		// - bootstrap config
+		b("bootstrap-config").
+			Produces(mime.YAML).
+			Doc("Get the " + ws.hostDoc + "'s bootstrap configuration"),
+		b("bootstrap-config.json").
+			Doc("Get the " + ws.hostDoc + "'s bootstrap configuration (as JSON)"),
+		// - bootstrap
+		b("bootstrap.tar").
+			Produces(mime.TAR).
+			Doc("Get the " + ws.hostDoc + "'s bootstrap seed archive"),
 	} {
 		alterRB(rb)
 		rws.Route(rb)
 	}
 }
 
-func (ws *wsHost) host(req *restful.Request, resp *restful.Response) (host *localconfig.Host, cfg *localconfig.Config) {
-	hostname := ws.getHost(req)
+func (ws wsHost) host(req *restful.Request, resp *restful.Response) (host *localconfig.Host, cfg *localconfig.Config) {
+	hostname, err := ws.getHost(req)
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
 	if hostname == "" {
-		wsNotFound(req, resp)
+		wsNotFound(resp)
 		return
 	}
 
-	cfg, err := readConfig()
+	cfg, err = readConfig()
 	if err != nil {
 		wsError(resp, err)
 		return
@@ -94,13 +117,13 @@ func (ws *wsHost) host(req *restful.Request, resp *restful.Response) (host *loca
 	host = cfg.Host(hostname)
 	if host == nil {
 		log.Print("no host named ", hostname)
-		wsNotFound(req, resp)
+		wsNotFound(resp)
 		return
 	}
 	return
 }
 
-func (ws *wsHost) get(req *restful.Request, resp *restful.Response) {
+func (ws wsHost) get(req *restful.Request, resp *restful.Response) {
 	host, _ := ws.host(req, resp)
 	if host == nil {
 		return
@@ -109,7 +132,7 @@ func (ws *wsHost) get(req *restful.Request, resp *restful.Response) {
 	resp.WriteEntity(host)
 }
 
-func (ws *wsHost) render(req *restful.Request, resp *restful.Response) {
+func (ws wsHost) render(req *restful.Request, resp *restful.Response) {
 	host, cfg := ws.host(req, resp)
 	if host == nil {
 		return
@@ -141,21 +164,26 @@ func renderHost(w http.ResponseWriter, r *http.Request, what string, host *local
 	case "kernel":
 		err = renderKernel(w, r, ctx)
 
+	// boot v2
+	case "bootstrap-config":
+		err = renderBootstrapConfig(w, r, ctx, false)
+	case "bootstrap-config.json":
+		err = renderBootstrapConfig(w, r, ctx, true)
 	case "initrd":
 		err = renderCtx(w, r, ctx, what, buildInitrd)
-
+	case "bootstrap.tar":
+		err = renderCtx(w, r, ctx, what, buildBootstrap)
 	case "boot.iso":
 		err = renderCtx(w, r, ctx, what, buildBootISO)
-
 	case "boot.tar":
 		err = renderCtx(w, r, ctx, what, buildBootTar)
+	case "boot-efi.tar":
+		err = renderCtx(w, r, ctx, what, buildBootEFITar)
 
 	case "boot.img":
 		err = renderCtx(w, r, ctx, what, buildBootImg)
-
 	case "boot.img.gz":
 		err = renderCtx(w, r, ctx, what, buildBootImgGZ)
-
 	case "boot.img.lz4":
 		err = renderCtx(w, r, ctx, what, buildBootImgLZ4)
 

cmd/dkl-local-server/ws-public.go

@@ -0,0 +1,195 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"io"
+	"io/fs"
+	"log"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	restful "github.com/emicklei/go-restful"
+	"m.cluseau.fr/go/httperr"
+	"novit.tech/direktil/local-server/secretstore"
+)
+
+type NamedPassphrase struct {
+	Name       string
+	Passphrase []byte
+}
+
+func wsUnlockStore(req *restful.Request, resp *restful.Response) {
+	np := NamedPassphrase{}
+	err := req.ReadEntity(&np)
+	if err != nil {
+		resp.WriteError(http.StatusBadRequest, err)
+		return
+	}
+
+	defer secretstore.Memzero(np.Passphrase)
+
+	if secStore.IsNew() {
+		if len(np.Name) == 0 {
+			wsBadRequest(resp, "no name given")
+			return
+		}
+	}
+
+	if len(np.Passphrase) == 0 {
+		wsBadRequest(resp, "no passphrase given")
+		return
+	}
+
+	if secStore.Unlocked() {
+		if secStore.HasKey(np.Passphrase) {
+			resp.WriteEntity(adminToken)
+		} else {
+			wsError(resp, ErrUnauthorized)
+		}
+		return
+	}
+
+	if err := unlockSecretStore(np.Name, np.Passphrase); err.Any() {
+		err.WriteJSON(resp.ResponseWriter)
+		return
+	}
+
+	resp.WriteEntity(adminToken)
+}
+
+func wsStoreDownload(req *restful.Request, resp *restful.Response) {
+	token := req.QueryParameter("token")
+	if token != wState.Get().Store.DownloadToken {
+		wsError(resp, ErrInvalidToken)
+		return
+	}
+
+	buf := new(bytes.Buffer)
+	arch := tar.NewWriter(buf)
+
+	root := os.DirFS(secStoreRoot())
+
+	err := fs.WalkDir(root, ".", func(path string, d fs.DirEntry, readErr error) (err error) {
+		if readErr != nil {
+			err = readErr
+			return
+		}
+
+		if path == "." {
+			return
+		}
+
+		fi, err := d.Info()
+		if err != nil {
+			return
+		}
+
+		hdr, err := tar.FileInfoHeader(fi, "")
+		if err != nil {
+			return
+		}
+
+		hdr.Name = path
+		hdr.Uid = 0
+		hdr.Gid = 0
+
+		err = arch.WriteHeader(hdr)
+		if err != nil {
+			return
+		}
+
+		if fi.IsDir() {
+			return
+		}
+
+		f, err := root.Open(path)
+		if err != nil {
+			return
+		}
+		defer f.Close()
+
+		io.Copy(arch, f)
+
+		return
+	})
+
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+
+	err = arch.Close()
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+
+	buf.WriteTo(resp)
+}
+
+func wsStoreUpload(req *restful.Request, resp *restful.Response) {
+	if !secStore.IsNew() {
+		wsError(resp, httperr.BadRequest("store is not new"))
+		return
+	}
+
+	buf := new(bytes.Buffer)
+
+	_, err := io.Copy(buf, req.Request.Body)
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+
+	arch := tar.NewReader(buf)
+
+	root := secStoreRoot()
+
+	for {
+		hdr, err := arch.Next()
+		if err == io.EOF {
+			err = nil
+			break
+		} else if err != nil {
+			wsError(resp, err)
+			return
+		}
+
+		log.Print(hdr.Name)
+
+		fullPath := filepath.Join(root, hdr.Name)
+
+		switch {
+		case hdr.FileInfo().IsDir():
+			err = os.MkdirAll(fullPath, 0700)
+			if err != nil {
+				wsError(resp, err)
+				return
+			}
+
+		default:
+			content, err := io.ReadAll(io.LimitReader(arch, hdr.Size))
+			if err != nil {
+				wsError(resp, err)
+				return
+			}
+
+			err = os.WriteFile(fullPath, content, 0600)
+			if err != nil {
+				wsError(resp, err)
+				return
+			}
+		}
+	}
+
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+
+	openSecretStore()
+
+	resp.WriteEntity(map[string]any{"ok": true})
+}

cmd/dkl-local-server/ws-ssh-acls.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"net/http"
+	"os"
+	"path/filepath"
+
+	restful "github.com/emicklei/go-restful"
+	yaml "gopkg.in/yaml.v2"
+)
+
+type SSH_ACL struct {
+	Keys     []string
+	Clusters []string
+	Groups   []string
+	Hosts    []string
+}
+
+func loadSSH_ACLs() (acls []SSH_ACL, err error) {
+	f, err := os.Open(filepath.Join(*dataDir, "ssh-acls.yaml"))
+	if err != nil {
+		return
+	}
+
+	defer f.Close()
+
+	err = yaml.NewDecoder(f).Decode(&acls)
+	return
+}
+
+func wsSSH_ACL_List(req *restful.Request, resp *restful.Response) {
+	// TODO
+	http.NotFound(resp.ResponseWriter, req.Request)
+}
+
+func wsSSH_ACL_Get(req *restful.Request, resp *restful.Response) {
+	// TODO
+	http.NotFound(resp.ResponseWriter, req.Request)
+}
+
+func wsSSH_ACL_Set(req *restful.Request, resp *restful.Response) {
+	// TODO
+	http.NotFound(resp.ResponseWriter, req.Request)
+}

cmd/dkl-local-server/ws-store.go

@@ -0,0 +1,80 @@
+package main
+
+import (
+	"strconv"
+	"strings"
+
+	restful "github.com/emicklei/go-restful"
+	"novit.tech/direktil/local-server/secretstore"
+)
+
+func wsStoreAddKey(req *restful.Request, resp *restful.Response) {
+	np := NamedPassphrase{}
+
+	err := req.ReadEntity(&np)
+	if err != nil {
+		wsBadRequest(resp, err.Error())
+		return
+	}
+
+	np.Name = strings.TrimSpace(np.Name)
+
+	if len(np.Name) == 0 {
+		wsBadRequest(resp, "no name given")
+		return
+	}
+
+	if len(np.Passphrase) == 0 {
+		wsBadRequest(resp, "no passphrase given")
+		return
+	}
+
+	secStore.AddKey(np.Name, np.Passphrase)
+	defer updateState()
+
+	for _, k := range secStore.Keys {
+		if k.Name == np.Name {
+			wsBadRequest(resp, "there's already a passphrase named "+strconv.Quote(np.Name))
+			return
+		}
+	}
+
+	err = secStore.SaveTo(secKeysStorePath())
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+}
+
+func wsStoreDelKey(req *restful.Request, resp *restful.Response) {
+	name := ""
+
+	err := req.ReadEntity(&name)
+	if err != nil {
+		wsBadRequest(resp, err.Error())
+		return
+	}
+
+	newKeys := make([]secretstore.KeyEntry, 0, len(secStore.Keys))
+	for _, k := range secStore.Keys {
+		if k.Name == name {
+			continue
+		}
+
+		newKeys = append(newKeys, k)
+	}
+
+	if len(newKeys) == 0 {
+		wsBadRequest(resp, "can't remove the last key from the store")
+		return
+	}
+
+	secStore.Keys = newKeys
+	defer updateState()
+
+	err = secStore.SaveTo(secKeysStorePath())
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+}

cmd/dkl-local-server/ws.go

@@ -1,62 +1,211 @@
 package main
 
 import (
+	"errors"
+	"fmt"
 	"log"
 	"net"
 	"net/http"
 	"strings"
+	"text/template"
 
+	cfsslconfig "github.com/cloudflare/cfssl/config"
 	"github.com/emicklei/go-restful"
-	"novit.nc/direktil/local-server/pkg/mime"
-	"novit.nc/direktil/pkg/localconfig"
+	"m.cluseau.fr/go/httperr"
+
+	"novit.tech/direktil/pkg/localconfig"
+
+	"novit.tech/direktil/local-server/pkg/mime"
 )
 
-func buildWS() *restful.WebService {
+func registerWS(rest *restful.Container) {
+	// public-level APIs
+	{
 		ws := &restful.WebService{}
+		ws.
+			Path("/public").
+			Produces(mime.JSON).
+			Consumes(mime.JSON).
+			Route(ws.POST("/unlock-store").To(wsUnlockStore).
+				Reads(NamedPassphrase{}).
+				Writes("").
+				Doc("Try to unlock the store")).
+			Route(ws.GET("/store.tar").To(wsStoreDownload).
+				Produces(mime.TAR).
+				Param(ws.QueryParameter("token", "the download token")).
+				Doc("Fetch the encrypted store")).
+			Route(ws.POST("/store.tar").To(wsStoreUpload).
+				Consumes(mime.TAR).
+				Doc("Upload an existing store")).
+			Route(ws.GET("/downloads/{token}/{asset}").To(wsDownload).
+				Param(ws.PathParameter("token", "the download token")).
+				Param(ws.PathParameter("asset", "the requested asset")).
+				Doc("Fetch an asset via a download token"))
 
-	// configs API
-	ws.Route(ws.POST("/configs").Filter(adminAuth).To(wsUploadConfig).
+		rest.Add(ws)
+	}
+
+	// Admin-level APIs
+	ws := (&restful.WebService{}).
+		Filter(requireSecStore).
+		Filter(adminAuth).
+		Param(restful.HeaderParameter("Authorization", "Admin bearer token").Required(true)).
+		Produces(mime.JSON)
+
+	// - store management
+	ws.Route(ws.POST("/store/add-key").To(wsStoreAddKey).
+		Consumes(mime.JSON).Reads(NamedPassphrase{}).
+		Doc("Add an unlock key to the store"))
+	ws.Route(ws.POST("/store/delete-key").To(wsStoreDelKey).
+		Consumes(mime.JSON).Reads("").
+		Doc("Remove an unlock key to the store (by its name)"))
+
+	// - downloads
+	ws.Route(ws.POST("/authorize-download").To(wsAuthorizeDownload).
+		Consumes(mime.JSON).Reads(DownloadSpec{}).
+		Produces(mime.JSON).
+		Doc("Create a download token for the given download"))
+
+	// - configs API
+	ws.Route(ws.POST("/configs").To(wsUploadConfig).
+		Consumes(mime.YAML).Param(ws.BodyParameter("config", "The new full configuration")).
+		Produces(mime.JSON).Writes(true).
 		Doc("Upload a new current configuration, archiving the previous one"))
 
-	// clusters API
-	ws.Route(ws.GET("/clusters").Filter(adminAuth).To(wsListClusters).
+	// - clusters API
+	ws.Route(ws.GET("/clusters").To(wsListClusters).
 		Doc("List clusters"))
 
-	ws.Route(ws.GET("/clusters/{cluster-name}").Filter(adminAuth).To(wsCluster).
-		Doc("Get cluster details"))
+	const (
+		GET = http.MethodGet
+		PUT = http.MethodPut
+	)
 
-	ws.Route(ws.GET("/clusters/{cluster-name}/addons").Filter(adminAuth).To(wsClusterAddons).
+	cluster := func(method, subPath string) *restful.RouteBuilder {
+		return ws.Method(method).Path("/clusters/{cluster-name}" + subPath).
+			Param(ws.PathParameter("cluster-name", "name of the cluster"))
+	}
+
+	for _, builder := range []*restful.RouteBuilder{
+		cluster(GET, "").To(wsCluster).
+			Doc("Get cluster details"),
+
+		cluster(GET, "/addons").To(wsClusterAddons).
 			Produces(mime.YAML).
 			Doc("Get cluster addons").
 			Returns(http.StatusOK, "OK", nil).
-		Returns(http.StatusNotFound, "The cluster does not exists or does not have addons defined", nil))
+			Returns(http.StatusNotFound, "The cluster does not exists or does not have addons defined", nil),
 
-	// hosts API
-	ws.Route(ws.GET("/hosts").Filter(hostsAuth).To(wsListHosts).
+		cluster(GET, "/tokens").To(wsClusterTokens).
+			Doc("List cluster's tokens"),
+		cluster(GET, "/tokens/{token-name}").To(wsClusterToken).
+			Doc("Get cluster's token"),
+
+		cluster(GET, "/passwords").To(wsClusterPasswords).
+			Doc("List cluster's passwords"),
+		cluster(GET, "/passwords/{password-name}").To(wsClusterPassword).
+			Doc("Get cluster's password"),
+		cluster(PUT, "/passwords/{password-name}").To(wsClusterSetPassword).
+			Doc("Set cluster's password"),
+
+		cluster(GET, "/CAs").To(wsClusterCAs).
+			Doc("Get cluster CAs"),
+		cluster(GET, "/CAs/{ca-name}/certificate").To(wsClusterCACert).
+			Produces(mime.CACERT).
+			Doc("Get cluster CA's certificate"),
+		cluster(GET, "/CAs/{ca-name}/signed").To(wsClusterSignedCert).
+			Produces(mime.CERT).
+			Param(ws.QueryParameter("name", "signed reference name").Required(true)).
+			Doc("Get cluster's certificate signed by the CA"),
+	} {
+		ws.Route(builder)
+	}
+
+	ws.Route(ws.GET("/hosts").To(wsListHosts).
 		Doc("List hosts"))
 
+	ws.Route(ws.GET("/ssh-acls").To(wsSSH_ACL_List))
+	ws.Route(ws.GET("/ssh-acls/{acl-name}").To(wsSSH_ACL_Get))
+	ws.Route(ws.PUT("/ssh-acls/{acl-name}").To(wsSSH_ACL_Set))
+
+	rest.Add(ws)
+
+	// Hosts API
+	ws = (&restful.WebService{}).
+		Filter(requireSecStore).
+		Filter(adminAuth).
+		Path("/hosts/{host-name}").
+		Param(ws.HeaderParameter("Authorization", "Host or admin bearer token"))
+
+	(&wsHost{
+		hostDoc: "given host",
+		getHost: func(req *restful.Request) (string, error) {
+			return req.PathParameter("host-name"), nil
+		},
+	}).register(ws, func(rb *restful.RouteBuilder) {
+		rb.Param(ws.PathParameter("host-name", "host's name"))
+	})
+
+	rest.Add(ws)
+
+	// Detected host API
+	ws = (&restful.WebService{}).
+		Filter(requireSecStore).
+		Path("/me").
+		Param(ws.HeaderParameter("Authorization", "Host or admin bearer token"))
+
 	(&wsHost{
-		prefix:  "/me",
 		hostDoc: "detected host",
 		getHost: detectHost,
 	}).register(ws, func(rb *restful.RouteBuilder) {
 		rb.Notes("In this case, the host is detected from the remote IP")
 	})
 
-	(&wsHost{
-		prefix:  "/hosts/{host-name}",
-		hostDoc: "given host",
-		getHost: func(req *restful.Request) string {
-			return req.PathParameter("host-name")
-		},
-	}).register(ws, func(rb *restful.RouteBuilder) {
-		rb.Filter(adminAuth)
-	})
+	// Hosts by token API
+	ws = (&restful.WebService{}).
+		Filter(requireSecStore).
+		Path("/hosts-by-token/{host-token}").
+		Param(ws.PathParameter("host-token", "host's download token"))
 
-	return ws
+	(&wsHost{
+		hostDoc: "token's host",
+		getHost: func(req *restful.Request) (host string, err error) {
+			reqToken := req.PathParameter("host-token")
+
+			data, err := hostDownloadTokens.Data()
+			if err != nil {
+				return
+			}
+
+			for h, token := range data {
+				if token == reqToken {
+					host = h
+					return
+				}
+			}
+
+			return
+		},
+	}).register(ws, func(rb *restful.RouteBuilder) {
+		rb.Notes("In this case, the host is detected from the token")
+	})
+
+	rest.Add(ws)
+}
+
+func requireSecStore(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) {
+	if !secStore.Unlocked() {
+		wsError(resp, ErrStoreLocked)
+		return
+	}
+	chain.ProcessFilter(req, resp)
+}
+
+func detectHost(req *restful.Request) (hostName string, err error) {
+	if !*allowDetectedHost {
+		return
 	}
 
-func detectHost(req *restful.Request) string {
 	r := req.Request
 	remoteAddr := r.RemoteAddr
 
@@ -74,17 +223,17 @@ func detectHost(req *restful.Request) string {
 
 	cfg, err := readConfig()
 	if err != nil {
-		return ""
+		return
 	}
 
 	host := cfg.HostByIP(hostIP)
 
 	if host == nil {
 		log.Print("no host found for IP ", hostIP)
-		return ""
+		return
 	}
 
-	return host.Name
+	return host.Name, nil
 }
 
 func wsReadConfig(resp *restful.Response) *localconfig.Config {
@@ -98,13 +247,36 @@ func wsReadConfig(resp *restful.Response) *localconfig.Config {
 	return cfg
 }
 
-func wsNotFound(req *restful.Request, resp *restful.Response) {
-	http.NotFound(resp.ResponseWriter, req.Request)
+func wsNotFound(resp *restful.Response) {
+	wsError(resp, ErrNotFound)
+}
+
+func wsBadRequest(resp *restful.Response, err string) {
+	httperr.New(http.StatusBadRequest, errors.New(err)).WriteJSON(resp.ResponseWriter)
 }
 
 func wsError(resp *restful.Response, err error) {
-	log.Print("request failed: ", err)
-	resp.WriteErrorString(
-		http.StatusInternalServerError,
-		http.StatusText(http.StatusInternalServerError))
+	log.Output(2, fmt.Sprint("request failed: ", err))
+
+	switch err := err.(type) {
+	case httperr.Error:
+		err.WriteJSON(resp.ResponseWriter)
+
+	default:
+		httperr.Internal(err).WriteJSON(resp.ResponseWriter)
+	}
+}
+
+func wsRender(resp *restful.Response, sslCfg *cfsslconfig.Config, tmplStr string, value interface{}) {
+	tmpl, err := template.New("wsRender").Funcs(templateFuncs(sslCfg)).Parse(tmplStr)
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
+
+	err = tmpl.Execute(resp, value)
+	if err != nil {
+		wsError(resp, err)
+		return
+	}
 }

gen-api-js.sh

@@ -0,0 +1 @@
+docker run --rm --net=host --user $(id -u) -v ${PWD}:/local swaggerapi/swagger-codegen-cli generate -i http://[::1]:7606/swagger.json -l javascript -o /local/js/api/

go.mod

@@ -1,34 +1,81 @@
-module novit.nc/direktil/local-server
+module novit.tech/direktil/local-server
+
+go 1.21
 
 require (
-	github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e
-	github.com/cloudflare/cfssl v0.0.0-20181213083726-b94e044bb51e
-	github.com/coreos/etcd v3.3.11+incompatible // indirect
-	github.com/emicklei/go-restful v2.8.1+incompatible
-	github.com/emicklei/go-restful-openapi v1.0.0
-	github.com/go-openapi/jsonpointer v0.18.0 // indirect
-	github.com/go-openapi/jsonreference v0.18.0 // indirect
-	github.com/go-openapi/spec v0.18.0 // indirect
-	github.com/go-openapi/swag v0.18.0 // indirect
-	github.com/gobuffalo/buffalo-plugins v1.12.1 // indirect
-	github.com/google/certificate-transparency-go v1.0.21 // indirect
-	github.com/json-iterator/go v1.1.5 // indirect
-	github.com/markbates/going v1.0.3 // indirect
-	github.com/mcluseau/go-swagger-ui v0.0.0-20190204031235-fc4ac9154422
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/cespare/xxhash v1.1.0
+	github.com/cloudflare/cfssl v1.6.4
+	github.com/dustin/go-humanize v1.0.1
+	github.com/emicklei/go-restful v2.16.0+incompatible
+	github.com/emicklei/go-restful-openapi v1.4.1
+	github.com/go-git/go-git/v5 v5.10.0
+	github.com/mcluseau/go-swagger-ui v0.0.0-20191019002626-fd9128c24a34
+	github.com/miolini/datacounter v1.0.3
 	github.com/oklog/ulid v1.3.1
-	github.com/pierrec/lz4 v2.0.5+incompatible
-	github.com/spf13/afero v1.2.1 // indirect
-	github.com/src-d/go-git v4.7.0+incompatible // indirect
-	github.com/ugorji/go/codec v0.0.0-20190128213124-ee1426cffec0 // indirect
-	golang.org/x/crypto v0.0.0-20190131182504-b8fe1690c613 // indirect
-	golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3 // indirect
-	golang.org/x/sys v0.0.0-20190203050204-7ae0202eb74c // indirect
-	golang.org/x/tools v0.0.0-20190202235157-7414d4c1f71c // indirect
-	gopkg.in/src-d/go-billy.v4 v4.3.0
-	gopkg.in/src-d/go-git.v4 v4.10.0
-	gopkg.in/yaml.v2 v2.2.2
-	k8s.io/apimachinery v0.0.0-20190201131811-df262fa1a1ba
-	novit.nc/direktil/pkg v0.0.0-20181210211743-9dc80cd34b09
+	github.com/pierrec/lz4 v2.6.1+incompatible
+	github.com/sergeymakinen/go-crypt v1.0.0-beta.0
+	golang.org/x/crypto v0.14.0
+	gopkg.in/src-d/go-billy.v4 v4.3.2
+	gopkg.in/src-d/go-git.v4 v4.13.1
+	gopkg.in/yaml.v2 v2.4.0
+	k8s.io/apimachinery v0.28.3
+	m.cluseau.fr/go v0.0.0-20230809064045-12c5a121c766
+	novit.tech/direktil/pkg v0.0.0-20230201224712-5e39572dc50e
+)
+
+replace github.com/zmap/zlint/v3 => github.com/zmap/zlint/v3 v3.3.1
+
+require (
+	dario.cat/mergo v1.0.0 // indirect
+	github.com/Microsoft/go-winio v0.6.1 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c // indirect
+	github.com/acomagu/bufpipe v1.0.4 // indirect
+	github.com/cavaliergopher/cpio v1.0.1 // indirect
+	github.com/cloudflare/circl v1.3.6 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/frankban/quicktest v1.5.0 // indirect
+	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
+	github.com/go-git/go-billy/v5 v5.5.0 // indirect
+	github.com/go-logr/logr v1.3.0 // indirect
+	github.com/go-openapi/jsonpointer v0.20.0 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/spec v0.20.9 // indirect
+	github.com/go-openapi/swag v0.22.4 // indirect
+	github.com/gobuffalo/envy v1.10.2 // indirect
+	github.com/gobuffalo/packd v1.0.2 // indirect
+	github.com/gobuffalo/packr v1.30.1 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/google/certificate-transparency-go v1.1.7 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/jmoiron/sqlx v1.3.5 // indirect
+	github.com/joho/godotenv v1.5.1 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
+	github.com/kisielk/sqlstruct v0.0.0-20210630145711-dae28ed37023 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pjbgf/sha1cd v0.3.0 // indirect
+	github.com/rogpeppe/go-internal v1.11.0 // indirect
+	github.com/sergi/go-diff v1.3.1 // indirect
+	github.com/skeema/knownhosts v1.2.1 // indirect
+	github.com/src-d/gcfg v1.4.0 // indirect
+	github.com/weppos/publicsuffix-go v0.30.2-0.20230730094716-a20f9abcc222 // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
+	github.com/zmap/zcrypto v0.0.0-20231102161736-a55ea7b96dbc // indirect
+	github.com/zmap/zlint/v3 v3.1.0 // indirect
+	golang.org/x/mod v0.13.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/tools v0.14.0 // indirect
+	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/klog/v2 v2.110.1 // indirect
+	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
 )

hack/build

@@ -0,0 +1,3 @@
+#! /bin/sh
+set -ex
+go build -o dist/ -trimpath -ldflags "-X main.Version=${GIT_TAG:-$(git describe --always --dirty)}" $*

hack/docker-build

@@ -0,0 +1,8 @@
+#! /bin/bash
+set -ex
+case "$1" in
+commit) tag=$(git describe --always --dirty) ;;
+"")     tag=latest ;;
+*)      tag=$1 ;;
+esac
+docker build -t novit.tech/direktil/local-server:$tag .

hack/install

@@ -0,0 +1,4 @@
+#! /bin/sh
+set -ex
+go install -trimpath -ldflags "-X main.Version=$(git describe --always --dirty)" \
+    ./cmd/dkl-dir2config

html/html.go

@@ -0,0 +1,6 @@
+package dlshtml
+
+import "embed"
+
+//go:embed favicon.ico ui
+var FS embed.FS

html/ui/app.css

@@ -0,0 +1,23 @@
+
+.downloads {
+    display: flex;
+    align-content: stretch;
+}
+
+.downloads > * {
+    margin-left: 6pt;
+}
+.downloads > *:first-child {
+    margin-left: 0;
+}
+
+.downloads > div {
+    display: flex;
+    flex-flow: column;
+    max-height: 100pt;
+    overflow: auto;
+}
+
+.cluster {
+    max-width: 50%;
+}

html/ui/index.html

@@ -0,0 +1,116 @@
+<!doctype html>
+<html>
+<head>
+<title>Direktil Local Server</title>
+<style>
+@import url('./style.css');
+@import url('./app.css');
+</style>
+<script src="js/jsonpatch.min.js" crossorigin="anonymous"></script>
+<script src="js/app.js" type="module" defer></script>
+<body>
+
+<div id="app">
+<header>
+    <div id="logo">
+        <img src="/favicon.ico" />
+        <span>Direktil Local Server</span>
+    </div>
+    <div class="utils">
+        <span id="login-hdr" v-if="session.token">
+            Logged in
+            <button class="link" @click="copyText(session.token)">&#x1F5D0;</button>
+        </span>
+
+        <span>server <code>{{ serverVersion || '-----' }}</code></span>
+        <span>ui     <code>{{ uiHash || '-----' }}</code></span>
+
+        <span :class="publicState ? 'green' : 'red'">&#x1F5F2;</span>
+    </div>
+</header>
+
+<div class="error" v-if="error">
+    <button class="btn-close" @click="error=null">&times;</button>
+    <div class="code" v-if="error.code">{{ error.code }}</div>
+    <div class="message">{{ error.message }}</div>
+</div>
+
+<template v-if="!publicState">
+    <p>Not connected.</p>
+</template>
+<template v-else-if="publicState.Store.New">
+    <p>Store is new.</p>
+    <p>Option 1: initialize a new store</p>
+    <form @submit="unlockStore">
+        <input type="text"     v-model="forms.store.name"  name="name" placeholder="Name" /><br/>
+        <input type="password" v-model="forms.store.pass1" name="passphrase" required placeholder="Passphrase" />
+        <input type="password" v-model="forms.store.pass2" required placeholder="Passphrase confirmation" />
+        <input type="submit" value="initialize" :disabled="!forms.store.pass1 || forms.store.pass1 != forms.store.pass2" />
+    </form>
+    <p>Option 2: upload a previously downloaded store</p>
+    <form @submit="uploadStore">
+        <input type="file" ref="storeUpload" />
+        <input type="submit" value="upload" />
+    </form>
+</template>
+<template v-else-if="!publicState.Store.Open">
+    <p>Store is not open.</p>
+    <form @submit="unlockStore">
+        <input type="password" name="passphrase" v-model="forms.store.pass1" required placeholder="Passphrase" />
+        <input type="submit" value="unlock" :disabled="!forms.store.pass1" />
+    </form>
+</template>
+<template v-else-if="!state">
+    <p v-if="!session.token">Not logged in.</p>
+    <p v-else>Invalid token</p>
+
+    <form @submit="setToken">
+        <input type="password" v-model="forms.setToken" required placeholder="Token" />
+        <input type="submit" value="set token"/>
+    </form>
+</template>
+<template v-else>
+  <div v-if="state.Clusters" id="clusters">
+    <h2>Clusters</h2>
+
+    <div class="sheets">
+        <Cluster v-for="c in state.Clusters" :cluster="c" :token="session.token" :state="state" />
+    </div>
+  </div>
+
+  <div v-if="state.Hosts" id="hosts">
+    <h2>Hosts</h2>
+
+    <div class="sheets">
+        <Host v-for="h in state.Hosts" :host="h" :token="session.token" :state="state" />
+    </div>
+  </div>
+
+  <h2>Admin actions</h2>
+  <h3>Config</h3>
+  <form @submit="uploadConfig">
+      <input type="file" ref="configUpload" required />
+      <input type="submit" value="upload config" />
+  </form>
+  <h3>Store</h3>
+  <p><a :href="'/public/store.tar?token='+state.Store.DownloadToken" target="_blank">Download</a></p>
+  <form @submit="storeAddKey" action="/store/add-key">
+      <p>Add an unlock phrase:</p>
+      <input type="text"     v-model="forms.store.name"  name="name" required placeholder="Name" /><br/>
+      <input type="password" v-model="forms.store.pass1" name="passphrase" autocomplete="new-password" required placeholder="Phrase" />
+      <input type="password" v-model="forms.store.pass2" autocomplete="new-password" required placeholder="Phrase confirmation" />
+      <input type="submit" value="add unlock phrase" :disabled="!forms.store.pass1 || forms.store.pass1 != forms.store.pass2" />
+  </form>
+  <form @submit="storeDelKey" action="/store/delete-key">
+      <p>Remove an unlock phrase:</p>
+      <input type="text" v-model="forms.delKey.name"  name="name" required placeholder="Name" />
+      <input type="submit" value="remove unlock phrase" />
+
+      <p v-if="state.Store.KeyNames">Available names:
+        <template v-for="k,i in state.Store.KeyNames">{{i?", ":""}}<code @click="forms.delKey.name=k">{{k}}</code></template>.</p>
+  </form>
+</template>
+</div>
+
+</body>
+</html>

html/ui/js/Cluster.js

@@ -0,0 +1,35 @@
+
+import Downloads from './Downloads.js';
+import GetCopy from './GetCopy.js';
+
+export default {
+    components: { Downloads, GetCopy },
+    props: [ 'cluster', 'token', 'state' ],
+    template: `
+<div class="cluster">
+  <div class="title">Cluster {{ cluster.Name }}</div>
+  <div class="section">Tokens</div>
+  <section class="links">
+    <GetCopy v-for="n in cluster.Tokens" :token="token" :name="n" :href="'/clusters/'+cluster.Name+'/tokens/'+n" />
+  </section>
+  <div class="section">Passwords</div>
+  <section class="links">
+    <GetCopy v-for="n in cluster.Passwords" :token="token" :name="n" :href="'/clusters/'+cluster.Name+'/passwords/'+n" />
+  </section>
+  <div class="section">Downloads</div>
+  <section class="downloads">
+    <Downloads :token="token" :state="state" kind="cluster" :name="cluster.Name" />
+  </section>
+  <div class="section">CAs</div>
+  <table><tr><th>Name</th><th>Certificate</th><th>Signed certificates</th></tr>
+  <tr v-for="ca in cluster.CAs">
+    <td>{{ ca.Name }}</td>
+    <td><GetCopy :token="token" name="cert" :href="'/clusters/'+cluster.Name+'/CAs/'+ca.Name+'/certificate'" /></td>
+    <td><template v-for="signed in ca.Signed">
+      {{" "}}
+      <GetCopy :token="token" :name="signed" :href="'/clusters/'+cluster.Name+'/CAs/'+ca.Name+'/signed?name='+signed" />
+    </template></td>
+  </tr></table>
+</div>
+`
+}

html/ui/js/Downloads.js

@@ -0,0 +1,64 @@
+
+export default {
+    props: [ 'kind', 'name', 'token', 'state' ],
+    data() {
+        return { createDisabled: false, selectedAssets: {} }
+    },
+    computed: {
+        availableAssets() {
+            return {
+                cluster: ['addons'],
+                host: [
+                    "kernel",
+                    "initrd",
+                    "bootstrap.tar",
+                    "boot.img.lz4",
+                    "boot.iso",
+                    "config",
+                    "bootstrap-config",
+                    "boot.tar",
+                    "boot-efi.tar",
+                    "boot.img.gz",
+                    "boot.img",
+                    "ipxe",
+                ],
+            }[this.kind]
+        },
+        downloads() {
+            let ret = []
+            Object.entries(this.state.Downloads)
+              .filter(e => { let d=e[1]; return d.Kind == this.kind && d.Name == this.name })
+              .forEach(e => {
+                let token= e[0], d = e[1]
+                d.Assets.forEach(asset => {
+                    ret.push({name: asset, url: '/public/downloads/'+token+'/'+asset})
+                })
+              })
+          return ret
+        },
+        assets() {
+            return this.availableAssets.filter(a => this.selectedAssets[a])
+        },
+    },
+    methods: {
+        createToken() {
+            event.preventDefault()
+            this.createDisabled = true
+
+            fetch('/authorize-download', {
+                method: 'POST',
+                body: JSON.stringify({Kind: this.kind, Name: this.name, Assets: this.assets}),
+                headers: { 'Authorization': 'Bearer ' + this.token, 'Content-Type': 'application/json' },
+            }).then((resp) => resp.json())
+              .then((token) => { this.selectedAssets = {}; this.createDisabled = false })
+              .catch((e) => { alert('failed to create link'); this.createDisabled = false })
+        },
+    },
+    template: `<div class="downloads">
+  <div class="options">
+    <span v-for="asset in availableAssets"><label><input type="checkbox" v-model="selectedAssets[asset]" />&nbsp;{{ asset }}</label></span>
+  </div>
+  <button :disabled="createDisabled || assets.length==0" @click="createToken">+</button>
+  <div><a v-for="d in downloads" target="_blank" :href="d.url">{{ d.name }}</a></div>
+</div>`
+}

html/ui/js/GetCopy.js

@@ -0,0 +1,32 @@
+export default {
+    props: [ 'name', 'href', 'token' ],
+    data() { return {showCopied: false} },
+    template: `<span class="notif"><div v-if="showCopied">copied!</div><a :href="href" @click="fetchAndSave()">{{name}}</a>&nbsp;<a href="#" class="copy" @click="fetchAndCopy()">&#x1F5D0;</a></span>`,
+    methods: {
+        fetch() {
+            event.preventDefault()
+            return fetch(this.href, {
+                method: 'GET',
+                headers: { 'Authorization': 'Bearer ' + this.token },
+            })
+        },
+        handleFetchError(e) {
+            console.log("failed to get value:", e)
+            alert('failed to get value')
+        },
+        fetchAndSave() {
+            this.fetch().then(resp => resp.blob()).then((value) => {
+                window.open(URL.createObjectURL(value), "_blank")
+            }).catch(this.handleFetchError)
+        },
+        fetchAndCopy() {
+            this.fetch()
+            .then((resp) => resp.headers.get("content-type") == "application/json" ? resp.json() : resp.text())
+            .then((value) => {
+                window.navigator.clipboard.writeText(value)
+                this.showCopied = true
+                setTimeout(() => { this.showCopied = false }, 1000)
+            }).catch(this.handleFetchError)
+        },
+    },
+}

html/ui/js/Host.js

@@ -0,0 +1,21 @@
+
+import Downloads from './Downloads.js';
+
+export default {
+    components: { Downloads },
+    props: [ 'host', 'token', 'state' ],
+    template: `
+<div class="host">
+  <div class="title">Host {{ host.Name }}</div>
+  <section>
+    <template v-for="ip in host.IPs">
+        {{ ip }}
+    </template>
+  </section>
+  <div class="section">Downloads</div>
+  <section>
+    <Downloads :token="token" :state="state" kind="host" :name="host.Name" />
+  </section>
+</div>
+`
+}

html/ui/js/app.js

@@ -0,0 +1,199 @@
+
+import { createApp } from './vue.esm-browser.js';
+
+import Cluster from './Cluster.js';
+import Host    from './Host.js';
+
+createApp({
+    components: { Cluster, Host },
+    data() {
+        return {
+            forms: {
+                store: {},
+                storeUpload: {},
+                delKey: {},
+            },
+            session: {},
+            error: null,
+            publicState: null,
+            serverVersion: null,
+            uiHash: null,
+            watchingState: false,
+            state: null,
+        }
+    },
+    mounted() {
+        this.session = JSON.parse(sessionStorage.state || "{}")
+        this.watchPublicState()
+    },
+    watch: {
+        session: {
+            deep: true,
+            handler(v) {
+                sessionStorage.state = JSON.stringify(v)
+
+                if (v.token && !this.watchingState) {
+                    this.watchState()
+                    this.watchingState = true
+                }
+            }
+        },
+        publicState: {
+            deep: true,
+            handler(v) {
+                if (v) {
+                    this.serverVersion = v.ServerVersion
+                    if (this.uiHash && v.UIHash != this.uiHash) {
+                        console.log("reloading")
+                        location.reload()
+                    } else {
+                        this.uiHash = v.UIHash
+                    }
+                }
+            },
+        }
+    },
+
+    methods: {
+        copyText(text) {
+            event.preventDefault()
+            window.navigator.clipboard.writeText(text)
+        },
+        setToken() {
+            event.preventDefault()
+            this.session.token = this.forms.setToken
+            this.forms.setToken = null
+        },
+        uploadStore() {
+            event.preventDefault()
+            this.apiPost('/public/store.tar', this.$refs.storeUpload.files[0], (v) => {
+                this.forms.store = {}
+            }, "application/tar")
+        },
+        namedPassphrase(name, passphrase) {
+            return {Name: this.forms.store.name, Passphrase: btoa(this.forms.store.pass1)}
+        },
+        storeAddKey() {
+            this.apiPost('/store/add-key', this.namedPassphrase(), (v) => {
+                this.forms.store = {}
+            })
+        },
+        storeDelKey() {
+            let name = this.forms.delKey.name
+
+            if (!confirm("Remove key named "+JSON.stringify(name)+"?")) {
+                return
+            }
+            this.apiPost('/store/delete-key', name , (v) => {
+                this.forms.delKey = {}
+            })
+        },
+        unlockStore() {
+            this.apiPost('/public/unlock-store', this.namedPassphrase(), (v) => {
+                this.forms.store = {}
+
+                if (v) {
+                    this.session.token = v
+                    if (!this.watchingState) {
+                        this.watchState()
+                        this.watchingState = true
+                    }
+                }
+            })
+        },
+        uploadConfig() {
+            event.preventDefault()
+            this.apiPost('/configs', this.$refs.configUpload.files[0], (v) => {}, "text/vnd.yaml")
+        },
+        apiPost(action, data, onload, contentType = 'application/json') {
+            event.preventDefault()
+
+            if (data === undefined) {
+                throw("action " + action + ": no data")
+            }
+
+            /* TODO
+            fetch(action, {
+                method: 'POST',
+                body: JSON.stringify(data),
+            })
+              .then((response) => response.json())
+              .then((result) => onload)
+            // */
+
+            var xhr = new XMLHttpRequest()
+
+            xhr.responseType = 'json'
+            // TODO spinner, pending action notification, or something
+            xhr.onerror = () => {
+                // this.actionResults.splice(idx, 1, {...item, done: true, failed: true })
+            }
+            xhr.onload = (r) => {
+                if (xhr.status != 200) {
+                    this.error = xhr.response
+                    return
+                }
+                // this.actionResults.splice(idx, 1, {...item, done: true, resp: xhr.responseText})
+                this.error = null
+                if (onload) {
+                    onload(xhr.response)
+                }
+            }
+
+            xhr.open("POST", action)
+            xhr.setRequestHeader('Accept', 'application/json')
+            xhr.setRequestHeader('Content-Type', contentType)
+            if (this.session.token) {
+                xhr.setRequestHeader('Authorization', 'Bearer '+this.session.token)
+            }
+
+            if (contentType == "application/json") {
+                xhr.send(JSON.stringify(data))
+            } else {
+                xhr.send(data)
+            }
+        },
+        download(url) {
+            event.target.target = '_blank'
+            event.target.href = this.downloadLink(url)
+        },
+        downloadLink(url) {
+            // TODO once-shot download link
+            return url + '?token=' + this.session.token
+        },
+        watchPublicState() {
+            this.watchStream('publicState', '/public-state')
+        },
+        watchState() {
+            this.watchStream('state', '/state', true)
+        },
+        watchStream(field, path, withToken) {
+            let evtSrc = new EventSource(path + (withToken ? '?token='+this.session.token : ''));
+            evtSrc.onmessage = (e) => {
+                let update = JSON.parse(e.data)
+
+                console.log("watch "+path+":", update)
+
+                if (update.err) {
+                    console.log("watch error from server:", err)
+                }
+                if (update.set) {
+                    this[field] = update.set
+                }
+                if (update.p) { // patch
+                    new jsonpatch.JSONPatch(update.p, true).apply(this[field])
+                }
+            }
+            evtSrc.onerror = (e) => {
+                // console.log("event source " + path + " error:", e)
+                if (evtSrc) evtSrc.close()
+
+                this[field] = null
+
+                window.setTimeout(() => { this.watchStream(field, path, withToken) }, 1000)
+            }
+        },
+    }
+
+}).mount('#app')
+

html/ui/style.css

@@ -0,0 +1,152 @@
+body {
+    background: white;
+}
+
+button[disabled] {
+    opacity: 0.5;
+}
+
+a[href], a[href]:visited, button.link {
+    border: none;
+    color: blue;
+    background: none;
+    cursor: pointer;
+    text-decoration: none;
+}
+
+table {
+    border-collapse: collapse;
+}
+th, td {
+    border-left:   dotted 1pt;
+    border-right:  dotted 1pt;
+    border-bottom: dotted 1pt;
+    padding: 2pt 4pt;
+}
+tr:first-child > th {
+    border-top: dotted 1pt;
+}
+th, tr:last-child > td {
+    border-bottom: solid 1pt;
+}
+
+.flat > * { margin-left: 1ex; }
+.flat > *:first-child { margin-left: 0; }
+
+.green { color: green; }
+.red   { color: red; }
+
+@media (prefers-color-scheme: dark) {
+  body {
+    background: black;
+    color: orange;
+  }
+  button, input[type=submit] {
+    background: #333;
+    color: #eee;
+  }
+  a[href], a[href]:visited, button.link {
+    border: none;
+    color: #31b0fa;
+  }
+
+  .red { color: #c00; }
+}
+
+header {
+    display: flex;
+    align-items: center;
+    border-bottom: 2pt solid;
+    margin: 0 0 1em 0;
+    padding: 1ex;
+    justify-content: space-between;
+}
+#logo > img {
+    vertical-align: middle;
+}
+header .utils > * {
+    margin-left: 1ex;
+}
+
+.error {
+    display: flex;
+    position: relative;
+    background: rgba(255,0,0,0.2);
+    border: 1pt solid red;
+    justify-content: space-between;
+    align-items: center;
+}
+.error .btn-close,
+.error .code {
+    background: #600;
+    color: white;
+    font-weight: bold;
+    border: none;
+    align-self: stretch;
+    padding: 1ex 1em;
+}
+.error .code {
+    order: 1;
+
+    display: flex;
+    align-items: center;
+    text-align: center;
+}
+.error .message {
+    order: 2;
+    padding: 1ex 2em;
+}
+.error .btn-close {
+    order: 3;
+}
+
+.sheets {
+    display: flex;
+    align-items: stretch;
+}
+.sheets > div {
+    margin: 0 1ex;
+    border: 1pt solid;
+    border-radius: 6pt;
+}
+.sheets .title {
+    text-align: center;
+    font-weight: bold;
+    font-size: large;
+    padding: 2pt 6pt;
+    background: rgba(127,127,127,0.5);
+}
+.sheets .section {
+    padding: 2pt 6pt 2pt 6pt;
+    font-weight: bold;
+    border-top: 1px dotted;
+}
+.sheets section {
+    margin: 2pt 6pt 6pt 6pt;
+}
+.sheets > *:last-child > table:last-child > tr:last-child > td {
+    border-bottom: none;
+}
+
+.notif {
+    display: inline-block;
+    position: relative;
+}
+.notif > div:first-child {
+    position: absolute;
+    min-width: 100%; height: 100%;
+    background: white;
+    opacity: 75%;
+    text-align: center;
+}
+
+.links > * { margin-left: 1ex; }
+.links > *:first-child { margin-left: 0; }
+
+@media (prefers-color-scheme: dark) {
+  .notif > div:first-child {
+    background: black;
+  }
+}
+
+.copy { font-size: small; }

install

@@ -0,0 +1,2 @@
+#! /bin/sh
+go install -trimpath ./cmd/...

modd.conf

@@ -1,6 +1,26 @@
-**/*.go Dockerfile {
+modd.conf {}
+
+**/*.go go.mod go.sum {
   prep: go test ./...
-  prep: go install ./cmd/...
-  prep: docker build -t dls .
+  prep: mkdir -p dist
+  prep: hack/build ./...
+  #prep: docker build --build-arg GOPROXY=$GOPROXY -t dls .
   #daemon +sigterm: /var/lib/direktil/test-run
 }
+
+html/**/* {
+  prep: hack/build ./cmd/dkl-local-server
+}
+
+dist/dkl-local-server {
+  prep: mkdir -p tmp
+  #daemon +sigterm: dist/dkl-local-server -data tmp -auto-unlock test
+}
+
+dist/dkl-dir2config {
+  prep: dist/dkl-dir2config --debug --in test-dir2config
+}
+
+**/*.proto !dist/**/* {
+  prep: for mod in @mods; do protoc -I ./ --go_out=plugins=grpc,paths=source_relative:. $mod; done
+}

pkg/clustersconfig/clustersconfig.go

@@ -1,22 +1,33 @@
 package clustersconfig
 
 import (
+	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
+	"log"
 	"net"
+	"os"
+	"path/filepath"
 	"strings"
 	"text/template"
 
 	yaml "gopkg.in/yaml.v2"
 )
 
+var (
+	templateDetailsDir = flag.String("template-details-dir",
+		filepath.Join(os.TempDir(), "dkl-dir2config"),
+		"write details of template execute in this dir")
+
+	templateID = 0
+)
+
 type Config struct {
 	Hosts        []*Host
-	Groups       []*Group
 	Clusters     []*Cluster
 	Configs      []*Template
-	StaticPods   []*Template `yaml:"static_pods"`
+	StaticPods   map[string][]*Template `yaml:"static_pods"`
 	Addons       map[string][]*Template
 	SSLConfig    string         `yaml:"ssl_config"`
 	CertRequests []*CertRequest `yaml:"cert_requests"`
@@ -76,15 +87,6 @@ func (c *Config) HostByMAC(mac string) *Host {
 	return nil
 }
 
-func (c *Config) Group(name string) *Group {
-	for _, group := range c.Groups {
-		if group.Name == name {
-			return group
-		}
-	}
-	return nil
-}
-
 func (c *Config) Cluster(name string) *Cluster {
 	for _, cluster := range c.Clusters {
 		if cluster.Name == name {
@@ -103,15 +105,6 @@ func (c *Config) ConfigTemplate(name string) *Template {
 	return nil
 }
 
-func (c *Config) StaticPodsTemplate(name string) *Template {
-	for _, s := range c.StaticPods {
-		if s.Name == name {
-			return s
-		}
-	}
-	return nil
-}
-
 func (c *Config) CSR(name string) *CertRequest {
 	for _, s := range c.CertRequests {
 		if s.Name == name {
@@ -133,17 +126,19 @@ func (c *Config) SaveTo(path string) error {
 type Template struct {
 	Name     string
 	Template string
-
-	parsedTemplate *template.Template
 }
 
-func (t *Template) Execute(wr io.Writer, data interface{}, extraFuncs map[string]interface{}) error {
-	if t.parsedTemplate == nil {
+func (t *Template) Execute(contextName, elementName string, wr io.Writer, data interface{}, extraFuncs map[string]interface{}) error {
 	var templateFuncs = map[string]interface{}{
 		"indent": func(indent, s string) (indented string) {
 			indented = indent + strings.Replace(s, "\n", "\n"+indent, -1)
 			return
 		},
+		"yaml": func(v any) (s string, err error) {
+			ba, err := yaml.Marshal(v)
+			s = string(ba)
+			return
+		},
 	}
 
 	for name, f := range extraFuncs {
@@ -156,35 +151,65 @@ func (t *Template) Execute(wr io.Writer, data interface{}, extraFuncs map[string
 	if err != nil {
 		return err
 	}
-		t.parsedTemplate = tmpl
+
+	if *templateDetailsDir != "" {
+		templateID++
+
+		base := filepath.Join(*templateDetailsDir, contextName, fmt.Sprintf("%s-%03d", elementName, templateID))
+		os.MkdirAll(base, 0700)
+
+		base += string(filepath.Separator)
+		log.Print("writing template details: ", base, "{in,data,out}")
+
+		if err := ioutil.WriteFile(base+"in", []byte(t.Template), 0600); err != nil {
+			return err
 		}
 
-	return t.parsedTemplate.Execute(wr, data)
+		yamlBytes, err := yaml.Marshal(data)
+		if err != nil {
+			return err
+		}
+
+		if err := ioutil.WriteFile(base+"data", yamlBytes, 0600); err != nil {
+			return err
+		}
+
+		out, err := os.Create(base + "out")
+		if err != nil {
+			return err
+		}
+
+		defer out.Close()
+
+		wr = io.MultiWriter(wr, out)
+	}
+
+	return tmpl.Execute(wr, data)
 }
 
 // Host represents a host served by this server.
 type Host struct {
 	WithRev
+
 	Name        string
+	Labels      map[string]string
+	Annotations map[string]string
+
 	MAC     string
 	IP      string
 	IPs     []string
 	Cluster string
 	Group   string
-	Vars    Vars
-}
 
-// Group represents a group of hosts and provides their configuration.
-type Group struct {
-	WithRev
-	Name       string
-	Master     bool
 	IPXE            string
 	Kernel          string
 	Initrd          string
+	BootstrapConfig string `yaml:"bootstrap_config"`
 	Config          string
-	StaticPods string `yaml:"static_pods"`
 	Versions        map[string]string
+
+	StaticPods string `yaml:"static_pods"`
+
 	Vars Vars
 }
 
@@ -194,13 +219,18 @@ type Vars map[string]interface{}
 // Cluster represents a cluster of hosts, allowing for cluster-wide variables.
 type Cluster struct {
 	WithRev
+
 	Name        string
+	Labels      map[string]string
+	Annotations map[string]string
+
 	Domain  string
-	Addons  string
+	Addons  []string
 	Subnets struct {
 		Services string
 		Pods     string
 	}
+
 	Vars Vars
 }
 
@@ -215,7 +245,7 @@ func (c *Cluster) DNSSvcIP() net.IP {
 func (c *Cluster) NthSvcIP(n byte) net.IP {
 	_, cidr, err := net.ParseCIDR(c.Subnets.Services)
 	if err != nil {
-		panic(fmt.Errorf("Invalid services CIDR: %v", err))
+		panic(fmt.Errorf("invalid services CIDR: %v", err))
 	}
 
 	ip := cidr.IP

pkg/clustersconfig/defaults.go

@@ -70,6 +70,7 @@ func (d *Defaults) Load(dir, suffix string, value Rev, data []byte) (err error)
 }
 
 func (d *Defaults) Open(rev, filePath string) (rd io.Reader, err error) {
+	log.Printf("openning defaults at %s:%s", rev, filePath)
 	tree, err := d.treeAt(rev)
 	if err != nil {
 		return
@@ -94,12 +95,17 @@ func (d *Defaults) ReadAll(rev, filePath string) (ba []byte, err error) {
 }
 
 func (d *Defaults) List(rev, dir string) (names []string, err error) {
+	log.Printf("listing defaults at %s:%s", rev, dir)
 	tree, err := d.treeAt(rev)
 	if err != nil {
 		return
 	}
 
+	dirPrefix := dir + "/"
 	err = tree.Files().ForEach(func(f *object.File) (err error) {
+		if !strings.HasPrefix(f.Name, dirPrefix) {
+			return
+		}
 		if !strings.HasSuffix(f.Name, ".yaml") {
 			return
 		}
@@ -126,6 +132,11 @@ func (d *Defaults) treeAt(rev string) (tree *object.Tree, err error) {
 			obj, err = o.Object()
 
 		case *object.Commit: // commit -> tree
+			msg := o.Message
+			if len(msg) > 30 {
+				msg = msg[:27] + "..."
+			}
+			log.Printf("open defaults at commit %s: %s", o.Hash.String()[:7], msg)
 			return o.Tree()
 
 		default:

pkg/clustersconfig/dir.go

@@ -3,7 +3,6 @@ package clustersconfig
 import (
 	"fmt"
 	"io/ioutil"
-	"log"
 	"os"
 	"path"
 	"path/filepath"
@@ -15,37 +14,38 @@ import (
 // Debug enables debug logs from this package.
 var Debug = false
 
-func FromDir(dirPath, defaultsPath string) (*Config, error) {
-	if Debug {
-		log.Printf("loading config from dir %s (defaults from %s)", dirPath, defaultsPath)
-	}
+func FromDir(
+	read func(path string) ([]byte, error),
+	assemble func(path string) ([]byte, error),
+	listBase func(path string) ([]string, error),
+	listMerged func(path string) ([]string, error),
+) (*Config, error) {
 
-	defaults, err := NewDefaults(defaultsPath)
+	load := func(dir, name string, out any) (err error) {
+		ba, err := assemble(filepath.Join(dir, name))
 		if err != nil {
-		return nil, fmt.Errorf("failed to load defaults: %v", err)
+			return
 		}
-
-	store := &dirStore{dirPath}
-	load := func(dir, name string, out Rev) error {
-		ba, err := store.Get(path.Join(dir, name))
+		err = yaml.UnmarshalStrict(ba, out)
 		if err != nil {
-			return fmt.Errorf("failed to load %s/%s from dir: %v", dir, name, err)
-		}
-		if err = defaults.Load(dir, ".yaml", out, ba); err != nil {
-			return fmt.Errorf("failed to enrich %s/%s from defaults: %v", dir, name, err)
+			return
 		}
 		return nil
 	}
 
-	config := &Config{Addons: make(map[string][]*Template)}
+	config := &Config{
+		Addons:     make(map[string][]*Template),
+		StaticPods: make(map[string][]*Template),
+	}
 
 	// load clusters
-	names, err := store.List("clusters")
+	names, err := listBase("clusters")
 	if err != nil {
 		return nil, fmt.Errorf("failed to list clusters: %v", err)
 	}
 
 	for _, name := range names {
+		name, _ = strings.CutSuffix(name, ".yaml")
 		cluster := &Cluster{Name: name}
 		if err := load("clusters", name, cluster); err != nil {
 			return nil, err
@@ -54,97 +54,14 @@ func FromDir(dirPath, defaultsPath string) (*Config, error) {
 		config.Clusters = append(config.Clusters, cluster)
 	}
 
-	// load groups
-	names, err = store.List("groups")
-	if err != nil {
-		return nil, fmt.Errorf("failed to list groups: %v", err)
-	}
-
-	read := func(rev, filePath string) (data []byte, fromDefaults bool, err error) {
-		data, err = store.Get(filePath)
-		if err != nil {
-			err = fmt.Errorf("faild to read %s: %v", filePath, err)
-			return
-		}
-
-		if data != nil {
-			return // ok
-		}
-
-		if len(rev) == 0 {
-			err = fmt.Errorf("entry not found: %s", filePath)
-			return
-		}
-
-		data, err = defaults.ReadAll(rev, filePath+".yaml")
-		if err != nil {
-			err = fmt.Errorf("failed to read %s:%s: %v", rev, filePath, err)
-			return
-		}
-
-		fromDefaults = true
-		return
-	}
-
-	template := func(rev, dir, name string, templates *[]*Template) (ref string, err error) {
-		ref = name
-		if len(name) == 0 {
-			return
-		}
-
-		ba, fromDefaults, err := read(rev, path.Join(dir, name))
-		if err != nil {
-			return
-		}
-
-		if fromDefaults {
-			ref = rev + ":" + name
-		}
-
-		if !hasTemplate(ref, *templates) {
-			if Debug {
-				log.Printf("new template in %s: %s", dir, ref)
-			}
-
-			*templates = append(*templates, &Template{
-				Name:     ref,
-				Template: string(ba),
-			})
-		}
-
-		return
-	}
-
-	for _, name := range names {
-		group := &Group{Name: name}
-		if err := load("groups", name, group); err != nil {
-			return nil, err
-		}
-
-		group.Config, err = template(group.Rev(), "configs", group.Config, &config.Configs)
-		if err != nil {
-			return nil, fmt.Errorf("failed to load config for group %q: %v", name, err)
-		}
-
-		if Debug {
-			log.Printf("group %q: config=%q static_pods=%q", group.Name, group.Config, group.StaticPods)
-		}
-
-		group.StaticPods, err = template(group.Rev(), "static-pods", group.StaticPods, &config.StaticPods)
-		if err != nil {
-			return nil, fmt.Errorf("failed to load static pods for group %q: %v", name, err)
-		}
-
-		config.Groups = append(config.Groups, group)
-	}
-
 	// load hosts
-	names, err = store.List("hosts")
+	names, err = listBase("hosts")
 	if err != nil {
 		return nil, fmt.Errorf("failed to list hosts: %v", err)
 	}
 
 	for _, name := range names {
+		name, _ = strings.CutSuffix(name, ".yaml")
 		o := &Host{Name: name}
 		if err := load("hosts", name, o); err != nil {
 			return nil, err
@@ -154,28 +71,20 @@ func FromDir(dirPath, defaultsPath string) (*Config, error) {
 	}
 
 	// load config templates
-	loadTemplates := func(rev, dir string, templates *[]*Template) error {
-		names, err := store.List(dir)
+	loadTemplates := func(dir string, templates *[]*Template) error {
+		names, err := listMerged(dir)
 		if err != nil {
 			return fmt.Errorf("failed to list %s: %v", dir, err)
 		}
 
-		if len(rev) != 0 {
-			var defaultsNames []string
-			defaultsNames, err = defaults.List(rev, dir)
-			if err != nil {
-				return fmt.Errorf("failed to list %s:%s: %v", rev, dir, err)
-			}
+		for _, fullName := range names {
+			name, _ := strings.CutSuffix(fullName, ".yaml")
 
-			names = append(names, defaultsNames...)
-		}
-
-		for _, name := range names {
 			if hasTemplate(name, *templates) {
 				continue
 			}
 
-			ba, _, err := read(rev, path.Join(dir, name))
+			ba, err := read(path.Join(dir, fullName))
 			if err != nil {
 				return err
 			}
@@ -189,33 +98,57 @@ func FromDir(dirPath, defaultsPath string) (*Config, error) {
 		return nil
 	}
 
+	loadTemplates("configs", &config.Configs)
+
+	// cluster addons
 	for _, cluster := range config.Clusters {
-		addonSet := cluster.Addons
-		if len(addonSet) == 0 {
+		addonSets := cluster.Addons
+		if len(addonSets) == 0 {
 			continue
 		}
 
+		for _, addonSet := range addonSets {
 			if _, ok := config.Addons[addonSet]; ok {
 				continue
 			}
 
 			templates := make([]*Template, 0)
-		if err = loadTemplates(cluster.Rev(), path.Join("addons", addonSet), &templates); err != nil {
+			if err = loadTemplates(path.Join("addons", addonSet), &templates); err != nil {
 				return nil, err
 			}
 
 			config.Addons[addonSet] = templates
 		}
+	}
+
+	// cluster static pods
+	for _, host := range config.Hosts {
+		bpSet := host.StaticPods
+		if bpSet == "" {
+			continue
+		}
+
+		if _, ok := config.StaticPods[bpSet]; ok {
+			continue
+		}
+
+		templates := make([]*Template, 0)
+		if err = loadTemplates(path.Join("static-pods", bpSet), &templates); err != nil {
+			return nil, err
+		}
+
+		config.StaticPods[bpSet] = templates
+	}
 
 	// load SSL configuration
-	if ba, err := ioutil.ReadFile(filepath.Join(dirPath, "ssl-config.json")); err == nil {
+	if ba, err := read("ssl-config.json"); err == nil {
 		config.SSLConfig = string(ba)
 
 	} else if !os.IsNotExist(err) {
 		return nil, err
 	}
 
-	if ba, err := ioutil.ReadFile(filepath.Join(dirPath, "cert-requests.yaml")); err == nil {
+	if ba, err := read("cert-requests.yaml"); err == nil {
 		reqs := make([]*CertRequest, 0)
 		if err = yaml.Unmarshal(ba, &reqs); err != nil {
 			return nil, err

pkg/initrdconfig/config.go

@@ -0,0 +1,61 @@
+package config
+
+type Config struct {
+	AntiPhishingCode string `json:"anti_phishing_code"`
+
+	Keymap  string
+	Modules string
+
+	Auths []Auth
+
+	Networks []struct {
+		Name       string
+		Interfaces []struct {
+			Var     string
+			N       int
+			Regexps []string
+		}
+		Script string
+	}
+
+	LVM       []LvmVG
+	Bootstrap Bootstrap
+}
+
+type Auth struct {
+	Name     string
+	SSHKey   string `yaml:"sshKey"`
+	Password string `yaml:"password"`
+}
+
+type LvmVG struct {
+	VG  string
+	PVs struct {
+		N       int
+		Regexps []string
+	}
+
+	Defaults struct {
+		FS   string
+		Raid *RaidConfig
+	}
+
+	LVs []struct {
+		Name    string
+		Crypt   string
+		FS      string
+		Raid    *RaidConfig
+		Size    string
+		Extents string
+	}
+}
+
+type RaidConfig struct {
+	Mirrors int
+	Stripes int
+}
+
+type Bootstrap struct {
+	Dev  string
+	Seed string
+}

pkg/mime/mime.go

@@ -1,10 +1,13 @@
 package mime
 
 const (
+	JSON   = "application/json"
 	YAML   = "text/vnd.yaml"
 	TAR    = "application/tar"
 	DISK   = "application/x-diskimage"
 	ISO    = "application/x-iso9660-image"
 	IPXE   = "text/x-ipxe"
 	OCTET  = "application/octet-stream"
+	CERT   = "application/x-x509-user-cert"
+	CACERT = "application/x-x509-ca-cert"
 )

pkg/utf16/utf16.go

@@ -0,0 +1,29 @@
+package utf16
+
+import (
+	"encoding/binary"
+    "fmt"
+	"unicode/utf8"
+)
+
+func FromUTF8(data []byte) (res []byte) {
+    endian := binary.LittleEndian
+
+	res = make([]byte, (len(data)+1)*2)
+
+    res = res[:2]
+    endian.PutUint16(res, 0xfeff)
+
+	for len(data) > 0 {
+		r, size := utf8.DecodeRune(data)
+		if r > 65535 {
+            panic(fmt.Errorf("r=0x%x > 0xffff", r))
+        }
+
+		slen := len(res)
+		res = res[:slen+2]
+		endian.PutUint16(res[slen:], uint16(r))
+		data = data[size:]
+	}
+	return
+}

secretstore/io.go

@@ -0,0 +1,30 @@
+package secretstore
+
+import (
+	"crypto/rand"
+	"encoding/binary"
+	"fmt"
+	"io"
+)
+
+func readFull(in io.Reader, ba []byte) (err error) {
+	_, err = io.ReadFull(in, ba)
+	return
+}
+
+func read[T any](in io.Reader) (v T, err error) {
+	err = binary.Read(in, binary.BigEndian, &v)
+	return
+}
+
+var readSize = read[uint16]
+
+func randRead(ba []byte) (err error) {
+	err = readFull(rand.Reader, ba)
+	if err != nil {
+		err = fmt.Errorf("failed to read random bytes: %w", err)
+		return
+	}
+
+	return
+}

secretstore/mem.go

@@ -0,0 +1,9 @@
+package secretstore
+
+func Memzero(ba []byte) { memzero(ba) }
+
+func memzero(ba []byte) {
+	for i := range ba {
+		ba[i] = 0
+	}
+}

secretstore/reader.go

@@ -0,0 +1,68 @@
+package secretstore
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"io"
+)
+
+func (s *Store) NewReader(reader io.Reader) (r io.Reader, err error) {
+	iv := [aes.BlockSize]byte{}
+
+	err = readFull(reader, iv[:])
+	if err != nil {
+		return
+	}
+
+	r = storeReader{reader, s.NewDecrypter(iv)}
+	return
+}
+
+type storeReader struct {
+	reader    io.Reader
+	decrypter cipher.Stream
+}
+
+func (r storeReader) Read(ba []byte) (n int, err error) {
+	n, err = r.reader.Read(ba)
+
+	if n > 0 {
+		r.decrypter.XORKeyStream(ba[:n], ba[:n])
+	}
+
+	return
+}
+
+func (s *Store) NewWriter(writer io.Writer) (r io.Writer, err error) {
+	iv := [aes.BlockSize]byte{}
+
+	if err = randRead(iv[:]); err != nil {
+		return
+	}
+
+	_, err = writer.Write(iv[:])
+	if err != nil {
+		return
+	}
+
+	r = storeWriter{writer, s.NewEncrypter(iv)}
+	return
+}
+
+type storeWriter struct {
+	writer    io.Writer
+	encrypter cipher.Stream
+}
+
+func (r storeWriter) Write(ba []byte) (n int, err error) {
+	if len(ba) == 0 {
+		return
+	}
+
+	encBA := make([]byte, len(ba))
+	r.encrypter.XORKeyStream(encBA, ba)
+
+	n, err = r.writer.Write(encBA)
+
+	return
+}

secretstore/secret-store.go

@@ -0,0 +1,278 @@
+package secretstore
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/sha512"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"strconv"
+	"syscall"
+
+	"golang.org/x/crypto/argon2"
+)
+
+type Store struct {
+	Salt [aes.BlockSize]byte
+	Keys []KeyEntry
+
+	unlocked bool
+	key      [32]byte
+}
+
+type KeyEntry struct {
+	Name   string
+	Hash   [64]byte
+	EncKey [32]byte
+}
+
+func New() (s *Store) {
+	s = &Store{}
+	syscall.Mlock(s.key[:])
+	return
+}
+
+func Open(path string) (s *Store, err error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return
+	}
+
+	defer f.Close()
+
+	s = New()
+	_, err = s.ReadFrom(bufio.NewReader(f))
+
+	return
+}
+
+func (s *Store) SaveTo(path string) (err error) {
+	f, err := os.OpenFile(path, syscall.O_CREAT|syscall.O_TRUNC|syscall.O_WRONLY, 0600)
+	if err != nil {
+		return
+	}
+
+	defer f.Close()
+
+	out := bufio.NewWriter(f)
+
+	_, err = s.WriteTo(out)
+	if err != nil {
+		return
+	}
+
+	err = out.Flush()
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func (s *Store) Close() {
+	memzero(s.key[:])
+	syscall.Munlock(s.key[:])
+	s.unlocked = false
+}
+
+func (s *Store) IsNew() bool {
+	return len(s.Keys) == 0
+}
+
+func (s *Store) Unlocked() bool {
+	return s.unlocked
+}
+
+func (s *Store) Init(name string, passphrase []byte) (err error) {
+	err = randRead(s.key[:])
+	if err != nil {
+		return
+	}
+	err = randRead(s.Salt[:])
+	if err != nil {
+		return
+	}
+
+	s.AddKey(name, passphrase)
+
+	s.unlocked = true
+
+	return
+}
+
+var jsonFormatHdr = []byte("{json}")
+
+func (s *Store) ReadFrom(in io.Reader) (n int64, err error) {
+	memzero(s.key[:])
+	s.unlocked = false
+
+	defer func() {
+		if err != nil {
+			log.Output(2, fmt.Sprintf("failed after %d bytes", n))
+		}
+	}()
+
+	readFull := func(ba []byte) {
+		var nr int
+		nr, err = io.ReadFull(in, ba)
+		n += int64(nr)
+	}
+
+	// read the file's start (json header or start of salt)
+
+	readFull(s.Salt[:len(jsonFormatHdr)])
+	if err != nil {
+		return
+	}
+
+	if !bytes.Equal(s.Salt[:len(jsonFormatHdr)], jsonFormatHdr) {
+		// old key file
+
+		// finish reading the salt
+		readFull(s.Salt[len(jsonFormatHdr):])
+		if err != nil {
+			return
+		}
+
+		// read the (encrypted) keys
+		s.Keys = make([]KeyEntry, 0)
+		for {
+			k := KeyEntry{Name: "key-" + strconv.Itoa(len(s.Keys))}
+			readFull(k.Hash[:])
+			if err != nil {
+				if err == io.EOF {
+					err = nil
+				}
+				return
+			}
+			readFull(k.EncKey[:])
+			if err != nil {
+				return
+			}
+
+			s.Keys = append(s.Keys, k)
+		}
+	}
+
+	err = json.NewDecoder(in).Decode(s)
+	return
+}
+
+func (s *Store) WriteTo(out io.Writer) (n int64, err error) {
+	_, err = out.Write(jsonFormatHdr)
+	if err != nil {
+		return
+	}
+	err = json.NewEncoder(out).Encode(s)
+	return
+}
+
+var ErrNoSuchKey = errors.New("no such key")
+
+func (s *Store) HasKey(passphrase []byte) bool {
+	key, hash := s.keyPairFromPassword(passphrase)
+	defer memzero(key[:])
+
+	for _, k := range s.Keys {
+		if k.Hash == hash {
+			return true
+		}
+	}
+
+	return false
+}
+
+func (s *Store) Unlock(passphrase []byte) (ok bool) {
+	key, hash := s.keyPairFromPassword(passphrase)
+	defer memzero(key[:])
+
+	var idx = -1
+	for i := range s.Keys {
+		if hash == s.Keys[i].Hash {
+			idx = i
+			break
+		}
+	}
+
+	if idx == -1 {
+		return
+	}
+
+	s.decryptTo(s.key[:], s.Keys[idx].EncKey[:], &key)
+
+	s.unlocked = true
+	return true
+}
+
+func (s *Store) AddKey(name string, passphrase []byte) {
+	key, hash := s.keyPairFromPassword(passphrase)
+	memzero(passphrase)
+
+	defer memzero(key[:])
+
+	k := KeyEntry{Name: name, Hash: hash}
+
+	encKey := s.encrypt(s.key[:], &key)
+	copy(k.EncKey[:], encKey)
+
+	s.Keys = append(s.Keys, k)
+}
+
+func (s *Store) keyPairFromPassword(password []byte) (key [32]byte, hash [64]byte) {
+	keySlice := argon2.IDKey(password, s.Salt[:], 1, 64*1024, 4, 32)
+
+	copy(key[:], keySlice)
+	memzero(keySlice)
+
+	hash = sha512.Sum512(key[:])
+
+	return
+}
+
+func (s *Store) NewEncrypter(iv [aes.BlockSize]byte) cipher.Stream {
+	if !s.unlocked {
+		panic("not unlocked")
+	}
+	return newEncrypter(iv, &s.key)
+}
+
+func (s *Store) NewDecrypter(iv [aes.BlockSize]byte) cipher.Stream {
+	if !s.unlocked {
+		panic("not unlocked")
+	}
+	return newDecrypter(iv, &s.key)
+}
+
+func (s *Store) encrypt(src []byte, key *[32]byte) (dst []byte) {
+	dst = make([]byte, len(src))
+	newEncrypter(s.Salt, key).XORKeyStream(dst, src)
+	return
+}
+
+func (s *Store) decryptTo(dst []byte, src []byte, key *[32]byte) {
+	newDecrypter(s.Salt, key).XORKeyStream(dst, src)
+}
+
+func newEncrypter(iv [aes.BlockSize]byte, key *[32]byte) cipher.Stream {
+	c, err := aes.NewCipher(key[:])
+	if err != nil {
+		panic(fmt.Errorf("failed to init AES: %w", err))
+	}
+
+	return cipher.NewCFBEncrypter(c, iv[:])
+}
+
+func newDecrypter(iv [aes.BlockSize]byte, key *[32]byte) cipher.Stream {
+	c, err := aes.NewCipher(key[:])
+	if err != nil {
+		panic(fmt.Errorf("failed to init AES: %w", err))
+	}
+
+	return cipher.NewCFBDecrypter(c, iv[:])
+}

update-boot.sh

@@ -0,0 +1,19 @@
+#! /bin/bash
+
+dls_url="$1"
+
+set -ex
+
+mount -o remount,rw /boot
+
+if [ -e /boot/previous ]; then
+    rm -fr /boot/previous
+fi
+
+if [ -e /boot/current ]; then
+    mv /boot/current /boot/previous
+fi
+
+curl $dls_url/me/boot.tar |tar xv -C /boot
+sync
+

vendor/github.com/PuerkitoBio/purell/.gitignore

@@ -1,5 +0,0 @@
-*.sublime-*
-.DS_Store
-*.swp
-*.swo
-tags

vendor/github.com/PuerkitoBio/purell/.travis.yml

@@ -1,7 +0,0 @@
-language: go
-
-go:
-    - 1.4
-    - 1.5
-    - 1.6
-    - tip

vendor/github.com/PuerkitoBio/purell/LICENSE

@@ -1,12 +0,0 @@
-Copyright (c) 2012, Martin Angers
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
-
-* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
-
-* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
-
-* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/PuerkitoBio/purell/README.md

@@ -1,187 +0,0 @@
-# Purell
-
-Purell is a tiny Go library to normalize URLs. It returns a pure URL. Pure-ell. Sanitizer and all. Yeah, I know...
-
-Based on the [wikipedia paper][wiki] and the [RFC 3986 document][rfc].
-
-[![build status](https://secure.travis-ci.org/PuerkitoBio/purell.png)](http://travis-ci.org/PuerkitoBio/purell)
-
-## Install
-
-`go get github.com/PuerkitoBio/purell`
-
-## Changelog
-
-*    **2016-11-14 (v1.1.0)** : IDN: Conform to RFC 5895: Fold character width (thanks to @beeker1121).
-*    **2016-07-27 (v1.0.0)** : Normalize IDN to ASCII (thanks to @zenovich).
-*    **2015-02-08** : Add fix for relative paths issue ([PR #5][pr5]) and add fix for unnecessary encoding of reserved characters ([see issue #7][iss7]).
-*    **v0.2.0** : Add benchmarks, Attempt IDN support.
-*    **v0.1.0** : Initial release.
-
-## Examples
-
-From `example_test.go` (note that in your code, you would import "github.com/PuerkitoBio/purell", and would prefix references to its methods and constants with "purell."):
-
-```go
-package purell
-
-import (
-  "fmt"
-  "net/url"
-)
-
-func ExampleNormalizeURLString() {
-  if normalized, err := NormalizeURLString("hTTp://someWEBsite.com:80/Amazing%3f/url/",
-    FlagLowercaseScheme|FlagLowercaseHost|FlagUppercaseEscapes); err != nil {
-    panic(err)
-  } else {
-    fmt.Print(normalized)
-  }
-  // Output: http://somewebsite.com:80/Amazing%3F/url/
-}
-
-func ExampleMustNormalizeURLString() {
-  normalized := MustNormalizeURLString("hTTpS://someWEBsite.com:443/Amazing%fa/url/",
-    FlagsUnsafeGreedy)
-  fmt.Print(normalized)
-
-  // Output: http://somewebsite.com/Amazing%FA/url
-}
-
-func ExampleNormalizeURL() {
-  if u, err := url.Parse("Http://SomeUrl.com:8080/a/b/.././c///g?c=3&a=1&b=9&c=0#target"); err != nil {
-    panic(err)
-  } else {
-    normalized := NormalizeURL(u, FlagsUsuallySafeGreedy|FlagRemoveDuplicateSlashes|FlagRemoveFragment)
-    fmt.Print(normalized)
-  }
-
-  // Output: http://someurl.com:8080/a/c/g?c=3&a=1&b=9&c=0
-}
-```
-
-## API
-
-As seen in the examples above, purell offers three methods, `NormalizeURLString(string, NormalizationFlags) (string, error)`, `MustNormalizeURLString(string, NormalizationFlags) (string)` and `NormalizeURL(*url.URL, NormalizationFlags) (string)`. They all normalize the provided URL based on the specified flags. Here are the available flags:
-
-```go
-const (
-	// Safe normalizations
-	FlagLowercaseScheme           NormalizationFlags = 1 << iota // HTTP://host -> http://host, applied by default in Go1.1
-	FlagLowercaseHost                                            // http://HOST -> http://host
-	FlagUppercaseEscapes                                         // http://host/t%ef -> http://host/t%EF
-	FlagDecodeUnnecessaryEscapes                                 // http://host/t%41 -> http://host/tA
-	FlagEncodeNecessaryEscapes                                   // http://host/!"#$ -> http://host/%21%22#$
-	FlagRemoveDefaultPort                                        // http://host:80 -> http://host
-	FlagRemoveEmptyQuerySeparator                                // http://host/path? -> http://host/path
-
-	// Usually safe normalizations
-	FlagRemoveTrailingSlash // http://host/path/ -> http://host/path
-	FlagAddTrailingSlash    // http://host/path -> http://host/path/ (should choose only one of these add/remove trailing slash flags)
-	FlagRemoveDotSegments   // http://host/path/./a/b/../c -> http://host/path/a/c
-
-	// Unsafe normalizations
-	FlagRemoveDirectoryIndex   // http://host/path/index.html -> http://host/path/
-	FlagRemoveFragment         // http://host/path#fragment -> http://host/path
-	FlagForceHTTP              // https://host -> http://host
-	FlagRemoveDuplicateSlashes // http://host/path//a///b -> http://host/path/a/b
-	FlagRemoveWWW              // http://www.host/ -> http://host/
-	FlagAddWWW                 // http://host/ -> http://www.host/ (should choose only one of these add/remove WWW flags)
-	FlagSortQuery              // http://host/path?c=3&b=2&a=1&b=1 -> http://host/path?a=1&b=1&b=2&c=3
-
-	// Normalizations not in the wikipedia article, required to cover tests cases
-	// submitted by jehiah
-	FlagDecodeDWORDHost           // http://1113982867 -> http://66.102.7.147
-	FlagDecodeOctalHost           // http://0102.0146.07.0223 -> http://66.102.7.147
-	FlagDecodeHexHost             // http://0x42660793 -> http://66.102.7.147
-	FlagRemoveUnnecessaryHostDots // http://.host../path -> http://host/path
-	FlagRemoveEmptyPortSeparator  // http://host:/path -> http://host/path
-
-	// Convenience set of safe normalizations
-	FlagsSafe NormalizationFlags = FlagLowercaseHost | FlagLowercaseScheme | FlagUppercaseEscapes | FlagDecodeUnnecessaryEscapes | FlagEncodeNecessaryEscapes | FlagRemoveDefaultPort | FlagRemoveEmptyQuerySeparator
-
-	// For convenience sets, "greedy" uses the "remove trailing slash" and "remove www. prefix" flags,
-	// while "non-greedy" uses the "add (or keep) the trailing slash" and "add www. prefix".
-
-	// Convenience set of usually safe normalizations (includes FlagsSafe)
-	FlagsUsuallySafeGreedy    NormalizationFlags = FlagsSafe | FlagRemoveTrailingSlash | FlagRemoveDotSegments
-	FlagsUsuallySafeNonGreedy NormalizationFlags = FlagsSafe | FlagAddTrailingSlash | FlagRemoveDotSegments
-
-	// Convenience set of unsafe normalizations (includes FlagsUsuallySafe)
-	FlagsUnsafeGreedy    NormalizationFlags = FlagsUsuallySafeGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagRemoveWWW | FlagSortQuery
-	FlagsUnsafeNonGreedy NormalizationFlags = FlagsUsuallySafeNonGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagAddWWW | FlagSortQuery
-
-	// Convenience set of all available flags
-	FlagsAllGreedy    = FlagsUnsafeGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator
-	FlagsAllNonGreedy = FlagsUnsafeNonGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator
-)
-```
-
-For convenience, the set of flags `FlagsSafe`, `FlagsUsuallySafe[Greedy|NonGreedy]`, `FlagsUnsafe[Greedy|NonGreedy]` and `FlagsAll[Greedy|NonGreedy]` are provided for the similarly grouped normalizations on [wikipedia's URL normalization page][wiki]. You can add (using the bitwise OR `|` operator) or remove (using the bitwise AND NOT `&^` operator) individual flags from the sets if required, to build your own custom set.
-
-The [full godoc reference is available on gopkgdoc][godoc].
-
-Some things to note:
-
-*    `FlagDecodeUnnecessaryEscapes`, `FlagEncodeNecessaryEscapes`, `FlagUppercaseEscapes` and `FlagRemoveEmptyQuerySeparator` are always implicitly set, because internally, the URL string is parsed as an URL object, which automatically decodes unnecessary escapes, uppercases and encodes necessary ones, and removes empty query separators (an unnecessary `?` at the end of the url). So this operation cannot **not** be done. For this reason, `FlagRemoveEmptyQuerySeparator` (as well as the other three) has been included in the `FlagsSafe` convenience set, instead of `FlagsUnsafe`, where Wikipedia puts it.
-
-*    The `FlagDecodeUnnecessaryEscapes` decodes the following escapes (*from -> to*):
-    -    %24 -> $
-    -    %26 -> &
-    -    %2B-%3B -> +,-./0123456789:;
-    -    %3D -> =
-    -    %40-%5A -> @ABCDEFGHIJKLMNOPQRSTUVWXYZ
-    -    %5F -> _
-    -    %61-%7A -> abcdefghijklmnopqrstuvwxyz
-    -    %7E -> ~
-
-
-*    When the `NormalizeURL` function is used (passing an URL object), this source URL object is modified (that is, after the call, the URL object will be modified to reflect the normalization).
-
-*    The *replace IP with domain name* normalization (`http://208.77.188.166/ → http://www.example.com/`) is obviously not possible for a library without making some network requests. This is not implemented in purell.
-
-*    The *remove unused query string parameters* and *remove default query parameters* are also not implemented, since this is a very case-specific normalization, and it is quite trivial to do with an URL object.
-
-### Safe vs Usually Safe vs Unsafe
-
-Purell allows you to control the level of risk you take while normalizing an URL. You can aggressively normalize, play it totally safe, or anything in between.
-
-Consider the following URL:
-
-`HTTPS://www.RooT.com/toto/t%45%1f///a/./b/../c/?z=3&w=2&a=4&w=1#invalid`
-
-Normalizing with the `FlagsSafe` gives:
-
-`https://www.root.com/toto/tE%1F///a/./b/../c/?z=3&w=2&a=4&w=1#invalid`
-
-With the `FlagsUsuallySafeGreedy`:
-
-`https://www.root.com/toto/tE%1F///a/c?z=3&w=2&a=4&w=1#invalid`
-
-And with `FlagsUnsafeGreedy`:
-
-`http://root.com/toto/tE%1F/a/c?a=4&w=1&w=2&z=3`
-
-## TODOs
-
-*    Add a class/default instance to allow specifying custom directory index names? At the moment, removing directory index removes `(^|/)((?:default|index)\.\w{1,4})$`.
-
-## Thanks / Contributions
-
-@rogpeppe
-@jehiah
-@opennota
-@pchristopher1275
-@zenovich
-@beeker1121
-
-## License
-
-The [BSD 3-Clause license][bsd].
-
-[bsd]: http://opensource.org/licenses/BSD-3-Clause
-[wiki]: http://en.wikipedia.org/wiki/URL_normalization
-[rfc]: http://tools.ietf.org/html/rfc3986#section-6
-[godoc]: http://go.pkgdoc.org/github.com/PuerkitoBio/purell
-[pr5]: https://github.com/PuerkitoBio/purell/pull/5
-[iss7]: https://github.com/PuerkitoBio/purell/issues/7

vendor/github.com/PuerkitoBio/purell/purell.go

@@ -1,379 +0,0 @@
-/*
-Package purell offers URL normalization as described on the wikipedia page:
-http://en.wikipedia.org/wiki/URL_normalization
-*/
-package purell
-
-import (
-	"bytes"
-	"fmt"
-	"net/url"
-	"regexp"
-	"sort"
-	"strconv"
-	"strings"
-
-	"github.com/PuerkitoBio/urlesc"
-	"golang.org/x/net/idna"
-	"golang.org/x/text/unicode/norm"
-	"golang.org/x/text/width"
-)
-
-// A set of normalization flags determines how a URL will
-// be normalized.
-type NormalizationFlags uint
-
-const (
-	// Safe normalizations
-	FlagLowercaseScheme           NormalizationFlags = 1 << iota // HTTP://host -> http://host, applied by default in Go1.1
-	FlagLowercaseHost                                            // http://HOST -> http://host
-	FlagUppercaseEscapes                                         // http://host/t%ef -> http://host/t%EF
-	FlagDecodeUnnecessaryEscapes                                 // http://host/t%41 -> http://host/tA
-	FlagEncodeNecessaryEscapes                                   // http://host/!"#$ -> http://host/%21%22#$
-	FlagRemoveDefaultPort                                        // http://host:80 -> http://host
-	FlagRemoveEmptyQuerySeparator                                // http://host/path? -> http://host/path
-
-	// Usually safe normalizations
-	FlagRemoveTrailingSlash // http://host/path/ -> http://host/path
-	FlagAddTrailingSlash    // http://host/path -> http://host/path/ (should choose only one of these add/remove trailing slash flags)
-	FlagRemoveDotSegments   // http://host/path/./a/b/../c -> http://host/path/a/c
-
-	// Unsafe normalizations
-	FlagRemoveDirectoryIndex   // http://host/path/index.html -> http://host/path/
-	FlagRemoveFragment         // http://host/path#fragment -> http://host/path
-	FlagForceHTTP              // https://host -> http://host
-	FlagRemoveDuplicateSlashes // http://host/path//a///b -> http://host/path/a/b
-	FlagRemoveWWW              // http://www.host/ -> http://host/
-	FlagAddWWW                 // http://host/ -> http://www.host/ (should choose only one of these add/remove WWW flags)
-	FlagSortQuery              // http://host/path?c=3&b=2&a=1&b=1 -> http://host/path?a=1&b=1&b=2&c=3
-
-	// Normalizations not in the wikipedia article, required to cover tests cases
-	// submitted by jehiah
-	FlagDecodeDWORDHost           // http://1113982867 -> http://66.102.7.147
-	FlagDecodeOctalHost           // http://0102.0146.07.0223 -> http://66.102.7.147
-	FlagDecodeHexHost             // http://0x42660793 -> http://66.102.7.147
-	FlagRemoveUnnecessaryHostDots // http://.host../path -> http://host/path
-	FlagRemoveEmptyPortSeparator  // http://host:/path -> http://host/path
-
-	// Convenience set of safe normalizations
-	FlagsSafe NormalizationFlags = FlagLowercaseHost | FlagLowercaseScheme | FlagUppercaseEscapes | FlagDecodeUnnecessaryEscapes | FlagEncodeNecessaryEscapes | FlagRemoveDefaultPort | FlagRemoveEmptyQuerySeparator
-
-	// For convenience sets, "greedy" uses the "remove trailing slash" and "remove www. prefix" flags,
-	// while "non-greedy" uses the "add (or keep) the trailing slash" and "add www. prefix".
-
-	// Convenience set of usually safe normalizations (includes FlagsSafe)
-	FlagsUsuallySafeGreedy    NormalizationFlags = FlagsSafe | FlagRemoveTrailingSlash | FlagRemoveDotSegments
-	FlagsUsuallySafeNonGreedy NormalizationFlags = FlagsSafe | FlagAddTrailingSlash | FlagRemoveDotSegments
-
-	// Convenience set of unsafe normalizations (includes FlagsUsuallySafe)
-	FlagsUnsafeGreedy    NormalizationFlags = FlagsUsuallySafeGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagRemoveWWW | FlagSortQuery
-	FlagsUnsafeNonGreedy NormalizationFlags = FlagsUsuallySafeNonGreedy | FlagRemoveDirectoryIndex | FlagRemoveFragment | FlagForceHTTP | FlagRemoveDuplicateSlashes | FlagAddWWW | FlagSortQuery
-
-	// Convenience set of all available flags
-	FlagsAllGreedy    = FlagsUnsafeGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator
-	FlagsAllNonGreedy = FlagsUnsafeNonGreedy | FlagDecodeDWORDHost | FlagDecodeOctalHost | FlagDecodeHexHost | FlagRemoveUnnecessaryHostDots | FlagRemoveEmptyPortSeparator
-)
-
-const (
-	defaultHttpPort  = ":80"
-	defaultHttpsPort = ":443"
-)
-
-// Regular expressions used by the normalizations
-var rxPort = regexp.MustCompile(`(:\d+)/?$`)
-var rxDirIndex = regexp.MustCompile(`(^|/)((?:default|index)\.\w{1,4})$`)
-var rxDupSlashes = regexp.MustCompile(`/{2,}`)
-var rxDWORDHost = regexp.MustCompile(`^(\d+)((?:\.+)?(?:\:\d*)?)$`)
-var rxOctalHost = regexp.MustCompile(`^(0\d*)\.(0\d*)\.(0\d*)\.(0\d*)((?:\.+)?(?:\:\d*)?)$`)
-var rxHexHost = regexp.MustCompile(`^0x([0-9A-Fa-f]+)((?:\.+)?(?:\:\d*)?)$`)
-var rxHostDots = regexp.MustCompile(`^(.+?)(:\d+)?$`)
-var rxEmptyPort = regexp.MustCompile(`:+$`)
-
-// Map of flags to implementation function.
-// FlagDecodeUnnecessaryEscapes has no action, since it is done automatically
-// by parsing the string as an URL. Same for FlagUppercaseEscapes and FlagRemoveEmptyQuerySeparator.
-
-// Since maps have undefined traversing order, make a slice of ordered keys
-var flagsOrder = []NormalizationFlags{
-	FlagLowercaseScheme,
-	FlagLowercaseHost,
-	FlagRemoveDefaultPort,
-	FlagRemoveDirectoryIndex,
-	FlagRemoveDotSegments,
-	FlagRemoveFragment,
-	FlagForceHTTP, // Must be after remove default port (because https=443/http=80)
-	FlagRemoveDuplicateSlashes,
-	FlagRemoveWWW,
-	FlagAddWWW,
-	FlagSortQuery,
-	FlagDecodeDWORDHost,
-	FlagDecodeOctalHost,
-	FlagDecodeHexHost,
-	FlagRemoveUnnecessaryHostDots,
-	FlagRemoveEmptyPortSeparator,
-	FlagRemoveTrailingSlash, // These two (add/remove trailing slash) must be last
-	FlagAddTrailingSlash,
-}
-
-// ... and then the map, where order is unimportant
-var flags = map[NormalizationFlags]func(*url.URL){
-	FlagLowercaseScheme:           lowercaseScheme,
-	FlagLowercaseHost:             lowercaseHost,
-	FlagRemoveDefaultPort:         removeDefaultPort,
-	FlagRemoveDirectoryIndex:      removeDirectoryIndex,
-	FlagRemoveDotSegments:         removeDotSegments,
-	FlagRemoveFragment:            removeFragment,
-	FlagForceHTTP:                 forceHTTP,
-	FlagRemoveDuplicateSlashes:    removeDuplicateSlashes,
-	FlagRemoveWWW:                 removeWWW,
-	FlagAddWWW:                    addWWW,
-	FlagSortQuery:                 sortQuery,
-	FlagDecodeDWORDHost:           decodeDWORDHost,
-	FlagDecodeOctalHost:           decodeOctalHost,
-	FlagDecodeHexHost:             decodeHexHost,
-	FlagRemoveUnnecessaryHostDots: removeUnncessaryHostDots,
-	FlagRemoveEmptyPortSeparator:  removeEmptyPortSeparator,
-	FlagRemoveTrailingSlash:       removeTrailingSlash,
-	FlagAddTrailingSlash:          addTrailingSlash,
-}
-
-// MustNormalizeURLString returns the normalized string, and panics if an error occurs.
-// It takes an URL string as input, as well as the normalization flags.
-func MustNormalizeURLString(u string, f NormalizationFlags) string {
-	result, e := NormalizeURLString(u, f)
-	if e != nil {
-		panic(e)
-	}
-	return result
-}
-
-// NormalizeURLString returns the normalized string, or an error if it can't be parsed into an URL object.
-// It takes an URL string as input, as well as the normalization flags.
-func NormalizeURLString(u string, f NormalizationFlags) (string, error) {
-	parsed, err := url.Parse(u)
-	if err != nil {
-		return "", err
-	}
-
-	if f&FlagLowercaseHost == FlagLowercaseHost {
-		parsed.Host = strings.ToLower(parsed.Host)
-	}
-
-	// The idna package doesn't fully conform to RFC 5895
-	// (https://tools.ietf.org/html/rfc5895), so we do it here.
-	// Taken from Go 1.8 cycle source, courtesy of bradfitz.
-	// TODO: Remove when (if?) idna package conforms to RFC 5895.
-	parsed.Host = width.Fold.String(parsed.Host)
-	parsed.Host = norm.NFC.String(parsed.Host)
-	if parsed.Host, err = idna.ToASCII(parsed.Host); err != nil {
-		return "", err
-	}
-
-	return NormalizeURL(parsed, f), nil
-}
-
-// NormalizeURL returns the normalized string.
-// It takes a parsed URL object as input, as well as the normalization flags.
-func NormalizeURL(u *url.URL, f NormalizationFlags) string {
-	for _, k := range flagsOrder {
-		if f&k == k {
-			flags[k](u)
-		}
-	}
-	return urlesc.Escape(u)
-}
-
-func lowercaseScheme(u *url.URL) {
-	if len(u.Scheme) > 0 {
-		u.Scheme = strings.ToLower(u.Scheme)
-	}
-}
-
-func lowercaseHost(u *url.URL) {
-	if len(u.Host) > 0 {
-		u.Host = strings.ToLower(u.Host)
-	}
-}
-
-func removeDefaultPort(u *url.URL) {
-	if len(u.Host) > 0 {
-		scheme := strings.ToLower(u.Scheme)
-		u.Host = rxPort.ReplaceAllStringFunc(u.Host, func(val string) string {
-			if (scheme == "http" && val == defaultHttpPort) || (scheme == "https" && val == defaultHttpsPort) {
-				return ""
-			}
-			return val
-		})
-	}
-}
-
-func removeTrailingSlash(u *url.URL) {
-	if l := len(u.Path); l > 0 {
-		if strings.HasSuffix(u.Path, "/") {
-			u.Path = u.Path[:l-1]
-		}
-	} else if l = len(u.Host); l > 0 {
-		if strings.HasSuffix(u.Host, "/") {
-			u.Host = u.Host[:l-1]
-		}
-	}
-}
-
-func addTrailingSlash(u *url.URL) {
-	if l := len(u.Path); l > 0 {
-		if !strings.HasSuffix(u.Path, "/") {
-			u.Path += "/"
-		}
-	} else if l = len(u.Host); l > 0 {
-		if !strings.HasSuffix(u.Host, "/") {
-			u.Host += "/"
-		}
-	}
-}
-
-func removeDotSegments(u *url.URL) {
-	if len(u.Path) > 0 {
-		var dotFree []string
-		var lastIsDot bool
-
-		sections := strings.Split(u.Path, "/")
-		for _, s := range sections {
-			if s == ".." {
-				if len(dotFree) > 0 {
-					dotFree = dotFree[:len(dotFree)-1]
-				}
-			} else if s != "." {
-				dotFree = append(dotFree, s)
-			}
-			lastIsDot = (s == "." || s == "..")
-		}
-		// Special case if host does not end with / and new path does not begin with /
-		u.Path = strings.Join(dotFree, "/")
-		if u.Host != "" && !strings.HasSuffix(u.Host, "/") && !strings.HasPrefix(u.Path, "/") {
-			u.Path = "/" + u.Path
-		}
-		// Special case if the last segment was a dot, make sure the path ends with a slash
-		if lastIsDot && !strings.HasSuffix(u.Path, "/") {
-			u.Path += "/"
-		}
-	}
-}
-
-func removeDirectoryIndex(u *url.URL) {
-	if len(u.Path) > 0 {
-		u.Path = rxDirIndex.ReplaceAllString(u.Path, "$1")
-	}
-}
-
-func removeFragment(u *url.URL) {
-	u.Fragment = ""
-}
-
-func forceHTTP(u *url.URL) {
-	if strings.ToLower(u.Scheme) == "https" {
-		u.Scheme = "http"
-	}
-}
-
-func removeDuplicateSlashes(u *url.URL) {
-	if len(u.Path) > 0 {
-		u.Path = rxDupSlashes.ReplaceAllString(u.Path, "/")
-	}
-}
-
-func removeWWW(u *url.URL) {
-	if len(u.Host) > 0 && strings.HasPrefix(strings.ToLower(u.Host), "www.") {
-		u.Host = u.Host[4:]
-	}
-}
-
-func addWWW(u *url.URL) {
-	if len(u.Host) > 0 && !strings.HasPrefix(strings.ToLower(u.Host), "www.") {
-		u.Host = "www." + u.Host
-	}
-}
-
-func sortQuery(u *url.URL) {
-	q := u.Query()
-
-	if len(q) > 0 {
-		arKeys := make([]string, len(q))
-		i := 0
-		for k, _ := range q {
-			arKeys[i] = k
-			i++
-		}
-		sort.Strings(arKeys)
-		buf := new(bytes.Buffer)
-		for _, k := range arKeys {
-			sort.Strings(q[k])
-			for _, v := range q[k] {
-				if buf.Len() > 0 {
-					buf.WriteRune('&')
-				}
-				buf.WriteString(fmt.Sprintf("%s=%s", k, urlesc.QueryEscape(v)))
-			}
-		}
-
-		// Rebuild the raw query string
-		u.RawQuery = buf.String()
-	}
-}
-
-func decodeDWORDHost(u *url.URL) {
-	if len(u.Host) > 0 {
-		if matches := rxDWORDHost.FindStringSubmatch(u.Host); len(matches) > 2 {
-			var parts [4]int64
-
-			dword, _ := strconv.ParseInt(matches[1], 10, 0)
-			for i, shift := range []uint{24, 16, 8, 0} {
-				parts[i] = dword >> shift & 0xFF
-			}
-			u.Host = fmt.Sprintf("%d.%d.%d.%d%s", parts[0], parts[1], parts[2], parts[3], matches[2])
-		}
-	}
-}
-
-func decodeOctalHost(u *url.URL) {
-	if len(u.Host) > 0 {
-		if matches := rxOctalHost.FindStringSubmatch(u.Host); len(matches) > 5 {
-			var parts [4]int64
-
-			for i := 1; i <= 4; i++ {
-				parts[i-1], _ = strconv.ParseInt(matches[i], 8, 0)
-			}
-			u.Host = fmt.Sprintf("%d.%d.%d.%d%s", parts[0], parts[1], parts[2], parts[3], matches[5])
-		}
-	}
-}
-
-func decodeHexHost(u *url.URL) {
-	if len(u.Host) > 0 {
-		if matches := rxHexHost.FindStringSubmatch(u.Host); len(matches) > 2 {
-			// Conversion is safe because of regex validation
-			parsed, _ := strconv.ParseInt(matches[1], 16, 0)
-			// Set host as DWORD (base 10) encoded host
-			u.Host = fmt.Sprintf("%d%s", parsed, matches[2])
-			// The rest is the same as decoding a DWORD host
-			decodeDWORDHost(u)
-		}
-	}
-}
-
-func removeUnncessaryHostDots(u *url.URL) {
-	if len(u.Host) > 0 {
-		if matches := rxHostDots.FindStringSubmatch(u.Host); len(matches) > 1 {
-			// Trim the leading and trailing dots
-			u.Host = strings.Trim(matches[1], ".")
-			if len(matches) > 2 {
-				u.Host += matches[2]
-			}
-		}
-	}
-}
-
-func removeEmptyPortSeparator(u *url.URL) {
-	if len(u.Host) > 0 {
-		u.Host = rxEmptyPort.ReplaceAllString(u.Host, "")
-	}
-}

vendor/github.com/PuerkitoBio/urlesc/.travis.yml

@@ -1,15 +0,0 @@
-language: go
-
-go:
-  - 1.4.x
-  - 1.5.x
-  - 1.6.x
-  - 1.7.x
-  - 1.8.x
-  - tip
-
-install:
-  - go build .
-
-script:
-  - go test -v

vendor/github.com/PuerkitoBio/urlesc/LICENSE

@@ -1,27 +0,0 @@
-Copyright (c) 2012 The Go Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-   * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/PuerkitoBio/urlesc/README.md

@@ -1,16 +0,0 @@
-urlesc [![Build Status](https://travis-ci.org/PuerkitoBio/urlesc.svg?branch=master)](https://travis-ci.org/PuerkitoBio/urlesc) [![GoDoc](http://godoc.org/github.com/PuerkitoBio/urlesc?status.svg)](http://godoc.org/github.com/PuerkitoBio/urlesc)
-======
-
-Package urlesc implements query escaping as per RFC 3986.
-
-It contains some parts of the net/url package, modified so as to allow
-some reserved characters incorrectly escaped by net/url (see [issue 5684](https://github.com/golang/go/issues/5684)).
-
-## Install
-
-    go get github.com/PuerkitoBio/urlesc
-
-## License
-
-Go license (BSD-3-Clause)
-

vendor/github.com/PuerkitoBio/urlesc/urlesc.go

@@ -1,180 +0,0 @@
-// Copyright 2009 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// Package urlesc implements query escaping as per RFC 3986.
-// It contains some parts of the net/url package, modified so as to allow
-// some reserved characters incorrectly escaped by net/url.
-// See https://github.com/golang/go/issues/5684
-package urlesc
-
-import (
-	"bytes"
-	"net/url"
-	"strings"
-)
-
-type encoding int
-
-const (
-	encodePath encoding = 1 + iota
-	encodeUserPassword
-	encodeQueryComponent
-	encodeFragment
-)
-
-// Return true if the specified character should be escaped when
-// appearing in a URL string, according to RFC 3986.
-func shouldEscape(c byte, mode encoding) bool {
-	// §2.3 Unreserved characters (alphanum)
-	if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' {
-		return false
-	}
-
-	switch c {
-	case '-', '.', '_', '~': // §2.3 Unreserved characters (mark)
-		return false
-
-	// §2.2 Reserved characters (reserved)
-	case ':', '/', '?', '#', '[', ']', '@', // gen-delims
-		'!', '$', '&', '\'', '(', ')', '*', '+', ',', ';', '=': // sub-delims
-		// Different sections of the URL allow a few of
-		// the reserved characters to appear unescaped.
-		switch mode {
-		case encodePath: // §3.3
-			// The RFC allows sub-delims and : @.
-			// '/', '[' and ']' can be used to assign meaning to individual path
-			// segments.  This package only manipulates the path as a whole,
-			// so we allow those as well.  That leaves only ? and # to escape.
-			return c == '?' || c == '#'
-
-		case encodeUserPassword: // §3.2.1
-			// The RFC allows : and sub-delims in
-			// userinfo.  The parsing of userinfo treats ':' as special so we must escape
-			// all the gen-delims.
-			return c == ':' || c == '/' || c == '?' || c == '#' || c == '[' || c == ']' || c == '@'
-
-		case encodeQueryComponent: // §3.4
-			// The RFC allows / and ?.
-			return c != '/' && c != '?'
-
-		case encodeFragment: // §4.1
-			// The RFC text is silent but the grammar allows
-			// everything, so escape nothing but #
-			return c == '#'
-		}
-	}
-
-	// Everything else must be escaped.
-	return true
-}
-
-// QueryEscape escapes the string so it can be safely placed
-// inside a URL query.
-func QueryEscape(s string) string {
-	return escape(s, encodeQueryComponent)
-}
-
-func escape(s string, mode encoding) string {
-	spaceCount, hexCount := 0, 0
-	for i := 0; i < len(s); i++ {
-		c := s[i]
-		if shouldEscape(c, mode) {
-			if c == ' ' && mode == encodeQueryComponent {
-				spaceCount++
-			} else {
-				hexCount++
-			}
-		}
-	}
-
-	if spaceCount == 0 && hexCount == 0 {
-		return s
-	}
-
-	t := make([]byte, len(s)+2*hexCount)
-	j := 0
-	for i := 0; i < len(s); i++ {
-		switch c := s[i]; {
-		case c == ' ' && mode == encodeQueryComponent:
-			t[j] = '+'
-			j++
-		case shouldEscape(c, mode):
-			t[j] = '%'
-			t[j+1] = "0123456789ABCDEF"[c>>4]
-			t[j+2] = "0123456789ABCDEF"[c&15]
-			j += 3
-		default:
-			t[j] = s[i]
-			j++
-		}
-	}
-	return string(t)
-}
-
-var uiReplacer = strings.NewReplacer(
-	"%21", "!",
-	"%27", "'",
-	"%28", "(",
-	"%29", ")",
-	"%2A", "*",
-)
-
-// unescapeUserinfo unescapes some characters that need not to be escaped as per RFC3986.
-func unescapeUserinfo(s string) string {
-	return uiReplacer.Replace(s)
-}
-
-// Escape reassembles the URL into a valid URL string.
-// The general form of the result is one of:
-//
-//	scheme:opaque
-//	scheme://userinfo@host/path?query#fragment
-//
-// If u.Opaque is non-empty, String uses the first form;
-// otherwise it uses the second form.
-//
-// In the second form, the following rules apply:
-//	- if u.Scheme is empty, scheme: is omitted.
-//	- if u.User is nil, userinfo@ is omitted.
-//	- if u.Host is empty, host/ is omitted.
-//	- if u.Scheme and u.Host are empty and u.User is nil,
-//	   the entire scheme://userinfo@host/ is omitted.
-//	- if u.Host is non-empty and u.Path begins with a /,
-//	   the form host/path does not add its own /.
-//	- if u.RawQuery is empty, ?query is omitted.
-//	- if u.Fragment is empty, #fragment is omitted.
-func Escape(u *url.URL) string {
-	var buf bytes.Buffer
-	if u.Scheme != "" {
-		buf.WriteString(u.Scheme)
-		buf.WriteByte(':')
-	}
-	if u.Opaque != "" {
-		buf.WriteString(u.Opaque)
-	} else {
-		if u.Scheme != "" || u.Host != "" || u.User != nil {
-			buf.WriteString("//")
-			if ui := u.User; ui != nil {
-				buf.WriteString(unescapeUserinfo(ui.String()))
-				buf.WriteByte('@')
-			}
-			if h := u.Host; h != "" {
-				buf.WriteString(h)
-			}
-		}
-		if u.Path != "" && u.Path[0] != '/' && u.Host != "" {
-			buf.WriteByte('/')
-		}
-		buf.WriteString(escape(u.Path, encodePath))
-	}
-	if u.RawQuery != "" {
-		buf.WriteByte('?')
-		buf.WriteString(u.RawQuery)
-	}
-	if u.Fragment != "" {
-		buf.WriteByte('#')
-		buf.WriteString(escape(u.Fragment, encodeFragment))
-	}
-	return buf.String()
-}

vendor/github.com/cavaliercoder/go-cpio/.gitignore

@@ -1,3 +0,0 @@
-.fuzz/
-*.zip
-

vendor/github.com/cavaliercoder/go-cpio/.travis.yml

@@ -1,10 +0,0 @@
-language: go
-
-go:
-  - 1.4.3
-  - 1.5.4
-  - 1.6.4
-  - 1.7.6
-  - 1.8.3
-
-script: make check

vendor/github.com/cavaliercoder/go-cpio/LICENSE

@@ -1,26 +0,0 @@
-Copyright (c) 2017 Ryan Armstrong. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification,
-are permitted provided that the following conditions are met:
-
-1. Redistributions of source code must retain the above copyright notice, this
-   list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright notice,
-   this list of conditions and the following disclaimer in the documentation
-   and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its contributors
-   may be used to endorse or promote products derived from this software without
-   specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
-ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/cavaliercoder/go-cpio/Makefile

@@ -1,18 +0,0 @@
-PACKAGE = github.com/cavaliercoder/go-cpio
-
-all: check
-
-check:
-	go test -v
-
-cpio-fuzz.zip: *.go
-	go-fuzz-build $(PACKAGE)
-
-fuzz: cpio-fuzz.zip
-	go-fuzz -bin=./cpio-fuzz.zip -workdir=.fuzz/
-
-clean-fuzz:
-	rm -rf cpio-fuzz.zip .fuzz/crashers/* .fuzz/suppressions/*
-
-
-.PHONY: all check

vendor/github.com/cavaliercoder/go-cpio/README.md

@@ -1,62 +0,0 @@
-# go-cpio [![GoDoc](https://godoc.org/github.com/cavaliercoder/go-cpio?status.svg)](https://godoc.org/github.com/cavaliercoder/go-cpio) [![Build Status](https://travis-ci.org/cavaliercoder/go-cpio.svg?branch=master)](https://travis-ci.org/cavaliercoder/go-cpio) [![Go Report Card](https://goreportcard.com/badge/github.com/cavaliercoder/go-cpio)](https://goreportcard.com/report/github.com/cavaliercoder/go-cpio)
-
-This package provides a Go native implementation of the CPIO archive file
-format.
-
-Currently, only the SVR4 (New ASCII) format is supported, both with and without
-checksums.
-
-```go
-// Create a buffer to write our archive to.
-buf := new(bytes.Buffer)
-
-// Create a new cpio archive.
-w := cpio.NewWriter(buf)
-
-// Add some files to the archive.
-var files = []struct {
-  Name, Body string
-}{
-  {"readme.txt", "This archive contains some text files."},
-  {"gopher.txt", "Gopher names:\nGeorge\nGeoffrey\nGonzo"},
-  {"todo.txt", "Get animal handling license."},
-}
-for _, file := range files {
-  hdr := &cpio.Header{
-    Name: file.Name,
-    Mode: 0600,
-    Size: int64(len(file.Body)),
-  }
-  if err := w.WriteHeader(hdr); err != nil {
-    log.Fatalln(err)
-  }
-  if _, err := w.Write([]byte(file.Body)); err != nil {
-    log.Fatalln(err)
-  }
-}
-// Make sure to check the error on Close.
-if err := w.Close(); err != nil {
-  log.Fatalln(err)
-}
-
-// Open the cpio archive for reading.
-b := bytes.NewReader(buf.Bytes())
-r := cpio.NewReader(b)
-
-// Iterate through the files in the archive.
-for {
-  hdr, err := r.Next()
-  if err == io.EOF {
-    // end of cpio archive
-    break
-  }
-  if err != nil {
-    log.Fatalln(err)
-  }
-  fmt.Printf("Contents of %s:\n", hdr.Name)
-  if _, err := io.Copy(os.Stdout, r); err != nil {
-    log.Fatalln(err)
-  }
-  fmt.Println()
-}
-```

vendor/github.com/cavaliercoder/go-cpio/cpio.go

@@ -1,8 +0,0 @@
-/*
-Package cpio implements access to CPIO archives. Currently, only the SVR4 (New
-ASCII) format is supported, both with and without checksums.
-
-References:
-		https://www.freebsd.org/cgi/man.cgi?query=cpio&sektion=5
-*/
-package cpio

vendor/github.com/cavaliercoder/go-cpio/fileinfo.go

@@ -1,75 +0,0 @@
-package cpio
-
-import (
-	"os"
-	"path"
-	"time"
-)
-
-// headerFileInfo implements os.FileInfo.
-type headerFileInfo struct {
-	h *Header
-}
-
-// Name returns the base name of the file.
-func (fi headerFileInfo) Name() string {
-	if fi.IsDir() {
-		return path.Base(path.Clean(fi.h.Name))
-	}
-	return path.Base(fi.h.Name)
-}
-
-func (fi headerFileInfo) Size() int64        { return fi.h.Size }
-func (fi headerFileInfo) IsDir() bool        { return fi.Mode().IsDir() }
-func (fi headerFileInfo) ModTime() time.Time { return fi.h.ModTime }
-func (fi headerFileInfo) Sys() interface{}   { return fi.h }
-
-func (fi headerFileInfo) Mode() (mode os.FileMode) {
-	// Set file permission bits.
-	mode = os.FileMode(fi.h.Mode).Perm()
-
-	// Set setuid, setgid and sticky bits.
-	if fi.h.Mode&ModeSetuid != 0 {
-		// setuid
-		mode |= os.ModeSetuid
-	}
-	if fi.h.Mode&ModeSetgid != 0 {
-		// setgid
-		mode |= os.ModeSetgid
-	}
-	if fi.h.Mode&ModeSticky != 0 {
-		// sticky
-		mode |= os.ModeSticky
-	}
-
-	// Set file mode bits.
-	// clear perm, setuid, setgid and sticky bits.
-	m := os.FileMode(fi.h.Mode) & 0170000
-	if m == ModeDir {
-		// directory
-		mode |= os.ModeDir
-	}
-	if m == ModeNamedPipe {
-		// named pipe (FIFO)
-		mode |= os.ModeNamedPipe
-	}
-	if m == ModeSymlink {
-		// symbolic link
-		mode |= os.ModeSymlink
-	}
-	if m == ModeDevice {
-		// device file
-		mode |= os.ModeDevice
-	}
-	if m == ModeCharDevice {
-		// Unix character device
-		mode |= os.ModeDevice
-		mode |= os.ModeCharDevice
-	}
-	if m == ModeSocket {
-		// Unix domain socket
-		mode |= os.ModeSocket
-	}
-
-	return mode
-}

vendor/github.com/cavaliercoder/go-cpio/fuzz.go

@@ -1,35 +0,0 @@
-// +build gofuzz
-
-package cpio
-
-import "bytes"
-import "io"
-
-// Fuzz tests the parsing and error handling of random byte arrays using
-// https://github.com/dvyukov/go-fuzz.
-func Fuzz(data []byte) int {
-	r := NewReader(bytes.NewReader(data))
-	h := NewHash()
-	for {
-		hdr, err := r.Next()
-		if err != nil {
-			if hdr != nil {
-				panic("hdr != nil on error")
-			}
-			if err == io.EOF {
-				// everything worked with random input... interesting
-				return 1
-			}
-			// error returned for random input. Good!
-			return -1
-		}
-
-		// hash file
-		h.Reset()
-		io.CopyN(h, r, hdr.Size)
-		h.Sum32()
-
-		// convert file header
-		FileInfoHeader(hdr.FileInfo())
-	}
-}

vendor/github.com/cavaliercoder/go-cpio/hash.go

@@ -1,45 +0,0 @@
-package cpio
-
-import (
-	"encoding/binary"
-	"hash"
-)
-
-type digest struct {
-	sum uint32
-}
-
-// NewHash returns a new hash.Hash32 computing the SVR4 checksum.
-func NewHash() hash.Hash32 {
-	return &digest{}
-}
-
-func (d *digest) Write(p []byte) (n int, err error) {
-	for _, b := range p {
-		d.sum += uint32(b & 0xFF)
-	}
-
-	return len(p), nil
-}
-
-func (d *digest) Sum(b []byte) []byte {
-	out := [4]byte{}
-	binary.LittleEndian.PutUint32(out[:], d.sum)
-	return append(b, out[:]...)
-}
-
-func (d *digest) Sum32() uint32 {
-	return d.sum
-}
-
-func (d *digest) Reset() {
-	d.sum = 0
-}
-
-func (d *digest) Size() int {
-	return 4
-}
-
-func (d *digest) BlockSize() int {
-	return 1
-}

vendor/github.com/cavaliercoder/go-cpio/header.go

@@ -1,153 +0,0 @@
-package cpio
-
-import (
-	"errors"
-	"fmt"
-	"os"
-	"time"
-)
-
-// Mode constants from the cpio spec.
-const (
-	ModeSetuid     = 04000   // Set uid
-	ModeSetgid     = 02000   // Set gid
-	ModeSticky     = 01000   // Save text (sticky bit)
-	ModeDir        = 040000  // Directory
-	ModeNamedPipe  = 010000  // FIFO
-	ModeRegular    = 0100000 // Regular file
-	ModeSymlink    = 0120000 // Symbolic link
-	ModeDevice     = 060000  // Block special file
-	ModeCharDevice = 020000  // Character special file
-	ModeSocket     = 0140000 // Socket
-
-	ModeType = 0170000 // Mask for the type bits
-	ModePerm = 0777    // Unix permission bits
-)
-
-const (
-	// headerEOF is the value of the filename of the last header in a CPIO archive.
-	headerEOF = "TRAILER!!!"
-)
-
-var (
-	ErrHeader = errors.New("cpio: invalid cpio header")
-)
-
-// A FileMode represents a file's mode and permission bits.
-type FileMode int64
-
-func (m FileMode) String() string {
-	return fmt.Sprintf("%#o", m)
-}
-
-// IsDir reports whether m describes a directory. That is, it tests for the
-// ModeDir bit being set in m.
-func (m FileMode) IsDir() bool {
-	return m&ModeDir != 0
-}
-
-// IsRegular reports whether m describes a regular file. That is, it tests for
-// the ModeRegular bit being set in m.
-func (m FileMode) IsRegular() bool {
-	return m&^ModePerm == ModeRegular
-}
-
-// Perm returns the Unix permission bits in m.
-func (m FileMode) Perm() FileMode {
-	return m & ModePerm
-}
-
-// Checksum is the sum of all bytes	in the file data. This sum is computed
-// treating all bytes as unsigned values and using unsigned arithmetic. Only
-// the least-significant 32 bits of the sum are stored. Use NewHash to compute
-// the actual checksum of an archived file.
-type Checksum uint32
-
-func (c Checksum) String() string {
-	return fmt.Sprintf("%08X", uint32(c))
-}
-
-// A Header represents a single header in a CPIO archive.
-type Header struct {
-	DeviceID int
-	Inode    int64     // inode number
-	Mode     FileMode  // permission and mode bits
-	UID      int       // user id of the owner
-	GID      int       // group id of the owner
-	Links    int       // number of inbound links
-	ModTime  time.Time // modified time
-	Size     int64     // size in bytes
-	Name     string    // filename
-	Linkname string    // target name of link
-	Checksum Checksum  // computed checksum
-
-	pad int64 // bytes to pad before next header
-}
-
-// FileInfo returns an os.FileInfo for the Header.
-func (h *Header) FileInfo() os.FileInfo {
-	return headerFileInfo{h}
-}
-
-// FileInfoHeader creates a partially-populated Header from fi.
-// If fi describes a symlink, FileInfoHeader records link as the link target.
-// If fi describes a directory, a slash is appended to the name.
-// Because os.FileInfo's Name method returns only the base name of
-// the file it describes, it may be necessary to modify the Name field
-// of the returned header to provide the full path name of the file.
-func FileInfoHeader(fi os.FileInfo, link string) (*Header, error) {
-	if fi == nil {
-		return nil, errors.New("cpio: FileInfo is nil")
-	}
-
-	if sys, ok := fi.Sys().(*Header); ok {
-		// This FileInfo came from a Header (not the OS). Return a copy of the
-		// original Header.
-		h := &Header{}
-		*h = *sys
-		return h, nil
-	}
-
-	fm := fi.Mode()
-	h := &Header{
-		Name:    fi.Name(),
-		Mode:    FileMode(fi.Mode().Perm()), // or'd with Mode* constants later
-		ModTime: fi.ModTime(),
-		Size:    fi.Size(),
-	}
-
-	switch {
-	case fm.IsRegular():
-		h.Mode |= ModeRegular
-	case fi.IsDir():
-		h.Mode |= ModeDir
-		h.Name += "/"
-		h.Size = 0
-	case fm&os.ModeSymlink != 0:
-		h.Mode |= ModeSymlink
-		h.Linkname = link
-	case fm&os.ModeDevice != 0:
-		if fm&os.ModeCharDevice != 0 {
-			h.Mode |= ModeCharDevice
-		} else {
-			h.Mode |= ModeDevice
-		}
-	case fm&os.ModeNamedPipe != 0:
-		h.Mode |= ModeNamedPipe
-	case fm&os.ModeSocket != 0:
-		h.Mode |= ModeSocket
-	default:
-		return nil, fmt.Errorf("cpio: unknown file mode %v", fm)
-	}
-	if fm&os.ModeSetuid != 0 {
-		h.Mode |= ModeSetuid
-	}
-	if fm&os.ModeSetgid != 0 {
-		h.Mode |= ModeSetgid
-	}
-	if fm&os.ModeSticky != 0 {
-		h.Mode |= ModeSticky
-	}
-
-	return h, nil
-}

vendor/github.com/cavaliercoder/go-cpio/reader.go

@@ -1,72 +0,0 @@
-package cpio
-
-import (
-	"io"
-	"io/ioutil"
-)
-
-// A Reader provides sequential access to the contents of a CPIO archive. A CPIO
-// archive consists of a sequence of files. The Next method advances to the next
-// file in the archive (including the first), and then it can be treated as an
-// io.Reader to access the file's data.
-type Reader struct {
-	r   io.Reader // underlying file reader
-	hdr *Header   // current Header
-	eof int64     // bytes until the end of the current file
-}
-
-// NewReader creates a new Reader reading from r.
-func NewReader(r io.Reader) *Reader {
-	return &Reader{
-		r: r,
-	}
-}
-
-// Read reads from the current entry in the CPIO archive. It returns 0, io.EOF
-// when it reaches the end of that entry, until Next is called to advance to the
-// next entry.
-func (r *Reader) Read(p []byte) (n int, err error) {
-	if r.hdr == nil || r.eof == 0 {
-		return 0, io.EOF
-	}
-	rn := len(p)
-	if r.eof < int64(rn) {
-		rn = int(r.eof)
-	}
-	n, err = r.r.Read(p[0:rn])
-	r.eof -= int64(n)
-	return
-}
-
-// Next advances to the next entry in the CPIO archive.
-// io.EOF is returned at the end of the input.
-func (r *Reader) Next() (*Header, error) {
-	if r.hdr == nil {
-		return r.next()
-	}
-	skp := r.eof + r.hdr.pad
-	if skp > 0 {
-		_, err := io.CopyN(ioutil.Discard, r.r, skp)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return r.next()
-}
-
-func (r *Reader) next() (*Header, error) {
-	r.eof = 0
-	hdr, err := readHeader(r.r)
-	if err != nil {
-		return nil, err
-	}
-	r.hdr = hdr
-	r.eof = hdr.Size
-	return hdr, nil
-}
-
-// ReadHeader creates a new Header, reading from r.
-func readHeader(r io.Reader) (*Header, error) {
-	// currently only SVR4 format is supported
-	return readSVR4Header(r)
-}