Initial commit

boot-iso.go

@@ -0,0 +1,191 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+)
+
+func buildBootISO(out io.Writer, ctx *renderContext) error {
+	tempDir, err := ioutil.TempDir("/tmp", "iso-")
+	if err != nil {
+		return err
+	}
+
+	defer os.RemoveAll(tempDir)
+
+	cp := func(src, dst string) error {
+		log.Printf("iso: adding %s as %s", src, dst)
+		in, err := os.Open(src)
+		if err != nil {
+			return err
+		}
+
+		defer in.Close()
+
+		outPath := filepath.Join(tempDir, dst)
+
+		if err := os.MkdirAll(filepath.Dir(outPath), 0755); err != nil {
+			return err
+		}
+
+		out, err := os.Create(outPath)
+		if err != nil {
+			return err
+		}
+
+		defer out.Close()
+
+		_, err = io.Copy(out, in)
+		return err
+	}
+
+	err = func() error {
+		// grub
+
+		if err := os.MkdirAll(filepath.Join(tempDir, "grub"), 0755); err != nil {
+			return err
+		}
+		err = ioutil.WriteFile(filepath.Join(tempDir, "grub", "grub.cfg"), []byte(`
+search --set=root --file /config.yaml
+
+insmod all_video
+set timeout=3
+
+menuentry "Direktil" {
+    linux  /vmlinuz direktil.boot=DEVNAME=sr0 direktil.boot.fs=iso9660
+    initrd /initrd
+}
+`), 0644)
+		if err != nil {
+			return err
+		}
+
+		coreImgPath := filepath.Join(tempDir, "grub", "core.img")
+		grubCfgPath := filepath.Join(tempDir, "grub", "grub.cfg")
+
+		cmd := exec.Command("grub-mkstandalone",
+			"--format=i386-pc",
+			"--output="+coreImgPath,
+			"--install-modules=linux normal iso9660 biosdisk memdisk search tar ls",
+			"--modules=linux normal iso9660 biosdisk search",
+			"--locales=",
+			"--fonts=",
+			"boot/grub/grub.cfg="+grubCfgPath,
+		)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		if err := cmd.Run(); err != nil {
+			return err
+		}
+
+		defer os.Remove(coreImgPath)
+		defer os.Remove(grubCfgPath)
+
+		out, err := os.Create(filepath.Join(tempDir, "grub", "bios.img"))
+		if err != nil {
+			return err
+		}
+
+		defer out.Close()
+
+		b, err := ioutil.ReadFile("/usr/lib/grub/i386-pc/cdboot.img")
+		if err != nil {
+			return err
+		}
+
+		if _, err := out.Write(b); err != nil {
+			return err
+		}
+
+		b, err = ioutil.ReadFile(coreImgPath)
+		if err != nil {
+			return err
+		}
+
+		if _, err := out.Write(b); err != nil {
+			return err
+		}
+
+		return nil
+	}()
+	if err != nil {
+		return err
+	}
+
+	// config
+	cfgBytes, cfg, err := ctx.Config()
+	if err != nil {
+		return err
+	}
+
+	ioutil.WriteFile(filepath.Join(tempDir, "config.yaml"), cfgBytes, 0600)
+
+	// kernel and initrd
+	type distCopy struct {
+		Src []string
+		Dst string
+	}
+
+	copies := []distCopy{
+		{Src: []string{"kernels", ctx.Group.Kernel}, Dst: "vmlinuz"},
+		{Src: []string{"initrd", ctx.Group.Initrd}, Dst: "initrd"},
+	}
+
+	// layers
+	for _, layer := range cfg.Layers {
+		layerVersion := ctx.Group.Versions[layer]
+		if layerVersion == "" {
+			return fmt.Errorf("layer %q not mapped to a version", layer)
+		}
+
+		copies = append(copies,
+			distCopy{
+				Src: []string{"layers", layer, layerVersion},
+				Dst: filepath.Join("current", "layers", layer+".fs"),
+			})
+	}
+
+	for _, copy := range copies {
+		outPath, err := ctx.distFetch(copy.Src...)
+		if err != nil {
+			return err
+		}
+
+		err = cp(outPath, copy.Dst)
+		if err != nil {
+			return err
+		}
+	}
+
+	// build the ISO
+	mkisofs, err := exec.LookPath("genisoimage")
+	if err != nil {
+		mkisofs, err = exec.LookPath("mkisofs")
+	}
+	if err != nil {
+		return err
+	}
+
+	cmd := exec.Command(mkisofs,
+		"-quiet",
+		"-joliet",
+		"-joliet-long",
+		"-rock",
+		"-translation-table",
+		"-no-emul-boot",
+		"-boot-load-size", "4",
+		"-boot-info-table",
+		"-eltorito-boot", "grub/bios.img",
+		"-eltorito-catalog", "grub/boot.cat",
+		tempDir,
+	)
+	cmd.Stdout = out
+	cmd.Stderr = os.Stderr
+
+	return cmd.Run()
+}

cas-cleaner.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"flag"
+	"log"
+	"sort"
+	"time"
+)
+
+var (
+	cacheCleanDelay = flag.Duration("cache-clean-delay", 10*time.Minute, "Time between cache cleanups")
+)
+
+func casCleaner() {
+	for {
+		err := cleanCAS()
+		if err != nil {
+			log.Print("warn: couldn't clean cache: ", err)
+		}
+
+		time.Sleep(*cacheCleanDelay)
+	}
+}
+
+func cleanCAS() error {
+	cfg, err := readConfig()
+	if err != nil {
+		return err
+	}
+
+	activeTags := make([]string, len(cfg.Hosts))
+
+	for i, host := range cfg.Hosts {
+		ctx := newRenderContext(host, cfg)
+
+		tag, err := ctx.Tag()
+		if err != nil {
+			return err
+		}
+
+		activeTags[i] = tag
+	}
+
+	tags, err := casStore.Tags()
+	if err != nil {
+		return err
+	}
+
+	sort.Strings(activeTags)
+
+	for _, tag := range tags {
+		idx := sort.SearchStrings(activeTags, tag)
+
+		if idx < len(activeTags) && activeTags[idx] == tag {
+			continue
+		}
+
+		// tag is not present in active tags
+		log.Print("cache cleaner: removing tag ", tag)
+		if err := casStore.Remove(tag); err != nil {
+			log.Printf("cache cleaner: failed to remove tag %s: %v", tag, err)
+		}
+	}
+
+	return nil
+}

data.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"flag"
+	"log"
+	"path/filepath"
+
+	"novit.nc/direktil/pkg/clustersconfig"
+)
+
+var (
+	dataDir       = flag.String("data", "/var/lib/direktil", "Data dir")
+	configFromDir = flag.String("config-from-dir", "", "Build configuration from this directory")
+)
+
+func readConfig() (config *clustersconfig.Config, err error) {
+	if *configFromDir != "" {
+		config, err = clustersconfig.FromDir(*dataDir)
+		if err != nil {
+			log.Print("failed to load config: ", err)
+			return nil, err
+		}
+
+		if err = config.SaveTo(filepath.Join(*dataDir, "global-config.yaml")); err != nil {
+			return nil, err
+		}
+
+		return
+	}
+
+	return clustersconfig.FromFile(filepath.Join(*dataDir, "current-config.yaml"))
+}

errors.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"fmt"
+	"os"
+)
+
+type notFoundError struct {
+	ref string
+}
+
+func (e notFoundError) Error() string {
+	return fmt.Sprintf("not found: %s", e.ref)
+}
+
+var _ error = notFoundError{}
+
+func isNotFound(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	if os.IsNotExist(err) {
+		return true
+	}
+
+	_, ok := err.(notFoundError)
+	return ok
+}

http.go

@@ -0,0 +1,166 @@
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"log"
+	"net"
+	"net/http"
+	"path"
+	"regexp"
+	"strings"
+
+	"novit.nc/direktil/pkg/clustersconfig"
+)
+
+var (
+	reHost = regexp.MustCompile("^/hosts/([^/]+)/([^/]+)$")
+
+	trustXFF = flag.Bool("trust-xff", true, "Trust the X-Forwarded-For header")
+)
+
+func serveHostByIP(w http.ResponseWriter, r *http.Request) {
+	host, cfg := hostByIP(w, r)
+	if host == nil {
+		return
+	}
+
+	what := path.Base(r.URL.Path)
+
+	renderHost(w, r, what, host, cfg)
+}
+
+func hostByIP(w http.ResponseWriter, r *http.Request) (*clustersconfig.Host, *clustersconfig.Config) {
+	remoteAddr := r.RemoteAddr
+
+	if *trustXFF {
+		xff := r.Header.Get("X-Forwarded-For")
+		if xff != "" {
+			remoteAddr = strings.Split(xff, ",")[0]
+		}
+	}
+
+	hostIP, _, err := net.SplitHostPort(remoteAddr)
+
+	if err != nil {
+		hostIP = remoteAddr
+	}
+
+	cfg, err := readConfig()
+	if err != nil {
+		http.Error(w, "", http.StatusServiceUnavailable)
+		return nil, nil
+	}
+
+	host := cfg.HostByIP(hostIP)
+
+	if host == nil {
+		log.Print("no host found for IP ", hostIP)
+		http.NotFound(w, r)
+		return nil, nil
+	}
+
+	return host, cfg
+}
+
+func serveHosts(w http.ResponseWriter, r *http.Request) {
+	cfg, err := readConfig()
+	if err != nil {
+		http.Error(w, "", http.StatusServiceUnavailable)
+		return
+	}
+
+	hostNames := make([]string, len(cfg.Hosts))
+	for i, host := range cfg.Hosts {
+		hostNames[i] = host.Name
+	}
+
+	renderJSON(w, hostNames)
+}
+
+func serveHost(w http.ResponseWriter, r *http.Request) {
+	match := reHost.FindStringSubmatch(r.URL.Path)
+	if match == nil {
+		http.NotFound(w, r)
+		return
+	}
+
+	hostName, what := match[1], match[2]
+
+	cfg, err := readConfig()
+	if err != nil {
+		http.Error(w, "", http.StatusServiceUnavailable)
+		return
+	}
+
+	host := cfg.Host(hostName)
+
+	if host == nil {
+		host = cfg.HostByMAC(hostName)
+	}
+
+	if host == nil {
+		log.Printf("no host with name or MAC %q", hostName)
+		http.NotFound(w, r)
+		return
+	}
+
+	renderHost(w, r, what, host, cfg)
+}
+
+func renderHost(w http.ResponseWriter, r *http.Request, what string, host *clustersconfig.Host, cfg *clustersconfig.Config) {
+	ctx := newRenderContext(host, cfg)
+
+	switch what {
+	case "ipxe":
+		w.Header().Set("Content-Type", "text/x-ipxe")
+	case "config":
+		w.Header().Set("Content-Type", "text/vnd.yaml")
+	default:
+		w.Header().Set("Content-Type", "application/octet-stream")
+	}
+
+	var err error
+
+	switch what {
+	case "ipxe":
+		err = renderIPXE(w, ctx)
+
+	case "kernel":
+		err = renderKernel(w, r, ctx)
+
+	case "initrd":
+		err = renderCtx(w, r, ctx, "initrd", buildInitrd)
+
+	case "boot.iso":
+		err = renderCtx(w, r, ctx, "boot.iso", buildBootISO)
+
+	case "config":
+		err = renderConfig(w, r, ctx)
+
+	case "static-pods":
+		if ctx.Group.StaticPods == "" {
+			w.WriteHeader(http.StatusNoContent)
+			return
+		}
+		err = renderStaticPods(w, r, ctx)
+
+	default:
+		http.NotFound(w, r)
+	}
+
+	if err != nil {
+		if isNotFound(err) {
+			log.Printf("host %s: %s: %v", what, host.Name, err)
+			http.NotFound(w, r)
+		} else {
+			log.Printf("host %s: %s: failed to render: %v", what, host.Name, err)
+			http.Error(w, "", http.StatusServiceUnavailable)
+		}
+	}
+}
+
+func renderJSON(w http.ResponseWriter, v interface{}) {
+	w.Header().Add("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(v)
+}

initrd.go

@@ -0,0 +1,152 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"time"
+
+	cpio "github.com/cavaliercoder/go-cpio"
+	yaml "gopkg.in/yaml.v2"
+)
+
+func renderConfig(w http.ResponseWriter, r *http.Request, ctx *renderContext) error {
+	log.Printf("sending config for %q", ctx.Host.Name)
+
+	_, cfg, err := ctx.Config()
+	if err != nil {
+		return err
+	}
+
+	ba, err := yaml.Marshal(cfg)
+	if err != nil {
+		return err
+	}
+
+	w.Header().Set("Content-Type", "application/yaml")
+	http.ServeContent(w, r, "config.yaml", time.Unix(0, 0), bytes.NewReader(ba))
+
+	return nil
+}
+
+func renderStaticPods(w http.ResponseWriter, r *http.Request, ctx *renderContext) error {
+	log.Printf("sending static-pods for %q", ctx.Host.Name)
+
+	ba, err := ctx.StaticPods()
+	if err != nil {
+		return err
+	}
+
+	w.Header().Set("Content-Type", "application/yaml") // XXX can also be JSON
+	http.ServeContent(w, r, "static-pods", time.Unix(0, 0), bytes.NewReader(ba))
+
+	return nil
+}
+
+func renderCtx(w http.ResponseWriter, r *http.Request, ctx *renderContext, what string,
+	create func(out io.Writer, ctx *renderContext) error) error {
+	log.Printf("sending %s for %q", what, ctx.Host.Name)
+
+	tag, err := ctx.Tag()
+	if err != nil {
+		return err
+	}
+
+	// get it or create it
+	content, meta, err := casStore.GetOrCreate(tag, what, func(out io.Writer) error {
+		log.Printf("building %s for %q", what, ctx.Host.Name)
+		return create(out, ctx)
+	})
+
+	if err != nil {
+		return err
+	}
+
+	// serve it
+	http.ServeContent(w, r, what, meta.ModTime(), content)
+	return nil
+}
+
+func buildInitrd(out io.Writer, ctx *renderContext) error {
+	_, cfg, err := ctx.Config()
+
+	if err != nil {
+		return err
+	}
+
+	// send initrd basis
+	initrdPath, err := ctx.distFetch("initrd", ctx.Group.Initrd)
+	if err != nil {
+		return err
+	}
+
+	err = writeFile(out, initrdPath)
+	if err != nil {
+		return err
+	}
+
+	// and our extra archive
+	archive := cpio.NewWriter(out)
+
+	// - required dirs
+	for _, dir := range []string{
+		"boot",
+		"boot/current",
+		"boot/current/layers",
+	} {
+		archive.WriteHeader(&cpio.Header{
+			Name: dir,
+			Mode: 0600 | cpio.ModeDir,
+		})
+	}
+
+	// - the layers
+	for _, layer := range cfg.Layers {
+		layerVersion := ctx.Group.Versions[layer]
+		if layerVersion == "" {
+			return fmt.Errorf("layer %q not mapped to a version", layer)
+		}
+
+		path, err := ctx.distFetch("layers", layer, layerVersion)
+		if err != nil {
+			return err
+		}
+
+		stat, err := os.Stat(path)
+		if err != nil {
+			return err
+		}
+
+		archive.WriteHeader(&cpio.Header{
+			Name: "boot/current/layers/" + layer + ".fs",
+			Mode: 0600,
+			Size: stat.Size(),
+		})
+
+		if err = writeFile(archive, path); err != nil {
+			return err
+		}
+	}
+
+	// - the configuration
+	ba, err := yaml.Marshal(cfg)
+	if err != nil {
+		return err
+	}
+
+	archive.WriteHeader(&cpio.Header{
+		Name: "boot/config.yaml",
+		Mode: 0600,
+		Size: int64(len(ba)),
+	})
+
+	archive.Write(ba)
+
+	// finalize the archive
+	archive.Flush()
+	archive.Close()
+	return nil
+}

ipxe.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"bytes"
+	"io"
+	"log"
+	"text/template"
+)
+
+func renderIPXE(out io.Writer, ctx *renderContext) error {
+	log.Printf("sending IPXE code for %q", ctx.Host.Name)
+
+	tmpl, err := template.New("ipxe").Parse(ctx.Group.IPXE)
+	if err != nil {
+		return err
+	}
+
+	buf := bytes.NewBuffer(make([]byte, 0, 4096))
+	if err := tmpl.Execute(buf, ctx); err != nil {
+		return err
+	}
+
+	_, err = buf.WriteTo(out)
+	return err
+}

kernel.go

@@ -0,0 +1,17 @@
+package main
+
+import (
+	"log"
+	"net/http"
+)
+
+func renderKernel(w http.ResponseWriter, r *http.Request, ctx *renderContext) error {
+	path, err := ctx.distFetch("kernels", ctx.Group.Kernel)
+	if err != nil {
+		return err
+	}
+
+	log.Printf("sending kernel %s for %q", ctx.Group.Kernel, ctx.Host.Name)
+	http.ServeFile(w, r, path)
+	return nil
+}

main.go

@@ -0,0 +1,57 @@
+package main
+
+import (
+	"flag"
+	"log"
+	"net/http"
+	"path/filepath"
+
+	"novit.nc/direktil/pkg/cas"
+)
+
+const (
+	etcDir = "/etc/direktil"
+)
+
+var (
+	address    = flag.String("address", ":7606", "HTTP listen address")
+	tlsAddress = flag.String("tls-address", "", "HTTPS listen address")
+	certFile   = flag.String("tls-cert", etcDir+"/server.crt", "Server TLS certificate")
+	keyFile    = flag.String("tls-key", etcDir+"/server.key", "Server TLS key")
+
+	casStore cas.Store
+)
+
+func main() {
+	flag.Parse()
+
+	if *address == "" && *tlsAddress == "" {
+		log.Fatal("no listen address given")
+	}
+
+	secrets = &SecretsFile{filepath.Join(*dataDir, "secret.yaml")}
+	casStore = cas.NewDir(filepath.Join(*dataDir, "cache"))
+
+	go casCleaner()
+
+	http.HandleFunc("/ipxe", serveHostByIP)
+	http.HandleFunc("/kernel", serveHostByIP)
+	http.HandleFunc("/initrd", serveHostByIP)
+	http.HandleFunc("/boot.iso", serveHostByIP)
+	http.HandleFunc("/static-pods", serveHostByIP)
+
+	http.HandleFunc("/hosts", serveHosts)
+	http.HandleFunc("/hosts/", serveHost)
+
+	if *address != "" {
+		log.Print("HTTP listening on ", *address)
+		go log.Fatal(http.ListenAndServe(*address, nil))
+	}
+
+	if *tlsAddress != "" {
+		log.Print("HTTPS listening on ", *tlsAddress)
+		go log.Fatal(http.ListenAndServeTLS(*tlsAddress, *certFile, *keyFile, nil))
+	}
+
+	select {}
+}

render-context.go

@@ -0,0 +1,146 @@
+package main
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"log"
+	"path/filepath"
+
+	yaml "gopkg.in/yaml.v2"
+	"novit.nc/direktil/pkg/clustersconfig"
+	"novit.nc/direktil/pkg/config"
+)
+
+type renderContext struct {
+	Host               *clustersconfig.Host
+	Group              *clustersconfig.Group
+	Cluster            *clustersconfig.Cluster
+	Vars               map[string]interface{}
+	ConfigTemplate     *clustersconfig.Template
+	StaticPodsTemplate *clustersconfig.Template
+
+	clusterConfig *clustersconfig.Config
+}
+
+func newRenderContext(host *clustersconfig.Host, cfg *clustersconfig.Config) *renderContext {
+	group := cfg.Group(host.Group)
+	cluster := cfg.Cluster(host.Cluster)
+
+	vars := make(map[string]interface{})
+
+	for _, oVars := range []map[string]interface{}{
+		cluster.Vars,
+		group.Vars,
+		host.Vars,
+	} {
+		for k, v := range oVars {
+			vars[k] = v
+		}
+	}
+
+	return &renderContext{
+		Host:               host,
+		Group:              group,
+		Cluster:            cluster,
+		Vars:               vars,
+		ConfigTemplate:     cfg.ConfigTemplate(group.Config),
+		StaticPodsTemplate: cfg.StaticPodsTemplate(group.StaticPods),
+
+		clusterConfig: cfg,
+	}
+}
+
+func (ctx *renderContext) Config() (ba []byte, cfg *config.Config, err error) {
+	if ctx.ConfigTemplate == nil {
+		err = notFoundError{fmt.Sprintf("config %q", ctx.Group.Config)}
+		return
+	}
+
+	extraFuncs := map[string]interface{}{
+		"static_pods": func(name string) (string, error) {
+			t := ctx.clusterConfig.StaticPodsTemplate(name)
+			if t == nil {
+				return "", nil
+			}
+
+			buf := &bytes.Buffer{}
+			err := t.Execute(buf, ctx, nil)
+			if err != nil {
+				log.Printf("host %s: failed to render static pods: %v", ctx.Host.Name, err)
+				return "", err
+			}
+
+			return buf.String(), nil
+		},
+	}
+
+	buf := bytes.NewBuffer(make([]byte, 0, 4096))
+	if err = ctx.ConfigTemplate.Execute(buf, ctx, extraFuncs); err != nil {
+		return
+	}
+
+	ba = buf.Bytes()
+
+	cfg = &config.Config{}
+
+	if err = yaml.Unmarshal(buf.Bytes(), cfg); err != nil {
+		return
+	}
+
+	// bind secrets in config
+	for idx, file := range cfg.Files {
+		if file.Secret == "" {
+			continue
+		}
+
+		v, err2 := getSecret(file.Secret, ctx)
+		if err2 != nil {
+			err = err2
+			return
+		}
+
+		cfg.Files[idx].Content = v
+	}
+
+	return
+}
+
+func (ctx *renderContext) StaticPods() (ba []byte, err error) {
+	if ctx.StaticPodsTemplate == nil {
+		err = notFoundError{fmt.Sprintf("static-pods %q", ctx.Group.StaticPods)}
+		return
+	}
+
+	buf := bytes.NewBuffer(make([]byte, 0, 4096))
+	if err = ctx.StaticPodsTemplate.Execute(buf, ctx, nil); err != nil {
+		return
+	}
+
+	ba = buf.Bytes()
+	return
+}
+
+func (ctx *renderContext) distFilePath(path ...string) string {
+	return filepath.Join(append([]string{*dataDir, "dist"}, path...)...)
+}
+
+func (ctx *renderContext) Tag() (string, error) {
+	h := sha256.New()
+
+	_, cfg, err := ctx.Config()
+	if err != nil {
+		return "", err
+	}
+
+	enc := yaml.NewEncoder(h)
+
+	for _, o := range []interface{}{cfg, ctx} {
+		if err := enc.Encode(o); err != nil {
+			return "", err
+		}
+	}
+
+	return hex.EncodeToString(h.Sum(nil)), nil
+}

secrets.go

@@ -0,0 +1,138 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"strings"
+
+	yaml "gopkg.in/yaml.v2"
+)
+
+var (
+	secrets SecretBackend
+)
+
+type SecretBackend interface {
+	Get(ref string) (string, error)
+	Set(ref, value string) error
+}
+
+type SecretsFile struct {
+	Path string
+}
+
+func (sf *SecretsFile) readData() (map[string]string, error) {
+	ba, err := ioutil.ReadFile(sf.Path)
+	if err != nil {
+		return nil, err
+	}
+
+	data := map[string]string{}
+	yaml.Unmarshal(ba, &data)
+
+	return data, nil
+}
+
+func (sf *SecretsFile) Get(ref string) (string, error) {
+	data, err := sf.readData()
+
+	if os.IsNotExist(err) {
+		return "", nil
+
+	} else if err != nil {
+		log.Printf("secret file: failed to read: %v", err)
+		return "", err
+	}
+
+	return data[ref], nil
+}
+
+func (sf *SecretsFile) Set(ref, value string) (err error) {
+	data, err := sf.readData()
+
+	if os.IsNotExist(err) {
+		data = map[string]string{}
+
+	} else if err != nil {
+		log.Printf("secret file: failed to read: %v", err)
+		return
+	}
+
+	data[ref] = value
+
+	ba, err := yaml.Marshal(data)
+	if err != nil {
+		log.Printf("secret file: failed to encode: %v", err)
+		return
+	}
+
+	os.Rename(sf.Path, sf.Path+".old")
+
+	err = ioutil.WriteFile(sf.Path, ba, 0600)
+	if err != nil {
+		log.Printf("secret file: failed to write: %v", err)
+		return
+	}
+
+	return
+}
+
+func getSecret(ref string, ctx *renderContext) (string, error) {
+	fullRef := fmt.Sprintf("%s/%s", ctx.Cluster.Name, ref)
+
+	v, err := secrets.Get(fullRef)
+
+	if err != nil {
+		return "", err
+	}
+
+	if v != "" {
+		return v, nil
+	}
+
+	// no value, generate
+	split := strings.SplitN(ref, ":", 2)
+	kind, path := split[0], split[1]
+
+	switch kind {
+	case "tls-key":
+		_, ba := PrivateKeyPEM()
+		v = string(ba)
+
+	case "tls-self-signed-cert":
+		caKey, err := loadPrivateKey(path, ctx)
+		if err != nil {
+			return "", err
+		}
+
+		ba := SelfSignedCertificatePEM(5, caKey)
+		v = string(ba)
+
+	case "tls-host-cert":
+		hostKey, err := loadPrivateKey(path, ctx)
+		if err != nil {
+			return "", err
+		}
+
+		ba, err := HostCertificatePEM(3, hostKey, ctx)
+		if err != nil {
+			return "", err
+		}
+		v = string(ba)
+
+	default:
+		return "", fmt.Errorf("unknown secret kind: %q", kind)
+	}
+
+	if v == "" {
+		panic("value not generated?!")
+	}
+
+	if err := secrets.Set(fullRef, v); err != nil {
+		return "", err
+	}
+
+	return v, nil
+}

ssl.go

@@ -0,0 +1,187 @@
+package main
+
+import (
+	"bytes"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"fmt"
+	"log"
+	"math/big"
+	"net"
+	"time"
+)
+
+const (
+	// From Kubernetes:
+	// ECPrivateKeyBlockType is a possible value for pem.Block.Type.
+	ECPrivateKeyBlockType = "EC PRIVATE KEY"
+)
+
+func PrivateKeyPEM() (*ecdsa.PrivateKey, []byte) {
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		log.Fatal("Failed to generate the key: ", err)
+	}
+
+	b, err := x509.MarshalECPrivateKey(key)
+	if err != nil {
+		log.Fatal("Unable to mashal EC key: ", err)
+	}
+
+	buf := &bytes.Buffer{}
+
+	if err := pem.Encode(buf, &pem.Block{
+		Type:  ECPrivateKeyBlockType,
+		Bytes: b,
+	}); err != nil {
+		log.Fatal("Failed to write encode key: ", err)
+	}
+
+	return key, buf.Bytes()
+}
+
+func SelfSignedCertificatePEM(ttlYears int, key *ecdsa.PrivateKey) []byte {
+	notBefore := time.Now()
+	notAfter := notBefore.AddDate(ttlYears, 0, 0).Truncate(24 * time.Hour)
+
+	serialNumber, err := rand.Int(rand.Reader, big.NewInt(0xffffffff))
+	if err != nil {
+		log.Fatal("Failed to generate serial number: ", err)
+	}
+
+	certTemplate := &x509.Certificate{
+		SerialNumber:          serialNumber,
+		NotBefore:             notBefore,
+		NotAfter:              notAfter,
+		IsCA:                  true,
+		Subject:               pkix.Name{},
+		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{},
+		BasicConstraintsValid: true,
+	}
+	parentTemplate := certTemplate // self-signed
+	publicKey := key.Public()
+
+	derBytes, err := x509.CreateCertificate(rand.Reader, certTemplate, parentTemplate, publicKey, key)
+	if err != nil {
+		log.Fatal("Failed to generate certificate: ", err)
+	}
+
+	buf := &bytes.Buffer{}
+
+	if err := pem.Encode(buf, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
+		log.Fatal("Failed to write certificate: ", err)
+	}
+
+	return buf.Bytes()
+}
+
+func HostCertificatePEM(ttlYears int, key *ecdsa.PrivateKey, ctx *renderContext) ([]byte, error) {
+	caKey, err := loadPrivateKey("ca", ctx)
+	if err != nil {
+		return nil, err
+	}
+	caCrt, err := loadCertificate("ca", ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	notBefore := time.Now()
+	notAfter := notBefore.AddDate(ttlYears, 0, 0).Truncate(24 * time.Hour)
+
+	serialNumber, err := rand.Int(rand.Reader, big.NewInt(0xffffffff))
+	if err != nil {
+		log.Fatal("Failed to generate serial number: ", err)
+	}
+
+	dnsNames := []string{ctx.Host.Name}
+	ips := []net.IP{net.ParseIP(ctx.Host.IP)}
+
+	if ctx.Group.Master {
+		dnsNames = append(dnsNames,
+			"kubernetes",
+			"kubernetes.kube-system",
+			"kubernetes.kube-system.svc."+ctx.Cluster.Domain,
+		)
+		ips = append(ips, ctx.Cluster.KubernetesSvcIP())
+	}
+
+	certTemplate := &x509.Certificate{
+		SerialNumber: serialNumber,
+		NotBefore:    notBefore,
+		NotAfter:     notAfter,
+		IsCA:         false,
+		Subject: pkix.Name{
+			CommonName: ctx.Host.Name,
+		},
+		KeyUsage:    x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth},
+		DNSNames:    dnsNames,
+		IPAddresses: ips,
+	}
+	parentTemplate := caCrt
+	publicKey := key.Public()
+
+	derBytes, err := x509.CreateCertificate(rand.Reader, certTemplate, parentTemplate, publicKey, caKey)
+	if err != nil {
+		log.Fatal("Failed to generate certificate: ", err)
+	}
+
+	f := &bytes.Buffer{}
+	if err := pem.Encode(f, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil {
+		log.Fatal("Failed to write certificate: ", err)
+	}
+
+	return f.Bytes(), nil
+}
+
+func loadPrivateKey(path string, ctx *renderContext) (*ecdsa.PrivateKey, error) {
+	keyS, err := getSecret("tls-key:"+path, ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	keyBytes := []byte(keyS)
+	if len(keyBytes) == 0 {
+		return nil, fmt.Errorf("%s is empty", path)
+	}
+
+	p, _ := pem.Decode(keyBytes)
+	if p.Type != ECPrivateKeyBlockType {
+		return nil, fmt.Errorf("wrong type in %s: %s", path, p.Type)
+	}
+
+	key, err := x509.ParseECPrivateKey(p.Bytes)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse key in %s: %v", path, err)
+	}
+	return key, nil
+}
+
+func loadCertificate(path string, ctx *renderContext) (*x509.Certificate, error) {
+	crtS, err := getSecret("tls-self-signed-cert:"+path, ctx)
+	if err != nil {
+		return nil, err
+	}
+
+	crtBytes := []byte(crtS)
+	if len(crtBytes) == 0 {
+		return nil, fmt.Errorf("%s is empty", path)
+	}
+
+	p, _ := pem.Decode(crtBytes)
+	if p.Type != "CERTIFICATE" {
+		return nil, fmt.Errorf("wrong type in %s: %s", path, p.Type)
+	}
+
+	crt, err := x509.ParseCertificate(p.Bytes)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse certificate in %s: %v", path, err)
+	}
+
+	return crt, nil
+}

upstream.go

@@ -0,0 +1,80 @@
+package main
+
+import (
+	"flag"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	gopath "path"
+	"path/filepath"
+	"time"
+)
+
+var (
+	upstreamURL = flag.String("upstream", "https://direktil.novit.nc/dist", "Upstream server for dist elements")
+)
+
+func (ctx *renderContext) distFetch(path ...string) (outPath string, err error) {
+	outPath = ctx.distFilePath(path...)
+
+	if _, err = os.Stat(outPath); err == nil {
+		return
+	} else if !os.IsNotExist(err) {
+		return
+	}
+
+	subPath := gopath.Join(path...)
+
+	log.Print("need to fetch ", subPath)
+
+	if err = os.MkdirAll(filepath.Dir(outPath), 0755); err != nil {
+		return
+	}
+
+	fullURL := *upstreamURL + "/" + subPath
+
+	resp, err := http.Get(fullURL)
+	if err != nil {
+		return
+	}
+
+	tempOutPath := filepath.Join(filepath.Dir(outPath), "._part_"+filepath.Base(outPath))
+
+	done := make(chan error, 1)
+	go func() {
+		defer resp.Body.Close()
+		defer close(done)
+
+		out, err := os.Create(tempOutPath)
+		if err != nil {
+			done <- err
+			return
+		}
+
+		defer out.Close()
+
+		_, err = io.Copy(out, resp.Body)
+		done <- err
+	}()
+
+wait:
+	select {
+	case <-time.After(10 * time.Second):
+		log.Print("still fetching ", subPath, "...")
+		goto wait
+
+	case err = <-done:
+		if err != nil {
+			log.Print("fetch of ", subPath, " failed: ", err)
+			os.Remove(tempOutPath)
+			return
+		}
+
+		log.Print("fetch of ", subPath, " finished")
+	}
+
+	err = os.Rename(tempOutPath, outPath)
+
+	return
+}

utils.go

@@ -0,0 +1,18 @@
+package main
+
+import (
+	"io"
+	"os"
+)
+
+func writeFile(out io.Writer, path string) error {
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+
+	defer f.Close()
+
+	_, err = io.Copy(out, f)
+	return err
+}