secrets migration & restitution

Dockerfile

@@ -1,5 +1,5 @@
 # ------------------------------------------------------------------------
-from mcluseau/golang-builder:1.19.4 as build
+from mcluseau/golang-builder:1.20.0 as build
 
 # ------------------------------------------------------------------------
 from debian:stretch

cmd/dkl-local-server/httperr.go

@@ -0,0 +1,9 @@
+package main
+
+import (
+	"net/http"
+
+	"m.cluseau.fr/go/httperr"
+)
+
+var ErrNotFound = httperr.NewStd(404, http.StatusNotFound, "not found")

cmd/dkl-local-server/secret-store.go

@@ -8,9 +8,13 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
+    "sort"
+	"strings"
 	"sync"
 
+	restful "github.com/emicklei/go-restful"
 	"m.cluseau.fr/go/httperr"
+
 	"novit.tech/direktil/local-server/secretstore"
 )
 
@@ -126,6 +130,7 @@ func unlockSecretStore(passphrase []byte) *httperr.Error {
 	})
 
 	go updateState()
+	go migrateSecrets()
 
 	return nil
 }
@@ -147,7 +152,13 @@ func readSecret(name string, value any) (err error) {
 }
 
 func writeSecret(name string, value any) (err error) {
-	f, err := os.Create(secStorePath(name + ".data.new"))
+	path := secStorePath(name + ".data.new")
+
+	if err = os.MkdirAll(filepath.Dir(path), 0700); err != nil {
+		return
+	}
+
+	f, err := os.Create(path)
 	if err != nil {
 		return
 	}
@@ -167,5 +178,148 @@ func writeSecret(name string, value any) (err error) {
 		return
 	}
 
-	return os.Rename(f.Name(), secStorePath(name+".data"))
+	err = os.Rename(f.Name(), secStorePath(name+".data"))
+    if err != nil {
+    return
+    }
+
+    go updateState()
+    return
+}
+
+var secL sync.Mutex
+
+func updateSecret[T any](name string, update func(*T)) (err error) {
+	secL.Lock()
+	defer secL.Unlock()
+
+	v := new(T)
+	err = readSecret(name, v)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return
+		}
+		err = nil
+	}
+
+	update(v)
+
+	return writeSecret(name, *v)
+}
+
+func updateSecretWithKey[T any](name, key string, update func(v *T)) (err error) {
+	secL.Lock()
+	defer secL.Unlock()
+
+	kvs := map[string]*T{}
+
+	err = readSecret(name, &kvs)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return
+		}
+		err = nil
+	}
+
+	update(kvs[key])
+
+	return writeSecret(name, kvs)
+}
+
+type KVSecrets[T any] struct{ Name string }
+
+func (s KVSecrets[T]) Data() (kvs map[string]T, err error) {
+	kvs = make(map[string]T)
+	err = readSecret(s.Name, &kvs)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return
+		}
+		err = nil
+	}
+	return
+}
+
+func (s KVSecrets[T]) Keys(prefix string) (keys []string, err error) {
+	kvs, err := s.Data()
+	if err != nil {
+		return
+	}
+
+	keys = make([]string, 0, len(kvs))
+
+	for k := range kvs {
+		if !strings.HasPrefix(k, prefix) {
+			continue
+		}
+		keys = append(keys, k[len(prefix):])
+	}
+
+    sort.Strings(keys)
+
+	return
+}
+
+func (s KVSecrets[T]) Get(key string) (v T, found bool, err error) {
+	kvs, err := s.Data()
+	if err != nil {
+		return
+	}
+
+	v, found = kvs[key]
+	return
+}
+
+func (s KVSecrets[T]) Put(key string, v T) (err error) {
+	secL.Lock()
+	defer secL.Unlock()
+
+	kvs, err := s.Data()
+	if err != nil {
+		return
+	}
+
+	kvs[key] = v
+	err = writeSecret(s.Name, kvs)
+	return
+}
+
+func (s KVSecrets[T]) WsList(resp *restful.Response, prefix string) {
+	keys, err := s.Keys(prefix)
+	if err != nil {
+		httperr.New(http.StatusInternalServerError, err).WriteJSON(resp.ResponseWriter)
+		return
+	}
+
+	resp.WriteEntity(keys)
+}
+
+func (s KVSecrets[T]) WsGet(resp *restful.Response, key string) {
+	keys, found, err := s.Get(key)
+	if err != nil {
+		httperr.New(http.StatusInternalServerError, err).WriteJSON(resp.ResponseWriter)
+		return
+	}
+
+	if !found {
+		ErrNotFound.WriteJSON(resp.ResponseWriter)
+		return
+	}
+
+	resp.WriteEntity(keys)
+}
+
+func (s KVSecrets[T]) WsPut(req *restful.Request, resp *restful.Response, key string) {
+	v := new(T)
+	err := req.ReadEntity(v)
+	if err != nil {
+		httperr.New(http.StatusBadRequest, err).WriteJSON(resp.ResponseWriter)
+		return
+	}
+
+	err = s.Put(key, *v)
+	if err != nil {
+		httperr.New(http.StatusInternalServerError, err).WriteJSON(resp.ResponseWriter)
+		return
+	}
 }

cmd/dkl-local-server/secrets-migrate.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"log"
+	"os"
+
+	cfsslconfig "github.com/cloudflare/cfssl/config"
+)
+
+func migrateSecrets() {
+	if _, err := os.Stat(secretDataPath()); err != nil {
+		if os.IsNotExist(err) {
+			return
+		}
+
+		log.Print("not migrating old secrets: ", err)
+
+		return
+	}
+
+	log.Print("migrating old secrets")
+
+	log := log.New(log.Default().Writer(), "secrets migration: ", log.Flags()|log.Lmsgprefix)
+
+	// load secrets
+	cfg, err := readConfig()
+	if err != nil {
+		log.Fatal(err)
+		return
+	}
+
+	var sslCfg *cfsslconfig.Config
+
+	if len(cfg.SSLConfig) == 0 {
+		sslCfg = &cfsslconfig.Config{}
+	} else {
+		sslCfg, err = cfsslconfig.LoadConfig([]byte(cfg.SSLConfig))
+		if err != nil {
+			return
+		}
+	}
+
+	if err := loadSecretData(sslCfg); err != nil {
+		log.Fatal(err)
+		return
+	}
+
+	for clusterName, cluster := range secretData.clusters {
+		for k, v := range cluster.Tokens {
+			err = clusterTokens.Put(clusterName+"/"+k, v)
+			if err != nil {
+				log.Fatal(err)
+				return
+			}
+		}
+
+		for k, v := range cluster.Passwords {
+			err = clusterPasswords.Put(clusterName+"/"+k, v)
+			if err != nil {
+				log.Fatal(err)
+				return
+			}
+		}
+
+		for caName, ca := range cluster.CAs {
+			clusterCAs.Put(clusterName+"/"+caName, CA{Key: ca.Key, Cert: ca.Cert})
+
+			for signedName, signed := range ca.Signed {
+				clusterCASignedKeys.Put(clusterName+"/"+caName+"/"+signedName, *signed)
+			}
+		}
+
+		// TODO
+	}
+}

cmd/dkl-local-server/state.go

@@ -28,11 +28,11 @@ type State struct {
 }
 
 type ClusterState struct {
-	Name   string
+	Name      string
-	Addons bool
+	Addons    bool
-	// TODO CAs
+	Passwords []string
-	// TODO passwords
+	Tokens    []string
-	// TODO tokens
+	CAs       []CAState
 }
 
 type HostState struct {
@@ -41,6 +41,11 @@ type HostState struct {
 	IPs     []string
 }
 
+type CAState struct {
+	Name   string
+	Signed []string
+}
+
 var wState = watchable.New[State]()
 
 func init() {
@@ -68,6 +73,34 @@ func updateState() {
 			Name:   cluster.Name,
 			Addons: len(cluster.Addons) != 0,
 		}
+
+		c.Passwords, err = clusterPasswords.Keys(c.Name + "/")
+		if err != nil {
+			log.Print("failed to read cluster passwords: ", err)
+		}
+		c.Tokens, err = clusterTokens.Keys(c.Name + "/")
+		if err != nil {
+			log.Print("failed to read cluster tokens: ", err)
+		}
+
+		caNames, err := clusterCAs.Keys(c.Name + "/")
+		if err != nil {
+			log.Print("failed to read cluster CAs: ", err)
+		}
+		for _, caName := range caNames {
+			ca := CAState{Name: caName}
+
+			signedNames, err := clusterCASignedKeys.Keys(c.Name + "/" + caName + "/")
+			if err != nil {
+				log.Print("failed to read cluster CA signed keys: ", err)
+			}
+			for _, signedName := range signedNames {
+				ca.Signed = append(ca.Signed, signedName)
+			}
+
+			c.CAs = append(c.CAs, ca)
+		}
+
 		clusters = append(clusters, c)
 	}
 

cmd/dkl-local-server/ws-cluster-cas.go

@@ -0,0 +1,35 @@
+package main
+
+import (
+	restful "github.com/emicklei/go-restful"
+)
+
+var clusterCAs = newClusterSecretKV[CA]("CAs")
+
+func wsClusterCAs(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	clusterCAs.WsList(resp, clusterName+"/")
+}
+
+func wsClusterCA(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	name := req.PathParameter("ca-name")
+
+	clusterCAs.WsGet(resp, clusterName+"/"+name)
+}
+
+var clusterCASignedKeys = newClusterSecretKV[KeyCert]("CA-signed-keys")
+
+func wsClusterCASignedKeys(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	caName := req.PathParameter("ca-name")
+	clusterCASignedKeys.WsList(resp, clusterName+"/"+caName+"/")
+}
+
+func wsClusterCASignedKey(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	caName := req.PathParameter("ca-name")
+	name := req.PathParameter("signed-name")
+
+	clusterCASignedKeys.WsGet(resp, clusterName+"/"+caName+"/"+name)
+}

cmd/dkl-local-server/ws-cluster-passwords.go

@@ -0,0 +1,30 @@
+package main
+
+import (
+	restful "github.com/emicklei/go-restful"
+)
+
+var clusterPasswords = newClusterSecretKV[string]("passwords")
+
+func wsClusterPasswords(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	clusterPasswords.WsList(resp, clusterName+"/")
+}
+
+func wsClusterPassword(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	name := req.PathParameter("password-name")
+
+	clusterPasswords.WsGet(resp, clusterName+"/"+name)
+}
+
+func wsClusterSetPassword(req *restful.Request, resp *restful.Response) {
+	cluster := wsReadCluster(req, resp)
+	if cluster == nil {
+		return
+	}
+
+	name := req.PathParameter("password-name")
+
+	clusterPasswords.WsPut(req, resp, cluster.Name+"/"+name)
+}

cmd/dkl-local-server/ws-cluster-tokens.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	restful "github.com/emicklei/go-restful"
+)
+
+var clusterTokens = newClusterSecretKV[string]("tokens")
+
+func wsClusterTokens(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	clusterTokens.WsList(resp, clusterName+"/")
+}
+
+func wsClusterToken(req *restful.Request, resp *restful.Response) {
+	clusterName := req.PathParameter("cluster-name")
+	name := req.PathParameter("token-name")
+
+	clusterTokens.WsGet(resp, clusterName+"/"+name)
+}

cmd/dkl-local-server/ws-clusters.go

@@ -9,6 +9,13 @@ import (
 	"novit.tech/direktil/pkg/localconfig"
 )
 
+var clusterSecretKVs = []string{}
+
+func newClusterSecretKV[T any](name string) KVSecrets[T] {
+    clusterSecretKVs = append(clusterSecretKVs, name)
+    return KVSecrets[T]{"clusters/"+name}
+}
+
 func wsListClusters(req *restful.Request, resp *restful.Response) {
 	cfg := wsReadConfig(resp)
 	if cfg == nil {
@@ -64,97 +71,6 @@ func wsClusterAddons(req *restful.Request, resp *restful.Response) {
 	wsRender(resp, cluster.Addons, cluster)
 }
 
-func wsClusterPasswords(req *restful.Request, resp *restful.Response) {
-	cluster := wsReadCluster(req, resp)
-	if cluster == nil {
-		return
-	}
-
-	resp.WriteEntity(secretData.Passwords(cluster.Name))
-}
-
-func wsClusterPassword(req *restful.Request, resp *restful.Response) {
-	cluster := wsReadCluster(req, resp)
-	if cluster == nil {
-		return
-	}
-
-	name := req.PathParameter("password-name")
-
-	resp.WriteEntity(secretData.Password(cluster.Name, name))
-}
-
-func wsClusterSetPassword(req *restful.Request, resp *restful.Response) {
-	cluster := wsReadCluster(req, resp)
-	if cluster == nil {
-		return
-	}
-
-	name := req.PathParameter("password-name")
-
-	var password string
-	if err := req.ReadEntity(&password); err != nil {
-		wsError(resp, err) // FIXME this is a BadRequest
-		return
-	}
-
-	secretData.SetPassword(cluster.Name, name, password)
-
-	if err := secretData.Save(); err != nil {
-		wsError(resp, err)
-		return
-	}
-}
-
-func wsClusterToken(req *restful.Request, resp *restful.Response) {
-	cluster := wsReadCluster(req, resp)
-	if cluster == nil {
-		return
-	}
-
-	name := req.PathParameter("token-name")
-
-	token, err := secretData.Token(cluster.Name, name)
-	if err != nil {
-		wsError(resp, err)
-		return
-	}
-
-	resp.WriteEntity(token)
-}
-
-func wsClusterBootstrapPods(req *restful.Request, resp *restful.Response) {
-	cluster := wsReadCluster(req, resp)
-	if cluster == nil {
-		return
-	}
-
-	if len(cluster.BootstrapPods) == 0 {
-		log.Printf("cluster %q has no bootstrap pods defined", cluster.Name)
-		wsNotFound(req, resp)
-		return
-	}
-
-	wsRender(resp, cluster.BootstrapPods, cluster)
-}
-
-func wsClusterCAs(req *restful.Request, resp *restful.Response) {
-	cs := secretData.clusters[req.PathParameter("cluster-name")]
-	if cs == nil {
-		wsNotFound(req, resp)
-		return
-	}
-
-	keys := make([]string, 0, len(cs.CAs))
-	for k := range cs.CAs {
-		keys = append(keys, k)
-	}
-
-	sort.Strings(keys)
-
-	resp.WriteJson(keys, restful.MIME_JSON)
-}
-
 func wsClusterCACert(req *restful.Request, resp *restful.Response) {
 	cs := secretData.clusters[req.PathParameter("cluster-name")]
 	if cs == nil {

cmd/dkl-local-server/ws.go

@@ -53,40 +53,50 @@ func registerWS(rest *restful.Container) {
 	ws.Route(ws.GET("/clusters").To(wsListClusters).
 		Doc("List clusters"))
 
-	ws.Route(ws.GET("/clusters/{cluster-name}").To(wsCluster).
+	const (
-		Doc("Get cluster details"))
+		GET = http.MethodGet
+		PUT = http.MethodPut
+	)
 
-	ws.Route(ws.GET("/clusters/{cluster-name}/addons").To(wsClusterAddons).
+	cluster := func(method, subPath string) *restful.RouteBuilder {
-		Produces(mime.YAML).
+		return ws.Method(method).Path("/clusters/{cluster-name}" + subPath).
-		Doc("Get cluster addons").
+			Param(ws.PathParameter("cluster-name", "name of the cluster"))
-		Returns(http.StatusOK, "OK", nil).
+	}
-		Returns(http.StatusNotFound, "The cluster does not exists or does not have addons defined", nil))
 
-	ws.Route(ws.GET("/clusters/{cluster-name}/bootstrap-pods").To(wsClusterBootstrapPods).
+	for _, builder := range []*restful.RouteBuilder{
-		Produces(mime.YAML).
+		cluster(GET, "").To(wsCluster).
-		Doc("Get cluster bootstrap pods YAML definitions").
+			Doc("Get cluster details"),
-		Returns(http.StatusOK, "OK", nil).
-		Returns(http.StatusNotFound, "The cluster does not exists or does not have bootstrap pods defined", nil))
 
-	ws.Route(ws.GET("/clusters/{cluster-name}/passwords").To(wsClusterPasswords).
+		cluster(GET, "/addons").To(wsClusterAddons).
-		Doc("List cluster's passwords"))
+			Produces(mime.YAML).
-	ws.Route(ws.GET("/clusters/{cluster-name}/passwords/{password-name}").To(wsClusterPassword).
+			Doc("Get cluster addons").
-		Doc("Get cluster's password"))
+			Returns(http.StatusOK, "OK", nil).
-	ws.Route(ws.PUT("/clusters/{cluster-name}/passwords/{password-name}").To(wsClusterSetPassword).
+			Returns(http.StatusNotFound, "The cluster does not exists or does not have addons defined", nil),
-		Doc("Set cluster's password"))
 
-	ws.Route(ws.GET("/clusters/{cluster-name}/ca").To(wsClusterCAs).
+		cluster(GET, "/tokens").To(wsClusterTokens).
-		Doc("Get cluster CAs"))
+			Doc("List cluster's tokens"),
-	ws.Route(ws.GET("/clusters/{cluster-name}/ca/{ca-name}/certificate").To(wsClusterCACert).
+		cluster(GET, "/tokens/{token-name}").To(wsClusterToken).
-		Produces(mime.CACERT).
+			Doc("Get cluster's token"),
-		Doc("Get cluster CA's certificate"))
-	ws.Route(ws.GET("/clusters/{cluster-name}/ca/{ca-name}/signed").To(wsClusterSignedCert).
-		Produces(mime.CERT).
-		Param(ws.QueryParameter("name", "signed reference name").Required(true)).
-		Doc("Get cluster's certificate signed by the CA"))
 
-	ws.Route(ws.GET("/clusters/{cluster-name}/tokens/{token-name}").To(wsClusterToken).
+		cluster(GET, "/passwords").To(wsClusterPasswords).
-		Doc("Get cluster's token"))
+			Doc("List cluster's passwords"),
+		cluster(GET, "/passwords/{password-name}").To(wsClusterPassword).
+			Doc("Get cluster's password"),
+		cluster(PUT, "/passwords/{password-name}").To(wsClusterSetPassword).
+			Doc("Set cluster's password"),
+
+		cluster(GET, "/CAs").To(wsClusterCAs).
+			Doc("Get cluster CAs"),
+		cluster(GET, "/CAs/{ca-name}/certificate").To(wsClusterCACert).
+			Produces(mime.CACERT).
+			Doc("Get cluster CA's certificate"),
+		cluster(GET, "/CAs/{ca-name}/signed").To(wsClusterSignedCert).
+			Produces(mime.CERT).
+			Param(ws.QueryParameter("name", "signed reference name").Required(true)).
+			Doc("Get cluster's certificate signed by the CA"),
+	} {
+		ws.Route(builder)
+	}
 
 	ws.Route(ws.GET("/hosts").To(wsListHosts).
 		Doc("List hosts"))

go.mod

@@ -1,6 +1,6 @@
 module novit.tech/direktil/local-server
 
-go 1.19
+go 1.20
 
 require (
 	github.com/cavaliergopher/cpio v1.0.1

html/ui/js/Cluster.js

@@ -1,16 +1,37 @@
 
 import Downloads from './Downloads.js';
+import GetCopy from './GetCopy.js';
 
 export default {
-    components: { Downloads },
+    components: { Downloads, GetCopy },
     props: [ 'cluster', 'token', 'state' ],
     template: `
 <div class="cluster">
   <div class="title">Cluster {{ cluster.Name }}</div>
+  <div class="section">Tokens</div>
+  <section class="links">
+    <GetCopy v-for="n in cluster.Tokens" :token="token" :name="n" :href="'/clusters/'+cluster.Name+'/tokens/'+n" />
+  </section>
+  <div class="section">Passwords</div>
+  <section class="links">
+    <GetCopy v-for="n in cluster.Passwords" :token="token" :name="n" :href="'/clusters/'+cluster.Name+'/passwords/'+n" />
+  </section>
   <div class="section">Downloads</div>
   <section class="downloads">
     <Downloads :token="token" :state="state" kind="cluster" :name="cluster.Name" />
   </section>
+  <div class="section">CAs</div>
+  <section v-for="ca in cluster.CAs">
+    {{ ca.Name }}:
+        <GetCopy :token="token" name="cert" :href="'/clusters/'+cluster.Name+'/CAs/'+ca.Name+'/certificate'" />
+        <template v-if="ca.Signed">
+          {{" "}}signed
+          <template v-for="signed in ca.Signed">
+            {{" "}}
+            <GetCopy :token="token" :name="signed" :href="'/clusters/'+cluster.Name+'/CAs/'+ca.Name+'/signed?name='+signed" />
+          </template>
+        </template>
+  </section>
 </div>
 `
 }

html/ui/js/GetCopy.js

@@ -0,0 +1,21 @@
+export default {
+    props: [ 'name', 'href', 'token' ],
+    data() { return {showCopied: false} },
+    template: `<span class="notif"><div v-if="showCopied">copied!</div><a :href="href" @click="fetchAndCopy()">{{name}}<small>&nbsp;&#x1F5D0;</small></a></span>`,
+    methods: {
+        fetchAndCopy() {
+            event.preventDefault()
+
+            fetch(this.href, {
+                method: 'GET',
+                headers: { 'Authorization': 'Bearer ' + this.token },
+            }).then((resp) => resp.headers.get("content-type") == "application/json" ? resp.json() : resp.text())
+              .then((value) => {
+                window.navigator.clipboard.writeText(value)
+                this.showCopied = true
+                setTimeout(() => { this.showCopied = false }, 1000)
+              })
+              .catch((e) => { console.log("failed to get value:", e); alert('failed to get value') })
+        },
+    },
+}

html/ui/style.css

@@ -45,7 +45,7 @@ th, tr:last-child > td {
     background: #333;
     color: #eee;
   }
-  a[href], button.link {
+  a[href], a[href]:visited, button.link {
     border: none;
     color: #31b0fa;
   }
@@ -125,3 +125,23 @@ header .utils > * {
     margin: 2pt 6pt 6pt 6pt;
 }
 
+.notif {
+    display: inline-block;
+    position: relative;
+}
+.notif > div:first-child {
+    position: absolute;
+    min-width: 100%; height: 100%;
+    background: white;
+    opacity: 75%;
+    text-align: center;
+}
+
+.links > * { margin-left: 1ex; }
+.links > *:first-child { margin-left: 0; }
+
+@media (prefers-color-scheme: dark) {
+  .notif > div:first-child {
+    background: black;
+  }
+}