dir2config: switch to includes instead of ad-hoc defaults

cmd/dkl-dir2config/assemble.go

@@ -0,0 +1,138 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"strings"
+
+	"gopkg.in/yaml.v2"
+)
+
+func mergeIn(tgt, add map[any]any) {
+mergeLoop:
+	for k, v := range add {
+		switch v := v.(type) {
+		case map[any]any:
+			if tgtV, ok := tgt[k]; ok {
+				switch tgtV := tgtV.(type) {
+				case map[any]any:
+					mergeIn(tgtV, v)
+					continue mergeLoop
+				}
+			}
+		}
+
+		tgt[k] = v
+	}
+}
+
+func assemble(path string) (yamlBytes []byte, err error) {
+	obj := map[any]any{}
+
+	if Debug {
+		log.Printf("assemble %q", path)
+	}
+
+	err = eachFragment(path, searchList, func(r io.Reader) (err error) {
+		m := map[any]any{}
+		err = yaml.NewDecoder(r).Decode(&m)
+		if err != nil {
+			return
+		}
+
+		mergeIn(obj, m)
+		return
+	})
+	if err != nil {
+		err = fmt.Errorf("failed to assemble %q: %w", path, err)
+		return
+	}
+
+	yamlBytes, err = yaml.Marshal(obj)
+	if err != nil {
+		return
+	}
+
+	if Debug {
+		log.Printf("assemble %q result:\n%s", path, yamlBytes)
+	}
+
+	return
+}
+
+func eachFragment(path string, searchList []FS, walk func(io.Reader) error) (err error) {
+	var r io.ReadCloser
+
+	for len(searchList) != 0 {
+		fs := searchList[0]
+
+		r, err = fs.Open(path + ".yaml")
+		if os.IsNotExist(err) {
+			searchList = searchList[1:]
+			continue
+		}
+
+		if err != nil {
+			return
+		}
+
+		// found and open
+		break
+	}
+
+	if r == nil {
+		err = fmt.Errorf("%s: %w", path, os.ErrNotExist)
+		return
+	}
+
+	ba, err := io.ReadAll(r)
+	r.Close()
+
+	if err != nil {
+		return
+	}
+
+	if Debug {
+		log.Print("fragment:\n", string(ba))
+	}
+
+	in := bytes.NewBuffer(ba)
+
+	for {
+		var line string
+		line, err = in.ReadString('\n')
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return
+		}
+
+		line = strings.TrimSpace(line)
+		if len(line) == 0 {
+			continue
+		}
+
+		includePath, found := strings.CutPrefix(line, "#!include ")
+		if !found {
+			continue // or break?
+		}
+
+		includePath = strings.TrimSpace(includePath)
+		if Debug {
+			log.Print("#!include ", includePath)
+		}
+		err = eachFragment(includePath, searchList, walk)
+		if err != nil {
+			err = fmt.Errorf("include %q: %w", includePath, err)
+			return
+		}
+	}
+
+	in = bytes.NewBuffer(ba)
+	err = walk(in)
+	return
+}

cmd/dkl-dir2config/fs.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"io"
+	iofs "io/fs"
+)
+
+type FS interface {
+	Open(path string) (io.ReadCloser, error)
+	List(path string) ([]string, error)
+}
+
+type fsFS struct{ iofs.FS }
+
+func (fs fsFS) Open(path string) (io.ReadCloser, error) {
+	return fs.FS.Open(path)
+}
+
+func (fs fsFS) List(path string) (entries []string, err error) {
+	dirEnts, err := iofs.ReadDir(fs.FS, path)
+	if err != nil {
+		return
+	}
+
+	entries = make([]string, 0, len(dirEnts))
+	for _, ent := range dirEnts {
+		if ent.IsDir() {
+			continue
+		}
+		entries = append(entries, ent.Name())
+	}
+
+	return
+}

cmd/dkl-dir2config/git.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"io"
+	"sort"
+
+	"github.com/go-git/go-git/v5/plumbing/object"
+)
+
+type gitFS struct{ *object.Tree }
+
+func (fs gitFS) Open(path string) (r io.ReadCloser, err error) {
+	f, err := fs.Tree.File(path)
+	if err != nil {
+		return
+	}
+	return f.Reader()
+}
+
+func (fs gitFS) List(path string) (entries []string, err error) {
+	tree, err := fs.Tree.Tree(path)
+	if err != nil {
+		return
+	}
+
+	entries = make([]string, 0, len(tree.Entries))
+
+	for _, ent := range tree.Entries {
+		if !ent.Mode.IsFile() {
+			continue
+		}
+
+		entries = append(entries, ent.Name)
+	}
+
+	sort.Strings(entries)
+	return
+}

cmd/dkl-dir2config/main.go

@@ -2,9 +2,13 @@ package main
 
 import (
 	"flag"
+	"io/fs"
 	"log"
 	"os"
+	"path/filepath"
 
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
 	yaml "gopkg.in/yaml.v2"
 
 	"novit.tech/direktil/pkg/localconfig"
@@ -13,17 +17,24 @@ import (
 )
 
 var (
-	dir          = flag.String("in", ".", "Source directory")
-	outPath      = flag.String("out", "config.yaml", "Output file")
-	defaultsPath = flag.String("defaults", "defaults", "Path to the defaults")
+	Debug = false
+
+	dir     = flag.String("in", ".", "Source directory")
+	outPath = flag.String("out", "config.yaml", "Output file")
+
+	base fs.FS
 
 	src *clustersconfig.Config
 	dst *localconfig.Config
 )
 
+func init() {
+	flag.BoolVar(&Debug, "debug", Debug, "debug")
+}
+
 func loadSrc() {
 	var err error
-	src, err = clustersconfig.FromDir(*dir, *defaultsPath)
+	src, err = clustersconfig.FromDir(read, assemble, listBase, listMerged)
 	if err != nil {
 		log.Fatal("failed to load config from dir: ", err)
 	}
@@ -34,6 +45,16 @@ func main() {
 
 	log.SetFlags(log.Ltime | log.Lmicroseconds | log.Lshortfile)
 
+	base = os.DirFS(*dir)
+	searchList = append(searchList, fsFS{base})
+
+	openIncludes()
+
+	if false {
+		assemble("hosts/m1")
+		log.Fatal("--- debug: end ---")
+	}
+
 	loadSrc()
 
 	dst = &localconfig.Config{
@@ -50,8 +71,6 @@ func main() {
 
 	// ----------------------------------------------------------------------
 	for _, host := range src.Hosts {
-		loadSrc() // FIXME ugly fix of some template caching or something
-
 		log.Print("rendering host ", host.Name)
 		ctx, err := newRenderContext(host, src)
 
@@ -70,9 +89,9 @@ func main() {
 		}
 		ips = append(ips, host.IPs...)
 
-		if ctx.Group.Versions["modules"] == "" {
+		if ctx.Host.Versions["modules"] == "" {
 			// default modules' version to kernel's version
-			ctx.Group.Versions["modules"] = ctx.Group.Kernel
+			ctx.Host.Versions["modules"] = ctx.Host.Kernel
 		}
 
 		dst.Hosts = append(dst.Hosts, &localconfig.Host{
@@ -86,11 +105,11 @@ func main() {
 			MACs: macs,
 			IPs:  ips,
 
-			IPXE: ctx.Group.IPXE, // TODO render
+			IPXE: ctx.Host.IPXE, // TODO render
 
-			Kernel:   ctx.Group.Kernel,
-			Initrd:   ctx.Group.Initrd,
-			Versions: ctx.Group.Versions,
+			Kernel:   ctx.Host.Kernel,
+			Initrd:   ctx.Host.Initrd,
+			Versions: ctx.Host.Versions,
 
 			BootstrapConfig: ctx.BootstrapConfig(),
 			Config:          ctx.Config(),
@@ -110,3 +129,76 @@ func main() {
 	}
 
 }
+
+func cfgPath(subPath string) string { return filepath.Join(*dir, subPath) }
+
+func openIncludes() {
+	includesFile, err := base.Open("includes.yaml")
+
+	if os.IsNotExist(err) {
+		return
+	}
+	if err != nil {
+		log.Fatal("failed to open includes: ", err)
+	}
+
+	includes := make([]struct {
+		Path   string
+		Branch string
+		Tag    string
+	}, 0)
+
+	err = yaml.NewDecoder(includesFile).Decode(&includes)
+	if err != nil {
+		log.Fatal("failed to parse includes: ", err)
+	}
+
+	for _, include := range includes {
+		switch {
+		case include.Branch != "" || include.Tag != "":
+			p := cfgPath(include.Path) // FIXME parse git path to allow remote repos
+
+			var rev plumbing.Revision
+
+			switch {
+			case include.Branch != "":
+				log.Printf("opening include path %q as git, branch %q", p, include.Branch)
+				rev = plumbing.Revision(plumbing.NewBranchReferenceName(include.Branch))
+
+			case include.Tag != "":
+				log.Printf("opening include path %q as git, tag %q", p, include.Branch)
+				rev = plumbing.Revision(plumbing.NewTagReferenceName(include.Branch))
+			}
+
+			repo, err := git.PlainOpen(p)
+			if err != nil {
+				log.Fatal("failed to open: ", err)
+			}
+
+			revH, err := repo.ResolveRevision(rev)
+			if err != nil {
+				log.Fatalf("failed to resolve revision %s: %v", rev, err)
+			}
+
+			log.Print("  -> resolved to commit ", *revH)
+
+			commit, err := repo.CommitObject(*revH)
+			if err != nil {
+				log.Fatal("failed to get commit object: ", err)
+			}
+
+			tree, err := commit.Tree()
+			if err != nil {
+				log.Fatal("failed to open git tree: ", err)
+			}
+
+			searchList = append(searchList, gitFS{tree})
+
+		default:
+			p := cfgPath(include.Path)
+			log.Printf("opening include path %q as raw dir", p)
+
+			searchList = append(searchList, fsFS{os.DirFS(p)})
+		}
+	}
+}

cmd/dkl-dir2config/render-context.go

@@ -23,9 +23,8 @@ type renderContext struct {
 	Annotations map[string]string
 
 	Host    *clustersconfig.Host
-	Group   *clustersconfig.Group
 	Cluster *clustersconfig.Cluster
-	Vars    map[string]interface{}
+	Vars    map[string]any
 
 	BootstrapConfigTemplate *clustersconfig.Template
 	ConfigTemplate          *clustersconfig.Template
@@ -41,34 +40,25 @@ func newRenderContext(host *clustersconfig.Host, cfg *clustersconfig.Config) (ct
 		return
 	}
 
-	group := cfg.Group(host.Group)
-	if group == nil {
-		err = fmt.Errorf("no group named %q", host.Group)
-		return
-	}
+	vars := make(map[string]any)
 
-	vars := make(map[string]interface{})
-
-	for _, oVars := range []map[string]interface{}{
+	for _, oVars := range []map[string]any{
 		cluster.Vars,
-		group.Vars,
 		host.Vars,
 	} {
 		mapMerge(vars, oVars)
 	}
 
 	return &renderContext{
-		Labels:      mergeLabels(cluster.Labels, group.Labels, host.Labels),
-		Annotations: mergeLabels(cluster.Annotations, group.Annotations, host.Annotations),
+		Labels:      mergeLabels(cluster.Labels, host.Labels),
+		Annotations: mergeLabels(cluster.Annotations, host.Annotations),
 
 		Host:    host,
-		Group:   group,
 		Cluster: cluster,
 		Vars:    vars,
 
-		BootstrapConfigTemplate: cfg.ConfigTemplate(group.BootstrapConfig),
-		ConfigTemplate:          cfg.ConfigTemplate(group.Config),
-		StaticPodsTemplate:      cfg.StaticPodsTemplate(group.StaticPods),
+		BootstrapConfigTemplate: cfg.ConfigTemplate(host.BootstrapConfig),
+		ConfigTemplate:          cfg.ConfigTemplate(host.Config),
 
 		clusterConfig: cfg,
 	}, nil
@@ -132,8 +122,6 @@ func (ctx *renderContext) Name() string {
 	switch {
 	case ctx.Host != nil:
 		return "host:" + ctx.Host.Name
-	case ctx.Group != nil:
-		return "group:" + ctx.Group.Name
 	case ctx.Cluster != nil:
 		return "cluster:" + ctx.Cluster.Name
 	default:
@@ -143,14 +131,14 @@ func (ctx *renderContext) Name() string {
 
 func (ctx *renderContext) BootstrapConfig() string {
 	if ctx.BootstrapConfigTemplate == nil {
-		log.Fatalf("no such (bootstrap) config: %q", ctx.Group.BootstrapConfig)
+		log.Fatalf("no such (bootstrap) config: %q", ctx.Host.BootstrapConfig)
 	}
 	return ctx.renderConfig(ctx.BootstrapConfigTemplate)
 }
 
 func (ctx *renderContext) Config() string {
 	if ctx.ConfigTemplate == nil {
-		log.Fatalf("no such config: %q", ctx.Group.Config)
+		log.Fatalf("no such config: %q", ctx.Host.Config)
 	}
 	return ctx.renderConfig(ctx.ConfigTemplate)
 }
@@ -166,36 +154,8 @@ func (ctx *renderContext) renderConfigTo(buf io.Writer, configTemplate *clusters
 
 	ctxMap := ctx.asMap()
 
-	templateFuncs := ctx.templateFuncs(ctxMap)
-
-	render := func(what string, t *clustersconfig.Template) (s string, err error) {
-		buf := &bytes.Buffer{}
-		err = t.Execute(ctxName, what, buf, ctxMap, templateFuncs)
-		if err != nil {
-			log.Printf("host %s: failed to render %s [%q]: %v", ctx.Host.Name, what, t.Name, err)
-			return
-		}
-
-		s = buf.String()
-		return
-	}
-
 	extraFuncs := ctx.templateFuncs(ctxMap)
 
-	extraFuncs["static_pods"] = func() (string, error) {
-		name := ctx.Group.StaticPods
-		if len(name) == 0 {
-			return "", fmt.Errorf("group %q has no static pods defined", ctx.Group.Name)
-		}
-
-		t := ctx.clusterConfig.StaticPodsTemplate(name)
-		if t == nil {
-			return "", fmt.Errorf("no static pods template named %q", name)
-		}
-
-		return render("static-pods", t)
-	}
-
 	extraFuncs["bootstrap_pods_files"] = func(dir string) (string, error) {
 		namePods := ctx.renderBootstrapPods()
 
@@ -226,26 +186,10 @@ func (ctx *renderContext) renderConfigTo(buf io.Writer, configTemplate *clusters
 	}
 
 	if err := configTemplate.Execute(ctxName, "config", buf, ctxMap, extraFuncs); err != nil {
-		log.Fatalf("failed to render config %q for host %q: %v", ctx.Group.Config, ctx.Host.Name, err)
+		log.Fatalf("failed to render config %q for host %q: %v", ctx.Host.Config, ctx.Host.Name, err)
 	}
 }
 
-func (ctx *renderContext) StaticPods() (ba []byte, err error) {
-	if ctx.StaticPodsTemplate == nil {
-		log.Fatalf("no such static-pods: %q", ctx.Group.StaticPods)
-	}
-
-	ctxMap := ctx.asMap()
-
-	buf := bytes.NewBuffer(make([]byte, 0, 4096))
-	if err = ctx.StaticPodsTemplate.Execute(ctx.Name(), "static-pods", buf, ctxMap, ctx.templateFuncs(ctxMap)); err != nil {
-		return
-	}
-
-	ba = buf.Bytes()
-	return
-}
-
 func (ctx *renderContext) templateFuncs(ctxMap map[string]interface{}) map[string]interface{} {
 	cluster := ctx.Cluster.Name
 

cmd/dkl-dir2config/render-host.go

@@ -11,13 +11,13 @@ import (
 )
 
 func (ctx *renderContext) renderBootstrapPods() (pods []namePod) {
-	if ctx.Cluster.BootstrapPods == "" {
+	if ctx.Host.BootstrapPods == "" {
 		return
 	}
 
-	bootstrapPods, ok := src.BootstrapPods[ctx.Cluster.BootstrapPods]
+	bootstrapPods, ok := src.BootstrapPods[ctx.Host.BootstrapPods]
 	if !ok {
-		log.Fatalf("no bootstrap pods template named %q", ctx.Cluster.BootstrapPods)
+		log.Fatalf("no bootstrap pods template named %q", ctx.Host.BootstrapPods)
 	}
 
 	// render bootstrap pods

cmd/dkl-dir2config/search-list.go

@@ -0,0 +1,60 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"sort"
+)
+
+var searchList = make([]FS, 0)
+
+// read the first file matching path in the search list
+func read(path string) (ba []byte, err error) {
+	for _, fs := range searchList {
+		var r io.ReadCloser
+		r, err = fs.Open(path)
+		if os.IsNotExist(err) {
+			continue
+		}
+		if err != nil {
+			return
+		}
+
+		defer r.Close()
+		return ioutil.ReadAll(r)
+	}
+
+	err = fmt.Errorf("%s: %w", path, os.ErrNotExist)
+	return
+}
+
+func listBase(path string) ([]string, error) {
+	return fsFS{base}.List(path)
+}
+
+func listMerged(path string) (entries []string, err error) {
+	seen := map[string]bool{}
+	for _, fs := range searchList {
+		var fsEnts []string
+		fsEnts, err = fs.List(path)
+		if os.IsNotExist(err) {
+			err = nil
+			continue
+		}
+		if err != nil {
+			return
+		}
+
+		for _, ent := range fsEnts {
+			if !seen[ent] {
+				entries = append(entries, ent)
+				seen[ent] = true
+			}
+		}
+	}
+
+	sort.Strings(entries)
+	return
+}