feat(dir2config): defaults

pkg/clustersconfig/dir.go

@@ -1,7 +1,9 @@
 package clustersconfig
 
 import (
+	"fmt"
 	"io/ioutil"
+	"log"
 	"os"
 	"path"
 	"path/filepath"
@@ -10,18 +12,31 @@ import (
 	yaml "gopkg.in/yaml.v2"
 )
 
-func FromDir(dirPath string) (*Config, error) {
-	config := &Config{Addons: make(map[string][]*Template)}
+// Debug enables debug logs from this package.
+var Debug = false
 
-	store := dirStore{dirPath}
-	load := func(dir, name string, out interface{}) error {
+func FromDir(dirPath, defaultsPath string) (*Config, error) {
+	if Debug {
+		log.Printf("loading config from dir %s (defaults from %s)", dirPath, defaultsPath)
+	}
+
+	defaults, err := NewDefaults(defaultsPath)
+	if err != nil {
+		return nil, err
+	}
+
+	store := &dirStore{dirPath}
+	load := func(dir, name string, out Rev) error {
 		ba, err := store.Get(path.Join(dir, name))
 		if err != nil {
 			return err
 		}
-		return yaml.Unmarshal(ba, out)
+
+		return defaults.Load(dir, ".yaml", out, ba)
 	}
 
+	config := &Config{Addons: make(map[string][]*Template)}
+
 	// load clusters
 	names, err := store.List("clusters")
 	if err != nil {
@@ -43,13 +58,79 @@ func FromDir(dirPath string) (*Config, error) {
 		return nil, err
 	}
 
+	read := func(rev, filePath string) (data []byte, fromDefaults bool, err error) {
+		data, err = store.Get(filePath)
+		if err != nil {
+			return
+		}
+
+		if data == nil {
+			if len(rev) == 0 {
+				err = fmt.Errorf("entry not found: %s", filePath)
+				return
+			}
+
+			data, err = defaults.ReadAll(rev, filePath+".yaml")
+			if err != nil {
+				return
+			}
+
+			fromDefaults = true
+		}
+
+		return
+	}
+
+	template := func(rev, dir, name string, templates *[]*Template) (ref string, err error) {
+		ref = name
+		if len(name) == 0 {
+			return
+		}
+
+		ba, fromDefaults, err := read(rev, path.Join(dir, name))
+		if err != nil {
+			return
+		}
+
+		if fromDefaults {
+			ref = rev + ":" + name
+		}
+
+		if !hasTemplate(ref, *templates) {
+			if Debug {
+				log.Printf("new template in %s: %s", dir, ref)
+			}
+
+			*templates = append(*templates, &Template{
+				Name:     ref,
+				Template: string(ba),
+			})
+		}
+
+		return
+	}
+
 	for _, name := range names {
-		o := &Group{Name: name}
-		if err := load("groups", name, o); err != nil {
+		group := &Group{Name: name}
+		if err := load("groups", name, group); err != nil {
 			return nil, err
 		}
 
-		config.Groups = append(config.Groups, o)
+		group.Config, err = template(group.Rev(), "configs", group.Config, &config.Configs)
+		if err != nil {
+			return nil, fmt.Errorf("failed to load config for group %q: %v", name, err)
+		}
+
+		if Debug {
+			log.Printf("group %q: config=%q static_pods=%q", group.Name, group.Config, group.StaticPods)
+		}
+
+		group.StaticPods, err = template(group.Rev(), "static-pods", group.StaticPods, &config.StaticPods)
+		if err != nil {
+			return nil, fmt.Errorf("failed to load static pods for group %q: %v", name, err)
+		}
+
+		config.Groups = append(config.Groups, group)
 	}
 
 	// load hosts
@@ -68,47 +149,57 @@ func FromDir(dirPath string) (*Config, error) {
 	}
 
 	// load config templates
-	loadTemplates := func(dir string, templates *[]*Template) error {
-		names, err = store.List(dir)
+	loadTemplates := func(rev, dir string, templates *[]*Template) error {
+		names, err := store.List(dir)
 		if err != nil {
 			return err
 		}
 
-		for _, name := range names {
-			ba, err := store.Get(path.Join(dir, name))
+		if len(rev) != 0 {
+			var defaultsNames []string
+			defaultsNames, err = defaults.List(rev, dir)
 			if err != nil {
 				return err
 			}
 
-			o := &Template{Name: name, Template: string(ba)}
+			names = append(names, defaultsNames...)
+		}
 
-			*templates = append(*templates, o)
+		for _, name := range names {
+			if hasTemplate(name, *templates) {
+				continue
+			}
+
+			ba, _, err := read(rev, path.Join(dir, name))
+			if err != nil {
+				return err
+			}
+
+			*templates = append(*templates, &Template{
+				Name:     name,
+				Template: string(ba),
+			})
 		}
 
 		return nil
 	}
 
-	if err := loadTemplates("configs", &config.Configs); err != nil {
-		return nil, err
-	}
-	if err := loadTemplates("static-pods", &config.StaticPods); err != nil {
-		return nil, err
-	}
+	for _, cluster := range config.Clusters {
+		addonSet := cluster.Addons
+		if len(addonSet) == 0 {
+			continue
+		}
 
-	{
-		addonSets, err := store.listDir("addons")
-		if err != nil {
+		if _, ok := config.Addons[addonSet]; ok {
+			continue
+		}
+
+		templates := make([]*Template, 0)
+		if err = loadTemplates(cluster.Rev(), path.Join("addons", addonSet), &templates); err != nil {
 			return nil, err
 		}
 
-		for _, addonSet := range addonSets {
-			templates := make([]*Template, 0)
-			if err = loadTemplates(path.Join("addons", addonSet), &templates); err != nil {
-				return nil, err
-			}
-
-			config.Addons[addonSet] = templates
-		}
+		config.Addons[addonSet] = templates
 	}
 
 	// load SSL configuration
@@ -134,6 +225,15 @@ func FromDir(dirPath string) (*Config, error) {
 	return config, nil
 }
 
+func hasTemplate(name string, templates []*Template) bool {
+	for _, tmpl := range templates {
+		if tmpl.Name == name {
+			return true
+		}
+	}
+	return false
+}
+
 type dirStore struct {
 	path string
 }
@@ -187,6 +287,10 @@ func (b *dirStore) List(prefix string) ([]string, error) {
 }
 
 // Load is part of the DataBackend interface
-func (b *dirStore) Get(key string) ([]byte, error) {
-	return ioutil.ReadFile(filepath.Join(b.path, filepath.Join(path.Split(key))+".yaml"))
+func (b *dirStore) Get(key string) (ba []byte, err error) {
+	ba, err = ioutil.ReadFile(filepath.Join(b.path, filepath.Join(path.Split(key))+".yaml"))
+	if os.IsNotExist(err) {
+		return nil, nil
+	}
+	return
 }