global command

pkg/cmd/init/boot/boot.go

@@ -0,0 +1,43 @@
+package initboot
+
+import (
+	"log"
+
+	"github.com/spf13/cobra"
+
+	"novit.nc/direktil/inits/pkg/sys"
+)
+
+var (
+	doNetwork bool
+)
+
+func Command() (c *cobra.Command) {
+	c = &cobra.Command{
+		Use:   "boot",
+		Short: "boot stage",
+		Run:   run,
+	}
+
+	c.Flags().BoolVar(&doNetwork, "do-network", true, "setup network")
+
+	return
+}
+
+func run(c *cobra.Command, args []string) {
+	setupFiles()
+	setupModules()
+
+	if doNetwork {
+		setupNetworking()
+	}
+
+	setupLVM()
+}
+
+func setupModules() {
+	for _, mod := range sys.Config().Modules {
+		log.Print("loading module ", mod)
+		sys.Run("modprobe", mod)
+	}
+}

pkg/cmd/init/boot/files.go

@@ -0,0 +1,60 @@
+package initboot
+
+import (
+	"log"
+	"strconv"
+	"syscall"
+
+	"novit.nc/direktil/inits/pkg/apply"
+	"novit.nc/direktil/inits/pkg/sys"
+)
+
+func setupFiles() {
+	cfg := sys.Config()
+
+	// make root rshared (default in systemd, required by Kubernetes 1.10+)
+	// equivalent to "mount --make-rshared /"
+	// see kernel's Documentation/sharedsubtree.txt (search rshared)
+	if err := syscall.Mount("", "/", "", syscall.MS_SHARED|syscall.MS_REC, ""); err != nil {
+		log.Fatalf("FATAL: mount --make-rshared / failed: %v", err)
+	}
+
+	// - setup root user
+	if passwordHash := cfg.RootUser.PasswordHash; passwordHash == "" {
+		sys.MustRun("/usr/bin/passwd", "-d", "root")
+	} else {
+		sys.MustRun("/bin/sh", "-c", "chpasswd --encrypted <<EOF\nroot:"+passwordHash+"\nEOF")
+	}
+
+	// - groups
+	for _, group := range cfg.Groups {
+		opts := make([]string, 0)
+		opts = append(opts, "-r")
+		if group.Gid != 0 {
+			opts = append(opts, "-g", strconv.Itoa(group.Gid))
+		}
+		opts = append(opts, group.Name)
+
+		sys.MustRun("groupadd", opts...)
+	}
+
+	// - user
+	for _, user := range cfg.Users {
+		opts := make([]string, 0)
+		opts = append(opts, "-r")
+		if user.Gid != 0 {
+			opts = append(opts, "-g", strconv.Itoa(user.Gid))
+		}
+		if user.Uid != 0 {
+			opts = append(opts, "-u", strconv.Itoa(user.Uid))
+		}
+		opts = append(opts, user.Name)
+
+		sys.MustRun("useradd", opts...)
+	}
+
+	// - files
+	if err := apply.Files(cfg); err != nil {
+		log.Fatal("FATAL: ", err)
+	}
+}

pkg/cmd/init/boot/lvm.go

@@ -0,0 +1,186 @@
+package initboot
+
+import (
+	"bytes"
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+
+	"novit.nc/direktil/pkg/config"
+
+	"novit.nc/direktil/inits/pkg/sys"
+	"novit.nc/direktil/inits/pkg/vars"
+)
+
+func setupLVM() {
+	if !dmInProc() {
+		sys.MustRun("modprobe", "dm-mod")
+	}
+
+	// start lvmetad
+	sys.Mkdir("/run/lvm", 0700)
+	sys.Mkdir("/run/lock/lvm", 0700)
+	sys.Run("lvmetad")
+
+	sys.WaitFile("/run/lvm/lvmetad.socket", time.After(30*time.Second))
+
+	// scan devices
+	sys.Run("lvm", "pvscan")
+	sys.Run("lvm", "vgscan", "--mknodes")
+	sys.Run("lvm", "vgchange", "--sysinit", "-a", "ly")
+
+	cfg := sys.Config()
+
+	// setup storage
+	log.Print("checking storage")
+	if err := exec.Command("vgdisplay", "storage").Run(); err != nil {
+		log.Print("- creating VG storage")
+		setupVG(vars.BootArgValue("storage", cfg.Storage.UdevMatch))
+	}
+
+	for _, name := range cfg.Storage.RemoveVolumes {
+		dev := "/dev/storage/" + name
+
+		if _, err := os.Stat(dev); os.IsNotExist(err) {
+			continue
+
+		} else if err != nil {
+			log.Fatal("failed to stat ", dev, ": ", err)
+		}
+
+		log.Print("- removing LV ", name)
+		cmd := exec.Command("lvremove", "-f", "storage/"+name)
+		cmd.Stderr = os.Stderr
+		if err := cmd.Run(); err != nil {
+			log.Fatal("failed to remove LV ", name)
+		}
+	}
+
+	// setup volumes
+	for _, volume := range cfg.Storage.Volumes {
+		if err := exec.Command("lvdisplay", "storage/"+volume.Name).Run(); err != nil {
+			log.Print("- creating LV ", volume.Name)
+			setupLV(volume)
+		}
+
+		dev := "/dev/storage/" + volume.Name
+
+		sys.WaitFile(dev, time.After(30*time.Second))
+
+		log.Printf("checking filesystem on %s", dev)
+		sys.MustRun("fsck", "-p", dev)
+
+		sys.Mount(dev, volume.Mount.Path, volume.FS,
+			syscall.MS_NOATIME|syscall.MS_RELATIME,
+			volume.Mount.Options)
+	}
+}
+
+func dmInProc() bool {
+	for _, f := range []string{"devices", "misc"} {
+		c, err := ioutil.ReadFile("/proc/" + f)
+		if err != nil {
+			log.Fatalf("failed to read %s: %v", f, err)
+		}
+		if !bytes.Contains(c, []byte("device-mapper")) {
+			return false
+		}
+	}
+	return true
+}
+
+func setupVG(udevMatch string) {
+	const pDevName = "DEVNAME="
+
+	dev := ""
+	try := 0
+
+retry:
+	paths, err := filepath.Glob("/sys/class/block/*")
+	if err != nil {
+		log.Fatal("failed to list block devices: ", err)
+	}
+
+	for _, path := range paths {
+		// ignore loop devices
+		if strings.HasPrefix("loop", filepath.Base(path)) {
+			continue
+		}
+
+		// fetch udev informations
+		out, err := exec.Command("udevadm", "info", "-q", "property", path).CombinedOutput()
+		if err != nil {
+			log.Printf("WARNING: udev query of %q failed: %v\n%s", path, err, string(out))
+			continue
+		}
+
+		propertyLines := strings.Split(strings.TrimSpace(string(out)), "\n")
+
+		devPath := ""
+		matches := false
+
+		for _, line := range propertyLines {
+			if strings.HasPrefix(line, pDevName) {
+				devPath = line[len(pDevName):]
+			}
+
+			if matched, err := filepath.Match(udevMatch, line); err != nil {
+				log.Fatalf("FATAL: invalid match: %q: %v", udevMatch, err)
+
+			} else if matched {
+				matches = true
+			}
+
+			if devPath != "" && matches {
+				break
+			}
+		}
+
+		if devPath != "" && matches {
+			dev = devPath
+			break
+		}
+	}
+
+	if dev == "" {
+		time.Sleep(1 * time.Second)
+		try++
+		if try > 30 {
+			log.Fatal("FATAL: storage device not found after 30s: ", udevMatch)
+		}
+		goto retry
+	}
+
+	log.Print("found storage device at ", dev)
+
+	sys.MustRun("pvcreate", dev)
+	sys.MustRun("vgcreate", "storage", dev)
+}
+
+func setupLV(volume config.VolumeDef) {
+	if volume.Extents != "" {
+		sys.MustRun("lvcreate", "-l", volume.Extents, "-n", volume.Name, "storage")
+	} else {
+		sys.MustRun("lvcreate", "-L", volume.Size, "-n", volume.Name, "storage")
+	}
+
+	// wait the device link
+	devPath := "/dev/storage/" + volume.Name
+	sys.WaitFile(devPath, time.After(30*time.Second))
+
+	args := make([]string, 0)
+
+	switch volume.FS {
+	case "btrfs":
+		args = append(args, "-f")
+	case "ext4":
+		args = append(args, "-F")
+	}
+
+	sys.MustRun("mkfs."+volume.FS, append(args, devPath)...)
+}

pkg/cmd/init/boot/network.go

@@ -0,0 +1,155 @@
+package initboot
+
+import (
+	"bytes"
+	"log"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"time"
+
+	ping "github.com/sparrc/go-ping"
+	"novit.nc/direktil/pkg/config"
+
+	"novit.nc/direktil/inits/pkg/sys"
+	"novit.nc/direktil/inits/pkg/vars"
+)
+
+var networkStarted = map[string]bool{}
+
+func setupNetworking() {
+	cfg := sys.Config()
+	for idx, network := range cfg.Networks {
+		setupNetwork(idx, network)
+	}
+}
+
+func setupNetwork(idx int, network config.NetworkDef) {
+	tries := 0
+retry:
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		log.Fatalf("FATAL: failed to get network interfaces: %v", err)
+	}
+
+	match := false
+	for _, iface := range ifaces {
+		if networkStarted[iface.Name] {
+			continue
+		}
+
+		if network.Match.Name != "" {
+			if ok, err := filepath.Match(network.Match.Name, iface.Name); err != nil {
+				log.Fatalf("FATAL: network[%d] name match error: %v", idx, err)
+			} else if !ok {
+				continue
+			}
+		}
+
+		if network.Match.Ping != nil {
+			log.Printf("network[%d] ping check on %s", idx, iface.Name)
+
+			if ok, err := networkPingCheck(iface.Name, network); err != nil {
+				log.Printf("ERROR: network[%d] ping check failed: %v", idx, err)
+
+			} else if !ok {
+				continue
+			}
+		}
+
+		log.Printf("network[%d] matches interface %s", idx, iface.Name)
+		match = true
+
+		startNetwork(iface.Name, idx, network)
+
+		if !network.Match.All {
+			return
+		}
+	}
+
+	if !match {
+		log.Printf("WARNING: network[%d] did not match any interface", idx)
+
+		tries++
+		if network.Optional && tries > 3 {
+			return
+		}
+
+		time.Sleep(1 * time.Second)
+		log.Printf("WARNING: network[%d] retrying (try: %d)", idx, tries)
+		goto retry
+	}
+}
+
+func startNetwork(ifaceName string, idx int, network config.NetworkDef) {
+	cfg := sys.Config()
+
+	log.Printf("starting network[%d]", idx)
+
+	script := vars.Substitute([]byte(network.Script), cfg)
+
+	c := exec.Command("/bin/sh")
+	c.Stdin = bytes.NewBuffer(script)
+	c.Stdout = os.Stdout
+	c.Stderr = os.Stderr
+
+	// TODO doc
+	c.Env = append(append(make([]string, 0), os.Environ()...), "IFNAME="+ifaceName)
+
+	if err := c.Run(); err != nil {
+		links, _ := exec.Command("ip", "link", "ls").CombinedOutput()
+		log.Fatalf("FATAL: network setup failed (link list below): %v\n%s", err, string(links))
+	}
+
+	networkStarted[ifaceName] = true
+}
+
+func networkPingCheck(ifName string, network config.NetworkDef) (b bool, err error) {
+	check := network.Match.Ping
+
+	source := string(vars.Substitute([]byte(check.Source), sys.Config()))
+
+	if err = sys.Run("ip", "addr", "add", source, "dev", ifName); err != nil {
+		return
+	}
+	if err = sys.Run("ip", "link", "set", ifName, "up"); err != nil {
+		return
+	}
+
+	defer func() {
+		sys.MustRun("ip", "link", "set", ifName, "down")
+		sys.MustRun("ip", "addr", "del", source, "dev", ifName)
+	}()
+
+	count := 3
+	if check.Count != 0 {
+		count = check.Count
+	}
+
+	for n := 0; n < count; n++ {
+		// TODO probably better to use golang.org/x/net/icmp directly
+		pinger, e := ping.NewPinger(network.Match.Ping.Target)
+		if e != nil {
+			err = e
+			return
+		}
+
+		pinger.Count = 1
+
+		pinger.Timeout = 1 * time.Second
+		if check.Timeout > 0 {
+			pinger.Timeout = time.Duration(check.Timeout) * time.Second
+		}
+
+		pinger.SetPrivileged(true)
+		pinger.Run()
+
+		if pinger.Statistics().PacketsRecv > 0 {
+			b = true
+			return
+		}
+	}
+
+	return
+}

pkg/cmd/init/default/default.go

@@ -0,0 +1,16 @@
+package initdefault
+
+import "github.com/spf13/cobra"
+
+func Command() (c *cobra.Command) {
+	c = &cobra.Command{
+		Use:   "default",
+		Short: "default stage",
+		Run:   run,
+	}
+
+	return
+}
+
+func run(c *cobra.Command, args []string) {
+}

pkg/cmd/init/init.go

@@ -0,0 +1,34 @@
+package cmdinit
+
+import (
+	"os"
+	"strings"
+
+	"github.com/spf13/cobra"
+
+	initboot "novit.nc/direktil/inits/pkg/cmd/init/boot"
+	initdefault "novit.nc/direktil/inits/pkg/cmd/init/default"
+	initservice "novit.nc/direktil/inits/pkg/cmd/init/service"
+)
+
+func Command() (c *cobra.Command) {
+	c = &cobra.Command{
+		Use:   "init",
+		Short: "init stages",
+
+		PersistentPreRun: func(_ *cobra.Command, _ []string) {
+			// set a reasonable path
+			os.Setenv("PATH", strings.Join([]string{
+				"/usr/local/bin:/usr/local/sbin",
+				"/usr/bin:/usr/sbin",
+				"/bin:/sbin",
+			}, ":"))
+		},
+	}
+
+	c.AddCommand(initboot.Command())
+	c.AddCommand(initdefault.Command())
+	c.AddCommand(initservice.Command())
+
+	return
+}

pkg/cmd/init/service/service.go

@@ -0,0 +1,90 @@
+package initservices
+
+import (
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"time"
+
+	"github.com/spf13/cobra"
+	plog "novit.nc/direktil/pkg/log"
+)
+
+var (
+	delays = []time.Duration{
+		1 * time.Second,
+		2 * time.Second,
+		4 * time.Second,
+		8 * time.Second,
+	}
+
+	crashForgiveDelay = 10 * time.Minute
+)
+
+func Command() (c *cobra.Command) {
+	c = &cobra.Command{
+		Use:   "services",
+		Short: "run user services",
+		Run:   run,
+	}
+
+	return
+}
+
+func run(c *cobra.Command, args []string) {
+	paths, err := filepath.Glob("/etc/direktil/services/*")
+
+	if err != nil && !os.IsNotExist(err) {
+		log.Fatal("failed to list services: ", err)
+	}
+
+	for _, path := range paths {
+		stat, err := os.Stat(path)
+		if err != nil {
+			log.Fatalf("failed to stat %s: %v", path, err)
+		}
+
+		if stat.Mode()&0100 == 0 {
+			// not executable
+			continue
+		}
+
+		go runService(path)
+	}
+
+	select {}
+}
+
+func runService(svcPath string) {
+	svc := filepath.Base(svcPath)
+
+	logger := plog.Get(svc)
+	plog.EnableFiles()
+
+	n := 0
+	for {
+		lastStart := time.Now()
+
+		cmd := exec.Command(svcPath)
+		cmd.Stdout = logger
+		cmd.Stderr = logger
+		err := cmd.Run()
+
+		if time.Since(lastStart) > crashForgiveDelay {
+			n = 0
+		}
+
+		if err == nil {
+			logger.Taintf(plog.Error, "service exited (%v), waiting %v", err, delays[n])
+		} else {
+			logger.Taintf(plog.Error, "service exited on error (%v), waiting %v", err, delays[n])
+		}
+
+		time.Sleep(delays[n])
+
+		if n+1 < len(delays) {
+			n++
+		}
+	}
+}